mirror of https://github.com/scrapy/scrapy.git
default httpcache to rfc2616 policy and improve storage and policy tests
This commit is contained in:
Daniel Graña
parent
cf5f0203b7
commit
cdecc760ee
|
|
@ -1,61 +1,167 @@
|
|||
import os
|
||||
from os.path import join, exists
|
||||
from time import time
|
||||
import cPickle as pickle
|
||||
|
||||
from w3lib.http import headers_dict_to_raw, headers_raw_to_dict
|
||||
|
||||
from email.utils import formatdate
|
||||
from weakref import WeakKeyDictionary
|
||||
from scrapy import signals
|
||||
from scrapy.http import Headers
|
||||
from scrapy.http.request import Request
|
||||
from scrapy.exceptions import NotConfigured, IgnoreRequest
|
||||
from scrapy.responsetypes import responsetypes
|
||||
from scrapy.utils.request import request_fingerprint
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.misc import load_object
|
||||
from scrapy.utils.project import data_path
|
||||
from scrapy.contrib.httpcache import rfc1123_to_epoch, parse_cachecontrol
|
||||
|
||||
|
||||
class DummyPolicy(object):
|
||||
|
||||
def __init__(self, settings):
|
||||
self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES')
|
||||
self.ignore_http_codes = map(int, settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES'))
|
||||
|
||||
def should_cache_response(self, response):
|
||||
return response.status not in self.ignore_http_codes
|
||||
|
||||
def should_cache_request(self, request):
|
||||
return urlparse_cached(request).scheme not in self.ignore_schemes
|
||||
|
||||
def should_cache_response(self, response, request):
|
||||
return response.status not in self.ignore_http_codes
|
||||
|
||||
def is_cached_response_fresh(self, response, request):
|
||||
return True
|
||||
|
||||
def is_cached_response_valid(self, cachedresponse, response, request):
|
||||
return True
|
||||
|
||||
|
||||
class RFC2616Policy(object):
|
||||
|
||||
MAXAGE = 3600 * 24 * 365 # one year
|
||||
|
||||
class RFC2616Policy(DummyPolicy):
|
||||
def __init__(self, settings):
|
||||
super(RFC2616Policy, self).__init__(settings)
|
||||
self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES')
|
||||
self._cc_parsed = WeakKeyDictionary()
|
||||
|
||||
def should_cache_response(self, response):
|
||||
retval = super(RFC2616Policy, self).should_cache_response(response)
|
||||
|
||||
if response.headers.has_key('cache-control'):
|
||||
retval = retval and (response.headers['cache-control'].lower().find('no-store') == -1)
|
||||
#retval = retval and self.policy_response(response)
|
||||
return retval
|
||||
def _parse_cachecontrol(self, r):
|
||||
if r not in self._cc_parsed:
|
||||
cch = r.headers.get('Cache-Control', '')
|
||||
self._cc_parsed[r] = parse_cachecontrol(cch)
|
||||
return self._cc_parsed[r]
|
||||
|
||||
def should_cache_request(self, request):
|
||||
retval = super(RFC2616Policy, self).should_cache_request(request)
|
||||
|
||||
if request.headers.has_key('cache-control'):
|
||||
retval = retval and (request.headers['cache-control'].lower().find('no-store') == -1)
|
||||
#retval = retval and self.policy_request(request)
|
||||
return retval
|
||||
if urlparse_cached(request).scheme in self.ignore_schemes:
|
||||
return False
|
||||
cc = self._parse_cachecontrol(request)
|
||||
# obey user-agent directive "Cache-Control: no-store"
|
||||
if 'no-store' in cc:
|
||||
return False
|
||||
# Any other is eligible for caching
|
||||
return True
|
||||
|
||||
def should_cache_response(self, response, request):
|
||||
# What is cacheable - http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec14.9.1
|
||||
# Response cacheability - http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4
|
||||
# Status code 206 is not included because cache can not deal with partial contents
|
||||
cc = self._parse_cachecontrol(response)
|
||||
# obey directive "Cache-Control: no-store"
|
||||
if 'no-store' in cc:
|
||||
return False
|
||||
# Never cache 304 (Not Modified) responses
|
||||
elif response.status == 304:
|
||||
return False
|
||||
# Any hint on response expiration is good
|
||||
elif 'max-age' in cc or 'Expires' in response.headers:
|
||||
return True
|
||||
# Firefox fallbacks this statuses to one year expiration if none is set
|
||||
elif response.status in (300, 301, 308):
|
||||
return True
|
||||
# Other statuses without expiration requires at least one validator
|
||||
elif response.status in (200, 203, 401):
|
||||
return 'Last-Modified' in response.headers or 'ETag' in response.headers
|
||||
# Any other is probably not eligible for caching
|
||||
# Makes no sense to cache responses that does not contain expiration
|
||||
# info and can not be revalidated
|
||||
else:
|
||||
return False
|
||||
|
||||
def is_cached_response_fresh(self, cachedresponse, request):
|
||||
cc = self._parse_cachecontrol(cachedresponse)
|
||||
if 'no-cache' in cc:
|
||||
return False
|
||||
|
||||
now = time()
|
||||
freshnesslifetime = self._compute_freshness_lifetime(cachedresponse, request, now)
|
||||
currentage = self._compute_current_age(cachedresponse, request, now)
|
||||
if currentage < freshnesslifetime:
|
||||
return True
|
||||
# Cached response is stale, try to set validators if any
|
||||
self._set_conditional_validators(request, cachedresponse)
|
||||
return False
|
||||
|
||||
def is_cached_response_valid(self, cachedresponse, response, request):
|
||||
return response.status == 304
|
||||
|
||||
def _set_conditional_validators(self, request, cachedresponse):
|
||||
if 'Last-Modified' in cachedresponse.headers:
|
||||
request.headers['If-Modified-Since'] = cachedresponse.headers['Last-Modified']
|
||||
|
||||
if 'ETag' in cachedresponse.headers:
|
||||
request.headers['If-None-Match'] = cachedresponse.headers['ETag']
|
||||
|
||||
def _compute_freshness_lifetime(self, response, request, now):
|
||||
# Reference nsHttpResponseHead::ComputeFresshnessLifetime
|
||||
# http://dxr.mozilla.org/mozilla-central/netwerk/protocol/http/nsHttpResponseHead.cpp.html#l259
|
||||
cc = self._parse_cachecontrol(response)
|
||||
if 'max-age' in cc:
|
||||
try:
|
||||
return max(0, int(cc['max-age']))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# Parse date header or synthesize it if none exists
|
||||
date = rfc1123_to_epoch(response.headers.get('Date')) or now
|
||||
|
||||
# Try HTTP/1.0 Expires header
|
||||
if 'Expires' in response.headers:
|
||||
expires = rfc1123_to_epoch(response.headers['Expires'])
|
||||
# When parsing Expires header fails RFC 2616 section 14.21 says we
|
||||
# should treat this as an expiration time in the past.
|
||||
return max(0, expires - date) if expires else 0
|
||||
|
||||
# Fallback to heuristic using last-modified header
|
||||
# This is not in RFC but on Firefox caching implementation
|
||||
lastmodified = rfc1123_to_epoch(response.headers.get('Last-Modified'))
|
||||
if lastmodified and lastmodified <= date:
|
||||
return (date - lastmodified) / 10
|
||||
|
||||
# This request can be cached indefinitely
|
||||
if response.status in (300, 301, 308):
|
||||
return self.MAXAGE
|
||||
|
||||
# Insufficient information to compute fresshness lifetime
|
||||
return 0
|
||||
|
||||
def _compute_current_age(self, response, request, now):
|
||||
# Reference nsHttpResponseHead::ComputeCurrentAge
|
||||
# http://dxr.mozilla.org/mozilla-central/netwerk/protocol/http/nsHttpResponseHead.cpp.html
|
||||
currentage = 0
|
||||
# If Date header is not set we assume it is a fast connection, and
|
||||
# clock is in sync with the server
|
||||
date = rfc1123_to_epoch(response.headers.get('Date')) or now
|
||||
if now > date:
|
||||
currentage = now - date
|
||||
|
||||
if 'Age' in response.headers:
|
||||
try:
|
||||
age = int(response.headers['Age'])
|
||||
currentage = max(currentage, age)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return currentage
|
||||
|
||||
|
||||
class HttpCacheMiddleware(object):
|
||||
|
||||
def __init__(self, settings, stats):
|
||||
if not settings.getbool('HTTPCACHE_ENABLED'):
|
||||
raise NotConfigured
|
||||
self.policy = load_object(settings['HTTPCACHE_POLICY'])(settings)
|
||||
self.storage = load_object(settings['HTTPCACHE_STORAGE'])(settings)
|
||||
self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING')
|
||||
self.policy = load_object(settings['HTTPCACHE_POLICY'])(settings)
|
||||
self.stats = stats
|
||||
|
||||
@classmethod
|
||||
|
|
@ -72,112 +178,72 @@ class HttpCacheMiddleware(object):
|
|||
self.storage.close_spider(spider)
|
||||
|
||||
def process_request(self, request, spider):
|
||||
# Skip uncacheable requests
|
||||
if not self.policy.should_cache_request(request):
|
||||
return
|
||||
response = self.storage.retrieve_response(spider, request)
|
||||
|
||||
# Response cached, but stale
|
||||
if response and type(response) is Request:
|
||||
# Return None so that Scrapy continues processing
|
||||
self.stats.inc_value('httpcache/revalidation', spider=spider)
|
||||
request.meta['_dont_cache'] = True # flag as uncacheable
|
||||
return
|
||||
|
||||
if response and self.policy.should_cache_response(response):
|
||||
# Look for cached response and check if expired
|
||||
cachedresponse = self.storage.retrieve_response(spider, request)
|
||||
if cachedresponse is None:
|
||||
self.stats.inc_value('httpcache/miss', spider=spider)
|
||||
if self.ignore_missing:
|
||||
self.stats.inc_value('httpcache/ignore', spider=spider)
|
||||
raise IgnoreRequest("Ignored request not in cache: %s" % request)
|
||||
return # first time request
|
||||
|
||||
# Return cached response only if not expired
|
||||
cachedresponse.flags.append('cached')
|
||||
if self.policy.is_cached_response_fresh(cachedresponse, request):
|
||||
self.stats.inc_value('httpcache/hit', spider=spider)
|
||||
if isinstance(self.policy, RFC2616Policy):
|
||||
# Response cached and fresh
|
||||
raise IgnoreRequest("Ignored request already in cache: %s" % request)
|
||||
else:
|
||||
response.flags.append('cached')
|
||||
return response
|
||||
return cachedresponse
|
||||
|
||||
# Response not cached
|
||||
self.stats.inc_value('httpcache/miss', spider=spider)
|
||||
if self.ignore_missing:
|
||||
raise IgnoreRequest("Ignored request not in cache: %s" % request)
|
||||
# Keep a reference to cached response to avoid a second cache lookup on
|
||||
# process_response hook
|
||||
request.meta['cached_response'] = cachedresponse
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
if (self.policy.should_cache_request(request)
|
||||
and self.policy.should_cache_response(response)):
|
||||
if isinstance(self.policy, RFC2616Policy):
|
||||
if response.status != 304:
|
||||
self.storage.store_response(spider, request, response)
|
||||
self.stats.inc_value('httpcache/store', spider=spider)
|
||||
else:
|
||||
response.flags.append('cached')
|
||||
self.stats.inc_value('httpcache/hit', spider=spider)
|
||||
else:
|
||||
if 'cached' not in response.flags:
|
||||
self.storage.store_response(spider, request, response)
|
||||
self.stats.inc_value('httpcache/store', spider=spider)
|
||||
# Skip cached responses and uncacheable requests
|
||||
if 'cached' in response.flags or '_dont_cache' in request.meta:
|
||||
request.meta.pop('_dont_cache', None)
|
||||
return response
|
||||
|
||||
# RFC2616 requires origin server to set Date header,
|
||||
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.18
|
||||
if 'Date' not in response.headers:
|
||||
response.headers['Date'] = formatdate(usegmt=1)
|
||||
|
||||
# Do not validate first-hand responses
|
||||
cachedresponse = request.meta.pop('cached_response', None)
|
||||
if cachedresponse is None:
|
||||
self.stats.inc_value('httpcache/firsthand', spider=spider)
|
||||
self._cache_response(spider, response, request, cachedresponse)
|
||||
return response
|
||||
|
||||
if self.policy.is_cached_response_valid(cachedresponse, response, request):
|
||||
self.stats.inc_value('httpcache/revalidate', spider=spider)
|
||||
return cachedresponse
|
||||
|
||||
self.stats.inc_value('httpcache/invalidate', spider=spider)
|
||||
self._cache_response(spider, response, request, cachedresponse)
|
||||
return response
|
||||
|
||||
def _cache_response(self, spider, response, request, cachedresponse):
|
||||
if self.policy.should_cache_response(response, request):
|
||||
self.stats.inc_value('httpcache/store', spider=spider)
|
||||
self.storage.store_response(spider, request, response)
|
||||
else:
|
||||
self.stats.inc_value('httpcache/uncacheable', spider=spider)
|
||||
|
||||
class FilesystemCacheStorage(object):
|
||||
|
||||
def __init__(self, settings):
|
||||
self.cachedir = data_path(settings['HTTPCACHE_DIR'])
|
||||
self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
|
||||
from scrapy.contrib.httpcache import FilesystemCacheStorage as _FilesystemCacheStorage
|
||||
class FilesystemCacheStorage(_FilesystemCacheStorage):
|
||||
|
||||
def open_spider(self, spider):
|
||||
pass
|
||||
|
||||
def close_spider(self, spider):
|
||||
pass
|
||||
|
||||
def retrieve_response(self, spider, request):
|
||||
"""Return response if present in cache, or None otherwise."""
|
||||
metadata = self._read_meta(spider, request)
|
||||
if metadata is None:
|
||||
return # not cached
|
||||
rpath = self._get_request_path(spider, request)
|
||||
with open(join(rpath, 'response_body'), 'rb') as f:
|
||||
body = f.read()
|
||||
with open(join(rpath, 'response_headers'), 'rb') as f:
|
||||
rawheaders = f.read()
|
||||
url = metadata.get('response_url')
|
||||
status = metadata['status']
|
||||
headers = Headers(headers_raw_to_dict(rawheaders))
|
||||
respcls = responsetypes.from_args(headers=headers, url=url)
|
||||
response = respcls(url=url, headers=headers, status=status, body=body)
|
||||
return response
|
||||
|
||||
def store_response(self, spider, request, response):
|
||||
"""Store the given response in the cache."""
|
||||
rpath = self._get_request_path(spider, request)
|
||||
if not exists(rpath):
|
||||
os.makedirs(rpath)
|
||||
metadata = {
|
||||
'url': request.url,
|
||||
'method': request.method,
|
||||
'status': response.status,
|
||||
'response_url': response.url,
|
||||
'timestamp': time(),
|
||||
}
|
||||
with open(join(rpath, 'meta'), 'wb') as f:
|
||||
f.write(repr(metadata))
|
||||
with open(join(rpath, 'pickled_meta'), 'wb') as f:
|
||||
pickle.dump(metadata, f, protocol=2)
|
||||
with open(join(rpath, 'response_headers'), 'wb') as f:
|
||||
f.write(headers_dict_to_raw(response.headers))
|
||||
with open(join(rpath, 'response_body'), 'wb') as f:
|
||||
f.write(response.body)
|
||||
with open(join(rpath, 'request_headers'), 'wb') as f:
|
||||
f.write(headers_dict_to_raw(request.headers))
|
||||
with open(join(rpath, 'request_body'), 'wb') as f:
|
||||
f.write(request.body)
|
||||
|
||||
def _get_request_path(self, spider, request):
|
||||
key = request_fingerprint(request)
|
||||
return join(self.cachedir, spider.name, key[0:2], key)
|
||||
|
||||
def _read_meta(self, spider, request):
|
||||
rpath = self._get_request_path(spider, request)
|
||||
metapath = join(rpath, 'pickled_meta')
|
||||
if not exists(metapath):
|
||||
return # not found
|
||||
mtime = os.stat(rpath).st_mtime
|
||||
if 0 < self.expiration_secs < time() - mtime:
|
||||
return # expired
|
||||
with open(metapath, 'rb') as f:
|
||||
return pickle.load(f)
|
||||
def __init__(self, *args, **kwargs):
|
||||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn('Importing FilesystemCacheStorage from '
|
||||
'scrapy.contrib.downloadermiddlware.httpcache is '
|
||||
'deprecated, use scrapy.contrib.httpcache instead.',
|
||||
category=ScrapyDeprecationWarning, stacklevel=1)
|
||||
super(_FilesystemCacheStorage, self).__init__(*args, **kwargs)
|
||||
|
|
|
|||
|
|
@ -1,12 +1,9 @@
|
|||
import os
|
||||
import calendar
|
||||
import email.utils
|
||||
from time import time
|
||||
import cPickle as pickle
|
||||
|
||||
from scrapy import log
|
||||
from email.utils import mktime_tz, parsedate_tz
|
||||
from w3lib.http import headers_raw_to_dict, headers_dict_to_raw
|
||||
from scrapy.http import Headers
|
||||
from scrapy.exceptions import IgnoreRequest
|
||||
from scrapy.responsetypes import responsetypes
|
||||
from scrapy.utils.request import request_fingerprint
|
||||
from scrapy.utils.project import data_path
|
||||
|
|
@ -30,7 +27,7 @@ class DbmCacheStorage(object):
|
|||
def retrieve_response(self, spider, request):
|
||||
data = self._read_data(spider, request)
|
||||
if data is None:
|
||||
return # not cached
|
||||
return # not cached
|
||||
url = data['url']
|
||||
status = data['status']
|
||||
headers = Headers(data['headers'])
|
||||
|
|
@ -55,149 +52,107 @@ class DbmCacheStorage(object):
|
|||
db = self.db
|
||||
tkey = '%s_time' % key
|
||||
if not db.has_key(tkey):
|
||||
return # not found
|
||||
return # not found
|
||||
ts = db[tkey]
|
||||
if 0 < self.expiration_secs < time() - float(ts):
|
||||
return # expired
|
||||
return # expired
|
||||
return pickle.loads(db['%s_data' % key])
|
||||
|
||||
def _request_key(self, request):
|
||||
return request_fingerprint(request)
|
||||
|
||||
|
||||
class BaseRealCacheStorage(object):
|
||||
"""
|
||||
Most of the code was taken from the httplib2 (MIT License)
|
||||
https://code.google.com/p/httplib2/source/browse/python2/httplib2/__init__.py
|
||||
"""
|
||||
|
||||
def parse_cache_control(self, headers):
|
||||
retval = {}
|
||||
if headers.has_key('cache-control'):
|
||||
parts = headers['cache-control'].split(',')
|
||||
parts_with_args = [tuple([x.strip().lower() for x in part.split("=", 1)]) for part in parts if -1 != part.find("=")]
|
||||
parts_wo_args = [(name.strip().lower(), 1) for name in parts if -1 == name.find("=")]
|
||||
retval = dict(parts_with_args + parts_wo_args)
|
||||
return retval
|
||||
|
||||
|
||||
def get(self, response_headers, request_headers):
|
||||
"""Determine freshness from the Date, Expires and Cache-Control headers.
|
||||
class FilesystemCacheStorage(object):
|
||||
|
||||
We don't handle the following:
|
||||
|
||||
1. Cache-Control: max-stale
|
||||
2. Age: headers are not used in the calculations.
|
||||
|
||||
Not that this algorithm is simpler than you might think
|
||||
because we are operating as a private (non-shared) cache.
|
||||
This lets us ignore 's-maxage'. We can also ignore
|
||||
'proxy-invalidate' since we aren't a proxy.
|
||||
We will never return a stale document as
|
||||
fresh as a design decision, and thus the non-implementation
|
||||
of 'max-stale'. This also lets us safely ignore 'must-revalidate'
|
||||
since we operate as if every server has sent 'must-revalidate'.
|
||||
Since we are private we get to ignore both 'public' and
|
||||
'private' parameters. We also ignore 'no-transform' since
|
||||
we don't do any transformations.
|
||||
The 'no-store' parameter is handled at a higher level.
|
||||
So the only Cache-Control parameters we look at are:
|
||||
|
||||
no-cache
|
||||
only-if-cached
|
||||
max-age
|
||||
min-fresh
|
||||
"""
|
||||
|
||||
retval = "STALE"
|
||||
cc = self.parse_cache_control(request_headers)
|
||||
cc_response = self.parse_cache_control(response_headers)
|
||||
|
||||
if request_headers.has_key('pragma') and request_headers['pragma'].lower().find('no-cache') != -1:
|
||||
retval = "TRANSPARENT"
|
||||
if 'cache-control' not in request_headers:
|
||||
request_headers['cache-control'] = 'no-cache'
|
||||
elif cc.has_key('no-cache'):
|
||||
retval = "TRANSPARENT"
|
||||
elif cc_response.has_key('no-cache'):
|
||||
retval = "STALE"
|
||||
elif cc.has_key('only-if-cached'):
|
||||
retval = "FRESH"
|
||||
elif response_headers.has_key('date'):
|
||||
date = calendar.timegm(email.utils.parsedate_tz(response_headers['date']))
|
||||
now = time()
|
||||
current_age = max(0, now - date)
|
||||
if cc_response.has_key('max-age'):
|
||||
try:
|
||||
freshness_lifetime = int(cc_response['max-age'])
|
||||
except ValueError:
|
||||
freshness_lifetime = 0
|
||||
elif response_headers.has_key('expires'):
|
||||
expires = email.utils.parsedate_tz(response_headers['expires'])
|
||||
if None == expires:
|
||||
freshness_lifetime = 0
|
||||
else:
|
||||
freshness_lifetime = max(0, calendar.timegm(expires) - date)
|
||||
else:
|
||||
freshness_lifetime = 0
|
||||
if cc.has_key('max-age'):
|
||||
try:
|
||||
freshness_lifetime = int(cc['max-age'])
|
||||
except ValueError:
|
||||
freshness_lifetime = 0
|
||||
if cc.has_key('min-fresh'):
|
||||
try:
|
||||
min_fresh = int(cc['min-fresh'])
|
||||
except ValueError:
|
||||
min_fresh = 0
|
||||
current_age += min_fresh
|
||||
if freshness_lifetime > current_age:
|
||||
retval = "FRESH"
|
||||
return retval
|
||||
|
||||
def retrieve_cache(self, spider, request, response_headers, response_status, response_url='', response_body=''):
|
||||
# Determine our course of action:
|
||||
# Is the cached entry fresh or stale?
|
||||
#
|
||||
# There seems to be three possible answers:
|
||||
# 1. [FRESH] Return the Response object
|
||||
# 2. [STALE] Update the Request object with cache validators if available
|
||||
# 3. [TRANSPARENT] Don't update the Request with cache validators (Cache-Control: no-cache)
|
||||
entry_disposition = self.get(Headers(response_headers), Headers(request.headers))
|
||||
|
||||
# Per the RFC, requests should not be repeated in these situations
|
||||
if response_status in [400, 401, 403, 410]:
|
||||
raise IgnoreRequest("Ignored request because cached response status is %d." % response_status)
|
||||
|
||||
if entry_disposition == "FRESH":
|
||||
log.msg("Cache is FRESH", level=log.DEBUG, spider=spider)
|
||||
|
||||
headers = Headers(response_headers)
|
||||
respcls = responsetypes.from_args(headers=headers, url=response_url)
|
||||
response = respcls(url=response_url, headers=headers, status=response_status, body=response_body)
|
||||
|
||||
return response
|
||||
|
||||
new_request = request.copy()
|
||||
if entry_disposition == "STALE":
|
||||
log.msg("Cache is STALE, updating Request object with cache validators", level=log.DEBUG, spider=spider)
|
||||
if response_headers.has_key('ETag') and not 'If-None-Match' in request.headers:
|
||||
new_request.headers['If-None-Match'] = response_headers['ETag']
|
||||
if response_headers.has_key('Last-Modified') and not 'Last-Modified' in request.headers:
|
||||
new_request.headers['If-Modified-Since'] = response_headers['Last-Modified']
|
||||
elif entry_disposition == "TRANSPARENT":
|
||||
log.msg("Cache is TRANSPARENT, not adding cache validators to Request object", level=log.DEBUG, spider=spider)
|
||||
|
||||
return new_request
|
||||
|
||||
|
||||
class DbmRealCacheStorage(DbmCacheStorage, BaseRealCacheStorage):
|
||||
def __init__(self, settings):
|
||||
super(DbmRealCacheStorage, self).__init__(settings)
|
||||
|
||||
self.cachedir = data_path(settings['HTTPCACHE_DIR'])
|
||||
self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
|
||||
|
||||
def open_spider(self, spider):
|
||||
pass
|
||||
|
||||
def close_spider(self, spider):
|
||||
pass
|
||||
|
||||
def retrieve_response(self, spider, request):
|
||||
data = self._read_data(spider, request)
|
||||
if data is None:
|
||||
return # not cached
|
||||
else:
|
||||
return self.retrieve_cache(spider, request, data['headers'], data['status'], data['url'], data['body'])
|
||||
"""Return response if present in cache, or None otherwise."""
|
||||
metadata = self._read_meta(spider, request)
|
||||
if metadata is None:
|
||||
return # not cached
|
||||
rpath = self._get_request_path(spider, request)
|
||||
with open(os.path.join(rpath, 'response_body'), 'rb') as f:
|
||||
body = f.read()
|
||||
with open(os.path.join(rpath, 'response_headers'), 'rb') as f:
|
||||
rawheaders = f.read()
|
||||
url = metadata.get('response_url')
|
||||
status = metadata['status']
|
||||
headers = Headers(headers_raw_to_dict(rawheaders))
|
||||
respcls = responsetypes.from_args(headers=headers, url=url)
|
||||
response = respcls(url=url, headers=headers, status=status, body=body)
|
||||
return response
|
||||
|
||||
def store_response(self, spider, request, response):
|
||||
"""Store the given response in the cache."""
|
||||
rpath = self._get_request_path(spider, request)
|
||||
if not os.path.exists(rpath):
|
||||
os.makedirs(rpath)
|
||||
metadata = {
|
||||
'url': request.url,
|
||||
'method': request.method,
|
||||
'status': response.status,
|
||||
'response_url': response.url,
|
||||
'timestamp': time(),
|
||||
}
|
||||
with open(os.path.join(rpath, 'meta'), 'wb') as f:
|
||||
f.write(repr(metadata))
|
||||
with open(os.path.join(rpath, 'pickled_meta'), 'wb') as f:
|
||||
pickle.dump(metadata, f, protocol=2)
|
||||
with open(os.path.join(rpath, 'response_headers'), 'wb') as f:
|
||||
f.write(headers_dict_to_raw(response.headers))
|
||||
with open(os.path.join(rpath, 'response_body'), 'wb') as f:
|
||||
f.write(response.body)
|
||||
with open(os.path.join(rpath, 'request_headers'), 'wb') as f:
|
||||
f.write(headers_dict_to_raw(request.headers))
|
||||
with open(os.path.join(rpath, 'request_body'), 'wb') as f:
|
||||
f.write(request.body)
|
||||
|
||||
def _get_request_path(self, spider, request):
|
||||
key = request_fingerprint(request)
|
||||
return os.path.join(self.cachedir, spider.name, key[0:2], key)
|
||||
|
||||
def _read_meta(self, spider, request):
|
||||
rpath = self._get_request_path(spider, request)
|
||||
metapath = os.path.join(rpath, 'pickled_meta')
|
||||
if not os.path.exists(metapath):
|
||||
return # not found
|
||||
mtime = os.stat(rpath).st_mtime
|
||||
if 0 < self.expiration_secs < time() - mtime:
|
||||
return # expired
|
||||
with open(metapath, 'rb') as f:
|
||||
return pickle.load(f)
|
||||
|
||||
|
||||
def parse_cachecontrol(header):
|
||||
"""Parse Cache-Control header
|
||||
|
||||
http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9
|
||||
|
||||
>>> cachecontrol_directives('public, max-age=3600')
|
||||
{'public': None, 'max-age': '3600'}
|
||||
>>> cachecontrol_directives('')
|
||||
{}
|
||||
|
||||
"""
|
||||
directives = {}
|
||||
for directive in header.split(','):
|
||||
key, sep, val = directive.strip().partition('=')
|
||||
if key:
|
||||
directives[key.lower()] = val if sep else None
|
||||
return directives
|
||||
|
||||
|
||||
def rfc1123_to_epoch(date_str):
|
||||
try:
|
||||
return mktime_tz(parsedate_tz(date_str))
|
||||
except Exception:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -10,31 +10,27 @@ from scrapy.spider import BaseSpider
|
|||
from scrapy.settings import Settings
|
||||
from scrapy.exceptions import IgnoreRequest
|
||||
from scrapy.utils.test import get_crawler
|
||||
from scrapy.contrib.downloadermiddleware.httpcache import \
|
||||
FilesystemCacheStorage, HttpCacheMiddleware
|
||||
from scrapy.contrib.httpcache import DbmRealCacheStorage
|
||||
from scrapy.contrib.downloadermiddleware.httpcache import HttpCacheMiddleware
|
||||
|
||||
|
||||
class HttpCacheMiddlewareTest(unittest.TestCase):
|
||||
class _BaseTest(unittest.TestCase):
|
||||
|
||||
storage_class = FilesystemCacheStorage
|
||||
realcache_storage_class = DbmRealCacheStorage
|
||||
|
||||
dummy_policy = 'scrapy.contrib.downloadermiddleware.httpcache.DummyPolicy'
|
||||
rfc2616_policy = 'scrapy.contrib.downloadermiddleware.httpcache.RFC2616Policy'
|
||||
|
||||
yesterday = email.utils.formatdate(time.time() - 1 * 24 * 60 * 60)
|
||||
now = email.utils.formatdate()
|
||||
tomorrow = email.utils.formatdate(time.time() + 1 * 24 * 60 * 60)
|
||||
storage_class = 'scrapy.contrib.httpcache.DbmCacheStorage'
|
||||
policy_class = 'scrapy.contrib.downloadermiddleware.httpcache.DummyPolicy'
|
||||
|
||||
def setUp(self):
|
||||
self.yesterday = email.utils.formatdate(time.time() - 86400)
|
||||
self.today = email.utils.formatdate()
|
||||
self.tomorrow = email.utils.formatdate(time.time() + 86400)
|
||||
self.crawler = get_crawler()
|
||||
self.spider = BaseSpider('example.com')
|
||||
self.tmpdir = tempfile.mkdtemp()
|
||||
self.request = Request('http://www.example.com',
|
||||
headers={'User-Agent': 'test'})
|
||||
self.response = Response('http://www.example.com', headers=
|
||||
{'Content-Type': 'text/html'}, body='test body', status=202)
|
||||
headers={'User-Agent': 'test'})
|
||||
self.response = Response('http://www.example.com',
|
||||
headers={'Content-Type': 'text/html'},
|
||||
body='test body',
|
||||
status=202)
|
||||
self.crawler.stats.open_spider(self.spider)
|
||||
|
||||
def tearDown(self):
|
||||
|
|
@ -47,23 +43,21 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
|
|||
'HTTPCACHE_DIR': self.tmpdir,
|
||||
'HTTPCACHE_EXPIRATION_SECS': 1,
|
||||
'HTTPCACHE_IGNORE_HTTP_CODES': [],
|
||||
'HTTPCACHE_POLICY': self.dummy_policy
|
||||
'HTTPCACHE_POLICY': self.policy_class,
|
||||
'HTTPCACHE_STORAGE': self.storage_class,
|
||||
}
|
||||
settings.update(new_settings)
|
||||
return Settings(settings)
|
||||
|
||||
@contextmanager
|
||||
def _storage(self, **new_settings):
|
||||
settings = self._get_settings(**new_settings)
|
||||
if settings.get('HTTPCACHE_POLICY') == self.dummy_policy:
|
||||
storage = self.storage_class(settings)
|
||||
else:
|
||||
storage = self.realcache_storage_class(settings)
|
||||
storage.open_spider(self.spider)
|
||||
try:
|
||||
yield storage
|
||||
finally:
|
||||
storage.close_spider(self.spider)
|
||||
with self._middleware(**new_settings) as mw:
|
||||
yield mw.storage
|
||||
|
||||
@contextmanager
|
||||
def _policy(self, **new_settings):
|
||||
with self._middleware(**new_settings) as mw:
|
||||
yield mw.policy
|
||||
|
||||
@contextmanager
|
||||
def _middleware(self, **new_settings):
|
||||
|
|
@ -75,6 +69,27 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
|
|||
finally:
|
||||
mw.spider_closed(self.spider)
|
||||
|
||||
def assertEqualResponse(self, response1, response2):
|
||||
self.assertEqual(response1.url, response2.url)
|
||||
self.assertEqual(response1.status, response2.status)
|
||||
self.assertEqual(response1.headers, response2.headers)
|
||||
self.assertEqual(response1.body, response2.body)
|
||||
|
||||
def assertEqualRequest(self, request1, request2):
|
||||
self.assertEqual(request1.url, request2.url)
|
||||
self.assertEqual(request1.headers, request2.headers)
|
||||
self.assertEqual(request1.body, request2.body)
|
||||
|
||||
def assertEqualRequestButWithCacheValidators(self, request1, request2):
|
||||
self.assertEqual(request1.url, request2.url)
|
||||
assert not 'If-None-Match' in request1.headers
|
||||
assert not 'If-Modified-Since' in request1.headers
|
||||
assert any(h in request2.headers for h in ('If-None-Match', 'If-Modified-Since'))
|
||||
self.assertEqual(request1.body, request2.body)
|
||||
|
||||
|
||||
class DefaultStorageTest(_BaseTest):
|
||||
|
||||
def test_storage(self):
|
||||
with self._storage() as storage:
|
||||
request2 = self.request.copy()
|
||||
|
|
@ -95,11 +110,23 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
|
|||
time.sleep(0.5) # give the chance to expire
|
||||
assert storage.retrieve_response(self.spider, self.request)
|
||||
|
||||
|
||||
class DbmStorageTest(DefaultStorageTest):
|
||||
|
||||
storage_class = 'scrapy.contrib.httpcache.DbmCacheStorage'
|
||||
|
||||
|
||||
class FilesystemStorageTest(DefaultStorageTest):
|
||||
|
||||
storage_class = 'scrapy.contrib.httpcache.FilesystemCacheStorage'
|
||||
|
||||
|
||||
class DefaultMiddlewaretest(_BaseTest):
|
||||
|
||||
def test_middleware(self):
|
||||
with self._middleware() as mw:
|
||||
assert mw.process_request(self.request, self.spider) is None
|
||||
mw.process_response(self.request, self.response, self.spider)
|
||||
|
||||
response = mw.process_request(self.request, self.spider)
|
||||
assert isinstance(response, HtmlResponse)
|
||||
self.assertEqualResponse(self.response, response)
|
||||
|
|
@ -109,10 +136,8 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
|
|||
with self._middleware() as mw:
|
||||
req = Request('http://host.com/path')
|
||||
res = Response('http://host2.net/test.html')
|
||||
|
||||
assert mw.process_request(req, self.spider) is None
|
||||
mw.process_response(req, res, self.spider)
|
||||
|
||||
cached = mw.process_request(req, self.spider)
|
||||
assert isinstance(cached, Response)
|
||||
self.assertEqualResponse(res, cached)
|
||||
|
|
@ -185,192 +210,167 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
|
|||
self.assertEqualResponse(self.response, response)
|
||||
assert 'cached' in response.flags
|
||||
|
||||
def test_middleware_rfc2616policy_response304_not_cached(self):
|
||||
# test response is not cached because the status is 304 Not Modified
|
||||
# (so it should be cached already)
|
||||
with self._middleware(HTTPCACHE_POLICY=self.rfc2616_policy) as mw:
|
||||
assert mw.process_request(self.request, self.spider) is None
|
||||
response = Response('http://www.example.com', status=304)
|
||||
mw.process_response(self.request, response, self.spider)
|
||||
|
||||
assert 'cached' in response.flags
|
||||
assert mw.storage.retrieve_response(self.spider, self.request) is None
|
||||
assert mw.process_request(self.request, self.spider) is None
|
||||
class DummyMiddlewareTest(DefaultStorageTest):
|
||||
|
||||
def test_middleware_rfc2616policy_response_nostore_not_cached(self):
|
||||
# test response is not cached because of the Cache-Control 'no-store' directive
|
||||
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2
|
||||
with self._middleware(HTTPCACHE_POLICY=self.rfc2616_policy) as mw:
|
||||
assert mw.process_request(self.request, self.spider) is None
|
||||
response = Response('http://www.example.com', headers=
|
||||
{'Content-Type': 'text/html', 'Cache-Control': 'no-store'},
|
||||
body='test body', status=200)
|
||||
mw.process_response(self.request, response, self.spider)
|
||||
policy_class = 'scrapy.contrib.downloadermiddleware.httpcache.DummyPolicy'
|
||||
|
||||
assert mw.storage.retrieve_response(self.spider, self.request) is None
|
||||
assert mw.process_request(self.request, self.spider) is None
|
||||
|
||||
def test_middleware_rfc2616policy_request_nostore_not_cached(self):
|
||||
# test response is not cached because of the request's Cache-Control 'no-store' directive
|
||||
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2
|
||||
with self._middleware(HTTPCACHE_POLICY=self.rfc2616_policy) as mw:
|
||||
request = Request('http://www.example.com',
|
||||
headers={'User-Agent': 'test', 'Cache-Control': 'no-store'})
|
||||
assert mw.process_request(request, self.spider) is None
|
||||
mw.process_response(request, self.response, self.spider)
|
||||
class RFC2616MiddlewareTest(DefaultStorageTest):
|
||||
|
||||
assert mw.storage.retrieve_response(self.spider, request) is None
|
||||
assert mw.process_request(request, self.spider) is None
|
||||
policy_class = 'scrapy.contrib.downloadermiddleware.httpcache.RFC2616Policy'
|
||||
|
||||
def test_middleware_rfc2616policy_response_cached_and_fresh(self):
|
||||
# test response cached and fresh
|
||||
with self._middleware(HTTPCACHE_POLICY=self.rfc2616_policy) as mw:
|
||||
response = mw.process_response(self.request, self.response, self.spider)
|
||||
self.assertRaises(IgnoreRequest, mw.process_request, self.request, self.spider)
|
||||
assert 'cached' not in response.flags
|
||||
def _process_requestresponse(self, mw, request, response):
|
||||
try:
|
||||
result = mw.process_request(request, self.spider)
|
||||
if result:
|
||||
assert isinstance(result, (Request, Response))
|
||||
return result
|
||||
else:
|
||||
result = mw.process_response(request, response, self.spider)
|
||||
assert isinstance(result, Response)
|
||||
return result
|
||||
except Exception:
|
||||
print 'Request', request
|
||||
print 'Response', response
|
||||
print 'Result', result
|
||||
raise
|
||||
|
||||
def test_middleware_rfc2616policy_response_cached_and_stale(self):
|
||||
# test response cached but stale
|
||||
with self._middleware(HTTPCACHE_POLICY=self.rfc2616_policy,
|
||||
HTTPCACHE_STORAGE = 'scrapy.contrib.httpcache.DbmRealCacheStorage') as mw:
|
||||
response = Response('http://www.example.com', headers=
|
||||
{'Content-Type': 'text/html', 'Cache-Control': 'no-cache'},
|
||||
body='test body', status=200)
|
||||
mw.process_response(self.request, response, self.spider)
|
||||
assert mw.process_request(self.request, self.spider) is None
|
||||
def test_request_cacheability(self):
|
||||
res0 = Response(self.request.url, status=200,
|
||||
headers={'Expires': self.tomorrow})
|
||||
req0 = Request('http://example.com')
|
||||
req1 = req0.replace(headers={'Cache-Control': 'no-store'})
|
||||
with self._middleware() as mw:
|
||||
res1 = self._process_requestresponse(mw, req1, res0)
|
||||
self.assertEqualResponse(res1, res0)
|
||||
assert mw.storage.retrieve_response(self.spider, req1) is None
|
||||
res2 = self._process_requestresponse(mw, req0, res0)
|
||||
assert 'cached' not in res2.flags
|
||||
res3 = mw.process_request(req0, self.spider)
|
||||
assert 'cached' in res3.flags
|
||||
self.assertEqualResponse(res2, res3)
|
||||
|
||||
response = mw.storage.retrieve_response(self.spider, self.request)
|
||||
assert isinstance(response, Request)
|
||||
def test_response_cacheability(self):
|
||||
responses = [
|
||||
# 304 is not cacheable no matter what servers sends
|
||||
(False, 304, {}),
|
||||
(False, 304, {'Last-Modified': self.yesterday}),
|
||||
(False, 304, {'Expires': self.tomorrow}),
|
||||
(False, 304, {'Etag': 'bar'}),
|
||||
(False, 304, {'Cache-Control': 'max-age=3600'}),
|
||||
# Always obey no-store cache control
|
||||
(False, 200, {'Cache-Control': 'no-store'}),
|
||||
(False, 200, {'Cache-Control': 'no-store, max-age=300'}), # invalid
|
||||
(False, 200, {'Cache-Control': 'no-store', 'Expires': self.tomorrow}), # invalid
|
||||
# Ignore responses missing expiration and/or validation headers
|
||||
(False, 200, {}),
|
||||
(False, 302, {}),
|
||||
(False, 307, {}),
|
||||
(False, 404, {}),
|
||||
# Cache responses with expiration and/or validation headers
|
||||
(True, 200, {'Last-Modified': self.yesterday}),
|
||||
(True, 203, {'Last-Modified': self.yesterday}),
|
||||
(True, 300, {'Last-Modified': self.yesterday}),
|
||||
(True, 301, {'Last-Modified': self.yesterday}),
|
||||
(True, 401, {'Last-Modified': self.yesterday}),
|
||||
(True, 404, {'Cache-Control': 'public, max-age=600'}),
|
||||
(True, 302, {'Expires': self.tomorrow}),
|
||||
(True, 200, {'Etag': 'foo'}),
|
||||
]
|
||||
with self._middleware() as mw:
|
||||
for idx, (shouldcache, status, headers) in enumerate(responses):
|
||||
req0 = Request('http://example-%d.com' % idx)
|
||||
res0 = Response(req0.url, status=status, headers=headers)
|
||||
res1 = self._process_requestresponse(mw, req0, res0)
|
||||
res304 = res0.replace(status=304)
|
||||
res2 = self._process_requestresponse(mw, req0, res304 if shouldcache else res0)
|
||||
self.assertEqualResponse(res1, res0)
|
||||
self.assertEqualResponse(res2, res0)
|
||||
resc = mw.storage.retrieve_response(self.spider, req0)
|
||||
if shouldcache:
|
||||
self.assertEqualResponse(resc, res1)
|
||||
assert 'cached' in res2.flags and res2.status != 304
|
||||
else:
|
||||
self.assertFalse(resc)
|
||||
assert 'cached' not in res2.flags
|
||||
|
||||
def test_storage_rfc2616policy_response_cached_and_fresh(self):
|
||||
# test response is cached and is fresh
|
||||
# (response requested should be same as response received)
|
||||
with self._storage(HTTPCACHE_POLICY=self.rfc2616_policy) as storage:
|
||||
assert storage.retrieve_response(self.spider, self.request) is None
|
||||
def test_cached_and_fresh(self):
|
||||
sampledata = [
|
||||
(200, {'Date': self.yesterday, 'Expires': self.tomorrow}),
|
||||
(200, {'Date': self.yesterday, 'Cache-Control': 'max-age=86405'}),
|
||||
(200, {'Age': '299', 'Cache-Control': 'max-age=300'}),
|
||||
# Obey max-age if present over any others
|
||||
(200, {'Date': self.today,
|
||||
'Age': '86405',
|
||||
'Cache-Control': 'max-age=' + str(86400 * 3),
|
||||
'Expires': self.yesterday,
|
||||
'Last-Modified': self.yesterday,
|
||||
}),
|
||||
# obey Expires if max-age is not present
|
||||
(200, {'Date': self.yesterday,
|
||||
'Age': '86400',
|
||||
'Cache-Control': 'public',
|
||||
'Expires': self.tomorrow,
|
||||
'Last-Modified': self.yesterday,
|
||||
}),
|
||||
# Default missing Date header to right now
|
||||
(200, {'Expires': self.tomorrow}),
|
||||
# Firefox - Expires if age is greater than 10% of (Date - Last-Modified)
|
||||
(200, {'Date': self.today, 'Last-Modified': self.yesterday, 'Age': str(86400 / 10 - 1)}),
|
||||
# Firefox - Set one year maxage to permanent redirects missing expiration info
|
||||
(300, {}), (301, {}), (308, {}),
|
||||
]
|
||||
with self._middleware() as mw:
|
||||
for idx, (status, headers) in enumerate(sampledata):
|
||||
req0 = Request('http://example-%d.com' % idx)
|
||||
res0 = Response(req0.url, status=status, headers=headers)
|
||||
# cache fresh response
|
||||
res1 = self._process_requestresponse(mw, req0, res0)
|
||||
self.assertEqualResponse(res1, res0)
|
||||
assert 'cached' not in res1.flags
|
||||
# return fresh cached response without network interaction
|
||||
res2 = self._process_requestresponse(mw, req0, None)
|
||||
self.assertEqualResponse(res1, res2)
|
||||
assert 'cached' in res2.flags
|
||||
|
||||
response = Response('http://www.example.com', headers=
|
||||
{'Content-Type': 'text/html', 'Date': self.yesterday, 'Expires': self.tomorrow},
|
||||
body='test body', status=200)
|
||||
storage.store_response(self.spider, self.request, response)
|
||||
response2 = storage.retrieve_response(self.spider, self.request)
|
||||
self.assertEqualResponse(response, response2)
|
||||
def test_cached_and_stale(self):
|
||||
sampledata = [
|
||||
(200, {'Date': self.today, 'Expires': self.yesterday}),
|
||||
(200, {'Date': self.today, 'Expires': self.yesterday, 'Last-Modified': self.yesterday}),
|
||||
(200, {'Expires': self.yesterday}),
|
||||
(200, {'Expires': self.yesterday, 'ETag': 'foo'}),
|
||||
(200, {'Expires': self.yesterday, 'Last-Modified': self.yesterday}),
|
||||
(200, {'Expires': self.tomorrow, 'Age': '86405'}),
|
||||
(200, {'Cache-Control': 'max-age=86400', 'Age': '86405'}),
|
||||
# no-cache forces expiration, also revalidation if validators exists
|
||||
(200, {'Cache-Control': 'no-cache'}),
|
||||
(200, {'Cache-Control': 'no-cache', 'ETag': 'foo'}),
|
||||
(200, {'Cache-Control': 'no-cache', 'Last-Modified': self.yesterday}),
|
||||
]
|
||||
with self._middleware() as mw:
|
||||
for idx, (status, headers) in enumerate(sampledata):
|
||||
req0 = Request('http://example-%d.com' % idx)
|
||||
res0a = Response(req0.url, status=status, headers=headers)
|
||||
# cache expired response
|
||||
res1 = self._process_requestresponse(mw, req0, res0a)
|
||||
self.assertEqualResponse(res1, res0a)
|
||||
assert 'cached' not in res1.flags
|
||||
# Same request but as cached response is stale a new response must
|
||||
# be returned
|
||||
res0b = res0a.replace(body='bar')
|
||||
res2 = self._process_requestresponse(mw, req0, res0b)
|
||||
self.assertEqualResponse(res2, res0b)
|
||||
assert 'cached' not in res2.flags
|
||||
# Previous response expired too, subsequent request to same
|
||||
# resource must revalidate and succeed on 304 if validators
|
||||
# are present
|
||||
if 'ETag' in headers or 'Last-Modified' in headers:
|
||||
res0c = res0b.replace(status=304)
|
||||
res3 = self._process_requestresponse(mw, req0, res0c)
|
||||
self.assertEqualResponse(res3, res0b)
|
||||
assert 'cached' in res3.flags
|
||||
|
||||
def test_storage_rfc2616policy_response403_cached_and_further_requests_ignored(self):
|
||||
# test response is cached but further requests are ignored
|
||||
# because response status is 403 (as per the RFC)
|
||||
with self._storage(HTTPCACHE_POLICY=self.rfc2616_policy) as storage:
|
||||
assert storage.retrieve_response(self.spider, self.request) is None
|
||||
|
||||
response = Response('http://www.example.com', headers=
|
||||
{'Content-Type': 'text/html', 'Date': self.yesterday, 'Expires': self.tomorrow},
|
||||
body='test body', status=403)
|
||||
storage.store_response(self.spider, self.request, response)
|
||||
self.assertRaises(IgnoreRequest, storage.retrieve_response,
|
||||
self.spider, self.request)
|
||||
|
||||
def test_storage_rfc2616policy_response_cached_and_stale(self):
|
||||
# test response is cached and is stale (no cache validators inserted)
|
||||
# (request should be same as response received)
|
||||
with self._storage(HTTPCACHE_POLICY=self.rfc2616_policy) as storage:
|
||||
assert storage.retrieve_response(self.spider, self.request) is None
|
||||
|
||||
response = Response('http://www.example.com', headers=
|
||||
{'Content-Type': 'text/html', 'Date': self.now, 'Expires': self.yesterday},
|
||||
body='test body', status=200)
|
||||
storage.store_response(self.spider, self.request, response)
|
||||
response2 = storage.retrieve_response(self.spider, self.request)
|
||||
assert isinstance(response2, Request)
|
||||
self.assertEqualRequest(self.request, response2)
|
||||
|
||||
def test_storage_rfc2616policy_response_cached_and_stale_with_cache_validators(self):
|
||||
# test response is cached and is stale and cache validators are inserted
|
||||
with self._storage(HTTPCACHE_POLICY=self.rfc2616_policy) as storage:
|
||||
assert storage.retrieve_response(self.spider, self.request) is None
|
||||
|
||||
response = Response('http://www.example.com', headers=
|
||||
{'Content-Type': 'text/html', 'Date': self.now, 'Expires': self.yesterday,
|
||||
'Last-Modified': self.yesterday}, body='test body', status=200)
|
||||
storage.store_response(self.spider, self.request, response)
|
||||
response2 = storage.retrieve_response(self.spider, self.request)
|
||||
assert isinstance(response2, Request)
|
||||
self.assertEqualRequestButWithCacheValidators(self.request, response2)
|
||||
|
||||
def test_storage_rfc2616policy_response_cached_and_transparent(self):
|
||||
# test response is not cached because of the request's Cache-Control 'no-cache' directive
|
||||
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2
|
||||
with self._storage(HTTPCACHE_POLICY=self.rfc2616_policy) as storage:
|
||||
request = Request('http://www.example.com',
|
||||
headers={'User-Agent': 'test', 'Cache-Control': 'no-cache'})
|
||||
assert storage.retrieve_response(self.spider, request) is None
|
||||
storage.store_response(self.spider, request, self.response)
|
||||
response = storage.retrieve_response(self.spider, request)
|
||||
assert isinstance(response, Request)
|
||||
self.assertEqualRequest(request, response)
|
||||
|
||||
def test_http_cache_request_policy(self):
|
||||
callback = lambda r: (r.url == 'http://www.example.com/')
|
||||
|
||||
# Should be cached
|
||||
req, res = Request('http://www.example.com/'), Response('http://www.example.com/')
|
||||
with self._middleware(HTTPCACHE_POLICY_REQUEST=callback) as mw:
|
||||
assert mw.process_request(req, self.spider) is None
|
||||
mw.process_response(req, res, self.spider)
|
||||
|
||||
cached = mw.process_request(req, self.spider)
|
||||
assert isinstance(cached, Response), type(cached)
|
||||
self.assertEqualResponse(res, cached)
|
||||
assert 'cached' in cached.flags
|
||||
|
||||
# Should not be cached
|
||||
req, res = Request('http://www.test.com/'), Response('http://www.test.com/')
|
||||
with self._middleware(HTTPCACHE_POLICY_REQUEST=callback) as mw:
|
||||
assert mw.process_request(req, self.spider) is None
|
||||
mw.process_response(req, res, self.spider)
|
||||
|
||||
assert mw.storage.retrieve_response(self.spider, req) is None
|
||||
assert mw.process_request(req, self.spider) is None
|
||||
|
||||
def test_http_cache_response_policy(self):
|
||||
callback = lambda r: (r.url == 'http://www.example.com/')
|
||||
|
||||
# Should be cached
|
||||
req, res = Request('http://www.example.com/'), Response('http://www.example.com/')
|
||||
with self._middleware(HTTPCACHE_POLICY_RESPONSE=callback) as mw:
|
||||
assert mw.process_request(req, self.spider) is None
|
||||
mw.process_response(req, res, self.spider)
|
||||
|
||||
cached = mw.process_request(req, self.spider)
|
||||
assert isinstance(cached, Response), type(cached)
|
||||
self.assertEqualResponse(res, cached)
|
||||
assert 'cached' in cached.flags
|
||||
|
||||
# Should not be cached
|
||||
req, res = Request('http://www.test.com/'), Response('http://www.test.com/')
|
||||
with self._middleware(HTTPCACHE_POLICY_RESPONSE=callback) as mw:
|
||||
assert mw.process_request(req, self.spider) is None
|
||||
mw.process_response(req, res, self.spider)
|
||||
|
||||
assert mw.storage.retrieve_response(self.spider, req) is None
|
||||
assert mw.process_request(req, self.spider) is None
|
||||
|
||||
def assertEqualResponse(self, response1, response2):
|
||||
self.assertEqual(response1.url, response2.url)
|
||||
self.assertEqual(response1.status, response2.status)
|
||||
self.assertEqual(response1.headers, response2.headers)
|
||||
self.assertEqual(response1.body, response2.body)
|
||||
|
||||
def assertEqualRequest(self, request1, request2):
|
||||
self.assertEqual(request1.url, request2.url)
|
||||
self.assertEqual(request1.headers, request2.headers)
|
||||
self.assertEqual(request1.body, request2.body)
|
||||
|
||||
def assertEqualRequestButWithCacheValidators(self, request1, request2):
|
||||
self.assertEqual(request1.url, request2.url)
|
||||
assert not request1.headers.has_key('If-None-Match')
|
||||
assert not request1.headers.has_key('If-Modified-Since')
|
||||
assert (request2.headers.has_key('If-None-Match') or \
|
||||
request2.headers.has_key('If-Modified-Since'))
|
||||
self.assertEqual(request1.body, request2.body)
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Reference in New Issue