From cdecc760eeef2c7c60b2c9883c5c0eee0cd38e6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Fri, 4 Jan 2013 03:43:25 -0200 Subject: [PATCH] default httpcache to rfc2616 policy and improve storage and policy tests --- .../contrib/downloadermiddleware/httpcache.py | 320 ++++++++------ scrapy/contrib/httpcache.py | 237 ++++------ .../test_downloadermiddleware_httpcache.py | 410 +++++++++--------- 3 files changed, 494 insertions(+), 473 deletions(-) diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index c267d1c8a..aa736708b 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -1,61 +1,167 @@ -import os -from os.path import join, exists from time import time -import cPickle as pickle - -from w3lib.http import headers_dict_to_raw, headers_raw_to_dict - +from email.utils import formatdate +from weakref import WeakKeyDictionary from scrapy import signals -from scrapy.http import Headers -from scrapy.http.request import Request from scrapy.exceptions import NotConfigured, IgnoreRequest -from scrapy.responsetypes import responsetypes -from scrapy.utils.request import request_fingerprint from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object -from scrapy.utils.project import data_path +from scrapy.contrib.httpcache import rfc1123_to_epoch, parse_cachecontrol class DummyPolicy(object): + def __init__(self, settings): self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') self.ignore_http_codes = map(int, settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES')) - - def should_cache_response(self, response): - return response.status not in self.ignore_http_codes def should_cache_request(self, request): return urlparse_cached(request).scheme not in self.ignore_schemes + def should_cache_response(self, response, request): + return response.status not in self.ignore_http_codes + + def is_cached_response_fresh(self, response, request): + return True + + def is_cached_response_valid(self, cachedresponse, response, request): + return True + + +class RFC2616Policy(object): + + MAXAGE = 3600 * 24 * 365 # one year -class RFC2616Policy(DummyPolicy): def __init__(self, settings): - super(RFC2616Policy, self).__init__(settings) + self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') + self._cc_parsed = WeakKeyDictionary() - def should_cache_response(self, response): - retval = super(RFC2616Policy, self).should_cache_response(response) - - if response.headers.has_key('cache-control'): - retval = retval and (response.headers['cache-control'].lower().find('no-store') == -1) - #retval = retval and self.policy_response(response) - return retval + def _parse_cachecontrol(self, r): + if r not in self._cc_parsed: + cch = r.headers.get('Cache-Control', '') + self._cc_parsed[r] = parse_cachecontrol(cch) + return self._cc_parsed[r] def should_cache_request(self, request): - retval = super(RFC2616Policy, self).should_cache_request(request) - - if request.headers.has_key('cache-control'): - retval = retval and (request.headers['cache-control'].lower().find('no-store') == -1) - #retval = retval and self.policy_request(request) - return retval + if urlparse_cached(request).scheme in self.ignore_schemes: + return False + cc = self._parse_cachecontrol(request) + # obey user-agent directive "Cache-Control: no-store" + if 'no-store' in cc: + return False + # Any other is eligible for caching + return True + + def should_cache_response(self, response, request): + # What is cacheable - http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec14.9.1 + # Response cacheability - http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4 + # Status code 206 is not included because cache can not deal with partial contents + cc = self._parse_cachecontrol(response) + # obey directive "Cache-Control: no-store" + if 'no-store' in cc: + return False + # Never cache 304 (Not Modified) responses + elif response.status == 304: + return False + # Any hint on response expiration is good + elif 'max-age' in cc or 'Expires' in response.headers: + return True + # Firefox fallbacks this statuses to one year expiration if none is set + elif response.status in (300, 301, 308): + return True + # Other statuses without expiration requires at least one validator + elif response.status in (200, 203, 401): + return 'Last-Modified' in response.headers or 'ETag' in response.headers + # Any other is probably not eligible for caching + # Makes no sense to cache responses that does not contain expiration + # info and can not be revalidated + else: + return False + + def is_cached_response_fresh(self, cachedresponse, request): + cc = self._parse_cachecontrol(cachedresponse) + if 'no-cache' in cc: + return False + + now = time() + freshnesslifetime = self._compute_freshness_lifetime(cachedresponse, request, now) + currentage = self._compute_current_age(cachedresponse, request, now) + if currentage < freshnesslifetime: + return True + # Cached response is stale, try to set validators if any + self._set_conditional_validators(request, cachedresponse) + return False + + def is_cached_response_valid(self, cachedresponse, response, request): + return response.status == 304 + + def _set_conditional_validators(self, request, cachedresponse): + if 'Last-Modified' in cachedresponse.headers: + request.headers['If-Modified-Since'] = cachedresponse.headers['Last-Modified'] + + if 'ETag' in cachedresponse.headers: + request.headers['If-None-Match'] = cachedresponse.headers['ETag'] + + def _compute_freshness_lifetime(self, response, request, now): + # Reference nsHttpResponseHead::ComputeFresshnessLifetime + # http://dxr.mozilla.org/mozilla-central/netwerk/protocol/http/nsHttpResponseHead.cpp.html#l259 + cc = self._parse_cachecontrol(response) + if 'max-age' in cc: + try: + return max(0, int(cc['max-age'])) + except ValueError: + pass + + # Parse date header or synthesize it if none exists + date = rfc1123_to_epoch(response.headers.get('Date')) or now + + # Try HTTP/1.0 Expires header + if 'Expires' in response.headers: + expires = rfc1123_to_epoch(response.headers['Expires']) + # When parsing Expires header fails RFC 2616 section 14.21 says we + # should treat this as an expiration time in the past. + return max(0, expires - date) if expires else 0 + + # Fallback to heuristic using last-modified header + # This is not in RFC but on Firefox caching implementation + lastmodified = rfc1123_to_epoch(response.headers.get('Last-Modified')) + if lastmodified and lastmodified <= date: + return (date - lastmodified) / 10 + + # This request can be cached indefinitely + if response.status in (300, 301, 308): + return self.MAXAGE + + # Insufficient information to compute fresshness lifetime + return 0 + + def _compute_current_age(self, response, request, now): + # Reference nsHttpResponseHead::ComputeCurrentAge + # http://dxr.mozilla.org/mozilla-central/netwerk/protocol/http/nsHttpResponseHead.cpp.html + currentage = 0 + # If Date header is not set we assume it is a fast connection, and + # clock is in sync with the server + date = rfc1123_to_epoch(response.headers.get('Date')) or now + if now > date: + currentage = now - date + + if 'Age' in response.headers: + try: + age = int(response.headers['Age']) + currentage = max(currentage, age) + except ValueError: + pass + + return currentage + class HttpCacheMiddleware(object): def __init__(self, settings, stats): if not settings.getbool('HTTPCACHE_ENABLED'): raise NotConfigured + self.policy = load_object(settings['HTTPCACHE_POLICY'])(settings) self.storage = load_object(settings['HTTPCACHE_STORAGE'])(settings) self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING') - self.policy = load_object(settings['HTTPCACHE_POLICY'])(settings) self.stats = stats @classmethod @@ -72,112 +178,72 @@ class HttpCacheMiddleware(object): self.storage.close_spider(spider) def process_request(self, request, spider): + # Skip uncacheable requests if not self.policy.should_cache_request(request): - return - response = self.storage.retrieve_response(spider, request) - - # Response cached, but stale - if response and type(response) is Request: - # Return None so that Scrapy continues processing - self.stats.inc_value('httpcache/revalidation', spider=spider) + request.meta['_dont_cache'] = True # flag as uncacheable return - if response and self.policy.should_cache_response(response): + # Look for cached response and check if expired + cachedresponse = self.storage.retrieve_response(spider, request) + if cachedresponse is None: + self.stats.inc_value('httpcache/miss', spider=spider) + if self.ignore_missing: + self.stats.inc_value('httpcache/ignore', spider=spider) + raise IgnoreRequest("Ignored request not in cache: %s" % request) + return # first time request + + # Return cached response only if not expired + cachedresponse.flags.append('cached') + if self.policy.is_cached_response_fresh(cachedresponse, request): self.stats.inc_value('httpcache/hit', spider=spider) - if isinstance(self.policy, RFC2616Policy): - # Response cached and fresh - raise IgnoreRequest("Ignored request already in cache: %s" % request) - else: - response.flags.append('cached') - return response + return cachedresponse - # Response not cached - self.stats.inc_value('httpcache/miss', spider=spider) - if self.ignore_missing: - raise IgnoreRequest("Ignored request not in cache: %s" % request) + # Keep a reference to cached response to avoid a second cache lookup on + # process_response hook + request.meta['cached_response'] = cachedresponse def process_response(self, request, response, spider): - if (self.policy.should_cache_request(request) - and self.policy.should_cache_response(response)): - if isinstance(self.policy, RFC2616Policy): - if response.status != 304: - self.storage.store_response(spider, request, response) - self.stats.inc_value('httpcache/store', spider=spider) - else: - response.flags.append('cached') - self.stats.inc_value('httpcache/hit', spider=spider) - else: - if 'cached' not in response.flags: - self.storage.store_response(spider, request, response) - self.stats.inc_value('httpcache/store', spider=spider) + # Skip cached responses and uncacheable requests + if 'cached' in response.flags or '_dont_cache' in request.meta: + request.meta.pop('_dont_cache', None) + return response + + # RFC2616 requires origin server to set Date header, + # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.18 + if 'Date' not in response.headers: + response.headers['Date'] = formatdate(usegmt=1) + + # Do not validate first-hand responses + cachedresponse = request.meta.pop('cached_response', None) + if cachedresponse is None: + self.stats.inc_value('httpcache/firsthand', spider=spider) + self._cache_response(spider, response, request, cachedresponse) + return response + + if self.policy.is_cached_response_valid(cachedresponse, response, request): + self.stats.inc_value('httpcache/revalidate', spider=spider) + return cachedresponse + + self.stats.inc_value('httpcache/invalidate', spider=spider) + self._cache_response(spider, response, request, cachedresponse) return response + def _cache_response(self, spider, response, request, cachedresponse): + if self.policy.should_cache_response(response, request): + self.stats.inc_value('httpcache/store', spider=spider) + self.storage.store_response(spider, request, response) + else: + self.stats.inc_value('httpcache/uncacheable', spider=spider) -class FilesystemCacheStorage(object): - def __init__(self, settings): - self.cachedir = data_path(settings['HTTPCACHE_DIR']) - self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS') +from scrapy.contrib.httpcache import FilesystemCacheStorage as _FilesystemCacheStorage +class FilesystemCacheStorage(_FilesystemCacheStorage): - def open_spider(self, spider): - pass - - def close_spider(self, spider): - pass - - def retrieve_response(self, spider, request): - """Return response if present in cache, or None otherwise.""" - metadata = self._read_meta(spider, request) - if metadata is None: - return # not cached - rpath = self._get_request_path(spider, request) - with open(join(rpath, 'response_body'), 'rb') as f: - body = f.read() - with open(join(rpath, 'response_headers'), 'rb') as f: - rawheaders = f.read() - url = metadata.get('response_url') - status = metadata['status'] - headers = Headers(headers_raw_to_dict(rawheaders)) - respcls = responsetypes.from_args(headers=headers, url=url) - response = respcls(url=url, headers=headers, status=status, body=body) - return response - - def store_response(self, spider, request, response): - """Store the given response in the cache.""" - rpath = self._get_request_path(spider, request) - if not exists(rpath): - os.makedirs(rpath) - metadata = { - 'url': request.url, - 'method': request.method, - 'status': response.status, - 'response_url': response.url, - 'timestamp': time(), - } - with open(join(rpath, 'meta'), 'wb') as f: - f.write(repr(metadata)) - with open(join(rpath, 'pickled_meta'), 'wb') as f: - pickle.dump(metadata, f, protocol=2) - with open(join(rpath, 'response_headers'), 'wb') as f: - f.write(headers_dict_to_raw(response.headers)) - with open(join(rpath, 'response_body'), 'wb') as f: - f.write(response.body) - with open(join(rpath, 'request_headers'), 'wb') as f: - f.write(headers_dict_to_raw(request.headers)) - with open(join(rpath, 'request_body'), 'wb') as f: - f.write(request.body) - - def _get_request_path(self, spider, request): - key = request_fingerprint(request) - return join(self.cachedir, spider.name, key[0:2], key) - - def _read_meta(self, spider, request): - rpath = self._get_request_path(spider, request) - metapath = join(rpath, 'pickled_meta') - if not exists(metapath): - return # not found - mtime = os.stat(rpath).st_mtime - if 0 < self.expiration_secs < time() - mtime: - return # expired - with open(metapath, 'rb') as f: - return pickle.load(f) + def __init__(self, *args, **kwargs): + import warnings + from scrapy.exceptions import ScrapyDeprecationWarning + warnings.warn('Importing FilesystemCacheStorage from ' + 'scrapy.contrib.downloadermiddlware.httpcache is ' + 'deprecated, use scrapy.contrib.httpcache instead.', + category=ScrapyDeprecationWarning, stacklevel=1) + super(_FilesystemCacheStorage, self).__init__(*args, **kwargs) diff --git a/scrapy/contrib/httpcache.py b/scrapy/contrib/httpcache.py index e815db3bc..d984ee8ed 100644 --- a/scrapy/contrib/httpcache.py +++ b/scrapy/contrib/httpcache.py @@ -1,12 +1,9 @@ import os -import calendar -import email.utils from time import time import cPickle as pickle - -from scrapy import log +from email.utils import mktime_tz, parsedate_tz +from w3lib.http import headers_raw_to_dict, headers_dict_to_raw from scrapy.http import Headers -from scrapy.exceptions import IgnoreRequest from scrapy.responsetypes import responsetypes from scrapy.utils.request import request_fingerprint from scrapy.utils.project import data_path @@ -30,7 +27,7 @@ class DbmCacheStorage(object): def retrieve_response(self, spider, request): data = self._read_data(spider, request) if data is None: - return # not cached + return # not cached url = data['url'] status = data['status'] headers = Headers(data['headers']) @@ -55,149 +52,107 @@ class DbmCacheStorage(object): db = self.db tkey = '%s_time' % key if not db.has_key(tkey): - return # not found + return # not found ts = db[tkey] if 0 < self.expiration_secs < time() - float(ts): - return # expired + return # expired return pickle.loads(db['%s_data' % key]) def _request_key(self, request): return request_fingerprint(request) -class BaseRealCacheStorage(object): - """ - Most of the code was taken from the httplib2 (MIT License) - https://code.google.com/p/httplib2/source/browse/python2/httplib2/__init__.py - """ - - def parse_cache_control(self, headers): - retval = {} - if headers.has_key('cache-control'): - parts = headers['cache-control'].split(',') - parts_with_args = [tuple([x.strip().lower() for x in part.split("=", 1)]) for part in parts if -1 != part.find("=")] - parts_wo_args = [(name.strip().lower(), 1) for name in parts if -1 == name.find("=")] - retval = dict(parts_with_args + parts_wo_args) - return retval - - - def get(self, response_headers, request_headers): - """Determine freshness from the Date, Expires and Cache-Control headers. +class FilesystemCacheStorage(object): - We don't handle the following: - - 1. Cache-Control: max-stale - 2. Age: headers are not used in the calculations. - - Not that this algorithm is simpler than you might think - because we are operating as a private (non-shared) cache. - This lets us ignore 's-maxage'. We can also ignore - 'proxy-invalidate' since we aren't a proxy. - We will never return a stale document as - fresh as a design decision, and thus the non-implementation - of 'max-stale'. This also lets us safely ignore 'must-revalidate' - since we operate as if every server has sent 'must-revalidate'. - Since we are private we get to ignore both 'public' and - 'private' parameters. We also ignore 'no-transform' since - we don't do any transformations. - The 'no-store' parameter is handled at a higher level. - So the only Cache-Control parameters we look at are: - - no-cache - only-if-cached - max-age - min-fresh - """ - - retval = "STALE" - cc = self.parse_cache_control(request_headers) - cc_response = self.parse_cache_control(response_headers) - - if request_headers.has_key('pragma') and request_headers['pragma'].lower().find('no-cache') != -1: - retval = "TRANSPARENT" - if 'cache-control' not in request_headers: - request_headers['cache-control'] = 'no-cache' - elif cc.has_key('no-cache'): - retval = "TRANSPARENT" - elif cc_response.has_key('no-cache'): - retval = "STALE" - elif cc.has_key('only-if-cached'): - retval = "FRESH" - elif response_headers.has_key('date'): - date = calendar.timegm(email.utils.parsedate_tz(response_headers['date'])) - now = time() - current_age = max(0, now - date) - if cc_response.has_key('max-age'): - try: - freshness_lifetime = int(cc_response['max-age']) - except ValueError: - freshness_lifetime = 0 - elif response_headers.has_key('expires'): - expires = email.utils.parsedate_tz(response_headers['expires']) - if None == expires: - freshness_lifetime = 0 - else: - freshness_lifetime = max(0, calendar.timegm(expires) - date) - else: - freshness_lifetime = 0 - if cc.has_key('max-age'): - try: - freshness_lifetime = int(cc['max-age']) - except ValueError: - freshness_lifetime = 0 - if cc.has_key('min-fresh'): - try: - min_fresh = int(cc['min-fresh']) - except ValueError: - min_fresh = 0 - current_age += min_fresh - if freshness_lifetime > current_age: - retval = "FRESH" - return retval - - def retrieve_cache(self, spider, request, response_headers, response_status, response_url='', response_body=''): - # Determine our course of action: - # Is the cached entry fresh or stale? - # - # There seems to be three possible answers: - # 1. [FRESH] Return the Response object - # 2. [STALE] Update the Request object with cache validators if available - # 3. [TRANSPARENT] Don't update the Request with cache validators (Cache-Control: no-cache) - entry_disposition = self.get(Headers(response_headers), Headers(request.headers)) - - # Per the RFC, requests should not be repeated in these situations - if response_status in [400, 401, 403, 410]: - raise IgnoreRequest("Ignored request because cached response status is %d." % response_status) - - if entry_disposition == "FRESH": - log.msg("Cache is FRESH", level=log.DEBUG, spider=spider) - - headers = Headers(response_headers) - respcls = responsetypes.from_args(headers=headers, url=response_url) - response = respcls(url=response_url, headers=headers, status=response_status, body=response_body) - - return response - - new_request = request.copy() - if entry_disposition == "STALE": - log.msg("Cache is STALE, updating Request object with cache validators", level=log.DEBUG, spider=spider) - if response_headers.has_key('ETag') and not 'If-None-Match' in request.headers: - new_request.headers['If-None-Match'] = response_headers['ETag'] - if response_headers.has_key('Last-Modified') and not 'Last-Modified' in request.headers: - new_request.headers['If-Modified-Since'] = response_headers['Last-Modified'] - elif entry_disposition == "TRANSPARENT": - log.msg("Cache is TRANSPARENT, not adding cache validators to Request object", level=log.DEBUG, spider=spider) - - return new_request - - -class DbmRealCacheStorage(DbmCacheStorage, BaseRealCacheStorage): def __init__(self, settings): - super(DbmRealCacheStorage, self).__init__(settings) - + self.cachedir = data_path(settings['HTTPCACHE_DIR']) + self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS') + + def open_spider(self, spider): + pass + + def close_spider(self, spider): + pass + def retrieve_response(self, spider, request): - data = self._read_data(spider, request) - if data is None: - return # not cached - else: - return self.retrieve_cache(spider, request, data['headers'], data['status'], data['url'], data['body']) + """Return response if present in cache, or None otherwise.""" + metadata = self._read_meta(spider, request) + if metadata is None: + return # not cached + rpath = self._get_request_path(spider, request) + with open(os.path.join(rpath, 'response_body'), 'rb') as f: + body = f.read() + with open(os.path.join(rpath, 'response_headers'), 'rb') as f: + rawheaders = f.read() + url = metadata.get('response_url') + status = metadata['status'] + headers = Headers(headers_raw_to_dict(rawheaders)) + respcls = responsetypes.from_args(headers=headers, url=url) + response = respcls(url=url, headers=headers, status=status, body=body) + return response + + def store_response(self, spider, request, response): + """Store the given response in the cache.""" + rpath = self._get_request_path(spider, request) + if not os.path.exists(rpath): + os.makedirs(rpath) + metadata = { + 'url': request.url, + 'method': request.method, + 'status': response.status, + 'response_url': response.url, + 'timestamp': time(), + } + with open(os.path.join(rpath, 'meta'), 'wb') as f: + f.write(repr(metadata)) + with open(os.path.join(rpath, 'pickled_meta'), 'wb') as f: + pickle.dump(metadata, f, protocol=2) + with open(os.path.join(rpath, 'response_headers'), 'wb') as f: + f.write(headers_dict_to_raw(response.headers)) + with open(os.path.join(rpath, 'response_body'), 'wb') as f: + f.write(response.body) + with open(os.path.join(rpath, 'request_headers'), 'wb') as f: + f.write(headers_dict_to_raw(request.headers)) + with open(os.path.join(rpath, 'request_body'), 'wb') as f: + f.write(request.body) + + def _get_request_path(self, spider, request): + key = request_fingerprint(request) + return os.path.join(self.cachedir, spider.name, key[0:2], key) + + def _read_meta(self, spider, request): + rpath = self._get_request_path(spider, request) + metapath = os.path.join(rpath, 'pickled_meta') + if not os.path.exists(metapath): + return # not found + mtime = os.stat(rpath).st_mtime + if 0 < self.expiration_secs < time() - mtime: + return # expired + with open(metapath, 'rb') as f: + return pickle.load(f) + + +def parse_cachecontrol(header): + """Parse Cache-Control header + + http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9 + + >>> cachecontrol_directives('public, max-age=3600') + {'public': None, 'max-age': '3600'} + >>> cachecontrol_directives('') + {} + + """ + directives = {} + for directive in header.split(','): + key, sep, val = directive.strip().partition('=') + if key: + directives[key.lower()] = val if sep else None + return directives + + +def rfc1123_to_epoch(date_str): + try: + return mktime_tz(parsedate_tz(date_str)) + except Exception: + return None diff --git a/scrapy/tests/test_downloadermiddleware_httpcache.py b/scrapy/tests/test_downloadermiddleware_httpcache.py index a38775ab2..28b3026f0 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcache.py +++ b/scrapy/tests/test_downloadermiddleware_httpcache.py @@ -10,31 +10,27 @@ from scrapy.spider import BaseSpider from scrapy.settings import Settings from scrapy.exceptions import IgnoreRequest from scrapy.utils.test import get_crawler -from scrapy.contrib.downloadermiddleware.httpcache import \ - FilesystemCacheStorage, HttpCacheMiddleware -from scrapy.contrib.httpcache import DbmRealCacheStorage +from scrapy.contrib.downloadermiddleware.httpcache import HttpCacheMiddleware -class HttpCacheMiddlewareTest(unittest.TestCase): +class _BaseTest(unittest.TestCase): - storage_class = FilesystemCacheStorage - realcache_storage_class = DbmRealCacheStorage - - dummy_policy = 'scrapy.contrib.downloadermiddleware.httpcache.DummyPolicy' - rfc2616_policy = 'scrapy.contrib.downloadermiddleware.httpcache.RFC2616Policy' - - yesterday = email.utils.formatdate(time.time() - 1 * 24 * 60 * 60) - now = email.utils.formatdate() - tomorrow = email.utils.formatdate(time.time() + 1 * 24 * 60 * 60) + storage_class = 'scrapy.contrib.httpcache.DbmCacheStorage' + policy_class = 'scrapy.contrib.downloadermiddleware.httpcache.DummyPolicy' def setUp(self): + self.yesterday = email.utils.formatdate(time.time() - 86400) + self.today = email.utils.formatdate() + self.tomorrow = email.utils.formatdate(time.time() + 86400) self.crawler = get_crawler() self.spider = BaseSpider('example.com') self.tmpdir = tempfile.mkdtemp() self.request = Request('http://www.example.com', - headers={'User-Agent': 'test'}) - self.response = Response('http://www.example.com', headers= - {'Content-Type': 'text/html'}, body='test body', status=202) + headers={'User-Agent': 'test'}) + self.response = Response('http://www.example.com', + headers={'Content-Type': 'text/html'}, + body='test body', + status=202) self.crawler.stats.open_spider(self.spider) def tearDown(self): @@ -47,23 +43,21 @@ class HttpCacheMiddlewareTest(unittest.TestCase): 'HTTPCACHE_DIR': self.tmpdir, 'HTTPCACHE_EXPIRATION_SECS': 1, 'HTTPCACHE_IGNORE_HTTP_CODES': [], - 'HTTPCACHE_POLICY': self.dummy_policy + 'HTTPCACHE_POLICY': self.policy_class, + 'HTTPCACHE_STORAGE': self.storage_class, } settings.update(new_settings) return Settings(settings) @contextmanager def _storage(self, **new_settings): - settings = self._get_settings(**new_settings) - if settings.get('HTTPCACHE_POLICY') == self.dummy_policy: - storage = self.storage_class(settings) - else: - storage = self.realcache_storage_class(settings) - storage.open_spider(self.spider) - try: - yield storage - finally: - storage.close_spider(self.spider) + with self._middleware(**new_settings) as mw: + yield mw.storage + + @contextmanager + def _policy(self, **new_settings): + with self._middleware(**new_settings) as mw: + yield mw.policy @contextmanager def _middleware(self, **new_settings): @@ -75,6 +69,27 @@ class HttpCacheMiddlewareTest(unittest.TestCase): finally: mw.spider_closed(self.spider) + def assertEqualResponse(self, response1, response2): + self.assertEqual(response1.url, response2.url) + self.assertEqual(response1.status, response2.status) + self.assertEqual(response1.headers, response2.headers) + self.assertEqual(response1.body, response2.body) + + def assertEqualRequest(self, request1, request2): + self.assertEqual(request1.url, request2.url) + self.assertEqual(request1.headers, request2.headers) + self.assertEqual(request1.body, request2.body) + + def assertEqualRequestButWithCacheValidators(self, request1, request2): + self.assertEqual(request1.url, request2.url) + assert not 'If-None-Match' in request1.headers + assert not 'If-Modified-Since' in request1.headers + assert any(h in request2.headers for h in ('If-None-Match', 'If-Modified-Since')) + self.assertEqual(request1.body, request2.body) + + +class DefaultStorageTest(_BaseTest): + def test_storage(self): with self._storage() as storage: request2 = self.request.copy() @@ -95,11 +110,23 @@ class HttpCacheMiddlewareTest(unittest.TestCase): time.sleep(0.5) # give the chance to expire assert storage.retrieve_response(self.spider, self.request) + +class DbmStorageTest(DefaultStorageTest): + + storage_class = 'scrapy.contrib.httpcache.DbmCacheStorage' + + +class FilesystemStorageTest(DefaultStorageTest): + + storage_class = 'scrapy.contrib.httpcache.FilesystemCacheStorage' + + +class DefaultMiddlewaretest(_BaseTest): + def test_middleware(self): with self._middleware() as mw: assert mw.process_request(self.request, self.spider) is None mw.process_response(self.request, self.response, self.spider) - response = mw.process_request(self.request, self.spider) assert isinstance(response, HtmlResponse) self.assertEqualResponse(self.response, response) @@ -109,10 +136,8 @@ class HttpCacheMiddlewareTest(unittest.TestCase): with self._middleware() as mw: req = Request('http://host.com/path') res = Response('http://host2.net/test.html') - assert mw.process_request(req, self.spider) is None mw.process_response(req, res, self.spider) - cached = mw.process_request(req, self.spider) assert isinstance(cached, Response) self.assertEqualResponse(res, cached) @@ -185,192 +210,167 @@ class HttpCacheMiddlewareTest(unittest.TestCase): self.assertEqualResponse(self.response, response) assert 'cached' in response.flags - def test_middleware_rfc2616policy_response304_not_cached(self): - # test response is not cached because the status is 304 Not Modified - # (so it should be cached already) - with self._middleware(HTTPCACHE_POLICY=self.rfc2616_policy) as mw: - assert mw.process_request(self.request, self.spider) is None - response = Response('http://www.example.com', status=304) - mw.process_response(self.request, response, self.spider) - assert 'cached' in response.flags - assert mw.storage.retrieve_response(self.spider, self.request) is None - assert mw.process_request(self.request, self.spider) is None +class DummyMiddlewareTest(DefaultStorageTest): - def test_middleware_rfc2616policy_response_nostore_not_cached(self): - # test response is not cached because of the Cache-Control 'no-store' directive - # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2 - with self._middleware(HTTPCACHE_POLICY=self.rfc2616_policy) as mw: - assert mw.process_request(self.request, self.spider) is None - response = Response('http://www.example.com', headers= - {'Content-Type': 'text/html', 'Cache-Control': 'no-store'}, - body='test body', status=200) - mw.process_response(self.request, response, self.spider) + policy_class = 'scrapy.contrib.downloadermiddleware.httpcache.DummyPolicy' - assert mw.storage.retrieve_response(self.spider, self.request) is None - assert mw.process_request(self.request, self.spider) is None - def test_middleware_rfc2616policy_request_nostore_not_cached(self): - # test response is not cached because of the request's Cache-Control 'no-store' directive - # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2 - with self._middleware(HTTPCACHE_POLICY=self.rfc2616_policy) as mw: - request = Request('http://www.example.com', - headers={'User-Agent': 'test', 'Cache-Control': 'no-store'}) - assert mw.process_request(request, self.spider) is None - mw.process_response(request, self.response, self.spider) +class RFC2616MiddlewareTest(DefaultStorageTest): - assert mw.storage.retrieve_response(self.spider, request) is None - assert mw.process_request(request, self.spider) is None + policy_class = 'scrapy.contrib.downloadermiddleware.httpcache.RFC2616Policy' - def test_middleware_rfc2616policy_response_cached_and_fresh(self): - # test response cached and fresh - with self._middleware(HTTPCACHE_POLICY=self.rfc2616_policy) as mw: - response = mw.process_response(self.request, self.response, self.spider) - self.assertRaises(IgnoreRequest, mw.process_request, self.request, self.spider) - assert 'cached' not in response.flags + def _process_requestresponse(self, mw, request, response): + try: + result = mw.process_request(request, self.spider) + if result: + assert isinstance(result, (Request, Response)) + return result + else: + result = mw.process_response(request, response, self.spider) + assert isinstance(result, Response) + return result + except Exception: + print 'Request', request + print 'Response', response + print 'Result', result + raise - def test_middleware_rfc2616policy_response_cached_and_stale(self): - # test response cached but stale - with self._middleware(HTTPCACHE_POLICY=self.rfc2616_policy, - HTTPCACHE_STORAGE = 'scrapy.contrib.httpcache.DbmRealCacheStorage') as mw: - response = Response('http://www.example.com', headers= - {'Content-Type': 'text/html', 'Cache-Control': 'no-cache'}, - body='test body', status=200) - mw.process_response(self.request, response, self.spider) - assert mw.process_request(self.request, self.spider) is None + def test_request_cacheability(self): + res0 = Response(self.request.url, status=200, + headers={'Expires': self.tomorrow}) + req0 = Request('http://example.com') + req1 = req0.replace(headers={'Cache-Control': 'no-store'}) + with self._middleware() as mw: + res1 = self._process_requestresponse(mw, req1, res0) + self.assertEqualResponse(res1, res0) + assert mw.storage.retrieve_response(self.spider, req1) is None + res2 = self._process_requestresponse(mw, req0, res0) + assert 'cached' not in res2.flags + res3 = mw.process_request(req0, self.spider) + assert 'cached' in res3.flags + self.assertEqualResponse(res2, res3) - response = mw.storage.retrieve_response(self.spider, self.request) - assert isinstance(response, Request) + def test_response_cacheability(self): + responses = [ + # 304 is not cacheable no matter what servers sends + (False, 304, {}), + (False, 304, {'Last-Modified': self.yesterday}), + (False, 304, {'Expires': self.tomorrow}), + (False, 304, {'Etag': 'bar'}), + (False, 304, {'Cache-Control': 'max-age=3600'}), + # Always obey no-store cache control + (False, 200, {'Cache-Control': 'no-store'}), + (False, 200, {'Cache-Control': 'no-store, max-age=300'}), # invalid + (False, 200, {'Cache-Control': 'no-store', 'Expires': self.tomorrow}), # invalid + # Ignore responses missing expiration and/or validation headers + (False, 200, {}), + (False, 302, {}), + (False, 307, {}), + (False, 404, {}), + # Cache responses with expiration and/or validation headers + (True, 200, {'Last-Modified': self.yesterday}), + (True, 203, {'Last-Modified': self.yesterday}), + (True, 300, {'Last-Modified': self.yesterday}), + (True, 301, {'Last-Modified': self.yesterday}), + (True, 401, {'Last-Modified': self.yesterday}), + (True, 404, {'Cache-Control': 'public, max-age=600'}), + (True, 302, {'Expires': self.tomorrow}), + (True, 200, {'Etag': 'foo'}), + ] + with self._middleware() as mw: + for idx, (shouldcache, status, headers) in enumerate(responses): + req0 = Request('http://example-%d.com' % idx) + res0 = Response(req0.url, status=status, headers=headers) + res1 = self._process_requestresponse(mw, req0, res0) + res304 = res0.replace(status=304) + res2 = self._process_requestresponse(mw, req0, res304 if shouldcache else res0) + self.assertEqualResponse(res1, res0) + self.assertEqualResponse(res2, res0) + resc = mw.storage.retrieve_response(self.spider, req0) + if shouldcache: + self.assertEqualResponse(resc, res1) + assert 'cached' in res2.flags and res2.status != 304 + else: + self.assertFalse(resc) + assert 'cached' not in res2.flags - def test_storage_rfc2616policy_response_cached_and_fresh(self): - # test response is cached and is fresh - # (response requested should be same as response received) - with self._storage(HTTPCACHE_POLICY=self.rfc2616_policy) as storage: - assert storage.retrieve_response(self.spider, self.request) is None + def test_cached_and_fresh(self): + sampledata = [ + (200, {'Date': self.yesterday, 'Expires': self.tomorrow}), + (200, {'Date': self.yesterday, 'Cache-Control': 'max-age=86405'}), + (200, {'Age': '299', 'Cache-Control': 'max-age=300'}), + # Obey max-age if present over any others + (200, {'Date': self.today, + 'Age': '86405', + 'Cache-Control': 'max-age=' + str(86400 * 3), + 'Expires': self.yesterday, + 'Last-Modified': self.yesterday, + }), + # obey Expires if max-age is not present + (200, {'Date': self.yesterday, + 'Age': '86400', + 'Cache-Control': 'public', + 'Expires': self.tomorrow, + 'Last-Modified': self.yesterday, + }), + # Default missing Date header to right now + (200, {'Expires': self.tomorrow}), + # Firefox - Expires if age is greater than 10% of (Date - Last-Modified) + (200, {'Date': self.today, 'Last-Modified': self.yesterday, 'Age': str(86400 / 10 - 1)}), + # Firefox - Set one year maxage to permanent redirects missing expiration info + (300, {}), (301, {}), (308, {}), + ] + with self._middleware() as mw: + for idx, (status, headers) in enumerate(sampledata): + req0 = Request('http://example-%d.com' % idx) + res0 = Response(req0.url, status=status, headers=headers) + # cache fresh response + res1 = self._process_requestresponse(mw, req0, res0) + self.assertEqualResponse(res1, res0) + assert 'cached' not in res1.flags + # return fresh cached response without network interaction + res2 = self._process_requestresponse(mw, req0, None) + self.assertEqualResponse(res1, res2) + assert 'cached' in res2.flags - response = Response('http://www.example.com', headers= - {'Content-Type': 'text/html', 'Date': self.yesterday, 'Expires': self.tomorrow}, - body='test body', status=200) - storage.store_response(self.spider, self.request, response) - response2 = storage.retrieve_response(self.spider, self.request) - self.assertEqualResponse(response, response2) + def test_cached_and_stale(self): + sampledata = [ + (200, {'Date': self.today, 'Expires': self.yesterday}), + (200, {'Date': self.today, 'Expires': self.yesterday, 'Last-Modified': self.yesterday}), + (200, {'Expires': self.yesterday}), + (200, {'Expires': self.yesterday, 'ETag': 'foo'}), + (200, {'Expires': self.yesterday, 'Last-Modified': self.yesterday}), + (200, {'Expires': self.tomorrow, 'Age': '86405'}), + (200, {'Cache-Control': 'max-age=86400', 'Age': '86405'}), + # no-cache forces expiration, also revalidation if validators exists + (200, {'Cache-Control': 'no-cache'}), + (200, {'Cache-Control': 'no-cache', 'ETag': 'foo'}), + (200, {'Cache-Control': 'no-cache', 'Last-Modified': self.yesterday}), + ] + with self._middleware() as mw: + for idx, (status, headers) in enumerate(sampledata): + req0 = Request('http://example-%d.com' % idx) + res0a = Response(req0.url, status=status, headers=headers) + # cache expired response + res1 = self._process_requestresponse(mw, req0, res0a) + self.assertEqualResponse(res1, res0a) + assert 'cached' not in res1.flags + # Same request but as cached response is stale a new response must + # be returned + res0b = res0a.replace(body='bar') + res2 = self._process_requestresponse(mw, req0, res0b) + self.assertEqualResponse(res2, res0b) + assert 'cached' not in res2.flags + # Previous response expired too, subsequent request to same + # resource must revalidate and succeed on 304 if validators + # are present + if 'ETag' in headers or 'Last-Modified' in headers: + res0c = res0b.replace(status=304) + res3 = self._process_requestresponse(mw, req0, res0c) + self.assertEqualResponse(res3, res0b) + assert 'cached' in res3.flags - def test_storage_rfc2616policy_response403_cached_and_further_requests_ignored(self): - # test response is cached but further requests are ignored - # because response status is 403 (as per the RFC) - with self._storage(HTTPCACHE_POLICY=self.rfc2616_policy) as storage: - assert storage.retrieve_response(self.spider, self.request) is None - - response = Response('http://www.example.com', headers= - {'Content-Type': 'text/html', 'Date': self.yesterday, 'Expires': self.tomorrow}, - body='test body', status=403) - storage.store_response(self.spider, self.request, response) - self.assertRaises(IgnoreRequest, storage.retrieve_response, - self.spider, self.request) - - def test_storage_rfc2616policy_response_cached_and_stale(self): - # test response is cached and is stale (no cache validators inserted) - # (request should be same as response received) - with self._storage(HTTPCACHE_POLICY=self.rfc2616_policy) as storage: - assert storage.retrieve_response(self.spider, self.request) is None - - response = Response('http://www.example.com', headers= - {'Content-Type': 'text/html', 'Date': self.now, 'Expires': self.yesterday}, - body='test body', status=200) - storage.store_response(self.spider, self.request, response) - response2 = storage.retrieve_response(self.spider, self.request) - assert isinstance(response2, Request) - self.assertEqualRequest(self.request, response2) - - def test_storage_rfc2616policy_response_cached_and_stale_with_cache_validators(self): - # test response is cached and is stale and cache validators are inserted - with self._storage(HTTPCACHE_POLICY=self.rfc2616_policy) as storage: - assert storage.retrieve_response(self.spider, self.request) is None - - response = Response('http://www.example.com', headers= - {'Content-Type': 'text/html', 'Date': self.now, 'Expires': self.yesterday, - 'Last-Modified': self.yesterday}, body='test body', status=200) - storage.store_response(self.spider, self.request, response) - response2 = storage.retrieve_response(self.spider, self.request) - assert isinstance(response2, Request) - self.assertEqualRequestButWithCacheValidators(self.request, response2) - - def test_storage_rfc2616policy_response_cached_and_transparent(self): - # test response is not cached because of the request's Cache-Control 'no-cache' directive - # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2 - with self._storage(HTTPCACHE_POLICY=self.rfc2616_policy) as storage: - request = Request('http://www.example.com', - headers={'User-Agent': 'test', 'Cache-Control': 'no-cache'}) - assert storage.retrieve_response(self.spider, request) is None - storage.store_response(self.spider, request, self.response) - response = storage.retrieve_response(self.spider, request) - assert isinstance(response, Request) - self.assertEqualRequest(request, response) - - def test_http_cache_request_policy(self): - callback = lambda r: (r.url == 'http://www.example.com/') - - # Should be cached - req, res = Request('http://www.example.com/'), Response('http://www.example.com/') - with self._middleware(HTTPCACHE_POLICY_REQUEST=callback) as mw: - assert mw.process_request(req, self.spider) is None - mw.process_response(req, res, self.spider) - - cached = mw.process_request(req, self.spider) - assert isinstance(cached, Response), type(cached) - self.assertEqualResponse(res, cached) - assert 'cached' in cached.flags - - # Should not be cached - req, res = Request('http://www.test.com/'), Response('http://www.test.com/') - with self._middleware(HTTPCACHE_POLICY_REQUEST=callback) as mw: - assert mw.process_request(req, self.spider) is None - mw.process_response(req, res, self.spider) - - assert mw.storage.retrieve_response(self.spider, req) is None - assert mw.process_request(req, self.spider) is None - - def test_http_cache_response_policy(self): - callback = lambda r: (r.url == 'http://www.example.com/') - - # Should be cached - req, res = Request('http://www.example.com/'), Response('http://www.example.com/') - with self._middleware(HTTPCACHE_POLICY_RESPONSE=callback) as mw: - assert mw.process_request(req, self.spider) is None - mw.process_response(req, res, self.spider) - - cached = mw.process_request(req, self.spider) - assert isinstance(cached, Response), type(cached) - self.assertEqualResponse(res, cached) - assert 'cached' in cached.flags - - # Should not be cached - req, res = Request('http://www.test.com/'), Response('http://www.test.com/') - with self._middleware(HTTPCACHE_POLICY_RESPONSE=callback) as mw: - assert mw.process_request(req, self.spider) is None - mw.process_response(req, res, self.spider) - - assert mw.storage.retrieve_response(self.spider, req) is None - assert mw.process_request(req, self.spider) is None - - def assertEqualResponse(self, response1, response2): - self.assertEqual(response1.url, response2.url) - self.assertEqual(response1.status, response2.status) - self.assertEqual(response1.headers, response2.headers) - self.assertEqual(response1.body, response2.body) - - def assertEqualRequest(self, request1, request2): - self.assertEqual(request1.url, request2.url) - self.assertEqual(request1.headers, request2.headers) - self.assertEqual(request1.body, request2.body) - - def assertEqualRequestButWithCacheValidators(self, request1, request2): - self.assertEqual(request1.url, request2.url) - assert not request1.headers.has_key('If-None-Match') - assert not request1.headers.has_key('If-Modified-Since') - assert (request2.headers.has_key('If-None-Match') or \ - request2.headers.has_key('If-Modified-Since')) - self.assertEqual(request1.body, request2.body) if __name__ == '__main__': unittest.main()