From 3e31d068725ad7b85f2ccdfa63c96f0b248fe5a9 Mon Sep 17 00:00:00 2001 From: Pedro Faustino Date: Fri, 28 Dec 2012 13:28:35 +0100 Subject: [PATCH] Implement single HTTP cache policy --- .../contrib/downloadermiddleware/httpcache.py | 56 +++++++++++-------- scrapy/settings/default_settings.py | 4 +- .../test_downloadermiddleware_httpcache.py | 26 ++++----- 3 files changed, 45 insertions(+), 41 deletions(-) diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index 5608ec2b9..b891c018c 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -16,23 +16,44 @@ from scrapy.utils.misc import load_object from scrapy.utils.project import data_path -class HttpCacheMiddleware(object): +class HttpCachePolicy(object): + def __init__(self, settings): + self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') + self.ignore_http_codes = map(int, settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES')) + self.policy = settings.get('HTTPCACHE_POLICY') + + if self.policy == 'dummy': + self.use_dummy_cache = True + else: + self.use_dummy_cache = False + + def should_cache_response(self, response): + retval = response.status not in self.ignore_http_codes + if not self.use_dummy_cache and response.headers.has_key('cache-control'): + retval = retval and (response.headers['cache-control'].lower().find('no-store') == -1) + #retval = retval and self.policy_response(response) + return retval + + def should_cache_request(self, request): + retval = urlparse_cached(request).scheme not in self.ignore_schemes + if not self.use_dummy_cache and request.headers.has_key('cache-control'): + retval = retval and (request.headers['cache-control'].lower().find('no-store') == -1) + #retval = retval and self.policy_request(request) + return retval + +class HttpCacheMiddleware(HttpCachePolicy): def __init__(self, settings, stats): if not settings.getbool('HTTPCACHE_ENABLED'): raise NotConfigured self.storage = load_object(settings['HTTPCACHE_STORAGE'])(settings) self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING') - self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') - self.ignore_http_codes = map(int, settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES')) - self.use_dummy_cache = settings.getbool('HTTPCACHE_USE_DUMMY') self.stats = stats - self.policy_request = settings.get('HTTPCACHE_POLICY_REQUEST') - self.policy_response = settings.get('HTTPCACHE_POLICY_RESPONSE') + super(HttpCacheMiddleware, self).__init__(settings) @classmethod def from_crawler(cls, crawler): - o = cls(crawler.settings, crawler.stats) + o = cls.from_settings(crawler.settings, crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o @@ -44,7 +65,7 @@ class HttpCacheMiddleware(object): self.storage.close_spider(spider) def process_request(self, request, spider): - if not self.is_cacheable(request): + if not self.should_cache_request(request): return response = self.storage.retrieve_response(spider, request) @@ -54,7 +75,7 @@ class HttpCacheMiddleware(object): self.stats.inc_value('httpcache/revalidation', spider=spider) return - if response and self.is_cacheable_response(response): + if response and self.should_cache_response(response): self.stats.inc_value('httpcache/hit', spider=spider) if self.use_dummy_cache: response.flags.append('cached') @@ -69,8 +90,8 @@ class HttpCacheMiddleware(object): raise IgnoreRequest("Ignored request not in cache: %s" % request) def process_response(self, request, response, spider): - if (self.is_cacheable(request) - and self.is_cacheable_response(response)): + if (self.should_cache_request(request) + and self.should_cache_response(response)): if self.use_dummy_cache: if 'cached' not in response.flags: self.storage.store_response(spider, request, response) @@ -84,19 +105,6 @@ class HttpCacheMiddleware(object): self.stats.inc_value('httpcache/hit', spider=spider) return response - def is_cacheable_response(self, response): - retval = response.status not in self.ignore_http_codes - if not self.use_dummy_cache and response.headers.has_key('cache-control'): - retval = retval and (response.headers['cache-control'].lower().find('no-store') == -1) - retval = retval and self.policy_response(response) - return retval - - def is_cacheable(self, request): - retval = urlparse_cached(request).scheme not in self.ignore_schemes - if not self.use_dummy_cache and request.headers.has_key('cache-control'): - retval = retval and (request.headers['cache-control'].lower().find('no-store') == -1) - retval = retval and self.policy_request(request) - return retval class FilesystemCacheStorage(object): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 1dd88219c..f494dc448 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -134,7 +134,6 @@ FEED_EXPORTERS_BASE = { } HTTPCACHE_ENABLED = False -HTTPCACHE_USE_DUMMY = True HTTPCACHE_DIR = 'httpcache' HTTPCACHE_IGNORE_MISSING = False HTTPCACHE_STORAGE = 'scrapy.contrib.httpcache.DbmCacheStorage' @@ -142,8 +141,7 @@ HTTPCACHE_EXPIRATION_SECS = 0 HTTPCACHE_IGNORE_HTTP_CODES = [] HTTPCACHE_IGNORE_SCHEMES = ['file'] HTTPCACHE_DBM_MODULE = 'anydbm' -HTTPCACHE_POLICY_REQUEST = lambda request : True -HTTPCACHE_POLICY_RESPONSE = lambda response : True +HTTPCACHE_POLICY = 'dummy' ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager' diff --git a/scrapy/tests/test_downloadermiddleware_httpcache.py b/scrapy/tests/test_downloadermiddleware_httpcache.py index 176a9d86a..7d505e4f9 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcache.py +++ b/scrapy/tests/test_downloadermiddleware_httpcache.py @@ -41,12 +41,10 @@ class HttpCacheMiddlewareTest(unittest.TestCase): def _get_settings(self, **new_settings): settings = { 'HTTPCACHE_ENABLED': True, - 'HTTPCACHE_USE_DUMMY': True, 'HTTPCACHE_DIR': self.tmpdir, 'HTTPCACHE_EXPIRATION_SECS': 1, 'HTTPCACHE_IGNORE_HTTP_CODES': [], - 'HTTPCACHE_POLICY_REQUEST': lambda request : True, - 'HTTPCACHE_POLICY_RESPONSE': lambda response : True, + 'HTTPCACHE_POLICY': 'dummy' } settings.update(new_settings) return Settings(settings) @@ -54,7 +52,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): @contextmanager def _storage(self, **new_settings): settings = self._get_settings(**new_settings) - if settings.getbool('HTTPCACHE_USE_DUMMY'): + if settings.get('HTTPCACHE_POLICY') == 'dummy': storage = self.storage_class(settings) else: storage = self.realcache_storage_class(settings) @@ -187,7 +185,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): def test_real_http_cache_middleware_response304_not_cached(self): # test response is not cached because the status is 304 Not Modified # (so it should be cached already) - with self._middleware(HTTPCACHE_USE_DUMMY=False) as mw: + with self._middleware(HTTPCACHE_POLICY='rfc2616') as mw: assert mw.process_request(self.request, self.spider) is None response = Response('http://www.example.com', status=304) mw.process_response(self.request, response, self.spider) @@ -199,7 +197,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): def test_real_http_cache_middleware_response_nostore_not_cached(self): # test response is not cached because of the Cache-Control 'no-store' directive # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2 - with self._middleware(HTTPCACHE_USE_DUMMY=False) as mw: + with self._middleware(HTTPCACHE_POLICY='rfc2616') as mw: assert mw.process_request(self.request, self.spider) is None response = Response('http://www.example.com', headers= {'Content-Type': 'text/html', 'Cache-Control': 'no-store'}, @@ -212,7 +210,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): def test_real_http_cache_middleware_request_nostore_not_cached(self): # test response is not cached because of the request's Cache-Control 'no-store' directive # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2 - with self._middleware(HTTPCACHE_USE_DUMMY=False) as mw: + with self._middleware(HTTPCACHE_POLICY='rfc2616') as mw: request = Request('http://www.example.com', headers={'User-Agent': 'test', 'Cache-Control': 'no-store'}) assert mw.process_request(request, self.spider) is None @@ -223,14 +221,14 @@ class HttpCacheMiddlewareTest(unittest.TestCase): def test_real_http_cache_middleware_response_cached_and_fresh(self): # test response cached and fresh - with self._middleware(HTTPCACHE_USE_DUMMY=False) as mw: + with self._middleware(HTTPCACHE_POLICY='rfc2616') as mw: response = mw.process_response(self.request, self.response, self.spider) self.assertRaises(IgnoreRequest, mw.process_request, self.request, self.spider) assert 'cached' not in response.flags def test_real_http_cache_middleware_response_cached_and_stale(self): # test response cached but stale - with self._middleware(HTTPCACHE_USE_DUMMY=False, + with self._middleware(HTTPCACHE_POLICY='rfc2616', HTTPCACHE_STORAGE = 'scrapy.contrib.httpcache.DbmRealCacheStorage') as mw: response = Response('http://www.example.com', headers= {'Content-Type': 'text/html', 'Cache-Control': 'no-cache'}, @@ -244,7 +242,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): def test_real_http_cache_storage_response_cached_and_fresh(self): # test response is cached and is fresh # (response requested should be same as response received) - with self._storage(HTTPCACHE_USE_DUMMY=False) as storage: + with self._storage(HTTPCACHE_POLICY='rfc2616') as storage: assert storage.retrieve_response(self.spider, self.request) is None response = Response('http://www.example.com', headers= @@ -257,7 +255,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): def test_real_http_cache_storage_response403_cached_and_further_requests_ignored(self): # test response is cached but further requests are ignored # because response status is 403 (as per the RFC) - with self._storage(HTTPCACHE_USE_DUMMY=False) as storage: + with self._storage(HTTPCACHE_POLICY='rfc2616') as storage: assert storage.retrieve_response(self.spider, self.request) is None response = Response('http://www.example.com', headers= @@ -270,7 +268,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): def test_real_http_cache_storage_response_cached_and_stale(self): # test response is cached and is stale (no cache validators inserted) # (request should be same as response received) - with self._storage(HTTPCACHE_USE_DUMMY=False) as storage: + with self._storage(HTTPCACHE_POLICY='rfc2616') as storage: assert storage.retrieve_response(self.spider, self.request) is None response = Response('http://www.example.com', headers= @@ -283,7 +281,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): def test_real_http_cache_storage_response_cached_and_stale_with_cache_validators(self): # test response is cached and is stale and cache validators are inserted - with self._storage(HTTPCACHE_USE_DUMMY=False) as storage: + with self._storage(HTTPCACHE_POLICY='rfc2616') as storage: assert storage.retrieve_response(self.spider, self.request) is None response = Response('http://www.example.com', headers= @@ -297,7 +295,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): def test_real_http_cache_storage_response_cached_and_transparent(self): # test response is not cached because of the request's Cache-Control 'no-cache' directive # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2 - with self._storage(HTTPCACHE_USE_DUMMY=False) as storage: + with self._storage(HTTPCACHE_POLICY='rfc2616') as storage: request = Request('http://www.example.com', headers={'User-Agent': 'test', 'Cache-Control': 'no-cache'}) assert storage.retrieve_response(self.spider, request) is None