From 93a1102189b5cc53b5816ba74568a0ab7e6f5e6f Mon Sep 17 00:00:00 2001 From: Hasnain Lakhani Date: Wed, 26 Dec 2012 16:29:48 -0800 Subject: [PATCH] Implemented policies for HTTP Cache --- docs/topics/downloader-middleware.rst | 23 +++++++++ .../contrib/downloadermiddleware/httpcache.py | 4 ++ scrapy/settings/default_settings.py | 2 + .../test_downloadermiddleware_httpcache.py | 48 +++++++++++++++++++ 4 files changed, 77 insertions(+) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index e42f85b64..48292e10c 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -495,6 +495,29 @@ Default: ``'anydbm'`` The database module to use in the :ref:`DBM storage backend `. This setting is specific to the DBM backend. +HTTPCACHE_POLICY_REQUEST +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 0.18 + +Default: ```lambda request: True``` + +A callback function used by the HTTP cache to decide whether a request +is cacheable. The function should take a :class:`~scrapy.http.Request` +object as a parameter and return ``True`` if a cached response can be returned; +or ``False`` if it should be fetched again. + +HTTPCACHE_POLICY_RESPONSE +^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 0.18 + +Default: ```lambda response: True``` + +A callback function used by the HTTP cache to decide whether a response +is cacheable. The function should take a :class:`~scrapy.http.Response` +object as a parameter and return ``True`` if a response can be cached; +or ``False`` if it should not be stored in the cache. HttpCompressionMiddleware ------------------------- diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index b1da47586..a5b7e5f05 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -27,6 +27,8 @@ class HttpCacheMiddleware(object): self.ignore_http_codes = map(int, settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES')) self.use_dummy_cache = settings.getbool('HTTPCACHE_USE_DUMMY') self.stats = stats + self.policy_request = settings.get('HTTPCACHE_POLICY_REQUEST') + self.policy_response = settings.get('HTTPCACHE_POLICY_RESPONSE') @classmethod def from_crawler(cls, crawler): @@ -86,12 +88,14 @@ class HttpCacheMiddleware(object): retval = response.status not in self.ignore_http_codes if not self.use_dummy_cache and response.headers.has_key('cache-control'): retval = retval and (response.headers['cache-control'].lower().find('no-store') == -1) + retval = retval and self.policy_response(response) return retval def is_cacheable(self, request): retval = urlparse_cached(request).scheme not in self.ignore_schemes if not self.use_dummy_cache and request.headers.has_key('cache-control'): retval = retval and (request.headers['cache-control'].lower().find('no-store') == -1) + retval = retval and self.policy_request(request) return retval class FilesystemCacheStorage(object): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 44b285f72..1dd88219c 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -142,6 +142,8 @@ HTTPCACHE_EXPIRATION_SECS = 0 HTTPCACHE_IGNORE_HTTP_CODES = [] HTTPCACHE_IGNORE_SCHEMES = ['file'] HTTPCACHE_DBM_MODULE = 'anydbm' +HTTPCACHE_POLICY_REQUEST = lambda request : True +HTTPCACHE_POLICY_RESPONSE = lambda response : True ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager' diff --git a/scrapy/tests/test_downloadermiddleware_httpcache.py b/scrapy/tests/test_downloadermiddleware_httpcache.py index d4c2b3dd7..176a9d86a 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcache.py +++ b/scrapy/tests/test_downloadermiddleware_httpcache.py @@ -45,6 +45,8 @@ class HttpCacheMiddlewareTest(unittest.TestCase): 'HTTPCACHE_DIR': self.tmpdir, 'HTTPCACHE_EXPIRATION_SECS': 1, 'HTTPCACHE_IGNORE_HTTP_CODES': [], + 'HTTPCACHE_POLICY_REQUEST': lambda request : True, + 'HTTPCACHE_POLICY_RESPONSE': lambda response : True, } settings.update(new_settings) return Settings(settings) @@ -303,6 +305,52 @@ class HttpCacheMiddlewareTest(unittest.TestCase): response = storage.retrieve_response(self.spider, request) assert isinstance(response, Request) self.assertEqualRequest(request, response) + + def test_http_cache_request_policy(self): + callback = lambda r: (r.url == 'http://www.example.com/') + + # Should be cached + req, res = Request('http://www.example.com/'), Response('http://www.example.com/') + with self._middleware(HTTPCACHE_POLICY_REQUEST=callback) as mw: + assert mw.process_request(req, self.spider) is None + mw.process_response(req, res, self.spider) + + cached = mw.process_request(req, self.spider) + assert isinstance(cached, Response), type(cached) + self.assertEqualResponse(res, cached) + assert 'cached' in cached.flags + + # Should not be cached + req, res = Request('http://www.test.com/'), Response('http://www.test.com/') + with self._middleware(HTTPCACHE_POLICY_REQUEST=callback) as mw: + assert mw.process_request(req, self.spider) is None + mw.process_response(req, res, self.spider) + + assert mw.storage.retrieve_response(self.spider, req) is None + assert mw.process_request(req, self.spider) is None + + def test_http_cache_response_policy(self): + callback = lambda r: (r.url == 'http://www.example.com/') + + # Should be cached + req, res = Request('http://www.example.com/'), Response('http://www.example.com/') + with self._middleware(HTTPCACHE_POLICY_RESPONSE=callback) as mw: + assert mw.process_request(req, self.spider) is None + mw.process_response(req, res, self.spider) + + cached = mw.process_request(req, self.spider) + assert isinstance(cached, Response), type(cached) + self.assertEqualResponse(res, cached) + assert 'cached' in cached.flags + + # Should not be cached + req, res = Request('http://www.test.com/'), Response('http://www.test.com/') + with self._middleware(HTTPCACHE_POLICY_RESPONSE=callback) as mw: + assert mw.process_request(req, self.spider) is None + mw.process_response(req, res, self.spider) + + assert mw.storage.retrieve_response(self.spider, req) is None + assert mw.process_request(req, self.spider) is None def assertEqualResponse(self, response1, response2): self.assertEqual(response1.url, response2.url)