Implemented policies for HTTP Cache

docs/topics/downloader-middleware.rst

@@ -495,6 +495,29 @@ Default: ``'anydbm'``
 The database module to use in the :ref:`DBM storage backend
 <httpcache-dbm-backend>`. This setting is specific to the DBM backend.
 
+HTTPCACHE_POLICY_REQUEST
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. versionadded:: 0.18
+
+Default: ```lambda request: True```
+
+A callback function used by the HTTP cache to decide whether a request
+is cacheable. The function should take a :class:`~scrapy.http.Request`
+object as a parameter and return ``True`` if a cached response can be returned;
+or ``False`` if it should be fetched again.
+
+HTTPCACHE_POLICY_RESPONSE
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. versionadded:: 0.18
+
+Default: ```lambda response: True```
+
+A callback function used by the HTTP cache to decide whether a response
+is cacheable. The function should take a :class:`~scrapy.http.Response`
+object as a parameter and return ``True`` if a response can be cached;
+or ``False`` if it should not be stored in the cache.
 
 HttpCompressionMiddleware
 -------------------------

scrapy/contrib/downloadermiddleware/httpcache.py

@@ -27,6 +27,8 @@ class HttpCacheMiddleware(object):
         self.ignore_http_codes = map(int, settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES'))
         self.use_dummy_cache = settings.getbool('HTTPCACHE_USE_DUMMY')
         self.stats = stats
+        self.policy_request = settings.get('HTTPCACHE_POLICY_REQUEST')
+        self.policy_response = settings.get('HTTPCACHE_POLICY_RESPONSE')
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -86,12 +88,14 @@ class HttpCacheMiddleware(object):
         retval = response.status not in self.ignore_http_codes
         if not self.use_dummy_cache and response.headers.has_key('cache-control'):
             retval = retval and (response.headers['cache-control'].lower().find('no-store') == -1)
+        retval = retval and self.policy_response(response)
         return retval
 
     def is_cacheable(self, request):
         retval = urlparse_cached(request).scheme not in self.ignore_schemes
         if not self.use_dummy_cache and request.headers.has_key('cache-control'):
             retval = retval and (request.headers['cache-control'].lower().find('no-store') == -1)
+        retval = retval and self.policy_request(request)
         return retval
 
 class FilesystemCacheStorage(object):

scrapy/settings/default_settings.py

@@ -142,6 +142,8 @@ HTTPCACHE_EXPIRATION_SECS = 0
 HTTPCACHE_IGNORE_HTTP_CODES = []
 HTTPCACHE_IGNORE_SCHEMES = ['file']
 HTTPCACHE_DBM_MODULE = 'anydbm'
+HTTPCACHE_POLICY_REQUEST = lambda request : True
+HTTPCACHE_POLICY_RESPONSE = lambda response : True
 
 ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager'
 

scrapy/tests/test_downloadermiddleware_httpcache.py

@@ -45,6 +45,8 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
             'HTTPCACHE_DIR': self.tmpdir,
             'HTTPCACHE_EXPIRATION_SECS': 1,
             'HTTPCACHE_IGNORE_HTTP_CODES': [],
+            'HTTPCACHE_POLICY_REQUEST': lambda request : True,
+            'HTTPCACHE_POLICY_RESPONSE': lambda response : True,
         }
         settings.update(new_settings)
         return Settings(settings)
@@ -303,6 +305,52 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
             response = storage.retrieve_response(self.spider, request)
             assert isinstance(response, Request)
             self.assertEqualRequest(request, response)
+    
+    def test_http_cache_request_policy(self):
+        callback = lambda r: (r.url == 'http://www.example.com/')
+        
+        # Should be cached
+        req, res = Request('http://www.example.com/'), Response('http://www.example.com/')
+        with self._middleware(HTTPCACHE_POLICY_REQUEST=callback) as mw:
+            assert mw.process_request(req, self.spider) is None
+            mw.process_response(req, res, self.spider)
+
+            cached = mw.process_request(req, self.spider)
+            assert isinstance(cached, Response), type(cached)
+            self.assertEqualResponse(res, cached)
+            assert 'cached' in cached.flags
+        
+        # Should not be cached
+        req, res = Request('http://www.test.com/'), Response('http://www.test.com/')
+        with self._middleware(HTTPCACHE_POLICY_REQUEST=callback) as mw:
+            assert mw.process_request(req, self.spider) is None
+            mw.process_response(req, res, self.spider)
+
+            assert mw.storage.retrieve_response(self.spider, req) is None
+            assert mw.process_request(req, self.spider) is None
+            
+    def test_http_cache_response_policy(self):
+        callback = lambda r: (r.url == 'http://www.example.com/')
+        
+        # Should be cached
+        req, res = Request('http://www.example.com/'), Response('http://www.example.com/')
+        with self._middleware(HTTPCACHE_POLICY_RESPONSE=callback) as mw:
+            assert mw.process_request(req, self.spider) is None
+            mw.process_response(req, res, self.spider)
+
+            cached = mw.process_request(req, self.spider)
+            assert isinstance(cached, Response), type(cached)
+            self.assertEqualResponse(res, cached)
+            assert 'cached' in cached.flags
+        
+        # Should not be cached
+        req, res = Request('http://www.test.com/'), Response('http://www.test.com/')
+        with self._middleware(HTTPCACHE_POLICY_RESPONSE=callback) as mw:
+            assert mw.process_request(req, self.spider) is None
+            mw.process_response(req, res, self.spider)
+
+            assert mw.storage.retrieve_response(self.spider, req) is None
+            assert mw.process_request(req, self.spider) is None
 
     def assertEqualResponse(self, response1, response2):
         self.assertEqual(response1.url, response2.url)