Make ignored schemes configurable in HttpCacheMiddleware. closes #224

--HG--
extra : rebase_source : 2e6e8b93c642290f9bd6eb634eb4c8cd6da07c75

docs/topics/downloader-middleware.rst

@@ -300,6 +300,17 @@ If enabled, requests not found in the cache will be ignored instead of downloade
 
 .. setting:: HTTPCACHE_STORAGE
 
+HTTPCACHE_IGNORE_SCHEMES
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. versionadded:: 0.10
+
+Default: ``['file']``
+
+Don't cache responses with these URI schemes.
+
+.. setting:: HTTPCACHE_IGNORE_MISSING
+
 HTTPCACHE_STORAGE
 ^^^^^^^^^^^^^^^^^
 

scrapy/conf/default_settings.py

@@ -144,6 +144,7 @@ HTTPCACHE_IGNORE_MISSING = False
 HTTPCACHE_STORAGE = 'scrapy.contrib.downloadermiddleware.httpcache.FilesystemCacheStorage'
 HTTPCACHE_EXPIRATION_SECS = 0
 HTTPCACHE_IGNORE_HTTP_CODES = []
+HTTPCACHE_IGNORE_SCHEMES = ['file']
 
 ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager'
 

scrapy/contrib/downloadermiddleware/httpcache.py

@@ -22,6 +22,7 @@ class HttpCacheMiddleware(object):
     def __init__(self, settings=conf.settings):
         self.storage = load_object(settings['HTTPCACHE_STORAGE'])(settings)
         self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING')
+        self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES')
         self.ignore_http_codes = map(int, settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES'))
         dispatcher.connect(self.spider_opened, signal=signals.spider_opened)
         dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
@@ -51,7 +52,7 @@ class HttpCacheMiddleware(object):
         return response.status not in self.ignore_http_codes
 
     def is_cacheable(self, request):
-        return urlparse_cached(request).scheme in ['http', 'https']
+        return urlparse_cached(request).scheme not in self.ignore_schemes
 
 
 class FilesystemCacheStorage(object):

scrapy/tests/test_downloadermiddleware_httpcache.py

@@ -88,6 +88,43 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
         self.assertEqualResponse(self.response, response)
         assert 'cached' in response.flags
 
+    def test_middleware_ignore_schemes(self):
+        # http responses are cached by default
+        req, res = Request('http://test.com/'), Response('http://test.com/')
+        mw = self._get_middleware()
+        assert mw.process_request(req, self.spider) is None
+        mw.process_response(req, res, self.spider)
+        cached = mw.process_request(req, self.spider)
+        assert isinstance(cached, Response), type(cached)
+        self.assertEqualResponse(res, cached)
+        assert 'cached' in cached.flags
+
+        # file response is not cached by default
+        req, res = Request('file:///tmp/t.txt'), Response('file:///tmp/t.txt')
+        mw = self._get_middleware()
+        assert mw.process_request(req, self.spider) is None
+        mw.process_response(req, res, self.spider)
+        assert mw.storage.retrieve_response(self.spider, req) is None
+        assert mw.process_request(req, self.spider) is None
+
+        # s3 scheme response is cached by default
+        req, res = Request('s3://bucket/key'), Response('http://bucket/key')
+        mw = self._get_middleware()
+        assert mw.process_request(req, self.spider) is None
+        mw.process_response(req, res, self.spider)
+        cached = mw.process_request(req, self.spider)
+        assert isinstance(cached, Response), type(cached)
+        self.assertEqualResponse(res, cached)
+        assert 'cached' in cached.flags
+
+        # ignore s3 scheme
+        req, res = Request('s3://bucket/key2'), Response('http://bucket/key2')
+        mw = self._get_middleware(HTTPCACHE_IGNORE_SCHEMES=['s3'])
+        assert mw.process_request(req, self.spider) is None
+        mw.process_response(req, res, self.spider)
+        assert mw.storage.retrieve_response(self.spider, req) is None
+        assert mw.process_request(req, self.spider) is None
+
     def test_middleware_ignore_http_codes(self):
         # test response is not cached
         mw = self._get_middleware(HTTPCACHE_IGNORE_HTTP_CODES=[202])