diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index eec67ea66..f394008be 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -300,6 +300,17 @@ If enabled, requests not found in the cache will be ignored instead of downloade .. setting:: HTTPCACHE_STORAGE +HTTPCACHE_IGNORE_SCHEMES +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 0.10 + +Default: ``['file']`` + +Don't cache responses with these URI schemes. + +.. setting:: HTTPCACHE_IGNORE_MISSING + HTTPCACHE_STORAGE ^^^^^^^^^^^^^^^^^ diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index b6fac9107..d07d4e996 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -144,6 +144,7 @@ HTTPCACHE_IGNORE_MISSING = False HTTPCACHE_STORAGE = 'scrapy.contrib.downloadermiddleware.httpcache.FilesystemCacheStorage' HTTPCACHE_EXPIRATION_SECS = 0 HTTPCACHE_IGNORE_HTTP_CODES = [] +HTTPCACHE_IGNORE_SCHEMES = ['file'] ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager' diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index f2a45256f..f7a83e657 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -22,6 +22,7 @@ class HttpCacheMiddleware(object): def __init__(self, settings=conf.settings): self.storage = load_object(settings['HTTPCACHE_STORAGE'])(settings) self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING') + self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') self.ignore_http_codes = map(int, settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES')) dispatcher.connect(self.spider_opened, signal=signals.spider_opened) dispatcher.connect(self.spider_closed, signal=signals.spider_closed) @@ -51,7 +52,7 @@ class HttpCacheMiddleware(object): return response.status not in self.ignore_http_codes def is_cacheable(self, request): - return urlparse_cached(request).scheme in ['http', 'https'] + return urlparse_cached(request).scheme not in self.ignore_schemes class FilesystemCacheStorage(object): diff --git a/scrapy/tests/test_downloadermiddleware_httpcache.py b/scrapy/tests/test_downloadermiddleware_httpcache.py index a5b6ce02f..4e569ea7a 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcache.py +++ b/scrapy/tests/test_downloadermiddleware_httpcache.py @@ -88,6 +88,43 @@ class HttpCacheMiddlewareTest(unittest.TestCase): self.assertEqualResponse(self.response, response) assert 'cached' in response.flags + def test_middleware_ignore_schemes(self): + # http responses are cached by default + req, res = Request('http://test.com/'), Response('http://test.com/') + mw = self._get_middleware() + assert mw.process_request(req, self.spider) is None + mw.process_response(req, res, self.spider) + cached = mw.process_request(req, self.spider) + assert isinstance(cached, Response), type(cached) + self.assertEqualResponse(res, cached) + assert 'cached' in cached.flags + + # file response is not cached by default + req, res = Request('file:///tmp/t.txt'), Response('file:///tmp/t.txt') + mw = self._get_middleware() + assert mw.process_request(req, self.spider) is None + mw.process_response(req, res, self.spider) + assert mw.storage.retrieve_response(self.spider, req) is None + assert mw.process_request(req, self.spider) is None + + # s3 scheme response is cached by default + req, res = Request('s3://bucket/key'), Response('http://bucket/key') + mw = self._get_middleware() + assert mw.process_request(req, self.spider) is None + mw.process_response(req, res, self.spider) + cached = mw.process_request(req, self.spider) + assert isinstance(cached, Response), type(cached) + self.assertEqualResponse(res, cached) + assert 'cached' in cached.flags + + # ignore s3 scheme + req, res = Request('s3://bucket/key2'), Response('http://bucket/key2') + mw = self._get_middleware(HTTPCACHE_IGNORE_SCHEMES=['s3']) + assert mw.process_request(req, self.spider) is None + mw.process_response(req, res, self.spider) + assert mw.storage.retrieve_response(self.spider, req) is None + assert mw.process_request(req, self.spider) is None + def test_middleware_ignore_http_codes(self): # test response is not cached mw = self._get_middleware(HTTPCACHE_IGNORE_HTTP_CODES=[202])