diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index dfd30d3b2..77f327b35 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -177,7 +177,9 @@ DefaultHeadersMiddleware .. class:: DefaultHeadersMiddleware This middleware sets all default requests headers specified in the - :setting:`DEFAULT_REQUEST_HEADERS` setting. + :setting:`DEFAULT_REQUEST_HEADERS` setting plus those found in spider + ``default_request_headers`` attribute. Spider headers has precedence over + global headers. HttpAuthMiddleware ------------------ @@ -276,6 +278,17 @@ Number of seconds to use for HTTP cache expiration. Requests that were cached before this time will be re-downloaded. If zero, cached requests will always expire. A negative number means requests will never expire. +.. setting:: HTTPCACHE_IGNORE_HTTP_CODES + +HTTPCACHE_IGNORE_HTTP_CODES +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 0.10 + +Default: ``[]`` + +Don't cache response with these HTTP codes. + .. setting:: HTTPCACHE_IGNORE_MISSING HTTPCACHE_IGNORE_MISSING diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 2db13a5dd..fd1628ef4 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -2,8 +2,8 @@ Scrapy - a screen scraping framework written in Python """ -version_info = (0, 9, 0, '') -__version__ = "0.9" +version_info = (0, 10, 0, 'dev') +__version__ = "0.10-dev" import sys, os, warnings diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index 809a62b98..e85f43d11 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -123,6 +123,7 @@ HTTPCACHE_DIR = '' HTTPCACHE_IGNORE_MISSING = False HTTPCACHE_STORAGE = 'scrapy.contrib.downloadermiddleware.httpcache.FilesystemCacheStorage' HTTPCACHE_EXPIRATION_SECS = 0 +HTTPCACHE_IGNORE_HTTP_CODES = [] ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager' diff --git a/scrapy/contrib/downloadermiddleware/defaultheaders.py b/scrapy/contrib/downloadermiddleware/defaultheaders.py index cebfe247b..1fab70858 100644 --- a/scrapy/contrib/downloadermiddleware/defaultheaders.py +++ b/scrapy/contrib/downloadermiddleware/defaultheaders.py @@ -3,15 +3,27 @@ DefaultHeaders downloader middleware See documentation in docs/topics/downloader-middleware.rst """ - from scrapy.conf import settings +from scrapy.xlib.pydispatch import dispatcher +from scrapy.core import signals + class DefaultHeadersMiddleware(object): def __init__(self): - self.default_headers = settings.get('DEFAULT_REQUEST_HEADERS') + self.global_default_headers = settings.get('DEFAULT_REQUEST_HEADERS') + self._default_headers = {} + dispatcher.connect(self.spider_opened, signal=signals.spider_opened) + dispatcher.connect(self.spider_closed, signal=signals.spider_closed) def process_request(self, request, spider): - for k, v in self.default_headers.iteritems(): + for k, v in self._default_headers[spider].iteritems(): if v: request.headers.setdefault(k, v) + + def spider_opened(self, spider): + self._default_headers[spider] = dict(self.global_default_headers, + **getattr(spider, 'default_request_headers', {})) + + def spider_closed(self, spider): + self._default_headers.pop(spider) diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index 341e9b5a1..d1af33d7a 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -22,6 +22,7 @@ class HttpCacheMiddleware(object): def __init__(self, settings=conf.settings): self.storage = load_object(settings['HTTPCACHE_STORAGE'])(settings) self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING') + self.ignore_http_codes = map(int, settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES')) dispatcher.connect(self.spider_opened, signal=signals.spider_opened) dispatcher.connect(self.spider_closed, signal=signals.spider_closed) @@ -35,17 +36,20 @@ class HttpCacheMiddleware(object): if not self.is_cacheable(request): return response = self.storage.retrieve_response(spider, request) - if response: + if response and self.is_cacheable_response(response): response.flags.append('cached') return response elif self.ignore_missing: raise IgnoreRequest("Ignored request not in cache: %s" % request) def process_response(self, request, response, spider): - if self.is_cacheable(request): + if self.is_cacheable(request) and self.is_cacheable_response(response): self.storage.store_response(spider, request, response) return response + def is_cacheable_response(self, response): + return response.status not in self.ignore_http_codes + def is_cacheable(self, request): return urlparse_cached(request).scheme in ['http', 'https'] diff --git a/scrapy/tests/test_downloadermiddleware_defaultheaders.py b/scrapy/tests/test_downloadermiddleware_defaultheaders.py index 805289787..d24882d02 100644 --- a/scrapy/tests/test_downloadermiddleware_defaultheaders.py +++ b/scrapy/tests/test_downloadermiddleware_defaultheaders.py @@ -2,7 +2,7 @@ from unittest import TestCase from scrapy.conf import settings from scrapy.contrib.downloadermiddleware.defaultheaders import DefaultHeadersMiddleware -from scrapy.http import Response, Request +from scrapy.http import Request from scrapy.spider import BaseSpider @@ -11,20 +11,38 @@ class TestDefaultHeadersMiddleware(TestCase): def setUp(self): self.spider = BaseSpider('foo') self.mw = DefaultHeadersMiddleware() - self.default_headers = dict([(k, [v]) for k, v in \ + self.default_request_headers = dict([(k, [v]) for k, v in \ settings.get('DEFAULT_REQUEST_HEADERS').iteritems()]) def test_process_request(self): req = Request('http://www.scrapytest.org') + self.mw.spider_opened(self.spider) self.mw.process_request(req, self.spider) - self.assertEquals(req.headers, self.default_headers) + self.mw.spider_closed(self.spider) + self.assertEquals(req.headers, self.default_request_headers) + + def test_spider_default_request_headers(self): + spider_headers = {'Unexistant-Header': ['value']} + # override one of the global default headers by spider + if self.default_request_headers: + k = set(self.default_request_headers).pop() + spider_headers[k] = ['__newvalue__'] + self.spider.default_request_headers = spider_headers + + req = Request('http://www.scrapytest.org') + self.mw.spider_opened(self.spider) + self.mw.process_request(req, self.spider) + self.mw.spider_closed(self.spider) + self.assertEquals(req.headers, dict(self.default_request_headers, **spider_headers)) def test_update_headers(self): headers = {'Accept-Language': ['es'], 'Test-Header': ['test']} req = Request('http://www.scrapytest.org', headers=headers) self.assertEquals(req.headers, headers) + self.mw.spider_opened(self.spider) self.mw.process_request(req, self.spider) - self.default_headers.update(headers) - self.assertEquals(req.headers, self.default_headers) + self.mw.spider_closed(self.spider) + self.default_request_headers.update(headers) + self.assertEquals(req.headers, self.default_request_headers) diff --git a/scrapy/tests/test_downloadermiddleware_httpcache.py b/scrapy/tests/test_downloadermiddleware_httpcache.py index b9331110c..230352804 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcache.py +++ b/scrapy/tests/test_downloadermiddleware_httpcache.py @@ -24,6 +24,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): settings = { 'HTTPCACHE_DIR': self.tmpdir, 'HTTPCACHE_EXPIRATION_SECS': 1, + 'HTTPCACHE_IGNORE_HTTP_CODES': [], } settings.update(new_settings) return Settings(settings) @@ -76,6 +77,22 @@ class HttpCacheMiddlewareTest(unittest.TestCase): self.assertEqualResponse(self.response, response) assert 'cached' in response.flags + def test_middleware_ignore_http_codes(self): + # test response is not cached + mw = self._get_middleware(HTTPCACHE_IGNORE_HTTP_CODES=[202]) + assert mw.process_request(self.request, self.spider) is None + mw.process_response(self.request, self.response, self.spider) + assert mw.storage.retrieve_response(self.spider, self.request) is None + assert mw.process_request(self.request, self.spider) is None + + # test response is cached + mw = self._get_middleware(HTTPCACHE_IGNORE_HTTP_CODES=[203]) + mw.process_response(self.request, self.response, self.spider) + response = mw.process_request(self.request, self.spider) + assert isinstance(response, HtmlResponse) + self.assertEqualResponse(self.response, response) + assert 'cached' in response.flags + def assertEqualResponse(self, response1, response2): self.assertEqual(response1.url, response2.url) self.assertEqual(response1.status, response2.status)