diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index fe0bfeb8a..77f327b35 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -177,7 +177,9 @@ DefaultHeadersMiddleware .. class:: DefaultHeadersMiddleware This middleware sets all default requests headers specified in the - :setting:`DEFAULT_REQUEST_HEADERS` setting. + :setting:`DEFAULT_REQUEST_HEADERS` setting plus those found in spider + ``default_request_headers`` attribute. Spider headers has precedence over + global headers. HttpAuthMiddleware ------------------ diff --git a/scrapy/contrib/downloadermiddleware/defaultheaders.py b/scrapy/contrib/downloadermiddleware/defaultheaders.py index b3fd3cb37..1fab70858 100644 --- a/scrapy/contrib/downloadermiddleware/defaultheaders.py +++ b/scrapy/contrib/downloadermiddleware/defaultheaders.py @@ -23,7 +23,7 @@ class DefaultHeadersMiddleware(object): def spider_opened(self, spider): self._default_headers[spider] = dict(self.global_default_headers, - **getattr(spider, 'default_headers', {})) + **getattr(spider, 'default_request_headers', {})) def spider_closed(self, spider): self._default_headers.pop(spider) diff --git a/scrapy/tests/test_downloadermiddleware_defaultheaders.py b/scrapy/tests/test_downloadermiddleware_defaultheaders.py index 129495dc7..d24882d02 100644 --- a/scrapy/tests/test_downloadermiddleware_defaultheaders.py +++ b/scrapy/tests/test_downloadermiddleware_defaultheaders.py @@ -11,7 +11,7 @@ class TestDefaultHeadersMiddleware(TestCase): def setUp(self): self.spider = BaseSpider('foo') self.mw = DefaultHeadersMiddleware() - self.default_headers = dict([(k, [v]) for k, v in \ + self.default_request_headers = dict([(k, [v]) for k, v in \ settings.get('DEFAULT_REQUEST_HEADERS').iteritems()]) def test_process_request(self): @@ -19,21 +19,21 @@ class TestDefaultHeadersMiddleware(TestCase): self.mw.spider_opened(self.spider) self.mw.process_request(req, self.spider) self.mw.spider_closed(self.spider) - self.assertEquals(req.headers, self.default_headers) + self.assertEquals(req.headers, self.default_request_headers) - def test_spider_default_headers(self): + def test_spider_default_request_headers(self): spider_headers = {'Unexistant-Header': ['value']} # override one of the global default headers by spider - if self.default_headers: - k = set(self.default_headers).pop() + if self.default_request_headers: + k = set(self.default_request_headers).pop() spider_headers[k] = ['__newvalue__'] - self.spider.default_headers = spider_headers + self.spider.default_request_headers = spider_headers req = Request('http://www.scrapytest.org') self.mw.spider_opened(self.spider) self.mw.process_request(req, self.spider) self.mw.spider_closed(self.spider) - self.assertEquals(req.headers, dict(self.default_headers, **spider_headers)) + self.assertEquals(req.headers, dict(self.default_request_headers, **spider_headers)) def test_update_headers(self): headers = {'Accept-Language': ['es'], 'Test-Header': ['test']} @@ -43,6 +43,6 @@ class TestDefaultHeadersMiddleware(TestCase): self.mw.spider_opened(self.spider) self.mw.process_request(req, self.spider) self.mw.spider_closed(self.spider) - self.default_headers.update(headers) - self.assertEquals(req.headers, self.default_headers) + self.default_request_headers.update(headers) + self.assertEquals(req.headers, self.default_request_headers)