RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update docs for defaultheaders middleware and change spider attribute to match global setting name

This commit is contained in:
Daniel Grana 2010-07-16 16:17:08 -03:00
parent 6883a99c1e
commit 3e013f564b
3 changed files with 13 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/topics/downloader-middleware.rst
View File

@ -177,7 +177,9 @@ DefaultHeadersMiddleware
.. class:: DefaultHeadersMiddleware
This middleware sets all default requests headers specified in the
:setting:`DEFAULT_REQUEST_HEADERS` setting.
:setting:`DEFAULT_REQUEST_HEADERS` setting plus those found in spider
``default_request_headers`` attribute. Spider headers has precedence over
global headers.
HttpAuthMiddleware
------------------

2
scrapy/contrib/downloadermiddleware/defaultheaders.py
View File

@ -23,7 +23,7 @@ class DefaultHeadersMiddleware(object):
def spider_opened(self, spider):
self._default_headers[spider] = dict(self.global_default_headers,
**getattr(spider, 'default_headers', {}))
**getattr(spider, 'default_request_headers', {}))
def spider_closed(self, spider):
self._default_headers.pop(spider)

18
scrapy/tests/test_downloadermiddleware_defaultheaders.py
View File

@ -11,7 +11,7 @@ class TestDefaultHeadersMiddleware(TestCase):
def setUp(self):
self.spider = BaseSpider('foo')
self.mw = DefaultHeadersMiddleware()
self.default_headers = dict([(k, [v]) for k, v in \
self.default_request_headers = dict([(k, [v]) for k, v in \
settings.get('DEFAULT_REQUEST_HEADERS').iteritems()])
def test_process_request(self):
@ -19,21 +19,21 @@ class TestDefaultHeadersMiddleware(TestCase):
self.mw.spider_opened(self.spider)
self.mw.process_request(req, self.spider)
self.mw.spider_closed(self.spider)
self.assertEquals(req.headers, self.default_headers)
self.assertEquals(req.headers, self.default_request_headers)
def test_spider_default_headers(self):
def test_spider_default_request_headers(self):
spider_headers = {'Unexistant-Header': ['value']}
# override one of the global default headers by spider
if self.default_headers:
k = set(self.default_headers).pop()
if self.default_request_headers:
k = set(self.default_request_headers).pop()
spider_headers[k] = ['__newvalue__']
self.spider.default_headers = spider_headers
self.spider.default_request_headers = spider_headers
req = Request('http://www.scrapytest.org')
self.mw.spider_opened(self.spider)
self.mw.process_request(req, self.spider)
self.mw.spider_closed(self.spider)
self.assertEquals(req.headers, dict(self.default_headers, **spider_headers))
self.assertEquals(req.headers, dict(self.default_request_headers, **spider_headers))
def test_update_headers(self):
headers = {'Accept-Language': ['es'], 'Test-Header': ['test']}
@ -43,6 +43,6 @@ class TestDefaultHeadersMiddleware(TestCase):
self.mw.spider_opened(self.spider)
self.mw.process_request(req, self.spider)
self.mw.spider_closed(self.spider)
self.default_headers.update(headers)
self.assertEquals(req.headers, self.default_headers)
self.default_request_headers.update(headers)
self.assertEquals(req.headers, self.default_request_headers)