Support default headers per spider. closes #181

--HG--
extra : rebase_source : 60162dffa4fbab525501e46b479dc272b8998942

scrapy/contrib/downloadermiddleware/defaultheaders.py

@@ -3,15 +3,27 @@ DefaultHeaders downloader middleware
 
 See documentation in docs/topics/downloader-middleware.rst
 """
-
 from scrapy.conf import settings
+from scrapy.xlib.pydispatch import dispatcher
+from scrapy.core import signals
+
 
 class DefaultHeadersMiddleware(object):
 
     def __init__(self):
-        self.default_headers = settings.get('DEFAULT_REQUEST_HEADERS')
+        self.global_default_headers = settings.get('DEFAULT_REQUEST_HEADERS')
+        self._default_headers = {}
+        dispatcher.connect(self.spider_opened, signal=signals.spider_opened)
+        dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
 
     def process_request(self, request, spider):
-        for k, v in self.default_headers.iteritems():
+        for k, v in self._default_headers[spider].iteritems():
             if v:
                 request.headers.setdefault(k, v)
+
+    def spider_opened(self, spider):
+        self._default_headers[spider] = dict(self.global_default_headers,
+                **getattr(spider, 'default_headers', {}))
+
+    def spider_closed(self, spider):
+        self._default_headers.pop(spider)

scrapy/tests/test_downloadermiddleware_defaultheaders.py

@@ -2,7 +2,7 @@ from unittest import TestCase
 
 from scrapy.conf import settings
 from scrapy.contrib.downloadermiddleware.defaultheaders import DefaultHeadersMiddleware
-from scrapy.http import Response, Request
+from scrapy.http import Request
 from scrapy.spider import BaseSpider
 
 
@@ -16,15 +16,33 @@ class TestDefaultHeadersMiddleware(TestCase):
 
     def test_process_request(self):
         req = Request('http://www.scrapytest.org')
+        self.mw.spider_opened(self.spider)
         self.mw.process_request(req, self.spider)
+        self.mw.spider_closed(self.spider)
         self.assertEquals(req.headers, self.default_headers)
 
+    def test_spider_default_headers(self):
+        spider_headers = {'Unexistant-Header': ['value']}
+        # override one of the global default headers by spider
+        if self.default_headers:
+            k = set(self.default_headers).pop()
+            spider_headers[k] = ['__newvalue__']
+        self.spider.default_headers = spider_headers
+
+        req = Request('http://www.scrapytest.org')
+        self.mw.spider_opened(self.spider)
+        self.mw.process_request(req, self.spider)
+        self.mw.spider_closed(self.spider)
+        self.assertEquals(req.headers, dict(self.default_headers, **spider_headers))
+
     def test_update_headers(self):
         headers = {'Accept-Language': ['es'], 'Test-Header': ['test']}
         req = Request('http://www.scrapytest.org', headers=headers)
         self.assertEquals(req.headers, headers)
 
+        self.mw.spider_opened(self.spider)
         self.mw.process_request(req, self.spider)
+        self.mw.spider_closed(self.spider)
         self.default_headers.update(headers)
         self.assertEquals(req.headers, self.default_headers)