Deprecated Common Downloader Middleware and added DefaultHeaders Downloader

Middleware

docs/intro/tutorial.rst

@@ -131,10 +131,11 @@ The ``crawl dmoz.org`` subcommand runs the spider for the ``dmoz.org`` domain,
 you'll get an output like this:: 
 
    [-] Log opened.
-   [dmoz] INFO: Enabled extensions: TelnetConsole, WebConsole
-   [dmoz] INFO: Enabled downloader middlewares: ErrorPagesMiddleware, CookiesMiddleware, HttpAuthMiddleware, UserAgentMiddleware, RetryMiddleware, CommonMiddleware, RedirectMiddleware, HttpCompressionMiddleware
-   [dmoz] INFO: Enabled spider middlewares: OffsiteMiddleware, RefererMiddleware, UrlLengthMiddleware, DepthMiddleware, UrlFilterMiddleware
-   [dmoz] INFO: Enabled item pipelines: 
+   [dmoz] INFO: Enabled extensions: ...
+   [dmoz] INFO: Enabled scheduler middlewares: ...
+   [dmoz] INFO: Enabled downloader middlewares: ...
+   [dmoz] INFO: Enabled spider middlewares: ...
+   [dmoz] INFO: Enabled item pipelines: ...
    [-] scrapy.management.web.WebConsole starting on 60738
    [-] scrapy.management.telnet.TelnetConsole starting on 51506
    [dmoz/dmoz.org] INFO: Domain opened

docs/ref/downloader-middleware.rst

@@ -12,33 +12,16 @@ middleware, see the :ref:`downloader middleware usage guide
 Available downloader middlewares
 ================================
 
-.. _ref-downloader-middleware-common:
+DefaultHeadersMiddleware
+------------------------
 
-"Common" downloader middleware
-------------------------------
+.. module:: scrapy.contrib.downloadermiddleware.defaultheaders
+   :synopsis: Default Headers Downloader Middleware
 
-.. module:: scrapy.contrib.downloadermiddleware.common
-   :synopsis: Downloader middleware for performing basic required tasks
+.. class:: scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware
 
-.. class:: scrapy.contrib.downloadermiddleware.common.CommonMiddleware
-
-This middleware performs some commonly required tasks over all requests, and
-thus it's recommended to leave it always enabled. Those tasks are:
-
-    * If the ``Accept`` request header is not already set, then set it to
-      :setting:`REQUEST_HEADER_ACCEPT`
-    
-    * If the ``Accept-Language`` request header is not already set, then set it
-      to :setting:`REQUEST_HEADER_ACCEPT_LANGUAGE` 
-
-    * If the request method is ``POST`` and the ``Content-Type`` header is not
-      set, then set it to ``'application/x-www-form-urlencoded'``, the `default
-      Form content type`_.
-
-    * If the request contains a body and the ``Content-Length`` headers it not
-      set, then set it to the ``len(body)``.
-    
-.. _default Form content type: http://www.w3.org/TR/html401/interact/forms.html#h-17.13.4.1
+This middleware sets all default requests headers specified in the
+:setting:`DEFAULT_REQUEST_HEADERS` setting.
 
 HttpCacheMiddleware
 -------------------

docs/ref/settings.rst

@@ -252,6 +252,21 @@ Default: ``'scrapy.item.ScrapedItem'``
 The default class that will be used for instantiating items in the :ref:`the
 Scrapy shell <topics-shell>`.
 
+.. setting:: DEFAULT_REQUEST_HEADERS
+
+DEFAULT_REQUEST_HEADERS
+-----------------------
+
+Default::
+
+    {
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Accept-Language': 'en',
+    }
+
+The default headers used for Scrapy HTTP Requests. They're populated in the
+DefaultHeadersMiddleware.
+
 .. setting:: DEFAULT_SPIDER
 
 DEFAULT_SPIDER
@@ -314,7 +329,7 @@ Default::
         'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware': 300,
         'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware': 400,
         'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware': 500,
-        'scrapy.contrib.downloadermiddleware.common.CommonMiddleware': 550,
+        'scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware': 550,
         'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware': 600,
         'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware': 700,
         'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 800,
@@ -655,30 +670,6 @@ Default: ``Not Defined``
 The name of the current project. It matches the project module name as created
 by ``startproject`` command, and is only defined by project settings file.
 
-.. setting:: REQUEST_HEADER_ACCEPT
-
-REQUEST_HEADER_ACCEPT
----------------------
-
-Default: ``'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'``
-
-Default value to use for the ``Accept`` request header (if not already set
-before). 
-
-See :ref:`ref-downloader-middleware-common`.
-
-.. setting:: REQUEST_HEADER_ACCEPT_LANGUAGE
-
-REQUEST_HEADER_ACCEPT_LANGUAGE
-------------------------------
-
-Default: ``'en'``
-
-Default value to use for the ``Accept-Language`` request header, if not already
-set before. 
-
-See :ref:`ref-downloader-middleware-common`.
-
 .. setting:: REQUESTS_QUEUE_SIZE
 
 REQUESTS_PER_DOMAIN

scrapy/conf/default_settings.py

@@ -44,6 +44,11 @@ COOKIES_DEBUG = False
 
 DEFAULT_ITEM_CLASS = 'scrapy.item.ScrapedItem'
 
+DEFAULT_REQUEST_HEADERS = {
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    'Accept-Language': 'en',
+}
+
 DEFAULT_SPIDER = None
 
 DEPTH_LIMIT = 0
@@ -63,7 +68,7 @@ DOWNLOADER_MIDDLEWARES_BASE = {
     'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware': 300,
     'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware': 400,
     'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware': 500,
-    'scrapy.contrib.downloadermiddleware.common.CommonMiddleware': 550,
+    'scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware': 550,
     'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware': 600,
     'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware': 700,
     'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 800,
@@ -138,9 +143,6 @@ PRIORITIZER = 'scrapy.core.prioritizers.RandomPrioritizer'
 
 REDIRECTMIDDLEWARE_MAX_TIMES = 20 # uses Firefox default setting
 
-REQUEST_HEADER_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
-REQUEST_HEADER_ACCEPT_LANGUAGE = 'en'
-
 REQUESTS_QUEUE_SIZE = 0
 REQUESTS_PER_DOMAIN = 8     # max simultaneous requests per domain
 

scrapy/contrib/downloadermiddleware/common.py

@@ -1,22 +1,10 @@
-"""
-Common downloader middleware
 
-See documentation in docs/ref/downloader-middleware.rst
-"""
+import warnings
 
-from scrapy.conf import settings
+from scrapy.contrib.downloadermiddleware.defaultheaders import DefaultHeadersMiddleware
 
-class CommonMiddleware(object):
+class CommonMiddleware(DefaultHeadersMiddleware):
 
     def __init__(self):
-        self.header_accept = settings.get('REQUEST_HEADER_ACCEPT')
-        self.header_accept_language = settings.get('REQUEST_HEADER_ACCEPT_LANGUAGE')
-
-    def process_request(self, request, spider):
-        request.headers.setdefault('Accept', self.header_accept)
-        request.headers.setdefault('Accept-Language', self.header_accept_language)
-        if request.method == 'POST':
-            request.headers.setdefault('Content-Type', 'application/x-www-form-urlencoded')
-            if request.body:
-                request.headers.setdefault('Content-Length', '%d' % len(request.body))
-
+        warnings.warn("scrapy.contrib.downloadermiddleware.common.CommonMiddleware has been replaced by scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware")
+        DefaultHeadersMiddleware.__init__(self)

scrapy/contrib/downloadermiddleware/defaultheaders.py

@@ -0,0 +1,17 @@
+"""
+DefaultHeaders downloader middleware
+
+See documentation in docs/ref/downloader-middleware.rst
+"""
+
+from scrapy.conf import settings
+
+class DefaultHeadersMiddleware(object):
+
+    def __init__(self):
+        self.default_headers = settings.get('DEFAULT_REQUEST_HEADERS')
+
+    def process_request(self, request, spider):
+        for k, v in self.default_headers.iteritems():
+            if v:
+                request.headers.setdefault(k, v)