added settings: REQUEST_HEADER_ACCEPT, REQUEST_HEADER_ACCEPT_LANGUAGE. started built-in downloader middleware reference

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40709

scrapy/trunk/docs/ref/downloader-middleware.rst

@@ -0,0 +1,39 @@
+.. _ref-downloader-middleware:
+
+========================================
+Built-in downloader middleware reference
+========================================
+
+This document explains all downloader middleware components that come with
+Scrapy. For information on how to use them and how to write your own downloader
+middleware, see the :ref:`downloader middleware usage guide
+<topics-downloader-middleware>`.
+
+Available downloader middlewares
+================================
+
+.. _ref-downloader-middleware-common:
+
+"Common" downloader middleware
+------------------------------
+
+.. module:: scrapy.contrib.downloadermiddleware.common
+   :synopsis: Downloader middleware for performing basic required tasks
+
+.. class:: scrapy.contrib.downloadermiddleware.common.CommonMiddleware
+
+This middleware performs some commonly required tasks over all requests, and
+thus it's recommended to leave it always enabled. Those tasks are:
+
+    * If the ``Accept`` request header is not already set, then set it to
+      :setting:`REQUEST_HEADER_ACCEPT`
+    
+    * If the ``Accept-Language`` request header is not already set, then set it
+      to :setting:`REQUEST_HEADER_ACCEPT_LANGUAGE` 
+
+    * If the request method is ``POST`` and the ``Content-Type`` header is not
+      set, then set it to ``'application/x-www-form-urlencoded'``, the `default
+      Form content type`_.
+    
+.. _default Form content type: http://www.w3.org/TR/html401/interact/forms.html#h-17.13.4.1
+

scrapy/trunk/docs/ref/index.rst

@@ -12,6 +12,7 @@ This section documents the API of Scrapy |version|. For more information see :re
    request-response
    extension-manager
    extensions
+   downloader-middleware
    settings
    signals
    logging

scrapy/trunk/docs/ref/settings.rst

@@ -574,6 +574,30 @@ Example::
 
     NEWSPIDER_MODULE = 'mybot.spiders_dev'
 
+.. setting:: REQUEST_HEADER_ACCEPT
+
+REQUEST_HEADER_ACCEPT
+---------------------
+
+Default: ``'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'``
+
+Default value to use for the ``Accept`` request header (if not already set
+before). 
+
+See :ref:`ref-downloader-middleware-common`.
+
+.. setting:: REQUEST_HEADER_ACCEPT_LANGUAGE
+
+REQUEST_HEADER_ACCEPT_LANGUAGE
+------------------------------
+
+Default: ``'en'``
+
+Default value to use for the ``Accept-Language`` request header, if not already
+set before. 
+
+See :ref:`ref-downloader-middleware-common`.
+
 .. setting:: REQUESTS_QUEUE_SIZE
 
 REQUESTS_PER_DOMAIN

scrapy/trunk/scrapy/conf/default_settings.py

@@ -129,6 +129,9 @@ NEWSPIDER_MODULE = ''
 
 PRIORITIZER = 'scrapy.core.prioritizers.RandomPrioritizer'
 
+REQUEST_HEADER_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
+REQUEST_HEADER_ACCEPT_LANGUAGE = 'en'
+
 REQUESTS_QUEUE_SIZE = 0
 REQUESTS_PER_DOMAIN = 8     # max simultaneous requests per domain
 

scrapy/trunk/scrapy/contrib/downloadermiddleware/common.py

@@ -1,10 +1,20 @@
+"""
+Common downloader middleare
+
+See documentation in docs/ref/downloader-middleware.rst
+"""
+
+from scrapy.conf import settings
+
 class CommonMiddleware(object):
-    """This middleware provides common/basic functionality, and should always
-    be enabled"""
+
+    def __init__(self):
+        self.header_accept = settings.get('REQUEST_HEADER_ACCEPT')
+        self.header_accept_language = settings.get('REQUEST_HEADER_ACCEPT_LANGUAGE')
 
     def process_request(self, request, spider):
-        request.headers.setdefault('Accept-Language', 'en')
-        request.headers.setdefault('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')
+        request.headers.setdefault('Accept', self.header_accept)
+        request.headers.setdefault('Accept-Language', self.header_accept_language)
         if request.method == 'POST':
             request.headers.setdefault('Content-Type', 'application/x-www-form-urlencoded')