diff --git a/scrapy/trunk/docs/ref/downloader-middleware.rst b/scrapy/trunk/docs/ref/downloader-middleware.rst new file mode 100644 index 000000000..c2285320c --- /dev/null +++ b/scrapy/trunk/docs/ref/downloader-middleware.rst @@ -0,0 +1,39 @@ +.. _ref-downloader-middleware: + +======================================== +Built-in downloader middleware reference +======================================== + +This document explains all downloader middleware components that come with +Scrapy. For information on how to use them and how to write your own downloader +middleware, see the :ref:`downloader middleware usage guide +`. + +Available downloader middlewares +================================ + +.. _ref-downloader-middleware-common: + +"Common" downloader middleware +------------------------------ + +.. module:: scrapy.contrib.downloadermiddleware.common + :synopsis: Downloader middleware for performing basic required tasks + +.. class:: scrapy.contrib.downloadermiddleware.common.CommonMiddleware + +This middleware performs some commonly required tasks over all requests, and +thus it's recommended to leave it always enabled. Those tasks are: + + * If the ``Accept`` request header is not already set, then set it to + :setting:`REQUEST_HEADER_ACCEPT` + + * If the ``Accept-Language`` request header is not already set, then set it + to :setting:`REQUEST_HEADER_ACCEPT_LANGUAGE` + + * If the request method is ``POST`` and the ``Content-Type`` header is not + set, then set it to ``'application/x-www-form-urlencoded'``, the `default + Form content type`_. + +.. _default Form content type: http://www.w3.org/TR/html401/interact/forms.html#h-17.13.4.1 + diff --git a/scrapy/trunk/docs/ref/index.rst b/scrapy/trunk/docs/ref/index.rst index da225360e..8160936c1 100644 --- a/scrapy/trunk/docs/ref/index.rst +++ b/scrapy/trunk/docs/ref/index.rst @@ -12,6 +12,7 @@ This section documents the API of Scrapy |version|. For more information see :re request-response extension-manager extensions + downloader-middleware settings signals logging diff --git a/scrapy/trunk/docs/ref/settings.rst b/scrapy/trunk/docs/ref/settings.rst index 83ce87128..eab7bc010 100644 --- a/scrapy/trunk/docs/ref/settings.rst +++ b/scrapy/trunk/docs/ref/settings.rst @@ -574,6 +574,30 @@ Example:: NEWSPIDER_MODULE = 'mybot.spiders_dev' +.. setting:: REQUEST_HEADER_ACCEPT + +REQUEST_HEADER_ACCEPT +--------------------- + +Default: ``'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'`` + +Default value to use for the ``Accept`` request header (if not already set +before). + +See :ref:`ref-downloader-middleware-common`. + +.. setting:: REQUEST_HEADER_ACCEPT_LANGUAGE + +REQUEST_HEADER_ACCEPT_LANGUAGE +------------------------------ + +Default: ``'en'`` + +Default value to use for the ``Accept-Language`` request header, if not already +set before. + +See :ref:`ref-downloader-middleware-common`. + .. setting:: REQUESTS_QUEUE_SIZE REQUESTS_PER_DOMAIN diff --git a/scrapy/trunk/scrapy/conf/default_settings.py b/scrapy/trunk/scrapy/conf/default_settings.py index a5a4ce764..4ddd21cf6 100644 --- a/scrapy/trunk/scrapy/conf/default_settings.py +++ b/scrapy/trunk/scrapy/conf/default_settings.py @@ -129,6 +129,9 @@ NEWSPIDER_MODULE = '' PRIORITIZER = 'scrapy.core.prioritizers.RandomPrioritizer' +REQUEST_HEADER_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' +REQUEST_HEADER_ACCEPT_LANGUAGE = 'en' + REQUESTS_QUEUE_SIZE = 0 REQUESTS_PER_DOMAIN = 8 # max simultaneous requests per domain diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/common.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/common.py index 063ea72b7..91d95496c 100644 --- a/scrapy/trunk/scrapy/contrib/downloadermiddleware/common.py +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/common.py @@ -1,10 +1,20 @@ +""" +Common downloader middleare + +See documentation in docs/ref/downloader-middleware.rst +""" + +from scrapy.conf import settings + class CommonMiddleware(object): - """This middleware provides common/basic functionality, and should always - be enabled""" + + def __init__(self): + self.header_accept = settings.get('REQUEST_HEADER_ACCEPT') + self.header_accept_language = settings.get('REQUEST_HEADER_ACCEPT_LANGUAGE') def process_request(self, request, spider): - request.headers.setdefault('Accept-Language', 'en') - request.headers.setdefault('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8') + request.headers.setdefault('Accept', self.header_accept) + request.headers.setdefault('Accept-Language', self.header_accept_language) if request.method == 'POST': request.headers.setdefault('Content-Type', 'application/x-www-form-urlencoded')