From b1dad251ae9602fa7c36e87ab1940f7b7508248a Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 25 May 2009 14:41:06 -0300 Subject: [PATCH] Deprecated Common Downloader Middleware and added DefaultHeaders Downloader Middleware --- docs/intro/tutorial.rst | 9 ++-- docs/ref/downloader-middleware.rst | 31 ++++---------- docs/ref/settings.rst | 41 ++++++++----------- scrapy/conf/default_settings.py | 10 +++-- scrapy/contrib/downloadermiddleware/common.py | 22 +++------- .../downloadermiddleware/defaultheaders.py | 17 ++++++++ 6 files changed, 56 insertions(+), 74 deletions(-) create mode 100644 scrapy/contrib/downloadermiddleware/defaultheaders.py diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 6d2533ff0..c4f832509 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -131,10 +131,11 @@ The ``crawl dmoz.org`` subcommand runs the spider for the ``dmoz.org`` domain, you'll get an output like this:: [-] Log opened. - [dmoz] INFO: Enabled extensions: TelnetConsole, WebConsole - [dmoz] INFO: Enabled downloader middlewares: ErrorPagesMiddleware, CookiesMiddleware, HttpAuthMiddleware, UserAgentMiddleware, RetryMiddleware, CommonMiddleware, RedirectMiddleware, HttpCompressionMiddleware - [dmoz] INFO: Enabled spider middlewares: OffsiteMiddleware, RefererMiddleware, UrlLengthMiddleware, DepthMiddleware, UrlFilterMiddleware - [dmoz] INFO: Enabled item pipelines: + [dmoz] INFO: Enabled extensions: ... + [dmoz] INFO: Enabled scheduler middlewares: ... + [dmoz] INFO: Enabled downloader middlewares: ... + [dmoz] INFO: Enabled spider middlewares: ... + [dmoz] INFO: Enabled item pipelines: ... [-] scrapy.management.web.WebConsole starting on 60738 [-] scrapy.management.telnet.TelnetConsole starting on 51506 [dmoz/dmoz.org] INFO: Domain opened diff --git a/docs/ref/downloader-middleware.rst b/docs/ref/downloader-middleware.rst index ed9d017f7..cb91c6932 100644 --- a/docs/ref/downloader-middleware.rst +++ b/docs/ref/downloader-middleware.rst @@ -12,33 +12,16 @@ middleware, see the :ref:`downloader middleware usage guide Available downloader middlewares ================================ -.. _ref-downloader-middleware-common: +DefaultHeadersMiddleware +------------------------ -"Common" downloader middleware ------------------------------- +.. module:: scrapy.contrib.downloadermiddleware.defaultheaders + :synopsis: Default Headers Downloader Middleware -.. module:: scrapy.contrib.downloadermiddleware.common - :synopsis: Downloader middleware for performing basic required tasks +.. class:: scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware -.. class:: scrapy.contrib.downloadermiddleware.common.CommonMiddleware - -This middleware performs some commonly required tasks over all requests, and -thus it's recommended to leave it always enabled. Those tasks are: - - * If the ``Accept`` request header is not already set, then set it to - :setting:`REQUEST_HEADER_ACCEPT` - - * If the ``Accept-Language`` request header is not already set, then set it - to :setting:`REQUEST_HEADER_ACCEPT_LANGUAGE` - - * If the request method is ``POST`` and the ``Content-Type`` header is not - set, then set it to ``'application/x-www-form-urlencoded'``, the `default - Form content type`_. - - * If the request contains a body and the ``Content-Length`` headers it not - set, then set it to the ``len(body)``. - -.. _default Form content type: http://www.w3.org/TR/html401/interact/forms.html#h-17.13.4.1 +This middleware sets all default requests headers specified in the +:setting:`DEFAULT_REQUEST_HEADERS` setting. HttpCacheMiddleware ------------------- diff --git a/docs/ref/settings.rst b/docs/ref/settings.rst index 31085b3bd..e1714eadd 100644 --- a/docs/ref/settings.rst +++ b/docs/ref/settings.rst @@ -252,6 +252,21 @@ Default: ``'scrapy.item.ScrapedItem'`` The default class that will be used for instantiating items in the :ref:`the Scrapy shell `. +.. setting:: DEFAULT_REQUEST_HEADERS + +DEFAULT_REQUEST_HEADERS +----------------------- + +Default:: + + { + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Accept-Language': 'en', + } + +The default headers used for Scrapy HTTP Requests. They're populated in the +DefaultHeadersMiddleware. + .. setting:: DEFAULT_SPIDER DEFAULT_SPIDER @@ -314,7 +329,7 @@ Default:: 'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware': 300, 'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware': 400, 'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware': 500, - 'scrapy.contrib.downloadermiddleware.common.CommonMiddleware': 550, + 'scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware': 550, 'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware': 600, 'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware': 700, 'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 800, @@ -655,30 +670,6 @@ Default: ``Not Defined`` The name of the current project. It matches the project module name as created by ``startproject`` command, and is only defined by project settings file. -.. setting:: REQUEST_HEADER_ACCEPT - -REQUEST_HEADER_ACCEPT ---------------------- - -Default: ``'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'`` - -Default value to use for the ``Accept`` request header (if not already set -before). - -See :ref:`ref-downloader-middleware-common`. - -.. setting:: REQUEST_HEADER_ACCEPT_LANGUAGE - -REQUEST_HEADER_ACCEPT_LANGUAGE ------------------------------- - -Default: ``'en'`` - -Default value to use for the ``Accept-Language`` request header, if not already -set before. - -See :ref:`ref-downloader-middleware-common`. - .. setting:: REQUESTS_QUEUE_SIZE REQUESTS_PER_DOMAIN diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index b7e742cdc..191c3e1e8 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -44,6 +44,11 @@ COOKIES_DEBUG = False DEFAULT_ITEM_CLASS = 'scrapy.item.ScrapedItem' +DEFAULT_REQUEST_HEADERS = { + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Accept-Language': 'en', +} + DEFAULT_SPIDER = None DEPTH_LIMIT = 0 @@ -63,7 +68,7 @@ DOWNLOADER_MIDDLEWARES_BASE = { 'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware': 300, 'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware': 400, 'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware': 500, - 'scrapy.contrib.downloadermiddleware.common.CommonMiddleware': 550, + 'scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware': 550, 'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware': 600, 'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware': 700, 'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 800, @@ -138,9 +143,6 @@ PRIORITIZER = 'scrapy.core.prioritizers.RandomPrioritizer' REDIRECTMIDDLEWARE_MAX_TIMES = 20 # uses Firefox default setting -REQUEST_HEADER_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' -REQUEST_HEADER_ACCEPT_LANGUAGE = 'en' - REQUESTS_QUEUE_SIZE = 0 REQUESTS_PER_DOMAIN = 8 # max simultaneous requests per domain diff --git a/scrapy/contrib/downloadermiddleware/common.py b/scrapy/contrib/downloadermiddleware/common.py index 188a1f05f..273ec7995 100644 --- a/scrapy/contrib/downloadermiddleware/common.py +++ b/scrapy/contrib/downloadermiddleware/common.py @@ -1,22 +1,10 @@ -""" -Common downloader middleware -See documentation in docs/ref/downloader-middleware.rst -""" +import warnings -from scrapy.conf import settings +from scrapy.contrib.downloadermiddleware.defaultheaders import DefaultHeadersMiddleware -class CommonMiddleware(object): +class CommonMiddleware(DefaultHeadersMiddleware): def __init__(self): - self.header_accept = settings.get('REQUEST_HEADER_ACCEPT') - self.header_accept_language = settings.get('REQUEST_HEADER_ACCEPT_LANGUAGE') - - def process_request(self, request, spider): - request.headers.setdefault('Accept', self.header_accept) - request.headers.setdefault('Accept-Language', self.header_accept_language) - if request.method == 'POST': - request.headers.setdefault('Content-Type', 'application/x-www-form-urlencoded') - if request.body: - request.headers.setdefault('Content-Length', '%d' % len(request.body)) - + warnings.warn("scrapy.contrib.downloadermiddleware.common.CommonMiddleware has been replaced by scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware") + DefaultHeadersMiddleware.__init__(self) diff --git a/scrapy/contrib/downloadermiddleware/defaultheaders.py b/scrapy/contrib/downloadermiddleware/defaultheaders.py new file mode 100644 index 000000000..dc95c2f9f --- /dev/null +++ b/scrapy/contrib/downloadermiddleware/defaultheaders.py @@ -0,0 +1,17 @@ +""" +DefaultHeaders downloader middleware + +See documentation in docs/ref/downloader-middleware.rst +""" + +from scrapy.conf import settings + +class DefaultHeadersMiddleware(object): + + def __init__(self): + self.default_headers = settings.get('DEFAULT_REQUEST_HEADERS') + + def process_request(self, request, spider): + for k, v in self.default_headers.iteritems(): + if v: + request.headers.setdefault(k, v)