From 49ac7de23162bd92dc0f2ba88967aa53bbc4af2b Mon Sep 17 00:00:00 2001 From: Raul Gallegos Date: Tue, 5 Jul 2016 15:38:17 -0500 Subject: [PATCH] prioritize default headers over user agent --- docs/topics/settings.rst | 6 +++--- scrapy/settings/default_settings.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 7f49aacdb..c845c59b9 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -459,9 +459,9 @@ Default:: 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100, 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300, 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350, - 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 400, - 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500, - 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 550, + 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400, + 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500, + 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550, 'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560, 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580, 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590, diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 8f064f81e..e563e56aa 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -93,9 +93,9 @@ DOWNLOADER_MIDDLEWARES_BASE = { 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100, 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300, 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350, - 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 400, - 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500, - 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 550, + 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400, + 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500, + 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550, 'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560, 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580, 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590,