diff --git a/docs/ref/settings.rst b/docs/ref/settings.rst index 2f3584547..107e3a4b8 100644 --- a/docs/ref/settings.rst +++ b/docs/ref/settings.rst @@ -296,26 +296,37 @@ Whether to enable the Downloader debugging mode. DOWNLOADER_MIDDLEWARES ---------------------- +Default:: ``{}`` + +A dict containing the downloader middlewares enabled in your project, and their +orders. For more info see :ref:`topics-downloader-middleware-setting`. + +.. setting:: DOWNLOADER_MIDDLEWARES_BASE + +DOWNLOADER_MIDDLEWARES_BASE +--------------------------- + Default:: - [ - 'scrapy.contrib.downloadermiddleware.robotstxt.RobotsTxtMiddleware', - 'scrapy.contrib.downloadermiddleware.errorpages.ErrorPagesMiddleware', - 'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware', - 'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware', - 'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware', - 'scrapy.contrib.downloadermiddleware.common.CommonMiddleware', - 'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware', - 'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware', - 'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware', - 'scrapy.contrib.downloadermiddleware.debug.CrawlDebug', - 'scrapy.contrib.downloadermiddleware.stats.DownloaderStats', - 'scrapy.contrib.downloadermiddleware.cache.CacheMiddleware', - ] + { + 'scrapy.contrib.downloadermiddleware.robotstxt.RobotsTxtMiddleware': 100, + 'scrapy.contrib.downloadermiddleware.errorpages.ErrorPagesMiddleware': 200, + 'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware': 300, + 'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware': 400, + 'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware': 500, + 'scrapy.contrib.downloadermiddleware.common.CommonMiddleware': 550, + 'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware': 600, + 'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware': 700, + 'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 800, + 'scrapy.contrib.downloadermiddleware.debug.CrawlDebug': 840, + 'scrapy.contrib.downloadermiddleware.stats.DownloaderStats': 850, + 'scrapy.contrib.downloadermiddleware.cache.CacheMiddleware': 900, + } -The list of enabled downloader middlewares. Keep in mind that some may need to -be enabled through a particular setting. The top (first) middleware is closer -to the engine, while the bottom (last) middleware is closer to the downloader. +A dict containing the downloader middlewares enabled by default in Scrapy. You +should never modify this setting in your project, modify +:setting:`DOWNLOADER_MIDDLEWARES` instead. For more info see +:ref:`topics-downloader-middleware-setting`. .. setting:: DOWNLOADER_STATS @@ -732,7 +743,17 @@ The order to use for the crawling scheduler. Available orders are: .. setting:: SCHEDULER_MIDDLEWARES SCHEDULER_MIDDLEWARES ----------------------- +--------------------- + +Default:: ``{}`` + +A dict containing the scheduler middlewares enabled in your project, and their +orders. + +.. setting:: SCHEDULER_MIDDLEWARES_BASE + +SCHEDULER_MIDDLEWARES_BASE +-------------------------- Default:: @@ -740,9 +761,10 @@ Default:: 'scrapy.contrib.schedulermiddleware.duplicatesfilter.DuplicatesFilterMiddleware', ] -The list of enabled scheduler middlewares. Keep in mind that some may need to -be enabled through a particular setting. The top (first) middleware is closer -to the engine, while the bottom (last) middleware is closer to the scheduler. + +A dict containing the scheduler middlewares enabled by default in Scrapy. You +should never modify this setting in your project, modify +:setting:`SCHEDULER_MIDDLEWARES` instead. .. setting:: SPIDERPROFILER_ENABLED @@ -759,21 +781,32 @@ performance. SPIDER_MIDDLEWARES ------------------ +Default:: ``{}`` + +A dict containing the spider middlewares enabled in your project, and their +orders. For more info see :ref:`topics-spider-middleware-setting`. + +.. setting:: SPIDER_MIDDLEWARES_BASE + +SPIDER_MIDDLEWARES_BASE +----------------------- + Default:: - [ - 'scrapy.contrib.itemsampler.ItemSamplerMiddleware', - 'scrapy.contrib.spidermiddleware.limit.RequestLimitMiddleware', - 'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware', - 'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware', - 'scrapy.contrib.spidermiddleware.referer.RefererMiddleware', - 'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware', - 'scrapy.contrib.spidermiddleware.depth.DepthMiddleware', - ] + { + 'scrapy.contrib.itemsampler.ItemSamplerMiddleware': 100, + 'scrapy.contrib.spidermiddleware.limit.RequestLimitMiddleware': 200, + 'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware': 300, + 'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': 500, + 'scrapy.contrib.spidermiddleware.referer.RefererMiddleware': 700, + 'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware': 800, + 'scrapy.contrib.spidermiddleware.depth.DepthMiddleware': 900, + } -The list of enabled spider middlewares. Keep in mind that some may need to be -enabled through a particular setting. The top (first) middleware is closer to -the engine, while the bottom (last) middleware is closer to the spider. +A dict containing the spider middlewares enabled by default in Scrapy. You +should never modify this setting in your project, modify +:setting:`SPIDER_MIDDLEWARES` instead. For more info see +:ref:`topics-spider-middleware-setting`. .. setting:: SPIDER_MODULES diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 71a102f08..22a4f9e89 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -6,22 +6,47 @@ Downloader Middleware The downloader middleware is a framework of hooks into Scrapy's request/response processing. It's a light, low-level system for globally -altering Scrapy's input and/or output. +altering Scrapy's requests and responses. + +.. _topics-downloader-middleware-setting: Activating a downloader middleware ================================== To activate a downloader middleware component, add it to the -:setting:`DOWNLOADER_MIDDLEWARES` list in your Scrapy settings. In -:setting:`DOWNLOADER_MIDDLEWARES`, each middleware component is represented by -a string: the full Python path to the middleware's class name. For example:: +:setting:`DOWNLOADER_MIDDLEWARES` setting, which is a dict whose keys are the +middleware class paths and their values are the middleware orders. - DOWNLOADER_MIDDLEWARES = [ - 'scrapy.contrib.middleware.common.SpiderMiddleware', - 'scrapy.contrib.middleware.common.CommonMiddleware', - 'scrapy.contrib.middleware.redirect.RedirectMiddleware', - 'scrapy.contrib.middleware.cache.CacheMiddleware', - ] +Here's an example:: + + DOWNLOADER_MIDDLEWARES = { + 'myproject.middlewares.CustomDownloaderMiddleware': 543, + } + +The :setting:`DOWNLOADER_MIDDLEWARES` setting is merged with the +:setting:`DOWNLOADER_MIDDLEWARES_BASE` setting defined in Scrapy (and not meant to +be overridden) and then sorted by order to get the final sorted list of enabled +middlewares: the first middleware is the one closer to the engine and the last +is the one closer to the downloader. + +To decide which order to assign to your middleware see the +:setting:`DOWNLOADER_MIDDLEWARES_BASE` setting and pick a value according to +where you want to insert the middleware. The order does matter because each +middleware performs a different action and your middleware could depend on some +previous (or subsequent) middleware being applied. + +If you want to disable a builtin middleware (the ones defined in +:setting:`DOWNLOADER_MIDDLEWARES_BASE` and enabled by default) you must define it +in your project :setting:`DOWNLOADER_MIDDLEWARES` setting and assign `None` +as its value. For example, if you want to disable the off-site middleware:: + + DOWNLOADER_MIDDLEWARES = { + 'myproject.middlewares.CustomDownloaderMiddleware': 543, + 'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware': None, + } + +Finally, keep in mind that some middlewares may need to be enabled through a +particular setting. See each middleware documentation for more info. Writing your own downloader middleware ====================================== diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 63278a09c..7e9db4809 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -9,25 +9,49 @@ mechanism where you can plug custom functionality to process the requests that are sent to :ref:`topics-spiders` for processing and to process the responses and item that are generated from spiders. +.. _topics-spider-middleware-setting: + Activating a spider middleware ============================== +To activate a downloader middleware component, add it to the +:setting:`SPIDER_MIDDLEWARES` setting, which is a dict whose keys are the +middleware class path and their values are the middleware orders. + To activate a middleware component, add it to the :setting:`SPIDER_MIDDLEWARES` -list in your Scrapy settings. In :setting:`SPIDER_MIDDLEWARES`, each -middleware component is represented by a string: the full Python path to the -middleware's class name. For example:: +setting, which is a dict whose keys are the middleware class path and their +values are the middleware orders. - SPIDER_MIDDLEWARES = [ - 'scrapy.contrib.spidermiddleware.limit.RequestLimitMiddleware', - 'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware', - 'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware', - 'scrapy.contrib.spidermiddleware.referer.RefererMiddleware', - 'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware', - 'scrapy.contrib.spidermiddleware.depth.DepthMiddleware', - ] +Here's an example:: -The first (top) middleware is the one closer to the engine, the last (bottom) -middleware is the one closer to the spider. + SPIDER_MIDDLEWARES = { + 'myproject.middlewares.CustomSpiderMiddleware': 543, + } + +The :setting:`SPIDER_MIDDLEWARES` setting is merged with the +:setting:`SPIDER_MIDDLEWARES_BASE` setting defined in Scrapy (and not meant to +be overridden) and then sorted by order to get the final sorted list of enabled +middlewares: the first middleware is the one closer to the engine and the last +is the one closer to the spider. + +To decide which order to assign to your middleware see the +:setting:`SPIDER_MIDDLEWARES_BASE` setting and pick a value according to where +you want to insert the middleware. The order does matter because each +middleware performs a different action and your middleware could depend on some +previous (or subsequent) middleware being applied. + +If you want to disable a builtin middleware (the ones defined in +:setting:`SPIDER_MIDDLEWARES_BASE`, and enabled by default) you must define it +in your project :setting:`SPIDER_MIDDLEWARES` setting and assign `None` as its +value. For example, if you want to disable the off-site middleware:: + + SPIDER_MIDDLEWARES = { + 'myproject.middlewares.CustomSpiderMiddleware': 543, + 'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': None, + } + +Finally, keep in mind that some middlewares may need to be enabled through a +particular setting. See each middleware documentation for more info. Writing your own spider middleware ================================== diff --git a/examples/experimental/googledir/googledir/settings.py b/examples/experimental/googledir/googledir/settings.py index 6424d7b2a..a5c324741 100644 --- a/examples/experimental/googledir/googledir/settings.py +++ b/examples/experimental/googledir/googledir/settings.py @@ -52,34 +52,6 @@ EXTENSIONS = ( 'scrapy.management.telnet.TelnetConsole', ) -DOWNLOADER_MIDDLEWARES = ( - # Engine side - 'scrapy.contrib.downloadermiddleware.errorpages.ErrorPagesMiddleware', - 'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware', - 'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware', - 'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware', - 'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware', - 'scrapy.contrib.downloadermiddleware.common.CommonMiddleware', - 'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware', - 'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware', - 'scrapy.contrib.downloadermiddleware.debug.CrawlDebug', - 'scrapy.contrib.downloadermiddleware.cache.CacheMiddleware', - # Downloader side -) - -SPIDER_MIDDLEWARES = ( - # Engine side - 'scrapy.contrib.spidermiddleware.limit.RequestLimitMiddleware', - 'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware', - 'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware', - 'scrapy.contrib.spidermiddleware.referer.RefererMiddleware', - 'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware', - 'scrapy.contrib.spidermiddleware.depth.DepthMiddleware', - 'scrapy.contrib.spidermiddleware.urlfilter.UrlFilterMiddleware', - 'scrapy.contrib.spidermiddleware.duplicatesfilter.DuplicatesFilterMiddleware', - # Spider side -) - ITEM_PIPELINES = ( 'googledir.pipelines.GoogledirPipeline', ) diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index 2ee605389..2206075a9 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -59,22 +59,24 @@ DOWNLOAD_TIMEOUT = 180 # 3mins DOWNLOADER_DEBUG = False -DOWNLOADER_MIDDLEWARES = [ +DOWNLOADER_MIDDLEWARES = {} + +DOWNLOADER_MIDDLEWARES_BASE = { # Engine side - 'scrapy.contrib.downloadermiddleware.robotstxt.RobotsTxtMiddleware', - 'scrapy.contrib.downloadermiddleware.errorpages.ErrorPagesMiddleware', - 'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware', - 'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware', - 'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware', - 'scrapy.contrib.downloadermiddleware.common.CommonMiddleware', - 'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware', - 'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware', - 'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware', - 'scrapy.contrib.downloadermiddleware.debug.CrawlDebug', - 'scrapy.contrib.downloadermiddleware.stats.DownloaderStats', - 'scrapy.contrib.downloadermiddleware.cache.CacheMiddleware', + 'scrapy.contrib.downloadermiddleware.robotstxt.RobotsTxtMiddleware': 100, + 'scrapy.contrib.downloadermiddleware.errorpages.ErrorPagesMiddleware': 200, + 'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware': 300, + 'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware': 400, + 'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware': 500, + 'scrapy.contrib.downloadermiddleware.common.CommonMiddleware': 550, + 'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware': 600, + 'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware': 700, + 'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 800, + 'scrapy.contrib.downloadermiddleware.debug.CrawlDebug': 840, + 'scrapy.contrib.downloadermiddleware.stats.DownloaderStats': 850, + 'scrapy.contrib.downloadermiddleware.cache.CacheMiddleware': 900, # Downloader side -] +} DOWNLOADER_STATS = True @@ -151,9 +153,11 @@ ROBOTSTXT_OBEY = False SCHEDULER = 'scrapy.core.scheduler.Scheduler' -SCHEDULER_MIDDLEWARES = [ - 'scrapy.contrib.schedulermiddleware.duplicatesfilter.DuplicatesFilterMiddleware', - ] +SCHEDULER_MIDDLEWARES = {} + +SCHEDULER_MIDDLEWARES_BASE = { + 'scrapy.contrib.schedulermiddleware.duplicatesfilter.DuplicatesFilterMiddleware': 500, +} SCHEDULER_ORDER = 'BFO' # available orders: BFO (default), DFO @@ -161,17 +165,19 @@ SPIDER_MODULES = [] SPIDERPROFILER_ENABLED = False -SPIDER_MIDDLEWARES = [ +SPIDER_MIDDLEWARES = {} + +SPIDER_MIDDLEWARES_BASE = { # Engine side - 'scrapy.contrib.itemsampler.ItemSamplerMiddleware', - 'scrapy.contrib.spidermiddleware.limit.RequestLimitMiddleware', - 'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware', - 'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware', - 'scrapy.contrib.spidermiddleware.referer.RefererMiddleware', - 'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware', - 'scrapy.contrib.spidermiddleware.depth.DepthMiddleware', + 'scrapy.contrib.itemsampler.ItemSamplerMiddleware': 100, + 'scrapy.contrib.spidermiddleware.limit.RequestLimitMiddleware': 200, + 'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware': 300, + 'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': 500, + 'scrapy.contrib.spidermiddleware.referer.RefererMiddleware': 700, + 'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware': 800, + 'scrapy.contrib.spidermiddleware.depth.DepthMiddleware': 900, # Spider side -] +} STATS_ENABLED = True STATS_CLEANUP = False diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 28b4b633d..68f7ef745 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -12,6 +12,7 @@ from scrapy.http import Request, Response from scrapy.core.exceptions import NotConfigured from scrapy.utils.misc import load_object from scrapy.utils.defer import mustbe_deferred +from scrapy.utils.middleware import build_middleware_list from scrapy.core.downloader.handlers import download_any from scrapy.conf import settings @@ -37,7 +38,9 @@ class DownloaderMiddlewareManager(object): """Load middleware defined in settings module """ mws = [] - for mwpath in settings.getlist('DOWNLOADER_MIDDLEWARES') or (): + mwlist = build_middleware_list(settings['DOWNLOADER_MIDDLEWARES_BASE'], + settings['DOWNLOADER_MIDDLEWARES']) + for mwpath in mwlist: cls = load_object(mwpath) if cls: try: diff --git a/scrapy/core/scheduler/middleware.py b/scrapy/core/scheduler/middleware.py index 5be834208..68e7080e1 100644 --- a/scrapy/core/scheduler/middleware.py +++ b/scrapy/core/scheduler/middleware.py @@ -12,6 +12,7 @@ from scrapy.http import Response from scrapy.core.exceptions import NotConfigured from scrapy.utils.misc import load_object from scrapy.utils.defer import mustbe_deferred +from scrapy.utils.middleware import build_middleware_list from scrapy.conf import settings class SchedulerMiddlewareManager(object): @@ -29,7 +30,9 @@ class SchedulerMiddlewareManager(object): def load(self): """Load middleware defined in settings module""" mws = [] - for mwpath in settings.getlist('SCHEDULER_MIDDLEWARES') or (): + mwlist = build_middleware_list(settings['SCHEDULER_MIDDLEWARES_BASE'], + settings['SCHEDULER_MIDDLEWARES']) + for mwpath in mwlist: cls = load_object(mwpath) if cls: try: diff --git a/scrapy/spider/middleware.py b/scrapy/spider/middleware.py index 34e189e2e..1d74c06b7 100644 --- a/scrapy/spider/middleware.py +++ b/scrapy/spider/middleware.py @@ -10,6 +10,7 @@ from scrapy import log from scrapy.core.exceptions import NotConfigured from scrapy.utils.misc import load_object, arg_to_iter from scrapy.utils.defer import mustbe_deferred, defer_result +from scrapy.utils.middleware import build_middleware_list from scrapy.conf import settings def _isiterable(possible_iterator): @@ -37,7 +38,9 @@ class SpiderMiddlewareManager(object): def load(self): """Load middleware defined in settings module""" mws = [] - for mwpath in settings.getlist('SPIDER_MIDDLEWARES') or (): + mwlist = build_middleware_list(settings['SPIDER_MIDDLEWARES_BASE'], + settings['SPIDER_MIDDLEWARES']) + for mwpath in mwlist: cls = load_object(mwpath) if cls: try: diff --git a/scrapy/tests/test_utils_middleware.py b/scrapy/tests/test_utils_middleware.py new file mode 100644 index 000000000..f0bdec84b --- /dev/null +++ b/scrapy/tests/test_utils_middleware.py @@ -0,0 +1,17 @@ +import unittest + +from scrapy.utils.middleware import build_middleware_list + +class UtilsMiddlewareTestCase(unittest.TestCase): + + def test_build_middleware_list(self): + base = {'one': 1, 'two': 2, 'three': 3, 'five': 5, 'six': None} + custom = {'two': None, 'three': 8, 'four': 4} + self.assertEqual(build_middleware_list(base, custom), + ['one', 'four', 'five', 'three']) + + custom = ['a', 'b', 'c'] + self.assertEqual(build_middleware_list(base, custom), custom) + +if __name__ == "__main__": + unittest.main() diff --git a/scrapy/utils/middleware.py b/scrapy/utils/middleware.py new file mode 100644 index 000000000..051e4c990 --- /dev/null +++ b/scrapy/utils/middleware.py @@ -0,0 +1,11 @@ +from operator import itemgetter + +def build_middleware_list(base, custom): + """Compose a middleware list based on a custom and base dict of + middlewares, unless custom is already a list, in which case it's returned. + """ + if isinstance(custom, (list, tuple)): + return custom + mwdict = base.copy() + mwdict.update(custom) + return [k for k, v in sorted(mwdict.items(), key=itemgetter(1)) if v is not None]