improved configuration of middlewares using dicts and orders (closes #85)

docs/ref/settings.rst

@@ -296,26 +296,37 @@ Whether to enable the Downloader debugging mode.
 DOWNLOADER_MIDDLEWARES
 ----------------------
 
+Default:: ``{}``
+
+A dict containing the downloader middlewares enabled in your project, and their
+orders. For more info see :ref:`topics-downloader-middleware-setting`.
+
+.. setting:: DOWNLOADER_MIDDLEWARES_BASE
+
+DOWNLOADER_MIDDLEWARES_BASE
+---------------------------
+
 Default:: 
 
-    [
-        'scrapy.contrib.downloadermiddleware.robotstxt.RobotsTxtMiddleware',
-        'scrapy.contrib.downloadermiddleware.errorpages.ErrorPagesMiddleware',
-        'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware',
-        'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware',
-        'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware',
-        'scrapy.contrib.downloadermiddleware.common.CommonMiddleware',
-        'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware',
-        'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware',
-        'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware',
-        'scrapy.contrib.downloadermiddleware.debug.CrawlDebug',
-        'scrapy.contrib.downloadermiddleware.stats.DownloaderStats',
-        'scrapy.contrib.downloadermiddleware.cache.CacheMiddleware',
-    ]
+    {
+        'scrapy.contrib.downloadermiddleware.robotstxt.RobotsTxtMiddleware': 100,
+        'scrapy.contrib.downloadermiddleware.errorpages.ErrorPagesMiddleware': 200,
+        'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware': 300,
+        'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware': 400,
+        'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware': 500,
+        'scrapy.contrib.downloadermiddleware.common.CommonMiddleware': 550,
+        'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware': 600,
+        'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware': 700,
+        'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 800,
+        'scrapy.contrib.downloadermiddleware.debug.CrawlDebug': 840,
+        'scrapy.contrib.downloadermiddleware.stats.DownloaderStats': 850,
+        'scrapy.contrib.downloadermiddleware.cache.CacheMiddleware': 900,
+    }
 
-The list of enabled downloader middlewares. Keep in mind that some may need to
-be enabled through a particular setting. The top (first) middleware is closer
-to the engine, while the bottom (last) middleware is closer to the downloader.
+A dict containing the downloader middlewares enabled by default in Scrapy. You
+should never modify this setting in your project, modify
+:setting:`DOWNLOADER_MIDDLEWARES` instead.  For more info see
+:ref:`topics-downloader-middleware-setting`.
 
 .. setting:: DOWNLOADER_STATS
 
@@ -732,7 +743,17 @@ The order to use for the crawling scheduler. Available orders are:
 .. setting:: SCHEDULER_MIDDLEWARES
 
 SCHEDULER_MIDDLEWARES
-----------------------
+---------------------
+
+Default:: ``{}``
+
+A dict containing the scheduler middlewares enabled in your project, and their
+orders. 
+
+.. setting:: SCHEDULER_MIDDLEWARES_BASE
+
+SCHEDULER_MIDDLEWARES_BASE
+--------------------------
 
 Default:: 
 
@@ -740,9 +761,10 @@ Default::
         'scrapy.contrib.schedulermiddleware.duplicatesfilter.DuplicatesFilterMiddleware',
     ]
 
-The list of enabled scheduler middlewares. Keep in mind that some may need to
-be enabled through a particular setting. The top (first) middleware is closer
-to the engine, while the bottom (last) middleware is closer to the scheduler.
+
+A dict containing the scheduler middlewares enabled by default in Scrapy. You
+should never modify this setting in your project, modify
+:setting:`SCHEDULER_MIDDLEWARES` instead. 
 
 .. setting:: SPIDERPROFILER_ENABLED
 
@@ -759,21 +781,32 @@ performance.
 SPIDER_MIDDLEWARES
 ------------------
 
+Default:: ``{}``
+
+A dict containing the spider middlewares enabled in your project, and their
+orders. For more info see :ref:`topics-spider-middleware-setting`.
+
+.. setting:: SPIDER_MIDDLEWARES_BASE
+
+SPIDER_MIDDLEWARES_BASE
+-----------------------
+
 Default::
 
-    [
-        'scrapy.contrib.itemsampler.ItemSamplerMiddleware',
-        'scrapy.contrib.spidermiddleware.limit.RequestLimitMiddleware',
-        'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware',
-        'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware',
-        'scrapy.contrib.spidermiddleware.referer.RefererMiddleware',
-        'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware',
-        'scrapy.contrib.spidermiddleware.depth.DepthMiddleware',
-    ]
+    {
+        'scrapy.contrib.itemsampler.ItemSamplerMiddleware': 100,
+        'scrapy.contrib.spidermiddleware.limit.RequestLimitMiddleware': 200,
+        'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware': 300,
+        'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': 500,
+        'scrapy.contrib.spidermiddleware.referer.RefererMiddleware': 700,
+        'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware': 800,
+        'scrapy.contrib.spidermiddleware.depth.DepthMiddleware': 900,
+    }
 
-The list of enabled spider middlewares. Keep in mind that some may need to be
-enabled through a particular setting. The top (first) middleware is closer to
-the engine, while the bottom (last) middleware is closer to the spider.
+A dict containing the spider middlewares enabled by default in Scrapy. You
+should never modify this setting in your project, modify
+:setting:`SPIDER_MIDDLEWARES` instead. For more info see
+:ref:`topics-spider-middleware-setting`.
 
 .. setting:: SPIDER_MODULES
 

docs/topics/downloader-middleware.rst

@@ -6,22 +6,47 @@ Downloader Middleware
 
 The downloader middleware is a framework of hooks into Scrapy's
 request/response processing.  It's a light, low-level system for globally
-altering Scrapy's input and/or output.
+altering Scrapy's requests and responses.
+
+.. _topics-downloader-middleware-setting:
 
 Activating a downloader middleware
 ==================================
 
 To activate a downloader middleware component, add it to the
-:setting:`DOWNLOADER_MIDDLEWARES` list in your Scrapy settings.  In
-:setting:`DOWNLOADER_MIDDLEWARES`, each middleware component is represented by
-a string: the full Python path to the middleware's class name. For example::
+:setting:`DOWNLOADER_MIDDLEWARES` setting, which is a dict whose keys are the
+middleware class paths and their values are the middleware orders.
 
-    DOWNLOADER_MIDDLEWARES = [
-            'scrapy.contrib.middleware.common.SpiderMiddleware',
-            'scrapy.contrib.middleware.common.CommonMiddleware',
-            'scrapy.contrib.middleware.redirect.RedirectMiddleware',
-            'scrapy.contrib.middleware.cache.CacheMiddleware',
-    ]
+Here's an example::
+
+    DOWNLOADER_MIDDLEWARES = {
+        'myproject.middlewares.CustomDownloaderMiddleware': 543,
+    }
+
+The :setting:`DOWNLOADER_MIDDLEWARES` setting is merged with the
+:setting:`DOWNLOADER_MIDDLEWARES_BASE` setting defined in Scrapy (and not meant to
+be overridden) and then sorted by order to get the final sorted list of enabled
+middlewares: the first middleware is the one closer to the engine and the last
+is the one closer to the downloader.
+
+To decide which order to assign to your middleware see the
+:setting:`DOWNLOADER_MIDDLEWARES_BASE` setting and pick a value according to
+where you want to insert the middleware. The order does matter because each
+middleware performs a different action and your middleware could depend on some
+previous (or subsequent) middleware being applied.
+
+If you want to disable a builtin middleware (the ones defined in
+:setting:`DOWNLOADER_MIDDLEWARES_BASE` and enabled by default) you must define it
+in your project :setting:`DOWNLOADER_MIDDLEWARES` setting and assign `None`
+as its value.  For example, if you want to disable the off-site middleware::
+
+    DOWNLOADER_MIDDLEWARES = {
+        'myproject.middlewares.CustomDownloaderMiddleware': 543,
+        'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware': None,
+    }
+
+Finally, keep in mind that some middlewares may need to be enabled through a
+particular setting. See each middleware documentation for more info.
 
 Writing your own downloader middleware
 ======================================

docs/topics/spider-middleware.rst

@@ -9,25 +9,49 @@ mechanism where you can plug custom functionality to process the requests that
 are sent to :ref:`topics-spiders` for processing and to process the responses
 and item that are generated from spiders. 
 
+.. _topics-spider-middleware-setting:
+
 Activating a spider middleware
 ==============================
 
+To activate a downloader middleware component, add it to the
+:setting:`SPIDER_MIDDLEWARES` setting, which is a dict whose keys are the
+middleware class path and their values are the middleware orders.
+
 To activate a middleware component, add it to the :setting:`SPIDER_MIDDLEWARES`
-list in your Scrapy settings.  In :setting:`SPIDER_MIDDLEWARES`, each
-middleware component is represented by a string: the full Python path to the
-middleware's class name. For example::
+setting, which is a dict whose keys are the middleware class path and their
+values are the middleware orders.
 
-    SPIDER_MIDDLEWARES = [
-        'scrapy.contrib.spidermiddleware.limit.RequestLimitMiddleware',
-        'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware',
-        'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware',
-        'scrapy.contrib.spidermiddleware.referer.RefererMiddleware',
-        'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware',
-        'scrapy.contrib.spidermiddleware.depth.DepthMiddleware',
-    ]
+Here's an example::
 
-The first (top) middleware is the one closer to the engine, the last (bottom)
-middleware is the one closer to the spider.
+    SPIDER_MIDDLEWARES = {
+        'myproject.middlewares.CustomSpiderMiddleware': 543,
+    }
+
+The :setting:`SPIDER_MIDDLEWARES` setting is merged with the
+:setting:`SPIDER_MIDDLEWARES_BASE` setting defined in Scrapy (and not meant to
+be overridden) and then sorted by order to get the final sorted list of enabled
+middlewares: the first middleware is the one closer to the engine and the last
+is the one closer to the spider.
+
+To decide which order to assign to your middleware see the
+:setting:`SPIDER_MIDDLEWARES_BASE` setting and pick a value according to where
+you want to insert the middleware. The order does matter because each
+middleware performs a different action and your middleware could depend on some
+previous (or subsequent) middleware being applied.
+
+If you want to disable a builtin middleware (the ones defined in
+:setting:`SPIDER_MIDDLEWARES_BASE`, and enabled by default) you must define it
+in your project :setting:`SPIDER_MIDDLEWARES` setting and assign `None` as its
+value.  For example, if you want to disable the off-site middleware::
+
+    SPIDER_MIDDLEWARES = {
+        'myproject.middlewares.CustomSpiderMiddleware': 543,
+        'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': None,
+    }
+
+Finally, keep in mind that some middlewares may need to be enabled through a
+particular setting. See each middleware documentation for more info.
 
 Writing your own spider middleware
 ==================================

examples/experimental/googledir/googledir/settings.py

@@ -52,34 +52,6 @@ EXTENSIONS = (
     'scrapy.management.telnet.TelnetConsole',
 )
 
-DOWNLOADER_MIDDLEWARES = (
-    # Engine side
-    'scrapy.contrib.downloadermiddleware.errorpages.ErrorPagesMiddleware',
-    'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware',
-    'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware',
-    'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware',
-    'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware',
-    'scrapy.contrib.downloadermiddleware.common.CommonMiddleware',
-    'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware',
-    'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware',
-    'scrapy.contrib.downloadermiddleware.debug.CrawlDebug',
-    'scrapy.contrib.downloadermiddleware.cache.CacheMiddleware',
-    # Downloader side
-)
-
-SPIDER_MIDDLEWARES = (
-    # Engine side
-    'scrapy.contrib.spidermiddleware.limit.RequestLimitMiddleware',
-    'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware',
-    'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware',
-    'scrapy.contrib.spidermiddleware.referer.RefererMiddleware',
-    'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware',
-    'scrapy.contrib.spidermiddleware.depth.DepthMiddleware',
-    'scrapy.contrib.spidermiddleware.urlfilter.UrlFilterMiddleware',
-    'scrapy.contrib.spidermiddleware.duplicatesfilter.DuplicatesFilterMiddleware',
-    # Spider side
-)
-
 ITEM_PIPELINES = (
     'googledir.pipelines.GoogledirPipeline',
 )

scrapy/conf/default_settings.py

@@ -59,22 +59,24 @@ DOWNLOAD_TIMEOUT = 180      # 3mins
 
 DOWNLOADER_DEBUG = False
 
-DOWNLOADER_MIDDLEWARES = [
+DOWNLOADER_MIDDLEWARES = {}
+
+DOWNLOADER_MIDDLEWARES_BASE = {
     # Engine side
-    'scrapy.contrib.downloadermiddleware.robotstxt.RobotsTxtMiddleware',
-    'scrapy.contrib.downloadermiddleware.errorpages.ErrorPagesMiddleware',
-    'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware',
-    'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware',
-    'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware',
-    'scrapy.contrib.downloadermiddleware.common.CommonMiddleware',
-    'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware',
-    'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware',
-    'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware',
-    'scrapy.contrib.downloadermiddleware.debug.CrawlDebug',
-    'scrapy.contrib.downloadermiddleware.stats.DownloaderStats',
-    'scrapy.contrib.downloadermiddleware.cache.CacheMiddleware',
+    'scrapy.contrib.downloadermiddleware.robotstxt.RobotsTxtMiddleware': 100,
+    'scrapy.contrib.downloadermiddleware.errorpages.ErrorPagesMiddleware': 200,
+    'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware': 300,
+    'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware': 400,
+    'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware': 500,
+    'scrapy.contrib.downloadermiddleware.common.CommonMiddleware': 550,
+    'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware': 600,
+    'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware': 700,
+    'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 800,
+    'scrapy.contrib.downloadermiddleware.debug.CrawlDebug': 840,
+    'scrapy.contrib.downloadermiddleware.stats.DownloaderStats': 850,
+    'scrapy.contrib.downloadermiddleware.cache.CacheMiddleware': 900,
     # Downloader side
-]
+}
 
 DOWNLOADER_STATS = True
 
@@ -151,9 +153,11 @@ ROBOTSTXT_OBEY = False
 
 SCHEDULER = 'scrapy.core.scheduler.Scheduler'
 
-SCHEDULER_MIDDLEWARES = [
-        'scrapy.contrib.schedulermiddleware.duplicatesfilter.DuplicatesFilterMiddleware',
-        ]
+SCHEDULER_MIDDLEWARES = {}
+
+SCHEDULER_MIDDLEWARES_BASE = {
+    'scrapy.contrib.schedulermiddleware.duplicatesfilter.DuplicatesFilterMiddleware': 500,
+}
 
 SCHEDULER_ORDER = 'BFO'   # available orders: BFO (default), DFO
 
@@ -161,17 +165,19 @@ SPIDER_MODULES = []
 
 SPIDERPROFILER_ENABLED = False
 
-SPIDER_MIDDLEWARES = [
+SPIDER_MIDDLEWARES = {}
+
+SPIDER_MIDDLEWARES_BASE = {
     # Engine side
-    'scrapy.contrib.itemsampler.ItemSamplerMiddleware',
-    'scrapy.contrib.spidermiddleware.limit.RequestLimitMiddleware',
-    'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware',
-    'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware',
-    'scrapy.contrib.spidermiddleware.referer.RefererMiddleware',
-    'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware',
-    'scrapy.contrib.spidermiddleware.depth.DepthMiddleware',
+    'scrapy.contrib.itemsampler.ItemSamplerMiddleware': 100,
+    'scrapy.contrib.spidermiddleware.limit.RequestLimitMiddleware': 200,
+    'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware': 300,
+    'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': 500,
+    'scrapy.contrib.spidermiddleware.referer.RefererMiddleware': 700,
+    'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware': 800,
+    'scrapy.contrib.spidermiddleware.depth.DepthMiddleware': 900,
     # Spider side
-]
+}
 
 STATS_ENABLED = True
 STATS_CLEANUP = False

scrapy/core/downloader/middleware.py

@@ -12,6 +12,7 @@ from scrapy.http import Request, Response
 from scrapy.core.exceptions import NotConfigured
 from scrapy.utils.misc import load_object
 from scrapy.utils.defer import mustbe_deferred
+from scrapy.utils.middleware import build_middleware_list
 from scrapy.core.downloader.handlers import download_any
 from scrapy.conf import settings
 
@@ -37,7 +38,9 @@ class DownloaderMiddlewareManager(object):
         """Load middleware defined in settings module
         """
         mws = []
-        for mwpath in settings.getlist('DOWNLOADER_MIDDLEWARES') or ():
+        mwlist = build_middleware_list(settings['DOWNLOADER_MIDDLEWARES_BASE'],
+                                       settings['DOWNLOADER_MIDDLEWARES'])
+        for mwpath in mwlist:
             cls = load_object(mwpath)
             if cls:
                 try:

scrapy/core/scheduler/middleware.py

@@ -12,6 +12,7 @@ from scrapy.http import Response
 from scrapy.core.exceptions import NotConfigured
 from scrapy.utils.misc import load_object
 from scrapy.utils.defer import mustbe_deferred
+from scrapy.utils.middleware import build_middleware_list
 from scrapy.conf import settings
 
 class SchedulerMiddlewareManager(object):
@@ -29,7 +30,9 @@ class SchedulerMiddlewareManager(object):
     def load(self):
         """Load middleware defined in settings module"""
         mws = []
-        for mwpath in settings.getlist('SCHEDULER_MIDDLEWARES') or ():
+        mwlist = build_middleware_list(settings['SCHEDULER_MIDDLEWARES_BASE'],
+                                       settings['SCHEDULER_MIDDLEWARES'])
+        for mwpath in mwlist:
             cls = load_object(mwpath)
             if cls:
                 try:

scrapy/spider/middleware.py

@@ -10,6 +10,7 @@ from scrapy import log
 from scrapy.core.exceptions import NotConfigured
 from scrapy.utils.misc import load_object, arg_to_iter
 from scrapy.utils.defer import mustbe_deferred, defer_result
+from scrapy.utils.middleware import build_middleware_list
 from scrapy.conf import settings
 
 def _isiterable(possible_iterator):
@@ -37,7 +38,9 @@ class SpiderMiddlewareManager(object):
     def load(self):
         """Load middleware defined in settings module"""
         mws = []
-        for mwpath in settings.getlist('SPIDER_MIDDLEWARES') or ():
+        mwlist = build_middleware_list(settings['SPIDER_MIDDLEWARES_BASE'],
+                                       settings['SPIDER_MIDDLEWARES'])
+        for mwpath in mwlist:
             cls = load_object(mwpath)
             if cls:
                 try:

scrapy/tests/test_utils_middleware.py

@@ -0,0 +1,17 @@
+import unittest
+
+from scrapy.utils.middleware import build_middleware_list
+
+class UtilsMiddlewareTestCase(unittest.TestCase):
+
+    def test_build_middleware_list(self):
+        base = {'one': 1, 'two': 2, 'three': 3, 'five': 5, 'six': None}
+        custom = {'two': None, 'three': 8, 'four': 4}
+        self.assertEqual(build_middleware_list(base, custom),
+                         ['one', 'four', 'five', 'three'])
+
+        custom = ['a', 'b', 'c']
+        self.assertEqual(build_middleware_list(base, custom), custom)
+
+if __name__ == "__main__":
+    unittest.main()

scrapy/utils/middleware.py

@@ -0,0 +1,11 @@
+from operator import itemgetter
+
+def build_middleware_list(base, custom):
+    """Compose a middleware list based on a custom and base dict of
+    middlewares, unless custom is already a list, in which case it's returned.
+    """
+    if isinstance(custom, (list, tuple)):
+        return custom
+    mwdict = base.copy()
+    mwdict.update(custom)
+    return [k for k, v in sorted(mwdict.items(), key=itemgetter(1)) if v is not None]