redirect mw: added REDIRECT_ENABLED setting and documented the other settings

docs/topics/downloader-middleware.rst

@@ -476,15 +476,48 @@ in the ``redirect_urls`` :attr:`Request.meta <scrapy.http.Request.meta>` key.
 The :class:`RedirectMiddleware` can be configured through the following
 settings (see the settings documentation for more info):
 
-* :setting:`REDIRECT_MAX_METAREFRESH_DELAY` - Maximum meta-refresh delay that a page is allowed to have for redirection.
-* :setting:`REDIRECT_MAX_TIMES` - Maximum number of redirects to perform on a request.
-* :setting:`REDIRECT_PRIORITY_ADJUST` - Adjusts the redirected request priority by this amount.
+* :setting:`REDIRECT_ENABLED`
+* :setting:`REDIRECT_MAX_TIMES`
+* :setting:`REDIRECT_MAX_METAREFRESH_DELAY`
 
 .. reqmeta:: dont_redirect
 
 If :attr:`Request.meta <scrapy.http.Request.meta>` contains the
 ``dont_redirect`` key, the request will be ignored by this middleware.
 
+
+RedirectMiddleware settings
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. setting:: REDIRECT_ENABLED
+
+REDIRECT_ENABLED
+^^^^^^^^^^^^^^^^
+
+.. versionadded:: 0.13
+
+Default: ``True``
+
+Whether the Redirect middleware will be enabled.
+
+.. setting:: REDIRECT_MAX_TIMES
+
+REDIRECT_MAX_TIMES
+^^^^^^^^^^^^^^^^^^
+
+Default: ``20``
+
+The maximum number of redirections that will be follow for a single request.
+
+.. setting:: REDIRECT_MAX_METAREFRESH_DELAY
+
+REDIRECT_MAX_METAREFRESH_DELAY
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Default: ``100``
+
+The maximum meta-refresh delay (in seconds) to follow the redirection.
+
 RetryMiddleware
 ---------------
 

scrapy/contrib/downloadermiddleware/redirect.py

@@ -11,6 +11,8 @@ class RedirectMiddleware(object):
     """Handle redirection of requests based on response status and meta-refresh html tag"""
 
     def __init__(self):
+        if not settings.getbool('REDIRECT_ENABLED'):
+            raise NotConfigured
         self.max_metarefresh_delay = settings.getint('REDIRECT_MAX_METAREFRESH_DELAY')
         self.max_redirect_times = settings.getint('REDIRECT_MAX_TIMES')
         self.priority_adjust = settings.getint('REDIRECT_PRIORITY_ADJUST')

scrapy/settings/default_settings.py

@@ -207,6 +207,7 @@ QUEUE_POLL_INTERVAL = 5
 
 RANDOMIZE_DOWNLOAD_DELAY = True
 
+REDIRECT_ENABLED = True
 REDIRECT_MAX_METAREFRESH_DELAY = 100
 REDIRECT_MAX_TIMES = 20 # uses Firefox default setting
 REDIRECT_PRIORITY_ADJUST = +2