Added dont_redirect request.meta key to make RedirectMiddleware ignore requests. Closes #233

docs/_ext/scrapydocs.py

@@ -14,3 +14,8 @@ def setup(app):
         rolename      = "command",
         indextemplate = "pair: %s; command",
     )
+    app.add_crossref_type(
+        directivename = "reqmeta",
+        rolename      = "reqmeta",
+        indextemplate = "pair: %s; reqmeta",
+    )

docs/topics/downloader-middleware.rst

@@ -377,12 +377,17 @@ RedirectMiddleware
    This middlware handles redirection of requests based on response status and
    meta-refresh html tag.
 
-   The :class:`RedirectMiddleware` can be configured through the following
-   settings (see the settings documentation for more info):
+The :class:`RedirectMiddleware` can be configured through the following
+settings (see the settings documentation for more info):
 
-   * :setting:`REDIRECT_MAX_METAREFRESH_DELAY` - Maximum meta-refresh delay that a page is allowed to have for redirection.
-   * :setting:`REDIRECT_MAX_TIMES` - Maximum number of redirects to perform on a request.
-   * :setting:`REDIRECT_PRIORITY_ADJUST` - Adjusts the redirected request priority by this amount.
+* :setting:`REDIRECT_MAX_METAREFRESH_DELAY` - Maximum meta-refresh delay that a page is allowed to have for redirection.
+* :setting:`REDIRECT_MAX_TIMES` - Maximum number of redirects to perform on a request.
+* :setting:`REDIRECT_PRIORITY_ADJUST` - Adjusts the redirected request priority by this amount.
+
+.. reqmeta:: dont_redirect
+
+If :attr:`Request.meta <scrapy.http.Request.meta>` contains the
+``dont_redirect`` key, the request will be ignored by this middleware.
 
 RetryMiddleware
 ---------------

docs/topics/request-response.rst

@@ -134,6 +134,9 @@ Request objects
         components (extensions, middlewares, etc). So the data contained in this
         dict depends on the extensions you have enabled.
 
+        See :ref:`topics-request-meta` for a list of special meta keys
+        recognized by Scrapy.
+
         This dict is `shallow copied`_ when the request is cloned using the
         ``copy()`` or ``replace()`` methods.
 
@@ -204,6 +207,18 @@ Using Request.meta::
         referer_url = response.request.meta['referer_url']
         self.log("Visited page %s from %s" % (response.url, referer_url))
 
+.. _topics-request-meta:
+
+Request.meta values
+===================
+
+The :attr:`Request.meta` attribute can contain any arbitrary data, but there
+are some special keys recognized by Scrapy and its built-in extensions.
+
+Those are:
+
+* :reqmeta:`dont_redirect`
+
 .. _topics-request-response-ref-request-subclasses:
 
 Request subclasses

scrapy/contrib/downloadermiddleware/redirect.py

@@ -15,6 +15,8 @@ class RedirectMiddleware(object):
         self.priority_adjust = settings.getint('REDIRECT_PRIORITY_ADJUST')
 
     def process_response(self, request, response, spider):
+        if 'dont_redirect' in request.meta:
+            return response
         if request.method.upper() == 'HEAD':
             if response.status in [301, 302, 303, 307] and 'Location' in response.headers:
                 redirected_url = urljoin_rfc(request.url, response.headers['location'])

scrapy/tests/test_downloadermiddleware_redirect.py

@@ -37,6 +37,16 @@ class RedirectMiddlewareTest(unittest.TestCase):
         _test('POST')
         _test('HEAD')
 
+    def test_dont_redirect(self):
+        url = 'http://www.example.com/301'
+        url2 = 'http://www.example.com/redirected'
+        req = Request(url, meta={'dont_redirect': True})
+        rsp = Response(url, headers={'Location': url2}, status=301)
+
+        r = self.mw.process_response(req, rsp, self.spider)
+        assert isinstance(r, Response)
+        assert r is rsp
+
     def test_redirect_302(self):
         url = 'http://www.example.com/302'
         url2 = 'http://www.example.com/redirected2'