diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index 6f370a5af..3cc916972 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -14,3 +14,8 @@ def setup(app): rolename = "command", indextemplate = "pair: %s; command", ) + app.add_crossref_type( + directivename = "reqmeta", + rolename = "reqmeta", + indextemplate = "pair: %s; reqmeta", + ) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 86eda9450..1c82b2c6e 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -377,12 +377,17 @@ RedirectMiddleware This middlware handles redirection of requests based on response status and meta-refresh html tag. - The :class:`RedirectMiddleware` can be configured through the following - settings (see the settings documentation for more info): +The :class:`RedirectMiddleware` can be configured through the following +settings (see the settings documentation for more info): - * :setting:`REDIRECT_MAX_METAREFRESH_DELAY` - Maximum meta-refresh delay that a page is allowed to have for redirection. - * :setting:`REDIRECT_MAX_TIMES` - Maximum number of redirects to perform on a request. - * :setting:`REDIRECT_PRIORITY_ADJUST` - Adjusts the redirected request priority by this amount. +* :setting:`REDIRECT_MAX_METAREFRESH_DELAY` - Maximum meta-refresh delay that a page is allowed to have for redirection. +* :setting:`REDIRECT_MAX_TIMES` - Maximum number of redirects to perform on a request. +* :setting:`REDIRECT_PRIORITY_ADJUST` - Adjusts the redirected request priority by this amount. + +.. reqmeta:: dont_redirect + +If :attr:`Request.meta ` contains the +``dont_redirect`` key, the request will be ignored by this middleware. RetryMiddleware --------------- diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index c35668f10..c7744ef0c 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -134,6 +134,9 @@ Request objects components (extensions, middlewares, etc). So the data contained in this dict depends on the extensions you have enabled. + See :ref:`topics-request-meta` for a list of special meta keys + recognized by Scrapy. + This dict is `shallow copied`_ when the request is cloned using the ``copy()`` or ``replace()`` methods. @@ -204,6 +207,18 @@ Using Request.meta:: referer_url = response.request.meta['referer_url'] self.log("Visited page %s from %s" % (response.url, referer_url)) +.. _topics-request-meta: + +Request.meta values +=================== + +The :attr:`Request.meta` attribute can contain any arbitrary data, but there +are some special keys recognized by Scrapy and its built-in extensions. + +Those are: + +* :reqmeta:`dont_redirect` + .. _topics-request-response-ref-request-subclasses: Request subclasses diff --git a/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/contrib/downloadermiddleware/redirect.py index 1e3e390ff..d4ea73ffc 100644 --- a/scrapy/contrib/downloadermiddleware/redirect.py +++ b/scrapy/contrib/downloadermiddleware/redirect.py @@ -15,6 +15,8 @@ class RedirectMiddleware(object): self.priority_adjust = settings.getint('REDIRECT_PRIORITY_ADJUST') def process_response(self, request, response, spider): + if 'dont_redirect' in request.meta: + return response if request.method.upper() == 'HEAD': if response.status in [301, 302, 303, 307] and 'Location' in response.headers: redirected_url = urljoin_rfc(request.url, response.headers['location']) diff --git a/scrapy/tests/test_downloadermiddleware_redirect.py b/scrapy/tests/test_downloadermiddleware_redirect.py index d07e86b9c..dc91626ab 100644 --- a/scrapy/tests/test_downloadermiddleware_redirect.py +++ b/scrapy/tests/test_downloadermiddleware_redirect.py @@ -37,6 +37,16 @@ class RedirectMiddlewareTest(unittest.TestCase): _test('POST') _test('HEAD') + def test_dont_redirect(self): + url = 'http://www.example.com/301' + url2 = 'http://www.example.com/redirected' + req = Request(url, meta={'dont_redirect': True}) + rsp = Response(url, headers={'Location': url2}, status=301) + + r = self.mw.process_response(req, rsp, self.spider) + assert isinstance(r, Response) + assert r is rsp + def test_redirect_302(self): url = 'http://www.example.com/302' url2 = 'http://www.example.com/redirected2'