From 2fd8b7c28c255148f6d7320cb97292ab2d569eff Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Wed, 27 Mar 2019 00:45:53 +0530 Subject: [PATCH] [MRG+1] redirect_reasons in Request.meta (#3687) --- docs/topics/downloader-middleware.rst | 16 ++++++++++++++-- docs/topics/request-response.rst | 1 + scrapy/downloadermiddlewares/redirect.py | 2 ++ tests/test_downloadermiddleware_redirect.py | 19 +++++++++++++++++++ 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 0d976077b..f2f3ef466 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -733,6 +733,17 @@ RedirectMiddleware The urls which the request goes through (while being redirected) can be found in the ``redirect_urls`` :attr:`Request.meta ` key. +.. reqmeta:: redirect_reasons + +The reason behind each redirect in :reqmeta:`redirect_urls` can be found in the +``redirect_reasons`` :attr:`Request.meta ` key. For +example: ``[301, 302, 307, 'meta refresh']``. + +The format of a reason depends on the middleware that handled the corresponding +redirect. For example, :class:`RedirectMiddleware` indicates the triggering +response status code as an integer, while :class:`MetaRefreshMiddleware` +always uses the ``'meta refresh'`` string as reason. + The :class:`RedirectMiddleware` can be configured through the following settings (see the settings documentation for more info): @@ -796,8 +807,9 @@ settings (see the settings documentation for more info): * :setting:`METAREFRESH_ENABLED` * :setting:`METAREFRESH_MAXDELAY` -This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect` -and :reqmeta:`redirect_urls` request meta keys as described for :class:`RedirectMiddleware` +This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect`, +:reqmeta:`redirect_urls` and :reqmeta:`redirect_reasons` request meta keys as described +for :class:`RedirectMiddleware` MetaRefreshMiddleware settings diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index ca59b46d8..ac6fe6e3f 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -299,6 +299,7 @@ Those are: * :reqmeta:`dont_merge_cookies` * :reqmeta:`cookiejar` * :reqmeta:`dont_cache` +* :reqmeta:`redirect_reasons` * :reqmeta:`redirect_urls` * :reqmeta:`bindaddress` * :reqmeta:`dont_obey_robotstxt` diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 30cae3fee..cb59d3fd2 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -34,6 +34,8 @@ class BaseRedirectMiddleware(object): redirected.meta['redirect_ttl'] = ttl - 1 redirected.meta['redirect_urls'] = request.meta.get('redirect_urls', []) + \ [request.url] + redirected.meta['redirect_reasons'] = request.meta.get('redirect_reasons', []) + \ + [reason] redirected.dont_filter = request.dont_filter redirected.priority = request.priority + self.priority_adjust logger.debug("Redirecting (%(reason)s) to %(redirected)s from %(request)s", diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 74137b4cd..6c81c94ca 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -139,6 +139,16 @@ class RedirectMiddlewareTest(unittest.TestCase): self.assertEqual(req3.url, 'http://scrapytest.org/redirected2') self.assertEqual(req3.meta['redirect_urls'], ['http://scrapytest.org/first', 'http://scrapytest.org/redirected']) + def test_redirect_reasons(self): + req1 = Request('http://scrapytest.org/first') + rsp1 = Response('http://scrapytest.org/first', headers={'Location': '/redirected1'}, status=301) + req2 = self.mw.process_response(req1, rsp1, self.spider) + rsp2 = Response('http://scrapytest.org/redirected1', headers={'Location': '/redirected2'}, status=301) + req3 = self.mw.process_response(req2, rsp2, self.spider) + + self.assertEqual(req2.meta['redirect_reasons'], [301]) + self.assertEqual(req3.meta['redirect_reasons'], [301, 301]) + def test_spider_handling(self): smartspider = self.crawler._create_spider('smarty') smartspider.handle_httpstatus_list = [404, 301, 302] @@ -259,6 +269,15 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): self.assertEqual(req3.url, 'http://scrapytest.org/redirected2') self.assertEqual(req3.meta['redirect_urls'], ['http://scrapytest.org/first', 'http://scrapytest.org/redirected']) + def test_redirect_reasons(self): + req1 = Request('http://scrapytest.org/first') + rsp1 = HtmlResponse('http://scrapytest.org/first', body=self._body(url='/redirected')) + req2 = self.mw.process_response(req1, rsp1, self.spider) + rsp2 = HtmlResponse('http://scrapytest.org/redirected', body=self._body(url='/redirected1')) + req3 = self.mw.process_response(req2, rsp2, self.spider) + + self.assertEqual(req2.meta['redirect_reasons'], ['meta refresh']) + self.assertEqual(req3.meta['redirect_reasons'], ['meta refresh', 'meta refresh']) if __name__ == "__main__": unittest.main()