From d164398a27736f75286cc435eca69b06ff7c1c06 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Fri, 21 Aug 2015 13:22:42 +0200 Subject: [PATCH] Fix RedirectMiddleware not honouring meta handle_httpstatus keys --- docs/topics/downloader-middleware.rst | 6 ++++++ scrapy/downloadermiddlewares/redirect.py | 4 +++- tests/test_downloadermiddleware_redirect.py | 11 +++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 6d986bbf7..73cc67423 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -724,6 +724,12 @@ responses (and pass them through to your spider) you can do this:: class MySpider(CrawlSpider): handle_httpstatus_list = [301, 302] +The ``handle_httpstatus_list`` key of :attr:`Request.meta +` can also be used to specify which response codes to +allow on a per-request basis. You can also set the meta key +``handle_httpstatus_all`` to ``True`` if you want to allow any response code +for a request. + RedirectMiddleware settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 363e56cb8..3cf8d2bee 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -55,7 +55,9 @@ class RedirectMiddleware(BaseRedirectMiddleware): def process_response(self, request, response, spider): if (request.meta.get('dont_redirect', False) or - response.status in getattr(spider, 'handle_httpstatus_list', [])): + response.status in getattr(spider, 'handle_httpstatus_list', []) or + response.status in request.meta.get('handle_httpstatus_list', []) or + request.meta.get('handle_httpstatus_all', False)): return response if request.method == 'HEAD': diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 9b00caa51..b3db7c42b 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -139,6 +139,17 @@ class RedirectMiddlewareTest(unittest.TestCase): r = self.mw.process_response(req, rsp, smartspider) self.assertIs(r, rsp) + def test_request_meta_handling(self): + url = 'http://www.example.com/301' + url2 = 'http://www.example.com/redirected' + def _test_passthrough(req): + rsp = Response(url, headers={'Location': url2}, status=301, request=req) + r = self.mw.process_response(req, rsp, self.spider) + self.assertIs(r, rsp) + _test_passthrough(Request(url, meta={'handle_httpstatus_list': + [404, 301, 302]})) + _test_passthrough(Request(url, meta={'handle_httpstatus_all': True})) + class MetaRefreshMiddlewareTest(unittest.TestCase):