From 13181ba7882f2aef3ea103a9c2391fd8f87fbb44 Mon Sep 17 00:00:00 2001 From: Jose Galdos Date: Thu, 23 Jul 2020 18:45:45 -0500 Subject: [PATCH 1/2] Improve http status all on http error middleware --- scrapy/spidermiddlewares/httperror.py | 2 +- tests/test_spidermiddleware_httperror.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 375042340..bf908d2f7 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -32,7 +32,7 @@ class HttpErrorMiddleware: if 200 <= response.status < 300: # common case return meta = response.meta - if 'handle_httpstatus_all' in meta: + if meta.get('handle_httpstatus_all', False): return if 'handle_httpstatus_list' in meta: allowed_statuses = meta['handle_httpstatus_list'] diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index e032b247c..f3e5478c4 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -139,6 +139,19 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider) + def test_httperror_allow_all_false(self): + crawler = get_crawler(_HttpErrorSpider) + mw = HttpErrorMiddleware.from_crawler(crawler) + request_httpstatus_false = Request('http://scrapytest.org', meta={'handle_httpstatus_all': False}) + request_httpstatus_true = Request('http://scrapytest.org', meta={'handle_httpstatus_all': True}) + res404 = self.res404.copy() + res404.request = request_httpstatus_false + res402 = self.res402.copy() + res402.request = request_httpstatus_true + + self.assertRaises(HttpError, mw.process_spider_input, res404, self.spider) + self.assertIsNone(mw.process_spider_input(res402, self.spider)) + class TestHttpErrorMiddlewareIntegrational(TrialTestCase): def setUp(self): From a41c205928aa2aa86233de0cfb694b0c7ded2297 Mon Sep 17 00:00:00 2001 From: Jose Galdos Date: Fri, 21 Aug 2020 12:16:37 -0500 Subject: [PATCH 2/2] Update httpstatus documentation. --- docs/topics/spider-middleware.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index c6cbdba76..28645fd53 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -255,7 +255,8 @@ this:: The ``handle_httpstatus_list`` key of :attr:`Request.meta ` can also be used to specify which response codes to allow on a per-request basis. You can also set the meta key ``handle_httpstatus_all`` -to ``True`` if you want to allow any response code for a request. +to ``True`` if you want to allow any response code for a request, and ``False`` to +disable the effects of the ``handle_httpstatus_all`` key. Keep in mind, however, that it's usually a bad idea to handle non-200 responses, unless you really know what you're doing.