Merge pull request #4694 from Jgaldos/improve-httpstatus-all-meta

Improve http status all on http error middleware

docs/topics/spider-middleware.rst

@@ -253,7 +253,8 @@ this::
 The ``handle_httpstatus_list`` key of :attr:`Request.meta
 <scrapy.http.Request.meta>` can also be used to specify which response codes to
 allow on a per-request basis. You can also set the meta key ``handle_httpstatus_all``
-to ``True`` if you want to allow any response code for a request.
+to ``True`` if you want to allow any response code for a request, and ``False`` to
+disable the effects of the ``handle_httpstatus_all`` key.
 
 Keep in mind, however, that it's usually a bad idea to handle non-200
 responses, unless you really know what you're doing.

scrapy/spidermiddlewares/httperror.py

@@ -32,7 +32,7 @@ class HttpErrorMiddleware:
         if 200 <= response.status < 300:  # common case
             return
         meta = response.meta
-        if 'handle_httpstatus_all' in meta:
+        if meta.get('handle_httpstatus_all', False):
             return
         if 'handle_httpstatus_list' in meta:
             allowed_statuses = meta['handle_httpstatus_list']

tests/test_spidermiddleware_httperror.py

@@ -139,6 +139,19 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
         self.assertIsNone(self.mw.process_spider_input(res404, self.spider))
         self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider)
 
+    def test_httperror_allow_all_false(self):
+        crawler = get_crawler(_HttpErrorSpider)
+        mw = HttpErrorMiddleware.from_crawler(crawler)
+        request_httpstatus_false = Request('http://scrapytest.org', meta={'handle_httpstatus_all': False})
+        request_httpstatus_true = Request('http://scrapytest.org', meta={'handle_httpstatus_all': True})
+        res404 = self.res404.copy()
+        res404.request = request_httpstatus_false
+        res402 = self.res402.copy()
+        res402.request = request_httpstatus_true
+
+        self.assertRaises(HttpError, mw.process_spider_input, res404, self.spider)
+        self.assertIsNone(mw.process_spider_input(res402, self.spider))
+
 
 class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
     def setUp(self):