RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improve http status all on http error middleware

This commit is contained in:
Jose Galdos 2020-07-23 18:45:45 -05:00
parent ce0c25fc85
commit 13181ba788
2 changed files with 14 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/spidermiddlewares/httperror.py
View File

@ -32,7 +32,7 @@ class HttpErrorMiddleware:
if 200 <= response.status < 300: # common case
return
meta = response.meta
if 'handle_httpstatus_all' in meta:
if meta.get('handle_httpstatus_all', False):
return
if 'handle_httpstatus_list' in meta:
allowed_statuses = meta['handle_httpstatus_list']

13
tests/test_spidermiddleware_httperror.py
View File

@ -139,6 +139,19 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
self.assertIsNone(self.mw.process_spider_input(res404, self.spider))
self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider)
def test_httperror_allow_all_false(self):
crawler = get_crawler(_HttpErrorSpider)
mw = HttpErrorMiddleware.from_crawler(crawler)
request_httpstatus_false = Request('http://scrapytest.org', meta={'handle_httpstatus_all': False})
request_httpstatus_true = Request('http://scrapytest.org', meta={'handle_httpstatus_all': True})
res404 = self.res404.copy()
res404.request = request_httpstatus_false
res402 = self.res402.copy()
res402.request = request_httpstatus_true
self.assertRaises(HttpError, mw.process_spider_input, res404, self.spider)
self.assertIsNone(mw.process_spider_input(res402, self.spider))
class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
def setUp(self):