RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

HttpErrorMiddleware: performance improvement and added support for 'handle_httpstatus_list' request meta key

This commit is contained in:
Pablo Hoffman 2009-08-15 17:14:35 -03:00
parent 8c2f62ba9c
commit 2463842acb
1 changed files with 9 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
scrapy/contrib/spidermiddleware/httperror.py
View File

@ -7,6 +7,12 @@ See documentation in docs/ref/spider-middleware.rst
class HttpErrorMiddleware(object):
def process_spider_input(self, response, spider):
if not (200 <= response.status < 300 or \
response.status in getattr(spider, 'handle_httpstatus_list', [])):
return []
if 200 <= response.status < 300: # common case
return
if 'handle_httpstatus_list' in response.request.meta:
allowed_statuses = response.request.meta['handle_httpstatus_list']
else:
allowed_statuses = getattr(spider, 'handle_httpstatus_list', ())
if response.status in allowed_statuses:
return
return []