RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

adding the process_spider_failure method to spider middlewares

This commit is contained in:
Raul Gallegos 2019-02-06 22:55:23 -05:00
parent 65d631329a
commit 9a0b909b9d
1 changed files with 18 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
scrapy/core/spidermw.py
View File

@ -26,8 +26,11 @@ class SpiderMiddlewareManager(MiddlewareManager):
self.methods['process_spider_input'].append(mw.process_spider_input)
if hasattr(mw, 'process_spider_output'):
self.methods['process_spider_output'].appendleft(mw.process_spider_output)
if hasattr(mw, 'process_spider_exception'):
self.methods['process_spider_exception'].appendleft(mw.process_spider_exception)
if hasattr(mw, 'process_spider_exception') or hasattr(mw, 'process_spider_failure'):
if hasattr(mw, 'process_spider_failure'):
self.methods['process_spider_failure'].appendleft(mw.process_spider_failure)
else:
self.methods['process_spider_exception'].appendleft(mw.process_spider_exception)
if hasattr(mw, 'process_start_requests'):
self.methods['process_start_requests'].appendleft(mw.process_start_requests)
@ -48,15 +51,23 @@ class SpiderMiddlewareManager(MiddlewareManager):
return scrape_func(Failure(), request, spider)
return scrape_func(response, request, spider)
def process_spider_exception(_failure):
exception = _failure.value
for method in self.methods['process_spider_exception']:
result = method(response=response, exception=exception, spider=spider)
def process_spider_failure(_failure):
for method in self.methods['process_spider_failure']:
result = method(response=response, failure=_failure, spider=spider)
assert result is None or _isiterable(result), \
'Middleware %s must returns None, or an iterable object, got %s ' % \
(fname(method), type(result))
if result is not None:
return result
for method in self.methods['process_spider_exception']:
result = method(response=response, exception=_failure.value, spider=spider)
assert result is None or _isiterable(result), \
'Middleware %s must returns None, or an iterable object, got %s ' % \
(fname(method), type(result))
if result is not None:
return result
return _failure
def process_spider_output(result):
@ -68,7 +79,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
return result
dfd = mustbe_deferred(process_spider_input, response)
dfd.addErrback(process_spider_exception)
dfd.addErrback(process_spider_failure)
dfd.addCallback(process_spider_output)
return dfd