RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Store output methods on the 'methods' dict

This commit is contained in:
Eugenio Lacuesta 2018-07-01 17:49:30 -03:00
parent 4fca9aba85
commit 985ab636cf
1 changed files with 18 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
scrapy/core/spidermw.py
View File

@ -17,11 +17,6 @@ class SpiderMiddlewareManager(MiddlewareManager):
component_name = 'spider middleware'
# List of dicts. Each dict represents a spider middleware and contains the
# 'process_spider_output', 'process_spider_exception' methods.
# The idea is to simulate the behaviour of a Twisted deferred's callback/errback chain
output_methods = []
@classmethod
def _get_mwlist_from_settings(cls, settings):
return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES'))
@ -32,10 +27,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
self.methods['process_spider_input'].append(mw.process_spider_input)
if hasattr(mw, 'process_start_requests'):
self.methods['process_start_requests'].insert(0, mw.process_start_requests)
self.output_methods.insert(0, dict(
process_spider_output=getattr(mw, 'process_spider_output', None),
process_spider_exception=getattr(mw, 'process_spider_exception', None),
))
self.methods['process_spider_output'].insert(0, getattr(mw, 'process_spider_output', None))
self.methods['process_spider_exception'].insert(0, getattr(mw, 'process_spider_exception', None))
def scrape_response(self, scrape_func, response, request, spider):
fname = lambda f:'%s.%s' % (
@ -53,54 +46,54 @@ class SpiderMiddlewareManager(MiddlewareManager):
return scrape_func(Failure(), request, spider)
return scrape_func(response, request, spider)
def process_spider_exception(_failure, mw_index):
def process_spider_exception(_failure, index):
exception = _failure.value
# don't handle _InvalidOutput exception
if isinstance(exception, _InvalidOutput):
return _failure
for index, mw in enumerate(self.output_methods):
if index < mw_index or mw['process_spider_exception'] is None:
for i, method in enumerate(self.methods['process_spider_exception']):
if i < index or method is None:
continue
result = mw['process_spider_exception'](response=response, exception=exception, spider=spider)
mw_index += 1
result = method(response=response, exception=exception, spider=spider)
index += 1
if _isiterable(result):
# stop exception handling by handing control over to the
# process_spider_output chain if an iterable has been returned
return process_spider_output(result, mw_index)
return process_spider_output(result, index)
elif result is None:
continue
else:
raise _InvalidOutput('Middleware {} must return None or an iterable, got {}' \
.format(fname(mw['process_spider_exception']), type(result)))
.format(fname(method), type(result)))
return _failure
def process_spider_output(result, mw_index):
def process_spider_output(result, index):
def wrapper(result_iterable):
try:
for r in result_iterable:
yield r
except Exception as ex:
# process the exception with the method from the next middleware
exception_result = process_spider_exception(Failure(ex), mw_index)
exception_result = process_spider_exception(Failure(ex), index)
if exception_result is None or isinstance(exception_result, Failure):
raise
for output in exception_result:
yield output
for index, mw in enumerate(self.output_methods):
if index < mw_index or mw['process_spider_output'] is None:
for i, method in enumerate(self.methods['process_spider_output']):
if i < index or method is None:
continue
result = mw['process_spider_output'](response=response, result=result, spider=spider)
mw_index += 1
result = method(response=response, result=result, spider=spider)
index += 1
if _isiterable(result):
result = wrapper(result)
else:
raise _InvalidOutput('Middleware {} must return an iterable, got {}' \
.format(fname(mw['process_spider_output']), type(result)))
.format(fname(method), type(result)))
return result
dfd = mustbe_deferred(process_spider_input, response)
dfd.addErrback(process_spider_exception, mw_index=0)
dfd.addCallback(process_spider_output, mw_index=0)
dfd.addErrback(process_spider_exception, index=0)
dfd.addCallback(process_spider_output, index=0)
return dfd
def process_start_requests(self, start_requests, spider):