RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

More robust sync/async middleware mix checking.

This commit is contained in:
Andrey Rakhmatullin 2021-04-02 23:06:29 +05:00
parent 30ed7fa349
commit 7bd1d888d4
1 changed files with 5 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
scrapy/core/spidermw.py
View File

@ -92,8 +92,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
def _process_spider_output(self, response, spider, result, start_index=0):
# items in this iterable do not need to go through the process_spider_output
# chain, they went through it already from the process_spider_exception method
result_async = isinstance(result, collections.abc.AsyncIterator)
if result_async:
last_result_async = isinstance(result, collections.abc.AsyncIterator)
if last_result_async:
recovered = MutableAsyncChain()
else:
recovered = MutableChain()
@ -116,11 +116,11 @@ class SpiderMiddlewareManager(MiddlewareManager):
msg = (f"Middleware {method.__qualname__} must return an "
f"iterable, got {type(result)}")
raise _InvalidOutput(msg)
if result_async and isinstance(result, collections.abc.Iterator):
if last_result_async and isinstance(result, collections.abc.Iterator):
raise TypeError(f"Synchronous {method.__qualname__} called with an async iterable")
last_result_async = isinstance(result, collections.abc.AsyncIterator)
# check this again as the middlewares could change "result" from sync to async
if isinstance(result, collections.abc.AsyncIterator):
if last_result_async:
return MutableAsyncChain(result, recovered)
else:
return MutableChain(result, recovered)