RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Some cleanup, make sync middlewares fail earlier.

This commit is contained in:
Andrey Rakhmatullin 2021-04-02 22:20:56 +05:00
parent 849472535e
commit 30ed7fa349
2 changed files with 11 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
scrapy/core/spidermw.py
View File

@ -20,10 +20,6 @@ def _isiterable(possible_iterator):
return hasattr(possible_iterator, '__iter__') or hasattr(possible_iterator, '__aiter__')
def _fname(f):
return f"{f.__self__.__class__.__name__}.{f.__func__.__name__}"
class SpiderMiddlewareManager(MiddlewareManager):
component_name = 'spider middleware'
@ -48,7 +44,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
try:
result = method(response=response, spider=spider)
if result is not None:
msg = (f"Middleware {_fname(method)} must return None "
msg = (f"Middleware {method.__qualname__} must return None "
f"or raise an exception, got {type(result)}")
raise _InvalidOutput(msg)
except _InvalidOutput:
@ -88,7 +84,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
elif result is None:
continue
else:
msg = (f"Middleware {_fname(method)} must return None "
msg = (f"Middleware {method.__qualname__} must return None "
f"or an iterable, got {type(result)}")
raise _InvalidOutput(msg)
return _failure
@ -96,7 +92,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
def _process_spider_output(self, response, spider, result, start_index=0):
# items in this iterable do not need to go through the process_spider_output
# chain, they went through it already from the process_spider_exception method
if isinstance(result, collections.abc.AsyncIterator):
result_async = isinstance(result, collections.abc.AsyncIterator)
if result_async:
recovered = MutableAsyncChain()
else:
recovered = MutableChain()
@ -116,9 +113,11 @@ class SpiderMiddlewareManager(MiddlewareManager):
if _isiterable(result):
result = self._evaluate_iterable(response, spider, result, method_index + 1, recovered)
else:
msg = (f"Middleware {_fname(method)} must return an "
msg = (f"Middleware {method.__qualname__} must return an "
f"iterable, got {type(result)}")
raise _InvalidOutput(msg)
if result_async and isinstance(result, collections.abc.Iterator):
raise TypeError(f"Synchronous {method.__qualname__} called with an async iterable")
# check this again as the middlewares could change "result" from sync to async
if isinstance(result, collections.abc.AsyncIterator):

18
tests/test_spidermiddleware.py
View File

@ -122,11 +122,6 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
result = yield self.mwman.scrape_response(self._scrape_func, self.response, self.request, self.spider)
return result
def assertAsyncGeneratorNotIterable(self, o):
with self.assertRaisesRegex(TypeError,
"'(async_generator|MutableAsyncChain)' object is not iterable"):
list(o)
@defer.inlineCallbacks
def _test_simple_base(self, *mw_classes):
result = yield self._get_middleware_result(*mw_classes)
@ -145,9 +140,8 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
@defer.inlineCallbacks
def _test_asyncgen_fail(self, *mw_classes):
result = yield self._get_middleware_result(*mw_classes)
self.assertIsInstance(result, collections.abc.Iterable)
self.assertAsyncGeneratorNotIterable(result)
with self.assertRaisesRegex(TypeError, "Synchronous .+ called with an async iterable"):
yield self._get_middleware_result(*mw_classes)
class ProcessSpiderOutputSimpleMiddleware:
@ -238,14 +232,10 @@ class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple):
""" Simple mw; cannot work """
return self._test_asyncgen_fail(self.MW_SIMPLE)
@defer.inlineCallbacks
def test_simple_asyncgen(self):
""" Simple mw -> asyncgen mw; cannot work """
result = yield self._get_middleware_result(
self.MW_ASYNCGEN,
self.MW_SIMPLE)
self.assertIsInstance(result, collections.abc.AsyncIterable)
self.assertAsyncGeneratorNotIterable(result)
return self._test_asyncgen_fail(self.MW_ASYNCGEN,
self.MW_SIMPLE)
def test_universal(self):
""" Universal mw """