diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 230332673..b24ccf6ae 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -20,10 +20,6 @@ def _isiterable(possible_iterator): return hasattr(possible_iterator, '__iter__') or hasattr(possible_iterator, '__aiter__') -def _fname(f): - return f"{f.__self__.__class__.__name__}.{f.__func__.__name__}" - - class SpiderMiddlewareManager(MiddlewareManager): component_name = 'spider middleware' @@ -48,7 +44,7 @@ class SpiderMiddlewareManager(MiddlewareManager): try: result = method(response=response, spider=spider) if result is not None: - msg = (f"Middleware {_fname(method)} must return None " + msg = (f"Middleware {method.__qualname__} must return None " f"or raise an exception, got {type(result)}") raise _InvalidOutput(msg) except _InvalidOutput: @@ -88,7 +84,7 @@ class SpiderMiddlewareManager(MiddlewareManager): elif result is None: continue else: - msg = (f"Middleware {_fname(method)} must return None " + msg = (f"Middleware {method.__qualname__} must return None " f"or an iterable, got {type(result)}") raise _InvalidOutput(msg) return _failure @@ -96,7 +92,8 @@ class SpiderMiddlewareManager(MiddlewareManager): def _process_spider_output(self, response, spider, result, start_index=0): # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method - if isinstance(result, collections.abc.AsyncIterator): + result_async = isinstance(result, collections.abc.AsyncIterator) + if result_async: recovered = MutableAsyncChain() else: recovered = MutableChain() @@ -116,9 +113,11 @@ class SpiderMiddlewareManager(MiddlewareManager): if _isiterable(result): result = self._evaluate_iterable(response, spider, result, method_index + 1, recovered) else: - msg = (f"Middleware {_fname(method)} must return an " + msg = (f"Middleware {method.__qualname__} must return an " f"iterable, got {type(result)}") raise _InvalidOutput(msg) + if result_async and isinstance(result, collections.abc.Iterator): + raise TypeError(f"Synchronous {method.__qualname__} called with an async iterable") # check this again as the middlewares could change "result" from sync to async if isinstance(result, collections.abc.AsyncIterator): diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 2584dec21..0a6b96c0c 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -122,11 +122,6 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): result = yield self.mwman.scrape_response(self._scrape_func, self.response, self.request, self.spider) return result - def assertAsyncGeneratorNotIterable(self, o): - with self.assertRaisesRegex(TypeError, - "'(async_generator|MutableAsyncChain)' object is not iterable"): - list(o) - @defer.inlineCallbacks def _test_simple_base(self, *mw_classes): result = yield self._get_middleware_result(*mw_classes) @@ -145,9 +140,8 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): @defer.inlineCallbacks def _test_asyncgen_fail(self, *mw_classes): - result = yield self._get_middleware_result(*mw_classes) - self.assertIsInstance(result, collections.abc.Iterable) - self.assertAsyncGeneratorNotIterable(result) + with self.assertRaisesRegex(TypeError, "Synchronous .+ called with an async iterable"): + yield self._get_middleware_result(*mw_classes) class ProcessSpiderOutputSimpleMiddleware: @@ -238,14 +232,10 @@ class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple): """ Simple mw; cannot work """ return self._test_asyncgen_fail(self.MW_SIMPLE) - @defer.inlineCallbacks def test_simple_asyncgen(self): """ Simple mw -> asyncgen mw; cannot work """ - result = yield self._get_middleware_result( - self.MW_ASYNCGEN, - self.MW_SIMPLE) - self.assertIsInstance(result, collections.abc.AsyncIterable) - self.assertAsyncGeneratorNotIterable(result) + return self._test_asyncgen_fail(self.MW_ASYNCGEN, + self.MW_SIMPLE) def test_universal(self): """ Universal mw """