diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 4b2520aa1..c7706bb7b 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -27,7 +27,7 @@ from scrapy.utils.defer import ( maybe_deferred_to_future, mustbe_deferred, ) -from scrapy.utils.python import MutableAsyncChain, MutableChain +from scrapy.utils.python import MutableAsyncChain, MutableChain, global_object_name if TYPE_CHECKING: from collections.abc import Generator @@ -51,10 +51,6 @@ def _isiterable(o: Any) -> bool: class SpiderMiddlewareManager(MiddlewareManager): component_name = "spider middleware" - def __init__(self, *middlewares: Any): - super().__init__(*middlewares) - self.downgrade_warning_done = False - @classmethod def _get_mwlist_from_settings(cls, settings: BaseSettings) -> list[Any]: return build_component_list(settings.getwithbase("SPIDER_MIDDLEWARES")) @@ -227,12 +223,13 @@ class SpiderMiddlewareManager(MiddlewareManager): # Iterable -> AsyncIterable result = as_async_generator(result) elif need_downgrade: - if not self.downgrade_warning_done: - logger.warning( - f"Async iterable passed to {method.__qualname__} " - f"was downgraded to a non-async one" - ) - self.downgrade_warning_done = True + logger.warning( + f"Async iterable passed to {method.__qualname__} was" + f" downgraded to a non-async one. This is deprecated and will" + f" stop working in a future version of Scrapy. Please see" + f" https://docs.scrapy.org/en/latest/topics/coroutines.html#mixing-synchronous-and-asynchronous-spider-middlewares" + f" for more information." + ) assert isinstance(result, AsyncIterable) # AsyncIterable -> Iterable result = yield deferred_from_coro(collect_asyncgen(result)) @@ -340,10 +337,19 @@ class SpiderMiddlewareManager(MiddlewareManager): methodname_async = methodname + "_async" async_method: Callable | None = getattr(mw, methodname_async, None) if not async_method: + if normal_method and not isasyncgenfunction(normal_method): + logger.warning( + f"Middleware {global_object_name(mw.__class__)} doesn't support" + f" asynchronous spider output, this is deprecated and will stop" + f" working in a future version of Scrapy. The middleware should" + f" be updated to support it. Please see" + f" https://docs.scrapy.org/en/latest/topics/coroutines.html#mixing-synchronous-and-asynchronous-spider-middlewares" + f" for more information." + ) return normal_method if not normal_method: logger.error( - f"Middleware {mw.__qualname__} has {methodname_async} " + f"Middleware {global_object_name(mw.__class__)} has {methodname_async} " f"without {methodname}, skipping this method." ) return None diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index ba64ba721..a8507c789 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -152,6 +152,10 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): self.assertEqual(len(result_list), self.RESULT_COUNT) self.assertIsInstance(result_list[0], self.ITEM_TYPE) self.assertEqual("downgraded to a non-async" in str(log), downgrade) + self.assertEqual( + "doesn't support asynchronous spider output" in str(log), + ProcessSpiderOutputSimpleMiddleware in mw_classes, + ) @defer.inlineCallbacks def _test_asyncgen_base( @@ -376,21 +380,21 @@ class UniversalMiddlewareManagerTest(TestCase): self.mwman = SpiderMiddlewareManager() def test_simple_mw(self): - mw = ProcessSpiderOutputSimpleMiddleware + mw = ProcessSpiderOutputSimpleMiddleware() self.mwman._add_middleware(mw) self.assertEqual( self.mwman.methods["process_spider_output"][0], mw.process_spider_output ) def test_async_mw(self): - mw = ProcessSpiderOutputAsyncGenMiddleware + mw = ProcessSpiderOutputAsyncGenMiddleware() self.mwman._add_middleware(mw) self.assertEqual( self.mwman.methods["process_spider_output"][0], mw.process_spider_output ) def test_universal_mw(self): - mw = ProcessSpiderOutputUniversalMiddleware + mw = ProcessSpiderOutputUniversalMiddleware() self.mwman._add_middleware(mw) self.assertEqual( self.mwman.methods["process_spider_output"][0], @@ -399,7 +403,7 @@ class UniversalMiddlewareManagerTest(TestCase): def test_universal_mw_no_sync(self): with LogCapture() as log: - self.mwman._add_middleware(UniversalMiddlewareNoSync) + self.mwman._add_middleware(UniversalMiddlewareNoSync()) self.assertIn( "UniversalMiddlewareNoSync has process_spider_output_async" " without process_spider_output", @@ -408,7 +412,7 @@ class UniversalMiddlewareManagerTest(TestCase): self.assertEqual(self.mwman.methods["process_spider_output"][0], None) def test_universal_mw_both_sync(self): - mw = UniversalMiddlewareBothSync + mw = UniversalMiddlewareBothSync() with LogCapture() as log: self.mwman._add_middleware(mw) self.assertIn( @@ -422,7 +426,7 @@ class UniversalMiddlewareManagerTest(TestCase): def test_universal_mw_both_async(self): with LogCapture() as log: - self.mwman._add_middleware(UniversalMiddlewareBothAsync) + self.mwman._add_middleware(UniversalMiddlewareBothAsync()) self.assertIn( "UniversalMiddlewareBothAsync.process_spider_output " "is an async generator function while process_spider_output_async exists",