mirror of https://github.com/scrapy/scrapy.git
Improve diagnostics for sync-only spider middlewares.
This commit is contained in:
Andrey Rakhmatullin
parent
f041f26a6f
commit
d8978d405c
|
|
@ -27,7 +27,7 @@ from scrapy.utils.defer import (
|
|||
maybe_deferred_to_future,
|
||||
mustbe_deferred,
|
||||
)
|
||||
from scrapy.utils.python import MutableAsyncChain, MutableChain
|
||||
from scrapy.utils.python import MutableAsyncChain, MutableChain, global_object_name
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Generator
|
||||
|
|
@ -51,10 +51,6 @@ def _isiterable(o: Any) -> bool:
|
|||
class SpiderMiddlewareManager(MiddlewareManager):
|
||||
component_name = "spider middleware"
|
||||
|
||||
def __init__(self, *middlewares: Any):
|
||||
super().__init__(*middlewares)
|
||||
self.downgrade_warning_done = False
|
||||
|
||||
@classmethod
|
||||
def _get_mwlist_from_settings(cls, settings: BaseSettings) -> list[Any]:
|
||||
return build_component_list(settings.getwithbase("SPIDER_MIDDLEWARES"))
|
||||
|
|
@ -227,12 +223,13 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
# Iterable -> AsyncIterable
|
||||
result = as_async_generator(result)
|
||||
elif need_downgrade:
|
||||
if not self.downgrade_warning_done:
|
||||
logger.warning(
|
||||
f"Async iterable passed to {method.__qualname__} "
|
||||
f"was downgraded to a non-async one"
|
||||
)
|
||||
self.downgrade_warning_done = True
|
||||
logger.warning(
|
||||
f"Async iterable passed to {method.__qualname__} was"
|
||||
f" downgraded to a non-async one. This is deprecated and will"
|
||||
f" stop working in a future version of Scrapy. Please see"
|
||||
f" https://docs.scrapy.org/en/latest/topics/coroutines.html#mixing-synchronous-and-asynchronous-spider-middlewares"
|
||||
f" for more information."
|
||||
)
|
||||
assert isinstance(result, AsyncIterable)
|
||||
# AsyncIterable -> Iterable
|
||||
result = yield deferred_from_coro(collect_asyncgen(result))
|
||||
|
|
@ -340,10 +337,19 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
methodname_async = methodname + "_async"
|
||||
async_method: Callable | None = getattr(mw, methodname_async, None)
|
||||
if not async_method:
|
||||
if normal_method and not isasyncgenfunction(normal_method):
|
||||
logger.warning(
|
||||
f"Middleware {global_object_name(mw.__class__)} doesn't support"
|
||||
f" asynchronous spider output, this is deprecated and will stop"
|
||||
f" working in a future version of Scrapy. The middleware should"
|
||||
f" be updated to support it. Please see"
|
||||
f" https://docs.scrapy.org/en/latest/topics/coroutines.html#mixing-synchronous-and-asynchronous-spider-middlewares"
|
||||
f" for more information."
|
||||
)
|
||||
return normal_method
|
||||
if not normal_method:
|
||||
logger.error(
|
||||
f"Middleware {mw.__qualname__} has {methodname_async} "
|
||||
f"Middleware {global_object_name(mw.__class__)} has {methodname_async} "
|
||||
f"without {methodname}, skipping this method."
|
||||
)
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -152,6 +152,10 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
|
|||
self.assertEqual(len(result_list), self.RESULT_COUNT)
|
||||
self.assertIsInstance(result_list[0], self.ITEM_TYPE)
|
||||
self.assertEqual("downgraded to a non-async" in str(log), downgrade)
|
||||
self.assertEqual(
|
||||
"doesn't support asynchronous spider output" in str(log),
|
||||
ProcessSpiderOutputSimpleMiddleware in mw_classes,
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _test_asyncgen_base(
|
||||
|
|
@ -376,21 +380,21 @@ class UniversalMiddlewareManagerTest(TestCase):
|
|||
self.mwman = SpiderMiddlewareManager()
|
||||
|
||||
def test_simple_mw(self):
|
||||
mw = ProcessSpiderOutputSimpleMiddleware
|
||||
mw = ProcessSpiderOutputSimpleMiddleware()
|
||||
self.mwman._add_middleware(mw)
|
||||
self.assertEqual(
|
||||
self.mwman.methods["process_spider_output"][0], mw.process_spider_output
|
||||
)
|
||||
|
||||
def test_async_mw(self):
|
||||
mw = ProcessSpiderOutputAsyncGenMiddleware
|
||||
mw = ProcessSpiderOutputAsyncGenMiddleware()
|
||||
self.mwman._add_middleware(mw)
|
||||
self.assertEqual(
|
||||
self.mwman.methods["process_spider_output"][0], mw.process_spider_output
|
||||
)
|
||||
|
||||
def test_universal_mw(self):
|
||||
mw = ProcessSpiderOutputUniversalMiddleware
|
||||
mw = ProcessSpiderOutputUniversalMiddleware()
|
||||
self.mwman._add_middleware(mw)
|
||||
self.assertEqual(
|
||||
self.mwman.methods["process_spider_output"][0],
|
||||
|
|
@ -399,7 +403,7 @@ class UniversalMiddlewareManagerTest(TestCase):
|
|||
|
||||
def test_universal_mw_no_sync(self):
|
||||
with LogCapture() as log:
|
||||
self.mwman._add_middleware(UniversalMiddlewareNoSync)
|
||||
self.mwman._add_middleware(UniversalMiddlewareNoSync())
|
||||
self.assertIn(
|
||||
"UniversalMiddlewareNoSync has process_spider_output_async"
|
||||
" without process_spider_output",
|
||||
|
|
@ -408,7 +412,7 @@ class UniversalMiddlewareManagerTest(TestCase):
|
|||
self.assertEqual(self.mwman.methods["process_spider_output"][0], None)
|
||||
|
||||
def test_universal_mw_both_sync(self):
|
||||
mw = UniversalMiddlewareBothSync
|
||||
mw = UniversalMiddlewareBothSync()
|
||||
with LogCapture() as log:
|
||||
self.mwman._add_middleware(mw)
|
||||
self.assertIn(
|
||||
|
|
@ -422,7 +426,7 @@ class UniversalMiddlewareManagerTest(TestCase):
|
|||
|
||||
def test_universal_mw_both_async(self):
|
||||
with LogCapture() as log:
|
||||
self.mwman._add_middleware(UniversalMiddlewareBothAsync)
|
||||
self.mwman._add_middleware(UniversalMiddlewareBothAsync())
|
||||
self.assertIn(
|
||||
"UniversalMiddlewareBothAsync.process_spider_output "
|
||||
"is an async generator function while process_spider_output_async exists",
|
||||
|
|
|
|||
Loading…
Reference in New Issue