From f0c933f0948fe6e5524d7f871d83e2e18669d6bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 12 Mar 2025 06:34:11 +0100 Subject: [PATCH] Implement spider middleware backward compatibility and deprecation warnings --- scrapy/core/engine.py | 1 + scrapy/core/spidermw.py | 84 +++++++- tests/test_spider_yield_seeds.py | 56 +++--- tests/test_spidermiddleware_process_seeds.py | 200 ++++++++++++++++--- 4 files changed, 277 insertions(+), 64 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 0f04667d1..60d959b2a 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -215,6 +215,7 @@ class ExecutionEngine: self.crawl(seed) else: self.scraper.start_itemproc(seed, response=None) + self.slot.nextcall.schedule() @inlineCallbacks def _start_next_requests(self) -> Generator[Deferred[Any], Any, None]: diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 188b8dcba..7c9932770 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -56,11 +56,55 @@ class SpiderMiddlewareManager(MiddlewareManager): def _get_mwlist_from_settings(cls, settings: BaseSettings) -> list[Any]: return build_component_list(settings.getwithbase("SPIDER_MIDDLEWARES")) + def __init__(self, *middlewares: Any) -> None: + self._check_deprecated_process_start_requests_use(middlewares) + super().__init__(*middlewares) + + def _check_deprecated_process_start_requests_use( + self, middlewares: list[Any] + ) -> None: + deprecated_middlewares = [ + middleware + for middleware in middlewares + if hasattr(middleware, "process_start_requests") + and not hasattr(middleware, "process_seeds") + ] + self._use_start_requests = bool(deprecated_middlewares) + if self._use_start_requests: + deprecated_middleware_list = ", ".join( + global_object_name(middleware.__class__) + for middleware in deprecated_middlewares + ) + warn( + f"The following enabled spider middlewares, directly or " + f"through their parent classes, define the deprecated " + f"process_start_requests() method: " + f"{deprecated_middleware_list}. process_start_requests() has " + f"been deprecated in favor of a new method, process_seeds(), " + f"to support asynchronous code execution. " + f"process_start_requests() will stop being called in a future " + f"version of Scrapy. If you use Scrapy VERSION or higher " + f"only, replace process_start_requests() with " + f"process_seeds(); note that process_seeds() is a coroutine " + f"(async def). If you need to maintain compatibility with " + f"lower Scrapy versions, when defining " + f"process_start_requests() in a spider middleware class, " + f"define process_seeds() as well. See the release notes of " + f"Scrapy VERSION for details: " + f"https://docs.scrapy.org/en/VERSION/news.html", + ScrapyDeprecationWarning, + ) + def _add_middleware(self, mw: Any) -> None: super()._add_middleware(mw) if hasattr(mw, "process_spider_input"): self.methods["process_spider_input"].append(mw.process_spider_input) - if hasattr(mw, "process_seeds"): + if self._use_start_requests: + if hasattr(mw, "process_start_requests"): + self.methods["process_start_requests"].appendleft( + mw.process_start_requests + ) + elif hasattr(mw, "process_seeds"): self.methods["process_seeds"].appendleft(mw.process_seeds) process_spider_output = self._get_async_method_pair(mw, "process_spider_output") self.methods["process_spider_output"].appendleft(process_spider_output) @@ -329,12 +373,16 @@ class SpiderMiddlewareManager(MiddlewareManager): self, spider: Spider ) -> Generator[Deferred[Any], Any, AsyncIterator[Any]]: self._check_deprecated_start_requests_use(spider) - seeds = yield self._iter_seeds(spider) - seeds = yield self._process_chain("process_seeds", seeds) + if self._use_start_requests: + seeds = iter(spider.start_requests()) + seeds = yield self._process_chain("process_start_requests", seeds, spider) + seeds = as_async_generator(seeds) + else: + seeds = yield self._iter_seeds(spider) + seeds = yield self._process_chain("process_seeds", seeds) return seeds - @staticmethod - def _check_deprecated_start_requests_use(spider: Spider): + def _check_deprecated_start_requests_use(self, spider: Spider): start_requests_cls = None yield_seeds_cls = None spidercls = spider.__class__ @@ -378,6 +426,32 @@ class SpiderMiddlewareManager(MiddlewareManager): ScrapyDeprecationWarning, ) + if ( + self._use_start_requests + and yield_seeds_cls is not Spider + and start_requests_cls is not yield_seeds_cls + and mro.index(yield_seeds_cls) < mro.index(start_requests_cls) + ): + src = global_object_name(yield_seeds_cls) + if yield_seeds_cls is not spidercls: + src += f" (inherited by {global_object_name(spidercls)})" + raise ValueError( + f"{src} does not define the deprecated start_requests() " + f"method. However, one or more of your enabled spider " + f"middlewares (reported in an earlier deprecation warning) " + f"define the process_start_requests() method, and not the " + f"process_seeds() method, making them only compatible with " + f"(deprecated) spiders that define the start_requests() " + f"method. To solve this issue, disable the offending spider " + f"middlewares, upgrade them as described in that earlier " + f"deprecation warning, or make your spider compatible with " + f"deprecated spider middlewares (and earlier Scrapy versions) " + f"by defining a sync start_requests() method that works " + f"similarly to its existing yield_seeds() method. See the " + f"release notes of Scrapy VERSION for details: " + f"https://docs.scrapy.org/en/VERSION/news.html" + ) + @staticmethod def _iter_seeds(spider: Spider): fn = spider.yield_seeds diff --git a/tests/test_spider_yield_seeds.py b/tests/test_spider_yield_seeds.py index 51b7c21ca..8f2a11afb 100644 --- a/tests/test_spider_yield_seeds.py +++ b/tests/test_spider_yield_seeds.py @@ -43,32 +43,6 @@ class MainTestCase(TestCase): assert crawler.stats.get_value("finish_reason") == "finished" assert actual_items == expected_items - async def _test_yield_seeds(self, yield_seeds_, expected_items=None): - class TestSpider(Spider): - name = "test" - yield_seeds = yield_seeds_ - - await self._test_spider(TestSpider, expected_items) - - @pytest.mark.only_asyncio - @deferred_f_from_coro_f - async def test_asyncio_delayed(self): - async def yield_seeds(spider): - await sleep(ASYNC_GEN_ERROR_MINIMUM_SECONDS) - yield ITEM_A - - await self._test_yield_seeds(yield_seeds, [ITEM_A]) - - @deferred_f_from_coro_f - async def test_twisted_delayed(self): - async def yield_seeds(spider): - await maybe_deferred_to_future( - twisted_sleep(ASYNC_GEN_ERROR_MINIMUM_SECONDS) - ) - yield ITEM_A - - await self._test_yield_seeds(yield_seeds, [ITEM_A]) - @deferred_f_from_coro_f async def test_start_urls(self): class TestSpider(Spider): @@ -102,7 +76,7 @@ class MainTestCase(TestCase): await self._test_spider(TestSpider, [ITEM_A]) @deferred_f_from_coro_f - async def test_start_requests(self): + async def test_deprecated(self): class TestSpider(Spider): name = "test" @@ -113,7 +87,7 @@ class MainTestCase(TestCase): await self._test_spider(TestSpider, [ITEM_A]) @deferred_f_from_coro_f - async def test_start_requests_subclass(self): + async def test_deprecated_subclass(self): class BaseSpider(Spider): def start_requests(self): yield ITEM_A @@ -151,3 +125,29 @@ class MainTestCase(TestCase): name = "test" await self._test_spider(TestSpider, [ITEM_A]) + + async def _test_yield_seeds(self, yield_seeds_, expected_items=None): + class TestSpider(Spider): + name = "test" + yield_seeds = yield_seeds_ + + await self._test_spider(TestSpider, expected_items) + + @pytest.mark.only_asyncio + @deferred_f_from_coro_f + async def test_asyncio_delayed(self): + async def yield_seeds(spider): + await sleep(ASYNC_GEN_ERROR_MINIMUM_SECONDS) + yield ITEM_A + + await self._test_yield_seeds(yield_seeds, [ITEM_A]) + + @deferred_f_from_coro_f + async def test_twisted_delayed(self): + async def yield_seeds(spider): + await maybe_deferred_to_future( + twisted_sleep(ASYNC_GEN_ERROR_MINIMUM_SECONDS) + ) + yield ITEM_A + + await self._test_yield_seeds(yield_seeds, [ITEM_A]) diff --git a/tests/test_spidermiddleware_process_seeds.py b/tests/test_spidermiddleware_process_seeds.py index 1affb30bb..57f28f1fb 100644 --- a/tests/test_spidermiddleware_process_seeds.py +++ b/tests/test_spidermiddleware_process_seeds.py @@ -1,17 +1,22 @@ from asyncio import sleep import pytest -from twisted.internet.defer import inlineCallbacks from twisted.trial.unittest import TestCase from scrapy import Spider, signals -from scrapy.utils.defer import maybe_deferred_to_future +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.test import get_crawler from .test_spider_yield_seeds import ASYNC_GEN_ERROR_MINIMUM_SECONDS, twisted_sleep +ITEM_A = {"id": "a"} +ITEM_B = {"id": "b"} +ITEM_C = {"id": "c"} +ITEM_D = {"id": "d"} -class AsyncioSpiderMiddleware: + +class AsyncioSleepSpiderMiddleware: async def process_seeds(self, seeds): await sleep(ASYNC_GEN_ERROR_MINIMUM_SECONDS) async for seed in seeds: @@ -24,24 +29,79 @@ class NoOpSpiderMiddleware: yield seed -class TwistedSpiderMiddleware: +class TwistedSleepSpiderMiddleware: async def process_seeds(self, seeds): await maybe_deferred_to_future(twisted_sleep(ASYNC_GEN_ERROR_MINIMUM_SECONDS)) async for seed in seeds: yield seed +class UniversalSpiderMiddleware: + async def process_seeds(self, seeds): + async for seed in seeds: + yield seed + + def process_start_requests(self, start_requests, spider): + raise NotImplementedError + + +# Spiders and spider middlewares for MainTestCase._test_wrap + + +class ModernWrapSpider(Spider): + name = "test" + + async def yield_seeds(self): + yield ITEM_B + + +class UniversalWrapSpider(Spider): + name = "test" + + async def yield_seeds(self): + yield ITEM_B + + def start_requests(self): + yield ITEM_D + + +class DeprecatedWrapSpider(Spider): + name = "test" + + def start_requests(self): + yield ITEM_B + + +class ModernWrapSpiderMiddleware: + async def process_seeds(self, seeds): + yield ITEM_A + async for seed in seeds: + yield seed + yield ITEM_C + + +class UniversalWrapSpiderMiddleware: + async def process_seeds(self, seeds): + yield ITEM_A + async for seed in seeds: + yield seed + yield ITEM_C + + def process_start_requests(self, seeds, spider): + yield ITEM_A + yield from seeds + yield ITEM_C + + +class DeprecatedWrapSpiderMiddleware: + def process_start_requests(self, seeds, spider): + yield ITEM_A + yield from seeds + yield ITEM_C + + class MainTestCase(TestCase): - @inlineCallbacks - def _test(self, spider_middlewares): - item = {"a": "b"} - - class TestSpider(Spider): - name = "test" - - async def yield_seeds(self): - yield item - + async def _test(self, spider_middlewares, spider_cls, expected_items): actual_items = [] def track_item(item, response, spider): @@ -50,30 +110,108 @@ class MainTestCase(TestCase): settings = { "SPIDER_MIDDLEWARES": {cls: n for n, cls in enumerate(spider_middlewares)}, } - crawler = get_crawler(TestSpider, settings_dict=settings) + crawler = get_crawler(spider_cls, settings_dict=settings) crawler.signals.connect(track_item, signals.item_scraped) - yield crawler.crawl() + await maybe_deferred_to_future(crawler.crawl()) assert crawler.stats.get_value("finish_reason") == "finished" - assert actual_items == [item] + assert actual_items == expected_items, f"{actual_items=} != {expected_items=}" + + async def _test_wrap(self, spider_middleware, spider_cls, expected_items=None): + expected_items = expected_items or [ITEM_A, ITEM_B, ITEM_C] + await self._test([spider_middleware], spider_cls, expected_items) + + @deferred_f_from_coro_f + async def test_modern_mw_modern_spider(self): + await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider) + + @deferred_f_from_coro_f + async def test_modern_mw_universal_spider(self): + await self._test_wrap(ModernWrapSpiderMiddleware, UniversalWrapSpider) + + @deferred_f_from_coro_f + async def test_modern_mw_deprecated_spider(self): + with pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" + ): + await self._test_wrap(ModernWrapSpiderMiddleware, DeprecatedWrapSpider) + + @deferred_f_from_coro_f + async def test_universal_mw_modern_spider(self): + await self._test_wrap(UniversalWrapSpiderMiddleware, ModernWrapSpider) + + @deferred_f_from_coro_f + async def test_universal_mw_universal_spider(self): + await self._test_wrap(UniversalWrapSpiderMiddleware, UniversalWrapSpider) + + @deferred_f_from_coro_f + async def test_universal_mw_deprecated_spider(self): + with pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" + ): + await self._test_wrap(UniversalWrapSpiderMiddleware, DeprecatedWrapSpider) + + @deferred_f_from_coro_f + async def test_deprecated_mw_modern_spider(self): + with ( + pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" + ), + pytest.raises( + ValueError, match=r"only compatible with \(deprecated\) spiders" + ), + ): + await self._test_wrap(DeprecatedWrapSpiderMiddleware, ModernWrapSpider) + + @deferred_f_from_coro_f + async def test_deprecated_mw_universal_spider(self): + with pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" + ): + await self._test_wrap( + DeprecatedWrapSpiderMiddleware, + UniversalWrapSpider, + [ITEM_A, ITEM_D, ITEM_C], + ) + + @deferred_f_from_coro_f + async def test_deprecated_mw_deprecated_spider(self): + with ( + pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" + ), + pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" + ), + ): + await self._test_wrap(DeprecatedWrapSpiderMiddleware, DeprecatedWrapSpider) + + async def _test_sleep(self, spider_middlewares): + class TestSpider(Spider): + name = "test" + + async def yield_seeds(self): + yield ITEM_A + + await self._test(spider_middlewares, TestSpider, [ITEM_A]) @pytest.mark.only_asyncio - @inlineCallbacks - def test_asyncio_delayed_single(self): - yield self._test([AsyncioSpiderMiddleware]) + @deferred_f_from_coro_f + async def test_asyncio_sleep_single(self): + await self._test_sleep([AsyncioSleepSpiderMiddleware]) @pytest.mark.only_asyncio - @inlineCallbacks - def test_asyncio_delayed_multiple(self): - yield self._test( - [NoOpSpiderMiddleware, AsyncioSpiderMiddleware, NoOpSpiderMiddleware] + @deferred_f_from_coro_f + async def test_asyncio_sleep_multiple(self): + await self._test_sleep( + [NoOpSpiderMiddleware, AsyncioSleepSpiderMiddleware, NoOpSpiderMiddleware] ) - @inlineCallbacks - def test_twisted_delayed_single(self): - yield self._test([TwistedSpiderMiddleware]) + @deferred_f_from_coro_f + async def test_twisted_sleep_single(self): + await self._test_sleep([TwistedSleepSpiderMiddleware]) - @inlineCallbacks - def test_twisted_delayed_multiple(self): - yield self._test( - [NoOpSpiderMiddleware, TwistedSpiderMiddleware, NoOpSpiderMiddleware] + @deferred_f_from_coro_f + async def test_twisted_sleep_multiple(self): + await self._test_sleep( + [NoOpSpiderMiddleware, TwistedSleepSpiderMiddleware, NoOpSpiderMiddleware] )