RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Implement spider middleware backward compatibility and deprecation warnings

This commit is contained in:
Adrián Chaves 2025-03-12 06:34:11 +01:00
parent 6ce483125f
commit f0c933f094
4 changed files with 277 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
scrapy/core/engine.py
View File

@ -215,6 +215,7 @@ class ExecutionEngine:
self.crawl(seed)
else:
self.scraper.start_itemproc(seed, response=None)
self.slot.nextcall.schedule()
@inlineCallbacks
def _start_next_requests(self) -> Generator[Deferred[Any], Any, None]:

84
scrapy/core/spidermw.py
View File

@ -56,11 +56,55 @@ class SpiderMiddlewareManager(MiddlewareManager):
def _get_mwlist_from_settings(cls, settings: BaseSettings) -> list[Any]:
return build_component_list(settings.getwithbase("SPIDER_MIDDLEWARES"))
def __init__(self, *middlewares: Any) -> None:
self._check_deprecated_process_start_requests_use(middlewares)
super().__init__(*middlewares)
def _check_deprecated_process_start_requests_use(
self, middlewares: list[Any]
) -> None:
deprecated_middlewares = [
middleware
for middleware in middlewares
if hasattr(middleware, "process_start_requests")
and not hasattr(middleware, "process_seeds")
]
self._use_start_requests = bool(deprecated_middlewares)
if self._use_start_requests:
deprecated_middleware_list = ", ".join(
global_object_name(middleware.__class__)
for middleware in deprecated_middlewares
)
warn(
f"The following enabled spider middlewares, directly or "
f"through their parent classes, define the deprecated "
f"process_start_requests() method: "
f"{deprecated_middleware_list}. process_start_requests() has "
f"been deprecated in favor of a new method, process_seeds(), "
f"to support asynchronous code execution. "
f"process_start_requests() will stop being called in a future "
f"version of Scrapy. If you use Scrapy VERSION or higher "
f"only, replace process_start_requests() with "
f"process_seeds(); note that process_seeds() is a coroutine "
f"(async def). If you need to maintain compatibility with "
f"lower Scrapy versions, when defining "
f"process_start_requests() in a spider middleware class, "
f"define process_seeds() as well. See the release notes of "
f"Scrapy VERSION for details: "
f"https://docs.scrapy.org/en/VERSION/news.html",
ScrapyDeprecationWarning,
)
def _add_middleware(self, mw: Any) -> None:
super()._add_middleware(mw)
if hasattr(mw, "process_spider_input"):
self.methods["process_spider_input"].append(mw.process_spider_input)
if hasattr(mw, "process_seeds"):
if self._use_start_requests:
if hasattr(mw, "process_start_requests"):
self.methods["process_start_requests"].appendleft(
mw.process_start_requests
)
elif hasattr(mw, "process_seeds"):
self.methods["process_seeds"].appendleft(mw.process_seeds)
process_spider_output = self._get_async_method_pair(mw, "process_spider_output")
self.methods["process_spider_output"].appendleft(process_spider_output)
@ -329,12 +373,16 @@ class SpiderMiddlewareManager(MiddlewareManager):
self, spider: Spider
) -> Generator[Deferred[Any], Any, AsyncIterator[Any]]:
self._check_deprecated_start_requests_use(spider)
seeds = yield self._iter_seeds(spider)
seeds = yield self._process_chain("process_seeds", seeds)
if self._use_start_requests:
seeds = iter(spider.start_requests())
seeds = yield self._process_chain("process_start_requests", seeds, spider)
seeds = as_async_generator(seeds)
else:
seeds = yield self._iter_seeds(spider)
seeds = yield self._process_chain("process_seeds", seeds)
return seeds
@staticmethod
def _check_deprecated_start_requests_use(spider: Spider):
def _check_deprecated_start_requests_use(self, spider: Spider):
start_requests_cls = None
yield_seeds_cls = None
spidercls = spider.__class__
@ -378,6 +426,32 @@ class SpiderMiddlewareManager(MiddlewareManager):
ScrapyDeprecationWarning,
)
if (
self._use_start_requests
and yield_seeds_cls is not Spider
and start_requests_cls is not yield_seeds_cls
and mro.index(yield_seeds_cls) < mro.index(start_requests_cls)
):
src = global_object_name(yield_seeds_cls)
if yield_seeds_cls is not spidercls:
src += f" (inherited by {global_object_name(spidercls)})"
raise ValueError(
f"{src} does not define the deprecated start_requests() "
f"method. However, one or more of your enabled spider "
f"middlewares (reported in an earlier deprecation warning) "
f"define the process_start_requests() method, and not the "
f"process_seeds() method, making them only compatible with "
f"(deprecated) spiders that define the start_requests() "
f"method. To solve this issue, disable the offending spider "
f"middlewares, upgrade them as described in that earlier "
f"deprecation warning, or make your spider compatible with "
f"deprecated spider middlewares (and earlier Scrapy versions) "
f"by defining a sync start_requests() method that works "
f"similarly to its existing yield_seeds() method. See the "
f"release notes of Scrapy VERSION for details: "
f"https://docs.scrapy.org/en/VERSION/news.html"
)
@staticmethod
def _iter_seeds(spider: Spider):
fn = spider.yield_seeds

56
tests/test_spider_yield_seeds.py
View File

@ -43,32 +43,6 @@ class MainTestCase(TestCase):
assert crawler.stats.get_value("finish_reason") == "finished"
assert actual_items == expected_items
async def _test_yield_seeds(self, yield_seeds_, expected_items=None):
class TestSpider(Spider):
name = "test"
yield_seeds = yield_seeds_
await self._test_spider(TestSpider, expected_items)
@pytest.mark.only_asyncio
@deferred_f_from_coro_f
async def test_asyncio_delayed(self):
async def yield_seeds(spider):
await sleep(ASYNC_GEN_ERROR_MINIMUM_SECONDS)
yield ITEM_A
await self._test_yield_seeds(yield_seeds, [ITEM_A])
@deferred_f_from_coro_f
async def test_twisted_delayed(self):
async def yield_seeds(spider):
await maybe_deferred_to_future(
twisted_sleep(ASYNC_GEN_ERROR_MINIMUM_SECONDS)
)
yield ITEM_A
await self._test_yield_seeds(yield_seeds, [ITEM_A])
@deferred_f_from_coro_f
async def test_start_urls(self):
class TestSpider(Spider):
@ -102,7 +76,7 @@ class MainTestCase(TestCase):
await self._test_spider(TestSpider, [ITEM_A])
@deferred_f_from_coro_f
async def test_start_requests(self):
async def test_deprecated(self):
class TestSpider(Spider):
name = "test"
@ -113,7 +87,7 @@ class MainTestCase(TestCase):
await self._test_spider(TestSpider, [ITEM_A])
@deferred_f_from_coro_f
async def test_start_requests_subclass(self):
async def test_deprecated_subclass(self):
class BaseSpider(Spider):
def start_requests(self):
yield ITEM_A
@ -151,3 +125,29 @@ class MainTestCase(TestCase):
name = "test"
await self._test_spider(TestSpider, [ITEM_A])
async def _test_yield_seeds(self, yield_seeds_, expected_items=None):
class TestSpider(Spider):
name = "test"
yield_seeds = yield_seeds_
await self._test_spider(TestSpider, expected_items)
@pytest.mark.only_asyncio
@deferred_f_from_coro_f
async def test_asyncio_delayed(self):
async def yield_seeds(spider):
await sleep(ASYNC_GEN_ERROR_MINIMUM_SECONDS)
yield ITEM_A
await self._test_yield_seeds(yield_seeds, [ITEM_A])
@deferred_f_from_coro_f
async def test_twisted_delayed(self):
async def yield_seeds(spider):
await maybe_deferred_to_future(
twisted_sleep(ASYNC_GEN_ERROR_MINIMUM_SECONDS)
)
yield ITEM_A
await self._test_yield_seeds(yield_seeds, [ITEM_A])

200
tests/test_spidermiddleware_process_seeds.py
View File

@ -1,17 +1,22 @@
from asyncio import sleep
import pytest
from twisted.internet.defer import inlineCallbacks
from twisted.trial.unittest import TestCase
from scrapy import Spider, signals
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
from scrapy.utils.test import get_crawler
from .test_spider_yield_seeds import ASYNC_GEN_ERROR_MINIMUM_SECONDS, twisted_sleep
ITEM_A = {"id": "a"}
ITEM_B = {"id": "b"}
ITEM_C = {"id": "c"}
ITEM_D = {"id": "d"}
class AsyncioSpiderMiddleware:
class AsyncioSleepSpiderMiddleware:
async def process_seeds(self, seeds):
await sleep(ASYNC_GEN_ERROR_MINIMUM_SECONDS)
async for seed in seeds:
@ -24,24 +29,79 @@ class NoOpSpiderMiddleware:
yield seed
class TwistedSpiderMiddleware:
class TwistedSleepSpiderMiddleware:
async def process_seeds(self, seeds):
await maybe_deferred_to_future(twisted_sleep(ASYNC_GEN_ERROR_MINIMUM_SECONDS))
async for seed in seeds:
yield seed
class UniversalSpiderMiddleware:
async def process_seeds(self, seeds):
async for seed in seeds:
yield seed
def process_start_requests(self, start_requests, spider):
raise NotImplementedError
# Spiders and spider middlewares for MainTestCase._test_wrap
class ModernWrapSpider(Spider):
name = "test"
async def yield_seeds(self):
yield ITEM_B
class UniversalWrapSpider(Spider):
name = "test"
async def yield_seeds(self):
yield ITEM_B
def start_requests(self):
yield ITEM_D
class DeprecatedWrapSpider(Spider):
name = "test"
def start_requests(self):
yield ITEM_B
class ModernWrapSpiderMiddleware:
async def process_seeds(self, seeds):
yield ITEM_A
async for seed in seeds:
yield seed
yield ITEM_C
class UniversalWrapSpiderMiddleware:
async def process_seeds(self, seeds):
yield ITEM_A
async for seed in seeds:
yield seed
yield ITEM_C
def process_start_requests(self, seeds, spider):
yield ITEM_A
yield from seeds
yield ITEM_C
class DeprecatedWrapSpiderMiddleware:
def process_start_requests(self, seeds, spider):
yield ITEM_A
yield from seeds
yield ITEM_C
class MainTestCase(TestCase):
@inlineCallbacks
def _test(self, spider_middlewares):
item = {"a": "b"}
class TestSpider(Spider):
name = "test"
async def yield_seeds(self):
yield item
async def _test(self, spider_middlewares, spider_cls, expected_items):
actual_items = []
def track_item(item, response, spider):
@ -50,30 +110,108 @@ class MainTestCase(TestCase):
settings = {
"SPIDER_MIDDLEWARES": {cls: n for n, cls in enumerate(spider_middlewares)},
}
crawler = get_crawler(TestSpider, settings_dict=settings)
crawler = get_crawler(spider_cls, settings_dict=settings)
crawler.signals.connect(track_item, signals.item_scraped)
yield crawler.crawl()
await maybe_deferred_to_future(crawler.crawl())
assert crawler.stats.get_value("finish_reason") == "finished"
assert actual_items == [item]
assert actual_items == expected_items, f"{actual_items=} != {expected_items=}"
async def _test_wrap(self, spider_middleware, spider_cls, expected_items=None):
expected_items = expected_items or [ITEM_A, ITEM_B, ITEM_C]
await self._test([spider_middleware], spider_cls, expected_items)
@deferred_f_from_coro_f
async def test_modern_mw_modern_spider(self):
await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider)
@deferred_f_from_coro_f
async def test_modern_mw_universal_spider(self):
await self._test_wrap(ModernWrapSpiderMiddleware, UniversalWrapSpider)
@deferred_f_from_coro_f
async def test_modern_mw_deprecated_spider(self):
with pytest.warns(
ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)"
):
await self._test_wrap(ModernWrapSpiderMiddleware, DeprecatedWrapSpider)
@deferred_f_from_coro_f
async def test_universal_mw_modern_spider(self):
await self._test_wrap(UniversalWrapSpiderMiddleware, ModernWrapSpider)
@deferred_f_from_coro_f
async def test_universal_mw_universal_spider(self):
await self._test_wrap(UniversalWrapSpiderMiddleware, UniversalWrapSpider)
@deferred_f_from_coro_f
async def test_universal_mw_deprecated_spider(self):
with pytest.warns(
ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)"
):
await self._test_wrap(UniversalWrapSpiderMiddleware, DeprecatedWrapSpider)
@deferred_f_from_coro_f
async def test_deprecated_mw_modern_spider(self):
with (
pytest.warns(
ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"
),
pytest.raises(
ValueError, match=r"only compatible with \(deprecated\) spiders"
),
):
await self._test_wrap(DeprecatedWrapSpiderMiddleware, ModernWrapSpider)
@deferred_f_from_coro_f
async def test_deprecated_mw_universal_spider(self):
with pytest.warns(
ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"
):
await self._test_wrap(
DeprecatedWrapSpiderMiddleware,
UniversalWrapSpider,
[ITEM_A, ITEM_D, ITEM_C],
)
@deferred_f_from_coro_f
async def test_deprecated_mw_deprecated_spider(self):
with (
pytest.warns(
ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"
),
pytest.warns(
ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)"
),
):
await self._test_wrap(DeprecatedWrapSpiderMiddleware, DeprecatedWrapSpider)
async def _test_sleep(self, spider_middlewares):
class TestSpider(Spider):
name = "test"
async def yield_seeds(self):
yield ITEM_A
await self._test(spider_middlewares, TestSpider, [ITEM_A])
@pytest.mark.only_asyncio
@inlineCallbacks
def test_asyncio_delayed_single(self):
yield self._test([AsyncioSpiderMiddleware])
@deferred_f_from_coro_f
async def test_asyncio_sleep_single(self):
await self._test_sleep([AsyncioSleepSpiderMiddleware])
@pytest.mark.only_asyncio
@inlineCallbacks
def test_asyncio_delayed_multiple(self):
yield self._test(
[NoOpSpiderMiddleware, AsyncioSpiderMiddleware, NoOpSpiderMiddleware]
@deferred_f_from_coro_f
async def test_asyncio_sleep_multiple(self):
await self._test_sleep(
[NoOpSpiderMiddleware, AsyncioSleepSpiderMiddleware, NoOpSpiderMiddleware]
)
@inlineCallbacks
def test_twisted_delayed_single(self):
yield self._test([TwistedSpiderMiddleware])
@deferred_f_from_coro_f
async def test_twisted_sleep_single(self):
await self._test_sleep([TwistedSleepSpiderMiddleware])
@inlineCallbacks
def test_twisted_delayed_multiple(self):
yield self._test(
[NoOpSpiderMiddleware, TwistedSpiderMiddleware, NoOpSpiderMiddleware]
@deferred_f_from_coro_f
async def test_twisted_sleep_multiple(self):
await self._test_sleep(
[NoOpSpiderMiddleware, TwistedSleepSpiderMiddleware, NoOpSpiderMiddleware]
)