diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index acd16ee41..0a1d85ae6 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -7,9 +7,11 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import logging +import warnings from typing import TYPE_CHECKING, Any, cast from scrapy import signals +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider @@ -124,10 +126,24 @@ class Spider(object_ref): .. seealso:: :ref:`start-requests` """ - for item_or_request in self.start_requests(): - yield item_or_request + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", category=ScrapyDeprecationWarning, module=r"^scrapy\.spiders$" + ) + for item_or_request in self.start_requests(): + yield item_or_request def start_requests(self) -> Iterable[Any]: + warnings.warn( + ( + "The Spider.start_requests() method is deprecated, use " + "Spider.start() instead. If you are calling " + "super().start_requests() from a Spider.start() override, " + "iterate super().start() instead." + ), + ScrapyDeprecationWarning, + stacklevel=2, + ) if not self.start_urls and hasattr(self, "start_url"): raise AttributeError( "Crawling could not start: 'start_urls' not found " diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index ca5f08e98..e5548b9fa 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -30,8 +30,12 @@ class InitSpider(Spider): ) async def start(self) -> AsyncIterator[Any]: - for item_or_request in self.start_requests(): - yield item_or_request + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", category=ScrapyDeprecationWarning, module=r"^scrapy\.spiders$" + ) + for item_or_request in self.start_requests(): + yield item_or_request def start_requests(self) -> Iterable[Request]: self._postinit_reqs: Iterable[Request] = super().start_requests() diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 7e7b83c59..1d9e34f35 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -21,10 +21,9 @@ from scrapy.utils.asyncgen import as_async_generator if TYPE_CHECKING: from collections.abc import Callable, Iterator from re import Pattern - from typing import Self - # typing.Concatenate and typing.ParamSpec require Python 3.10 - from typing_extensions import Concatenate, ParamSpec + # typing.Concatenate, typing.ParamSpec and typing.Self require Python 3.10 + from typing_extensions import Concatenate, ParamSpec, Self _P = ParamSpec("_P") diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py index b5eaa93f5..02f8770e9 100644 --- a/tests/test_spider_start.py +++ b/tests/test_spider_start.py @@ -1,3 +1,4 @@ +import warnings from asyncio import sleep import pytest @@ -47,7 +48,9 @@ class MainTestCase(TestCase): async def parse(self, response): yield ITEM_A - await self._test_spider(TestSpider, [ITEM_A]) + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_spider(TestSpider, [ITEM_A]) @deferred_f_from_coro_f async def test_start(self): @@ -57,7 +60,9 @@ class MainTestCase(TestCase): async def start(self): yield ITEM_A - await self._test_spider(TestSpider, [ITEM_A]) + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_spider(TestSpider, [ITEM_A]) @deferred_f_from_coro_f async def test_start_subclass(self): @@ -68,7 +73,9 @@ class MainTestCase(TestCase): class TestSpider(BaseSpider): name = "test" - await self._test_spider(TestSpider, [ITEM_A]) + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_spider(TestSpider, [ITEM_A]) @deferred_f_from_coro_f async def test_deprecated(self): @@ -105,7 +112,9 @@ class MainTestCase(TestCase): def start_requests(self): yield ITEM_B - await self._test_spider(TestSpider, [ITEM_A]) + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_spider(TestSpider, [ITEM_A]) @deferred_f_from_coro_f async def test_universal_subclass(self): @@ -119,7 +128,24 @@ class MainTestCase(TestCase): class TestSpider(BaseSpider): name = "test" - await self._test_spider(TestSpider, [ITEM_A]) + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_spider(TestSpider, [ITEM_A]) + + @deferred_f_from_coro_f + async def test_start_deprecated_super(self): + class TestSpider(Spider): + name = "test" + + async def start(self): + for item_or_request in super().start_requests(): + yield item_or_request + + with pytest.warns( + ScrapyDeprecationWarning, match=r"use Spider\.start\(\) instead" + ) as messages: + await self._test_spider(TestSpider, []) + assert messages[0].filename.endswith("test_spider_start.py") async def _test_start(self, start_, expected_items=None): class TestSpider(Spider): diff --git a/tox.ini b/tox.ini index 59572442d..089f48f86 100644 --- a/tox.ini +++ b/tox.ini @@ -44,7 +44,7 @@ install_command = python -I -m pip install -ctests/upper-constraints.txt {opts} {packages} [testenv:typing] -basepython = python3 +basepython = python3.9 deps = mypy==1.14.0 typing-extensions==4.12.2