Deprecate calls to Spider.start_requests

scrapy/spiders/__init__.py

@@ -7,9 +7,11 @@ See documentation in docs/topics/spiders.rst
 from __future__ import annotations
 
 import logging
+import warnings
 from typing import TYPE_CHECKING, Any, cast
 
 from scrapy import signals
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request, Response
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import url_is_from_spider
@@ -124,10 +126,24 @@ class Spider(object_ref):
 
         .. seealso:: :ref:`start-requests`
         """
-        for item_or_request in self.start_requests():
-            yield item_or_request
+        with warnings.catch_warnings():
+            warnings.filterwarnings(
+                "ignore", category=ScrapyDeprecationWarning, module=r"^scrapy\.spiders$"
+            )
+            for item_or_request in self.start_requests():
+                yield item_or_request
 
     def start_requests(self) -> Iterable[Any]:
+        warnings.warn(
+            (
+                "The Spider.start_requests() method is deprecated, use "
+                "Spider.start() instead. If you are calling "
+                "super().start_requests() from a Spider.start() override, "
+                "iterate super().start() instead."
+            ),
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
         if not self.start_urls and hasattr(self, "start_url"):
             raise AttributeError(
                 "Crawling could not start: 'start_urls' not found "

scrapy/spiders/init.py

@@ -30,8 +30,12 @@ class InitSpider(Spider):
         )
 
     async def start(self) -> AsyncIterator[Any]:
-        for item_or_request in self.start_requests():
-            yield item_or_request
+        with warnings.catch_warnings():
+            warnings.filterwarnings(
+                "ignore", category=ScrapyDeprecationWarning, module=r"^scrapy\.spiders$"
+            )
+            for item_or_request in self.start_requests():
+                yield item_or_request
 
     def start_requests(self) -> Iterable[Request]:
         self._postinit_reqs: Iterable[Request] = super().start_requests()

scrapy/utils/python.py

@@ -21,10 +21,9 @@ from scrapy.utils.asyncgen import as_async_generator
 if TYPE_CHECKING:
     from collections.abc import Callable, Iterator
     from re import Pattern
-    from typing import Self
 
-    # typing.Concatenate and typing.ParamSpec require Python 3.10
-    from typing_extensions import Concatenate, ParamSpec
+    # typing.Concatenate, typing.ParamSpec and typing.Self require Python 3.10
+    from typing_extensions import Concatenate, ParamSpec, Self
 
     _P = ParamSpec("_P")
 

tests/test_spider_start.py

@@ -1,3 +1,4 @@
+import warnings
 from asyncio import sleep
 
 import pytest
@@ -47,7 +48,9 @@ class MainTestCase(TestCase):
             async def parse(self, response):
                 yield ITEM_A
 
-        await self._test_spider(TestSpider, [ITEM_A])
+        with warnings.catch_warnings():
+            warnings.simplefilter("error")
+            await self._test_spider(TestSpider, [ITEM_A])
 
     @deferred_f_from_coro_f
     async def test_start(self):
@@ -57,7 +60,9 @@ class MainTestCase(TestCase):
             async def start(self):
                 yield ITEM_A
 
-        await self._test_spider(TestSpider, [ITEM_A])
+        with warnings.catch_warnings():
+            warnings.simplefilter("error")
+            await self._test_spider(TestSpider, [ITEM_A])
 
     @deferred_f_from_coro_f
     async def test_start_subclass(self):
@@ -68,7 +73,9 @@ class MainTestCase(TestCase):
         class TestSpider(BaseSpider):
             name = "test"
 
-        await self._test_spider(TestSpider, [ITEM_A])
+        with warnings.catch_warnings():
+            warnings.simplefilter("error")
+            await self._test_spider(TestSpider, [ITEM_A])
 
     @deferred_f_from_coro_f
     async def test_deprecated(self):
@@ -105,7 +112,9 @@ class MainTestCase(TestCase):
             def start_requests(self):
                 yield ITEM_B
 
-        await self._test_spider(TestSpider, [ITEM_A])
+        with warnings.catch_warnings():
+            warnings.simplefilter("error")
+            await self._test_spider(TestSpider, [ITEM_A])
 
     @deferred_f_from_coro_f
     async def test_universal_subclass(self):
@@ -119,7 +128,24 @@ class MainTestCase(TestCase):
         class TestSpider(BaseSpider):
             name = "test"
 
-        await self._test_spider(TestSpider, [ITEM_A])
+        with warnings.catch_warnings():
+            warnings.simplefilter("error")
+            await self._test_spider(TestSpider, [ITEM_A])
+
+    @deferred_f_from_coro_f
+    async def test_start_deprecated_super(self):
+        class TestSpider(Spider):
+            name = "test"
+
+            async def start(self):
+                for item_or_request in super().start_requests():
+                    yield item_or_request
+
+        with pytest.warns(
+            ScrapyDeprecationWarning, match=r"use Spider\.start\(\) instead"
+        ) as messages:
+            await self._test_spider(TestSpider, [])
+        assert messages[0].filename.endswith("test_spider_start.py")
 
     async def _test_start(self, start_, expected_items=None):
         class TestSpider(Spider):

tox.ini

@@ -44,7 +44,7 @@ install_command =
     python -I -m pip install -ctests/upper-constraints.txt {opts} {packages}
 
 [testenv:typing]
-basepython = python3
+basepython = python3.9
 deps =
     mypy==1.14.0
     typing-extensions==4.12.2