Implement a replacement for the removed test_graceful_crawl_error_handling

scrapy/core/engine.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 
 import logging
 from time import time
+from traceback import format_exc
 from typing import TYPE_CHECKING, Any, TypeVar, cast
 
 from twisted.internet.defer import Deferred, inlineCallbacks, succeed
@@ -175,12 +176,11 @@ class ExecutionEngine:
             item_or_request = yield deferred_from_coro(self._start.__anext__())
         except StopAsyncIteration:
             self._start = None
-        except Exception:
+        except Exception as exception:
             self._start = None
+            exception_traceback = format_exc()
             logger.error(
-                "Error while reading start items and requests",
-                exc_info=True,
-                extra={"spider": self.spider},
+                f"Error while reading start items and requests: {exception}.\n{exception_traceback}"
             )
         else:
             if isinstance(item_or_request, Request):

scrapy/core/spidermw.py

@@ -390,7 +390,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
     @inlineCallbacks
     def process_start(
         self, spider: Spider
-    ) -> Generator[Deferred[Any], Any, AsyncIterable[Any]]:
+    ) -> Generator[Deferred[Any], Any, AsyncIterable[Any] | None]:
         self._check_deprecated_start_requests_use(spider)
         if self._use_start_requests:
             sync_start = iter(spider.start_requests())

tests/test_spider_start.py

@@ -1,6 +1,7 @@
 from asyncio import sleep
 
 import pytest
+from testfixtures import LogCapture
 from twisted.internet.defer import Deferred
 from twisted.trial.unittest import TestCase
 
@@ -151,3 +152,24 @@ class MainTestCase(TestCase):
             yield ITEM_A
 
         await self._test_start(start, [ITEM_A])
+
+    # Exceptions
+
+    @deferred_f_from_coro_f
+    async def test_deprecated_non_generator_exception(self):
+        class TestSpider(Spider):
+            name = "test"
+
+            def start_requests(self):
+                raise RuntimeError
+
+        with (
+            LogCapture() as log,
+            pytest.warns(
+                ScrapyDeprecationWarning,
+                match=r"defines the deprecated start_requests\(\) method",
+            ),
+        ):
+            await self._test_spider(TestSpider, [])
+
+        assert "in start_requests\n    raise RuntimeError" in str(log), log