From 701f582f27af84acda85b0dd3b82965aaa8c01fc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 25 Mar 2025 17:51:50 +0100
Subject: [PATCH] Implement a replacement for the removed
 test_graceful_crawl_error_handling

---
 scrapy/core/engine.py      |  8 ++++----
 scrapy/core/spidermw.py    |  2 +-
 tests/test_spider_start.py | 22 ++++++++++++++++++++++
 3 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 7071d83db..e3eddd046 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -9,6 +9,7 @@ from __future__ import annotations
 
 import logging
 from time import time
+from traceback import format_exc
 from typing import TYPE_CHECKING, Any, TypeVar, cast
 
 from twisted.internet.defer import Deferred, inlineCallbacks, succeed
@@ -175,12 +176,11 @@ class ExecutionEngine:
             item_or_request = yield deferred_from_coro(self._start.__anext__())
         except StopAsyncIteration:
             self._start = None
-        except Exception:
+        except Exception as exception:
             self._start = None
+            exception_traceback = format_exc()
             logger.error(
-                "Error while reading start items and requests",
-                exc_info=True,
-                extra={"spider": self.spider},
+                f"Error while reading start items and requests: {exception}.\n{exception_traceback}"
             )
         else:
             if isinstance(item_or_request, Request):
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 2090d5506..bace52e4e 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -390,7 +390,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
     @inlineCallbacks
     def process_start(
         self, spider: Spider
-    ) -> Generator[Deferred[Any], Any, AsyncIterable[Any]]:
+    ) -> Generator[Deferred[Any], Any, AsyncIterable[Any] | None]:
         self._check_deprecated_start_requests_use(spider)
         if self._use_start_requests:
             sync_start = iter(spider.start_requests())
diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py
index 34f1d5e66..86a671f26 100644
--- a/tests/test_spider_start.py
+++ b/tests/test_spider_start.py
@@ -1,6 +1,7 @@
 from asyncio import sleep
 
 import pytest
+from testfixtures import LogCapture
 from twisted.internet.defer import Deferred
 from twisted.trial.unittest import TestCase
 
@@ -151,3 +152,24 @@ class MainTestCase(TestCase):
             yield ITEM_A
 
         await self._test_start(start, [ITEM_A])
+
+    # Exceptions
+
+    @deferred_f_from_coro_f
+    async def test_deprecated_non_generator_exception(self):
+        class TestSpider(Spider):
+            name = "test"
+
+            def start_requests(self):
+                raise RuntimeError
+
+        with (
+            LogCapture() as log,
+            pytest.warns(
+                ScrapyDeprecationWarning,
+                match=r"defines the deprecated start_requests\(\) method",
+            ),
+        ):
+            await self._test_spider(TestSpider, [])
+
+        assert "in start_requests\n    raise RuntimeError" in str(log), log