Merge branch 'minimal-async-seeds' into async-seeds

scrapy/core/engine.py

@@ -259,7 +259,7 @@ class ExecutionEngine:
             return
 
         while not self.needs_backout():
-            if self._start_scheduled_request() is None:
+            if not self._start_scheduled_request():
                 break
 
         if self.spider_is_idle() and self._slot.close_if_idle:
@@ -294,7 +294,7 @@ class ExecutionEngine:
             or self.scraper.slot.needs_backout()
         )
 
-    def _start_scheduled_request(self) -> Deferred[None] | None:
+    def _start_scheduled_request(self) -> bool:
         assert self._slot is not None  # typing
         assert self.spider is not None  # typing
         assert self.scheduler is not None  # typing
@@ -309,7 +309,7 @@ class ExecutionEngine:
             return None
         if request is None:
             self.signals.send_catch_log(signals.scheduler_empty)
-            return None
+            return False
 
         d: Deferred[Response | Request] = self._download(request)
         d.addBoth(self._handle_downloader_output, request)
@@ -342,7 +342,7 @@ class ExecutionEngine:
                 extra={"spider": self.spider},
             )
         )
-        return d2
+        return True
 
     def _handle_downloader_output(
         self, result: Request | Response | Failure, request: Request

tests/test_spider_start.py

@@ -3,7 +3,6 @@ from asyncio import sleep
 
 import pytest
 from testfixtures import LogCapture
-from twisted.internet.defer import Deferred
 from twisted.trial.unittest import TestCase
 
 from scrapy import Spider, signals
@@ -12,18 +11,12 @@ from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
 from scrapy.utils.test import get_crawler
 
 from .test_scheduler import MemoryScheduler
+from .utils import twisted_sleep
 
-SLEEP_SECONDS = 0.1
 ITEM_A = {"id": "a"}
 ITEM_B = {"id": "b"}
 
-
-def twisted_sleep(seconds):
-    from twisted.internet import reactor
-
-    d = Deferred()
-    reactor.callLater(seconds, d.callback, None)
-    return d
+SLEEP_SECONDS = 0.1
 
 
 class MainTestCase(TestCase):

tests/test_spidermiddleware_process_start.py

@@ -11,8 +11,9 @@ from scrapy import Spider, signals
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
 from scrapy.utils.test import get_crawler
+from tests.test_spider_start import SLEEP_SECONDS
 
-from .test_spider_start import SLEEP_SECONDS, twisted_sleep
+from .utils import twisted_sleep
 
 ITEM_A = {"id": "a"}
 ITEM_B = {"id": "b"}

tests/utils/__init__.py

@@ -0,0 +1,9 @@
+from twisted.internet.defer import Deferred
+
+
+def twisted_sleep(seconds):
+    from twisted.internet import reactor
+
+    d = Deferred()
+    reactor.callLater(seconds, d.callback, None)
+    return d