From 49b839b270253dcc547e0a69ab5c20eb52bbcfc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 26 Mar 2025 23:28:29 +0100 Subject: [PATCH] Reliability improvements --- scrapy/core/engine.py | 4 +++- tests/test_engine_loop.py | 5 +---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index d2fe6a813..4ef996464 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -198,6 +198,8 @@ class ExecutionEngine: exc_info=True, ) else: + if not self.spider: + return # spider already closed if isinstance(item_or_request, Request): self.crawl(item_or_request) else: @@ -219,7 +221,7 @@ class ExecutionEngine: self._slot.nextcall.schedule() self._slot.heartbeat.start(self._SLOT_HEARTBEAT_INTERVAL) - while self._start is not None: + while self._start and self.spider: await self._process_start_next() if not self.needs_backout(): # Give room for the outcome of self._process_start_next() to be diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index 845bbad3f..a0d337955 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -109,9 +109,6 @@ class RequestSendOrderTestCase(TestCase): that can be sent before the first callback requests are scheduled. """ - # Error out if any tests relies on the heartbeat. - timeout = ExecutionEngine._SLOT_HEARTBEAT_INTERVAL - @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -122,7 +119,7 @@ class RequestSendOrderTestCase(TestCase): cls.mockserver.__exit__(None, None, None) fast_seconds = 0.001 - slow_seconds = 1 # increase if flaky + slow_seconds = 2 # increase if flaky def _request(self, num, response_seconds, download_slots): url = self.mockserver.url(f"/delay?n={response_seconds}&{num}")