diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index e4ea3cbc2..e162f45cd 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -337,7 +337,7 @@ class RequestSendOrderTestCase(TestCase): async def test_lazy(self): start_nums = [1, 2, 4] cb_nums = [3] - response_seconds = self.seconds + response_seconds = self.seconds * 2**1 # increase if flaky download_slots = 1 async def start(spider): @@ -397,41 +397,38 @@ class RequestSendOrderTestCase(TestCase): expected_urls = [_url(letter) for letter in "abcd"] assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" - # Sleep handling + # Delay handling @deferred_f_from_coro_f - async def test_sleep(self): - """Neither asynchronous sleeps on Spider.start() nor the equivalent on - the scheduler (returning no requests while also returning True from - the has_pending_requests() method) should cause the spider to miss the - processing of any later requests.""" + async def test_delays(self): + """Delays in Spider.start() or in the scheduler (i.e. returning no + requests while also returning True from the has_pending_requests() + method) should cause the spider to miss the processing of any later + requests.""" seconds = ExecutionEngine._MIN_BACK_IN_SECONDS def _request(num): return self._request(num, seconds) - async def _sleep(): - await sleep(seconds) - async def start(spider): from twisted.internet import reactor yield _request(1) # Let request 1 be processed. - await _sleep() + await spider.crawler.signals.wait_for(signals.scheduler_empty) spider.crawler.engine._slot.scheduler.pause() spider.crawler.engine._slot.scheduler.enqueue_request(_request(2)) # During this time, the scheduler reports having requests but # returns None. - await _sleep() + await spider.crawler.signals.wait_for(signals.scheduler_empty) spider.crawler.engine._slot.scheduler.unpause() # The scheduler request is processed. - await _sleep() + await spider.crawler.signals.wait_for(signals.scheduler_empty) yield _request(3) @@ -442,7 +439,7 @@ class RequestSendOrderTestCase(TestCase): # delayed call below, proving that the start iteration can # finish before a scheduler “sleep” without causing the # scheduler to finish. - reactor.callLater(seconds, spider.crawler.engine._slot.scheduler.unpause) + reactor.callLater(0, spider.crawler.engine._slot.scheduler.unpause) await maybe_deferred_to_future( self._test_request_order( diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py index 3a20db922..d9e176ed8 100644 --- a/tests/test_spider_start.py +++ b/tests/test_spider_start.py @@ -11,7 +11,6 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.test import get_crawler -from . import TWISTED_KEEPS_TRACEBACKS from .test_scheduler import MemoryScheduler SLEEP_SECONDS = 0.1 @@ -256,10 +255,7 @@ class MainTestCase(TestCase): with LogCapture() as log: await self._test_start(start, []) - if TWISTED_KEEPS_TRACEBACKS: - assert "in start\n raise RuntimeError" in str(log), log - else: - assert "in _process_next_seed\n seed =" in str(log), log + assert "in start\n raise RuntimeError" in str(log), log @deferred_f_from_coro_f async def test_exception_after_yield(self): @@ -270,10 +266,7 @@ class MainTestCase(TestCase): with LogCapture() as log: await self._test_start(start, [ITEM_A]) - if TWISTED_KEEPS_TRACEBACKS: - assert "in start\n raise RuntimeError" in str(log), log - else: - assert "in _process_next_seed\n seed =" in str(log), log + assert "in start\n raise RuntimeError" in str(log), log @deferred_f_from_coro_f async def test_bad_definition_continuance(self): diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py index 5dcb9b73e..985a70cae 100644 --- a/tests/test_spidermiddleware_process_start.py +++ b/tests/test_spidermiddleware_process_start.py @@ -12,7 +12,6 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.test import get_crawler -from . import TWISTED_KEEPS_TRACEBACKS from .test_spider_start import SLEEP_SECONDS, twisted_sleep ITEM_A = {"id": "a"} @@ -386,10 +385,7 @@ class MainTestCase(TestCase): with LogCapture() as log: await self._test_process_start(process_start, []) - if TWISTED_KEEPS_TRACEBACKS: - assert "in process_start\n raise RuntimeError" in str(log), log - else: - assert "in _process_next_seed\n seed =" in str(log), log + assert "in process_start\n raise RuntimeError" in str(log), log @deferred_f_from_coro_f async def test_exception_after_yield(self): @@ -400,7 +396,4 @@ class MainTestCase(TestCase): with LogCapture() as log: await self._test_process_start(process_start, [ITEM_A]) - if TWISTED_KEEPS_TRACEBACKS: - assert "in process_start\n raise RuntimeError" in str(log), log - else: - assert "in _process_next_seed\n seed =" in str(log), log + assert "in process_start\n raise RuntimeError" in str(log), log