diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index a0d337955..8ef62d018 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -4,7 +4,6 @@ from twisted.internet.defer import Deferred from twisted.trial.unittest import TestCase from scrapy import Request, Spider, signals -from scrapy.core.engine import ExecutionEngine from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.test import get_crawler @@ -27,7 +26,7 @@ class MainTestCase(TestCase): the scheduler (returning no requests while also returning True from the has_pending_requests() method) should cause the spider to miss the processing of any later requests.""" - seconds = ExecutionEngine._SLOT_HEARTBEAT_INTERVAL + 0.01 + seconds = 2 class TestSpider(Spider): name = "test" @@ -109,6 +108,8 @@ class RequestSendOrderTestCase(TestCase): that can be sent before the first callback requests are scheduled. """ + seconds = 0.1 # increase if flaky + @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -116,10 +117,7 @@ class RequestSendOrderTestCase(TestCase): @classmethod def tearDownClass(cls): - cls.mockserver.__exit__(None, None, None) - - fast_seconds = 0.001 - slow_seconds = 2 # increase if flaky + cls.mockserver.__exit__(None, None, None) # increase if flaky def _request(self, num, response_seconds, download_slots): url = self.mockserver.url(f"/delay?n={response_seconds}&{num}") @@ -137,7 +135,7 @@ class RequestSendOrderTestCase(TestCase): start_fn=None, ): settings = settings or {} - response_seconds = response_seconds or self.slow_seconds + response_seconds = response_seconds or self.seconds if start_fn is None: @@ -171,249 +169,6 @@ class RequestSendOrderTestCase(TestCase): expected_nums = sorted(start_nums + cb_nums) assert actual_nums == expected_nums, f"{actual_nums=} != {expected_nums=}" - @deferred_f_from_coro_f - async def test_default(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 26, - 24, - 23, - 22, - 21, - 20, - 19, - 18, - 17, - ], - cb_nums=[25], - ) - ) - - @deferred_f_from_coro_f - async def test_conc1(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[1, 4, 2], - cb_nums=[3], - settings={"CONCURRENT_REQUESTS": 1}, - ) - ) - - @deferred_f_from_coro_f - async def test_conc2(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[1, 2, 6, 4, 3], - cb_nums=[5], - settings={"CONCURRENT_REQUESTS": 2}, - ) - ) - - @deferred_f_from_coro_f - async def test_conc8(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[1, 2, 3, 4, 5, 6, 7, 8, 18, 16, 15, 14, 13, 12, 11, 10, 9], - cb_nums=[17], - settings={"CONCURRENT_REQUESTS": 8}, - ) - ) - - @deferred_f_from_coro_f - async def test_conc16(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 34, - 32, - 31, - 30, - 29, - 28, - 27, - 26, - 25, - 24, - 23, - 22, - 21, - 20, - 19, - 18, - 17, - ], - cb_nums=[33], - settings={"CONCURRENT_REQUESTS_PER_DOMAIN": 16}, - ) - ) - - @deferred_f_from_coro_f - async def test_conc3_ds2(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[1, 2, 3, 8, 6, 5, 4], - cb_nums=[7], - settings={ - "CONCURRENT_REQUESTS": 3, - }, - download_slots=2, - ) - ) - - @deferred_f_from_coro_f - async def test_tconc3_dconc2(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[1, 2, 3, 7, 5, 4], - cb_nums=[6], - settings={ - "CONCURRENT_REQUESTS": 3, - "CONCURRENT_REQUESTS_PER_DOMAIN": 2, - }, - ) - ) - - @deferred_f_from_coro_f - async def test_tconc5_dconc3(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[1, 2, 3, 4, 5, 10, 8, 7, 6], - cb_nums=[9], - settings={ - "CONCURRENT_REQUESTS": 5, - "CONCURRENT_REQUESTS_PER_DOMAIN": 3, - }, - ) - ) - - @deferred_f_from_coro_f - async def test_tconc5_dconc2_ds3(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[1, 2, 3, 4, 5, 12, 10, 9, 8, 7, 6], - cb_nums=[11], - settings={ - "CONCURRENT_REQUESTS": 5, - "CONCURRENT_REQUESTS_PER_DOMAIN": 2, - }, - download_slots=3, - ) - ) - - @deferred_f_from_coro_f - async def test_tconc5_dconc3_ds2(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[1, 2, 3, 4, 5, 12, 10, 9, 8, 7, 6], - cb_nums=[11], - settings={ - "CONCURRENT_REQUESTS": 5, - "CONCURRENT_REQUESTS_PER_DOMAIN": 3, - }, - download_slots=2, - ) - ) - - @deferred_f_from_coro_f - async def test_tconc7_dconc2_ds3(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[1, 2, 3, 4, 5, 6, 7, 15, 13, 12, 11, 10, 9, 8], - cb_nums=[14], - settings={ - "CONCURRENT_REQUESTS": 7, - "CONCURRENT_REQUESTS_PER_DOMAIN": 2, - }, - download_slots=3, - ) - ) - - @deferred_f_from_coro_f - async def test_tconc7_dconc3_ds2(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[1, 2, 3, 4, 5, 6, 7, 15, 13, 12, 11, 10, 9, 8], - cb_nums=[14], - settings={ - "CONCURRENT_REQUESTS": 7, - "CONCURRENT_REQUESTS_PER_DOMAIN": 3, - }, - download_slots=2, - ) - ) - - @deferred_f_from_coro_f - async def test_fast(self): - """Very fast responses may increase the number of start requests sent - in reverse order before the first callback request.""" - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[1, 3, 2], - cb_nums=[4], - settings={"CONCURRENT_REQUESTS": 1}, - response_seconds=self.fast_seconds, - ) - ) - - @deferred_f_from_coro_f - async def test_await(self): - """Awaiting slow operations in Spider.start() may lower the number of - first start requests sent in order.""" - start_nums = [1, 3] - cb_nums = [2] - response_seconds = self.slow_seconds - download_slots = 1 - - async def start(spider): - assert len(start_nums) > 1 - for num in start_nums[:-1]: - yield self._request(num, response_seconds, download_slots) - await sleep(response_seconds * 2) - yield self._request(start_nums[-1], response_seconds, download_slots) - - await maybe_deferred_to_future( - self._test_request_order( - start_nums=start_nums, - cb_nums=cb_nums, - settings={"CONCURRENT_REQUESTS": 2}, - response_seconds=response_seconds, - start_fn=start, - ) - ) - # Examples from the “Start requests” section of the documentation about # spiders. @@ -421,7 +176,7 @@ class RequestSendOrderTestCase(TestCase): async def test_start_requests_first(self): start_nums = [1, 3, 2] cb_nums = [4] - response_seconds = self.slow_seconds + response_seconds = self.seconds download_slots = 1 async def start(spider): @@ -443,7 +198,7 @@ class RequestSendOrderTestCase(TestCase): async def test_start_requests_first_sorted(self): start_nums = [1, 2, 3] cb_nums = [4] - response_seconds = self.slow_seconds + response_seconds = self.seconds download_slots = 1 async def start(spider): @@ -467,7 +222,7 @@ class RequestSendOrderTestCase(TestCase): async def test_lazy(self): start_nums = [1, 2, 4] cb_nums = [3] - response_seconds = self.slow_seconds + response_seconds = self.seconds download_slots = 1 async def start(spider):