From 1a0b9d4f562c6cfd6839cf8557e6b01ad7e2264e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 25 Mar 2025 17:16:05 +0100 Subject: [PATCH] Cover start request sorting examples in the tests --- tests/test_engine_loop.py | 76 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index 3a9bcf6b9..9b7a06305 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -648,6 +648,7 @@ class RequestSendOrderTestCase(TestCase): """Awaiting slow operations in Spider.start() may lower the number of first start requests sent in order.""" start_nums = [1, 3] + cb_nums = [2] response_seconds = self.slow_seconds download_slots = 1 @@ -661,9 +662,82 @@ class RequestSendOrderTestCase(TestCase): await maybe_deferred_to_future( self._test_request_order( start_nums=start_nums, - cb_nums=[2], + cb_nums=cb_nums, settings={"CONCURRENT_REQUESTS": 2}, response_seconds=response_seconds, start_fn=start, ) ) + + # Examples from the “Start requests” section of the documentation about + # spiders. + + @pytest.mark.only_asyncio + @deferred_f_from_coro_f + async def test_ar_start_requests_first(self): + start_nums = [1, 3, 2] + cb_nums = [4] + response_seconds = self.slow_seconds + download_slots = 1 + + async def start(spider): + for num in start_nums: + request = self._request(num, response_seconds, download_slots) + yield request.replace(priority=1) + + await maybe_deferred_to_future( + self._test_request_order( + start_nums=start_nums, + cb_nums=cb_nums, + settings={"CONCURRENT_REQUESTS": 1}, + response_seconds=response_seconds, + start_fn=start, + ) + ) + + @pytest.mark.only_not_asyncio + @deferred_f_from_coro_f + async def test_dr_start_requests_first(self): + start_nums = [3, 2, 1] + cb_nums = [4] + response_seconds = self.slow_seconds + download_slots = 1 + + async def start(spider): + for num in start_nums: + request = self._request(num, response_seconds, download_slots) + yield request.replace(priority=1) + + await maybe_deferred_to_future( + self._test_request_order( + start_nums=start_nums, + cb_nums=cb_nums, + settings={"CONCURRENT_REQUESTS": 1}, + response_seconds=response_seconds, + start_fn=start, + ) + ) + + @deferred_f_from_coro_f + async def test_start_requests_first_sorted(self): + start_nums = [1, 2, 3] + cb_nums = [4] + response_seconds = self.slow_seconds + download_slots = 1 + + async def start(spider): + priority = len(start_nums) + for num in start_nums: + request = self._request(num, response_seconds, download_slots) + yield request.replace(priority=priority) + priority -= 1 + + await maybe_deferred_to_future( + self._test_request_order( + start_nums=start_nums, + cb_nums=cb_nums, + settings={"CONCURRENT_REQUESTS": 1}, + response_seconds=response_seconds, + start_fn=start, + ) + )