diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index bf7ed9d00..3a9bcf6b9 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -1,5 +1,6 @@ from collections import deque +import pytest from twisted.internet.defer import Deferred from twisted.trial.unittest import TestCase @@ -88,6 +89,8 @@ class RequestSendOrderTestCase(TestCase): It is a very unintuitive behavior, documented as “undefined” so that we may change it in the future without breaking the contract. + For the asyncio reactor: + 1. First, the first CONCURRENT_REQUESTS start requests are sent in order. Awaiting slow operations in Spider.start() can lower that. @@ -102,6 +105,9 @@ class RequestSendOrderTestCase(TestCase): but only when there are not enough pending requests yielded from callbacks to reach the configured concurrency. + For the default Twisted reactor, step 1 sends the last CONCURRENT_REQUESTS + start requests in reverse order instead. + The reverse order is because the scheduler uses a LIFO queue by default (SCHEDULER_MEMORY_QUEUE, SCHEDULER_DISK_QUEUE). The order of the first few requests is unnaffected because they are sent as soon as they are @@ -171,8 +177,11 @@ class RequestSendOrderTestCase(TestCase): expected_nums = sorted(start_nums + cb_nums) assert actual_nums == expected_nums, f"{actual_nums=} != {expected_nums=}" + # Asyncio reactor behavior + + @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_default(self): + async def test_ar_default(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[ @@ -206,8 +215,9 @@ class RequestSendOrderTestCase(TestCase): ) ) + @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_conc1(self): + async def test_ar_conc1(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 4, 2], @@ -216,8 +226,9 @@ class RequestSendOrderTestCase(TestCase): ) ) + @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_conc2(self): + async def test_ar_conc2(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 2, 6, 4, 3], @@ -226,8 +237,9 @@ class RequestSendOrderTestCase(TestCase): ) ) + @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_conc8(self): + async def test_ar_conc8(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 2, 3, 4, 5, 6, 7, 8, 18, 16, 15, 14, 13, 12, 11, 10, 9], @@ -236,8 +248,9 @@ class RequestSendOrderTestCase(TestCase): ) ) + @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_conc16(self): + async def test_ar_conc16(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[ @@ -280,8 +293,9 @@ class RequestSendOrderTestCase(TestCase): ) ) + @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_conc3_ds2(self): + async def test_ar_conc3_ds2(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 2, 3, 8, 6, 5, 4], @@ -293,8 +307,9 @@ class RequestSendOrderTestCase(TestCase): ) ) + @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_tconc3_dconc2(self): + async def test_ar_tconc3_dconc2(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 2, 3, 7, 5, 4], @@ -306,8 +321,9 @@ class RequestSendOrderTestCase(TestCase): ) ) + @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_tconc5_dconc3(self): + async def test_ar_tconc5_dconc3(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 2, 3, 4, 5, 10, 8, 7, 6], @@ -319,8 +335,9 @@ class RequestSendOrderTestCase(TestCase): ) ) + @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_tconc5_dconc2_ds3(self): + async def test_ar_tconc5_dconc2_ds3(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 2, 3, 4, 5, 12, 10, 9, 8, 7, 6], @@ -333,8 +350,9 @@ class RequestSendOrderTestCase(TestCase): ) ) + @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_tconc5_dconc3_ds2(self): + async def test_ar_tconc5_dconc3_ds2(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 2, 3, 4, 5, 12, 10, 9, 8, 7, 6], @@ -347,8 +365,9 @@ class RequestSendOrderTestCase(TestCase): ) ) + @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_tconc7_dconc2_ds3(self): + async def test_ar_tconc7_dconc2_ds3(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 2, 3, 4, 5, 6, 7, 15, 13, 12, 11, 10, 9, 8], @@ -361,8 +380,9 @@ class RequestSendOrderTestCase(TestCase): ) ) + @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_tconc7_dconc3_ds2(self): + async def test_ar_tconc7_dconc3_ds2(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 2, 3, 4, 5, 6, 7, 15, 13, 12, 11, 10, 9, 8], @@ -375,8 +395,9 @@ class RequestSendOrderTestCase(TestCase): ) ) + @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_fast(self): + async def test_ar_fast(self): """Very fast responses may increase the number of start requests sent in reverse order before the first callback request.""" await maybe_deferred_to_future( @@ -388,6 +409,240 @@ class RequestSendOrderTestCase(TestCase): ) ) + # Default reactor behavior + + @pytest.mark.only_not_asyncio + @deferred_f_from_coro_f + async def test_dr_default(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[ + 26, + 24, + 23, + 22, + 21, + 20, + 19, + 18, + 17, + 16, + 15, + 14, + 13, + 12, + 11, + 10, + 9, + 8, + 7, + 6, + 5, + 4, + 3, + 2, + 1, + ], + cb_nums=[25], + ) + ) + + @pytest.mark.only_not_asyncio + @deferred_f_from_coro_f + async def test_dr_conc1(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[4, 2, 1], + cb_nums=[3], + settings={"CONCURRENT_REQUESTS": 1}, + ) + ) + + @pytest.mark.only_not_asyncio + @deferred_f_from_coro_f + async def test_dr_conc2(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[6, 4, 3, 2, 1], + cb_nums=[5], + settings={"CONCURRENT_REQUESTS": 2}, + ) + ) + + @pytest.mark.only_not_asyncio + @deferred_f_from_coro_f + async def test_dr_conc8(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[18, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1], + cb_nums=[17], + settings={"CONCURRENT_REQUESTS": 8}, + ) + ) + + @pytest.mark.only_not_asyncio + @deferred_f_from_coro_f + async def test_dr_conc16(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[ + 34, + 32, + 31, + 30, + 29, + 28, + 27, + 26, + 25, + 24, + 23, + 22, + 21, + 20, + 19, + 18, + 17, + 16, + 15, + 14, + 13, + 12, + 11, + 10, + 9, + 8, + 7, + 6, + 5, + 4, + 3, + 2, + 1, + ], + cb_nums=[33], + settings={"CONCURRENT_REQUESTS_PER_DOMAIN": 16}, + ) + ) + + @pytest.mark.only_not_asyncio + @deferred_f_from_coro_f + async def test_dr_conc3_ds2(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[8, 6, 5, 4, 3, 2, 1], + cb_nums=[7], + settings={ + "CONCURRENT_REQUESTS": 3, + }, + download_slots=2, + ) + ) + + @pytest.mark.only_not_asyncio + @deferred_f_from_coro_f + async def test_dr_tconc3_dconc2(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[7, 5, 4, 3, 2, 1], + cb_nums=[6], + settings={ + "CONCURRENT_REQUESTS": 3, + "CONCURRENT_REQUESTS_PER_DOMAIN": 2, + }, + ) + ) + + @pytest.mark.only_not_asyncio + @deferred_f_from_coro_f + async def test_dr_tconc5_dconc3(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[10, 8, 7, 6, 5, 4, 3, 2, 1], + cb_nums=[9], + settings={ + "CONCURRENT_REQUESTS": 5, + "CONCURRENT_REQUESTS_PER_DOMAIN": 3, + }, + ) + ) + + @pytest.mark.only_not_asyncio + @deferred_f_from_coro_f + async def test_dr_tconc5_dconc2_ds3(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[12, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1], + cb_nums=[11], + settings={ + "CONCURRENT_REQUESTS": 5, + "CONCURRENT_REQUESTS_PER_DOMAIN": 2, + }, + download_slots=3, + ) + ) + + @pytest.mark.only_not_asyncio + @deferred_f_from_coro_f + async def test_dr_tconc5_dconc3_ds2(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[12, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1], + cb_nums=[11], + settings={ + "CONCURRENT_REQUESTS": 5, + "CONCURRENT_REQUESTS_PER_DOMAIN": 3, + }, + download_slots=2, + ) + ) + + @pytest.mark.only_not_asyncio + @deferred_f_from_coro_f + async def test_dr_tconc7_dconc2_ds3(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[15, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1], + cb_nums=[14], + settings={ + "CONCURRENT_REQUESTS": 7, + "CONCURRENT_REQUESTS_PER_DOMAIN": 2, + }, + download_slots=3, + ) + ) + + @pytest.mark.only_not_asyncio + @deferred_f_from_coro_f + async def test_dr_tconc7_dconc3_ds2(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[15, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1], + cb_nums=[14], + settings={ + "CONCURRENT_REQUESTS": 7, + "CONCURRENT_REQUESTS_PER_DOMAIN": 3, + }, + download_slots=2, + ) + ) + + @pytest.mark.only_not_asyncio + @deferred_f_from_coro_f + async def test_dr_fast(self): + """Very fast responses may increase the number of start requests sent + in reverse order before the first callback request.""" + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[3, 2, 1], + cb_nums=[4], + settings={"CONCURRENT_REQUESTS": 1}, + response_seconds=self.fast_seconds, + ) + ) + + # Behavior shared by both reactors + @deferred_f_from_coro_f async def test_await(self): """Awaiting slow operations in Spider.start() may lower the number of