diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 25ce81b2e..bc1c58ead 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -371,21 +371,24 @@ Spider start The way :meth:`~scrapy.Spider.start` works by default may be counterintuitive: -#. The first 8-16 start requests are sent in the order in which they are - yielded. +#. First, the first 16 start requests are sent in order. - That number depends on :setting:`CONCURRENT_REQUESTS`. - :ref:`Awaiting ` slow operations in :meth:`~scrapy.Spider.start` may - lower it. + That number depends on :setting:`CONCURRENT_REQUESTS`. :ref:`Awaiting + ` slow operations in :meth:`~scrapy.Spider.start` may lower it. -#. The last start request is sent. +#. Then, the last 8 start requests are sent in reverse order. - It could be more start requests for very low response times. If so, they - are the last start requests in reverse order. + That number depends on both :setting:`CONCURRENT_REQUESTS` and + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. Specifically, assuming an even + domain distribution in start requests (i.e. ABCABC, not AABBCC), it is: -#. The remaining start requests are also sent in reverse order, but only when - there are not enough pending requests yielded from callbacks to reach the - configured concurrency. + .. code-block:: python + + min(CONCURRENT_REQUESTS, CONCURRENT_REQUESTS_PER_DOMAIN * domain_count) + +#. Finally, the remaining start requests are also sent in reverse order, but + only when there are not enough pending requests yielded from callbacks to + reach the configured concurrency. .. note:: Response order is a different story: it is determined not only by request order, but also by response time. diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index fce20eda9..058deee8b 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -93,10 +93,6 @@ class MockServerTestCase(TestCase): # Verify the default behavior of the engine loop as described in the docs, # in the “Spider start” section of the page about spdiers. - # - # TODO: Check how combinations of the following parameters affect the - # behavior: response time (high/low), max concurrency (hihg/low), number of - # start requests (few/many), and number of domains (single/multiple). fast_seconds = 0.001 slow_seconds = 0.2 # increase if flaky @@ -142,10 +138,6 @@ class MockServerTestCase(TestCase): expected_nums = sorted(start_nums + cb_nums) assert actual_nums == expected_nums, f"{actual_nums=} != {expected_nums=}" - # TODO: Figure out why the behavior changes when CONCURRENT_REQUESTS is - # higher than CONCURRENT_REQUESTS_PER_DOMAIN. The number of requests sent - # before callback requests seems to depend on CONCURRENT_REQUESTS and not - # in CONCURRENT_REQUESTS_PER_DOMAIN. @deferred_f_from_coro_f async def test_default(self): await maybe_deferred_to_future( @@ -160,18 +152,24 @@ class MockServerTestCase(TestCase): 7, 8, 9, - 19, - 17, - 16, - 15, - 14, - 13, - 12, - 11, 10, + 11, + 12, + 13, + 14, + 15, + 16, + 26, + 24, + 23, + 22, + 21, + 20, + 19, + 18, + 17, ], - cb_nums=[18], - settings={"CONCURRENT_REQUESTS": 9}, + cb_nums=[25], ) ) @@ -250,14 +248,95 @@ class MockServerTestCase(TestCase): ) @deferred_f_from_coro_f - async def test_domains(self): + async def test_conc3_ds2(self): await maybe_deferred_to_future( self._test_request_order( - start_nums=[1, 2, 6, 4, 3], - cb_nums=[5], + start_nums=[1, 2, 3, 8, 6, 5, 4], + cb_nums=[7], settings={ - "CONCURRENT_REQUESTS": 2, - "CONCURRENT_REQUESTS_PER_DOMAIN": 1, + "CONCURRENT_REQUESTS": 3, + }, + download_slots=2, + ) + ) + + @deferred_f_from_coro_f + async def test_tconc3_dconc2(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[1, 2, 3, 7, 5, 4], + cb_nums=[6], + settings={ + "CONCURRENT_REQUESTS": 3, + "CONCURRENT_REQUESTS_PER_DOMAIN": 2, + }, + ) + ) + + @deferred_f_from_coro_f + async def test_tconc5_dconc3(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[1, 2, 3, 4, 5, 10, 8, 7, 6], + cb_nums=[9], + settings={ + "CONCURRENT_REQUESTS": 5, + "CONCURRENT_REQUESTS_PER_DOMAIN": 3, + }, + ) + ) + + @deferred_f_from_coro_f + async def test_tconc5_dconc2_ds3(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[1, 2, 3, 4, 5, 12, 10, 9, 8, 7, 6], + cb_nums=[11], + settings={ + "CONCURRENT_REQUESTS": 5, + "CONCURRENT_REQUESTS_PER_DOMAIN": 2, + }, + download_slots=3, + ) + ) + + @deferred_f_from_coro_f + async def test_tconc5_dconc3_ds2(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[1, 2, 3, 4, 5, 12, 10, 9, 8, 7, 6], + cb_nums=[11], + settings={ + "CONCURRENT_REQUESTS": 5, + "CONCURRENT_REQUESTS_PER_DOMAIN": 3, + }, + download_slots=2, + ) + ) + + @deferred_f_from_coro_f + async def test_tconc7_dconc2_ds3(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[1, 2, 3, 4, 5, 6, 7, 15, 13, 12, 11, 10, 9, 8], + cb_nums=[14], + settings={ + "CONCURRENT_REQUESTS": 7, + "CONCURRENT_REQUESTS_PER_DOMAIN": 2, + }, + download_slots=3, + ) + ) + + @deferred_f_from_coro_f + async def test_tconc7_dconc3_ds2(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[1, 2, 3, 4, 5, 6, 7, 15, 13, 12, 11, 10, 9, 8], + cb_nums=[14], + settings={ + "CONCURRENT_REQUESTS": 7, + "CONCURRENT_REQUESTS_PER_DOMAIN": 3, }, download_slots=2, ) @@ -273,3 +352,9 @@ class MockServerTestCase(TestCase): response_seconds=self.fast_seconds, ) ) + # TODO: Test how increasing concurrency behaves with fast responses. + + # TODO: Test claims: + # - :ref:`Awaiting ` slow operations in :meth:`~scrapy.Spider.start` may lower it. + # - If responses are very fast, it can be more than :setting:`CONCURRENT_REQUESTS`. + # - Otherwise, it can reach 16 (:setting:`CONCURRENT_REQUESTS`)