Support CONCURRENT_REQUESTS = 0 for unlimited concurrency (#7840)

docs/topics/settings.rst

@@ -575,7 +575,7 @@ CONCURRENT_REQUESTS
 Default: ``16``
 
 The maximum number of concurrent (i.e. simultaneous) requests that will be
-performed by the Scrapy downloader.
+performed by the Scrapy downloader. Use ``0`` for no limit.
 
 .. setting:: CONCURRENT_REQUESTS_PER_DOMAIN
 

scrapy/core/downloader/__init__.py

@@ -138,7 +138,8 @@ class Downloader:
             self.active.remove(request)
 
     def needs_backout(self) -> bool:
-        return len(self.active) >= self.total_concurrency
+        # A total concurrency of 0 means no limit.
+        return 0 < self.total_concurrency <= len(self.active)
 
     @_warn_spider_arg
     def _get_slot(

scrapy/core/downloader/handlers/_httpx.py

@@ -92,10 +92,11 @@ class HttpxDownloadHandler(_Base):
         self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings)
         self._bind_host: str | None = self._get_bind_address_host()
         self._limits: httpx.Limits = httpx.Limits(
-            # hard limit on simultaneous connections
-            max_connections=self._pool_size_total,
+            # hard limit on simultaneous connections (None for no limit, which
+            # is what a CONCURRENT_REQUESTS of 0 means)
+            max_connections=self._pool_size_total or None,
             # total number of idle connections in the pool (extra ones are closed)
-            max_keepalive_connections=self._pool_size_total,
+            max_keepalive_connections=self._pool_size_total or None,
         )
 
         self._default_client: httpx.AsyncClient = self._make_client()

tests/test_core_downloader.py

@@ -14,6 +14,7 @@ from twisted.web import server, static
 from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody
 from twisted.web.client import Response as TxResponse
 
+from scrapy import Request
 from scrapy.core.downloader import Downloader, Slot, tls
 from scrapy.core.downloader.contextfactory import (
     _load_context_factory_from_settings,
@@ -296,6 +297,23 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase):
         await self._assert_factory_works(server_url, client_context_factory)
 
 
+@pytest.mark.parametrize(
+    ("concurrency", "active", "expected"),
+    [
+        (2, 1, False),
+        (2, 2, True),
+        (0, 0, False),
+        (0, 2, False),
+    ],
+)
+def test_needs_backout(concurrency: int, active: int, expected: bool) -> None:
+    crawler = get_crawler(settings_dict={"CONCURRENT_REQUESTS": concurrency})
+    downloader = Downloader(crawler)
+    downloader.active = {Request(f"https://example.com/{i}") for i in range(active)}
+    assert downloader.needs_backout() is expected
+    downloader.close()
+
+
 @coroutine_test
 async def test_fetch_deprecated_spider_arg():
     class CustomDownloader(Downloader):

tests/test_downloader_handler_httpx.py

@@ -15,6 +15,8 @@ from scrapy.core.downloader.handlers._httpx import (
     HttpxDownloadHandler,
 )
 from scrapy.exceptions import DownloadFailedError
+from scrapy.utils.misc import build_from_crawler
+from scrapy.utils.test import get_crawler
 from tests.utils.bases.download_handlers_http import (
     TestHttpBase,
     TestHttpProxyBase,
@@ -161,3 +163,15 @@ class TestMitmProxy(HttpxDownloadHandlerMixin, TestMitmProxyBase):
 @pytest.mark.requires_internet
 class TestRealWebsite(HttpxDownloadHandlerMixin, TestRealWebsiteBase):
     pass
+
+
+@pytest.mark.parametrize(("concurrency", "expected"), [(16, 16), (0, None)])
+@coroutine_test
+async def test_pool_limits(concurrency: int, expected: int | None) -> None:
+    crawler = get_crawler(settings_dict={"CONCURRENT_REQUESTS": concurrency})
+    handler = build_from_crawler(HttpxDownloadHandler, crawler)
+    try:
+        assert handler._limits.max_connections == expected
+        assert handler._limits.max_keepalive_connections == expected
+    finally:
+        await handler.close()