WIP

docs/topics/broad-crawls.rst

@@ -48,7 +48,7 @@ Increase concurrency
 
 Concurrency is the number of requests that are processed in parallel. There is
 a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional limit that
-can be set per domain (:setting:`THROTTLING_SCOPE_CONCURRENCY`).
+can be set per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`).
 
 The default global concurrency limit in Scrapy is not suitable for crawling
 many different domains in parallel, so you will want to increase it. How much

docs/topics/request-response.rst

@@ -732,6 +732,8 @@ Those are:
 * :reqmeta:`redirect_reasons`
 * :reqmeta:`redirect_urls`
 * :reqmeta:`referrer_policy`
+* :reqmeta:`throttling_delay`
+* :reqmeta:`throttling_dont_track`
 * :reqmeta:`throttling_scopes`
 * :reqmeta:`verbatim_url`
 

scrapy/core/downloader/handlers/_base_streaming.py

@@ -84,19 +84,7 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, ABC, Generic[_Respon
             crawler.settings.get("DOWNLOAD_BIND_ADDRESS")
         )
         self._proxy_auth_encoding: str = crawler.settings.get("HTTPPROXY_AUTH_ENCODING")
-        # these are useful for many handlers but used in different ways by them
         self._pool_size_total: int = crawler.settings.getint("CONCURRENT_REQUESTS")
-        scope_concurrencies = [
-            scope["concurrency"]
-            for scope in crawler.settings.getdict("THROTTLING_SCOPES").values()
-            if "concurrency" in scope
-        ]
-        self._pool_size_per_host: int = max(
-            [
-                crawler.settings.getint("THROTTLING_SCOPE_CONCURRENCY"),
-                *scope_concurrencies,
-            ]
-        )
 
     @staticmethod
     @abstractmethod

scrapy/core/downloader/handlers/http11.py

@@ -92,6 +92,12 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler):
         from twisted.internet import reactor
 
         self._pool: HTTPConnectionPool = HTTPConnectionPool(reactor, persistent=True)
+        # Keep enough persistent connections per host to match the highest
+        # per-host concurrency the throttler may admit: the per-domain and
+        # per-IP limits, the default "other"-scope limit, and any explicit
+        # THROTTLING_SCOPES concurrency. Per-scope concurrency can grow beyond
+        # this at runtime (rampup); the excess simply uses non-persistent
+        # connections.
         scope_concurrencies = [
             scope["concurrency"]
             for scope in crawler.settings.getdict("THROTTLING_SCOPES").values()
@@ -99,6 +105,8 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler):
         ]
         self._pool.maxPersistentPerHost = max(
             [
+                crawler.settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN"),
+                crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP"),
                 crawler.settings.getint("THROTTLING_SCOPE_CONCURRENCY"),
                 *scope_concurrencies,
             ]

scrapy/throttling.py

@@ -1025,6 +1025,14 @@ class ThrottlingManager:
             conflicts.append(f"delay={config['delay']!r} < Crawl-delay {capped}")
         if config.get("concurrency") is not None and int(config["concurrency"]) > 1:
             conflicts.append(f"concurrency={config['concurrency']!r} > 1")
+        # A min_concurrency floor above 1 keeps the scope above the single-slot
+        # concurrency that a Crawl-delay implies, since set_concurrency(1) is
+        # clamped back up to it.
+        if (
+            config.get("min_concurrency") is not None
+            and int(config["min_concurrency"]) > 1
+        ):
+            conflicts.append(f"min_concurrency={config['min_concurrency']!r} > 1")
         if conflicts:
             logger.warning(
                 f"Throttling scope {scope_id!r} is configured with {' and '.join(conflicts)}, "
@@ -1421,11 +1429,17 @@ class ThrottlingScopeManager:
         if self._rampup_window_start is None:
             self._rampup_window_start = now
             return
-        while now - self._rampup_window_start >= self._window:
-            self._rampup_window_start += self._window
-            if self._rampup_backoffs < self._rampup_target[0]:
-                self._rampup_step()
-            self._rampup_backoffs = 0
+        if now - self._rampup_window_start < self._window:
+            return
+        # Catch up with elapsed time but apply at most one ramp step per call: a
+        # scope that stayed idle for several windows must not ramp up
+        # cumulatively once it becomes active again (that would collapse the
+        # delay or jump the concurrency limit by several steps at once).
+        elapsed_windows = int((now - self._rampup_window_start) // self._window)
+        self._rampup_window_start += elapsed_windows * self._window
+        if self._rampup_backoffs < self._rampup_target[0]:
+            self._rampup_step()
+        self._rampup_backoffs = 0
 
     def _rampup_step(self) -> None:
         # Backoff in progress: let it recover before probing again.