From ce35abbce70632d1993ac86a31a6cd4b55b99932 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Wed, 1 Jul 2026 09:14:10 +0200 Subject: [PATCH] WIP --- docs/topics/broad-crawls.rst | 2 +- docs/topics/request-response.rst | 2 ++ .../downloader/handlers/_base_streaming.py | 12 ---------- scrapy/core/downloader/handlers/http11.py | 8 +++++++ scrapy/throttling.py | 24 +++++++++++++++---- 5 files changed, 30 insertions(+), 18 deletions(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 81a4679ba..cace1f883 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -48,7 +48,7 @@ Increase concurrency Concurrency is the number of requests that are processed in parallel. There is a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional limit that -can be set per domain (:setting:`THROTTLING_SCOPE_CONCURRENCY`). +can be set per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`). The default global concurrency limit in Scrapy is not suitable for crawling many different domains in parallel, so you will want to increase it. How much diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index d7a76be17..0d8441900 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -732,6 +732,8 @@ Those are: * :reqmeta:`redirect_reasons` * :reqmeta:`redirect_urls` * :reqmeta:`referrer_policy` +* :reqmeta:`throttling_delay` +* :reqmeta:`throttling_dont_track` * :reqmeta:`throttling_scopes` * :reqmeta:`verbatim_url` diff --git a/scrapy/core/downloader/handlers/_base_streaming.py b/scrapy/core/downloader/handlers/_base_streaming.py index b7aba8fd0..6742781e6 100644 --- a/scrapy/core/downloader/handlers/_base_streaming.py +++ b/scrapy/core/downloader/handlers/_base_streaming.py @@ -84,19 +84,7 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, ABC, Generic[_Respon crawler.settings.get("DOWNLOAD_BIND_ADDRESS") ) self._proxy_auth_encoding: str = crawler.settings.get("HTTPPROXY_AUTH_ENCODING") - # these are useful for many handlers but used in different ways by them self._pool_size_total: int = crawler.settings.getint("CONCURRENT_REQUESTS") - scope_concurrencies = [ - scope["concurrency"] - for scope in crawler.settings.getdict("THROTTLING_SCOPES").values() - if "concurrency" in scope - ] - self._pool_size_per_host: int = max( - [ - crawler.settings.getint("THROTTLING_SCOPE_CONCURRENCY"), - *scope_concurrencies, - ] - ) @staticmethod @abstractmethod diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index f83c497bf..30b8d73e0 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -92,6 +92,12 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler): from twisted.internet import reactor self._pool: HTTPConnectionPool = HTTPConnectionPool(reactor, persistent=True) + # Keep enough persistent connections per host to match the highest + # per-host concurrency the throttler may admit: the per-domain and + # per-IP limits, the default "other"-scope limit, and any explicit + # THROTTLING_SCOPES concurrency. Per-scope concurrency can grow beyond + # this at runtime (rampup); the excess simply uses non-persistent + # connections. scope_concurrencies = [ scope["concurrency"] for scope in crawler.settings.getdict("THROTTLING_SCOPES").values() @@ -99,6 +105,8 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler): ] self._pool.maxPersistentPerHost = max( [ + crawler.settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN"), + crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP"), crawler.settings.getint("THROTTLING_SCOPE_CONCURRENCY"), *scope_concurrencies, ] diff --git a/scrapy/throttling.py b/scrapy/throttling.py index 0cfea5d6d..dcdfeb97d 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -1025,6 +1025,14 @@ class ThrottlingManager: conflicts.append(f"delay={config['delay']!r} < Crawl-delay {capped}") if config.get("concurrency") is not None and int(config["concurrency"]) > 1: conflicts.append(f"concurrency={config['concurrency']!r} > 1") + # A min_concurrency floor above 1 keeps the scope above the single-slot + # concurrency that a Crawl-delay implies, since set_concurrency(1) is + # clamped back up to it. + if ( + config.get("min_concurrency") is not None + and int(config["min_concurrency"]) > 1 + ): + conflicts.append(f"min_concurrency={config['min_concurrency']!r} > 1") if conflicts: logger.warning( f"Throttling scope {scope_id!r} is configured with {' and '.join(conflicts)}, " @@ -1421,11 +1429,17 @@ class ThrottlingScopeManager: if self._rampup_window_start is None: self._rampup_window_start = now return - while now - self._rampup_window_start >= self._window: - self._rampup_window_start += self._window - if self._rampup_backoffs < self._rampup_target[0]: - self._rampup_step() - self._rampup_backoffs = 0 + if now - self._rampup_window_start < self._window: + return + # Catch up with elapsed time but apply at most one ramp step per call: a + # scope that stayed idle for several windows must not ramp up + # cumulatively once it becomes active again (that would collapse the + # delay or jump the concurrency limit by several steps at once). + elapsed_windows = int((now - self._rampup_window_start) // self._window) + self._rampup_window_start += elapsed_windows * self._window + if self._rampup_backoffs < self._rampup_target[0]: + self._rampup_step() + self._rampup_backoffs = 0 def _rampup_step(self) -> None: # Backoff in progress: let it recover before probing again.