diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index b231e8586..d087ab302 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -166,9 +166,14 @@ until it is back to the configured value. A new trigger resets the countdown. This exponential-increase / linear-decrease pattern, similar to TCP congestion control, makes a scope back off quickly when a server is unhappy and return to -full speed gradually once it recovers. To keep a scope hovering around a target -rate instead of repeatedly probing and backing off, enable :ref:`rampup -`. +full speed gradually once it recovers. + +Backoff only ever *tightens* a scope, and recovery never goes past the +configured value: the delay can grow above the configured ``"delay"`` and then +recover back down to it, but never below it, and backoff never raises the +concurrency limit. So set the ``"delay"`` and ``"concurrency"`` you actually +want for a scope; backoff makes things gentler from there when a server pushes +back, and returns to those values once it recovers. Backoff triggers are detected by the :class:`~scrapy.downloadermiddlewares.backoff.BackoffMiddleware`, a built-in @@ -211,61 +216,6 @@ it. So a scope can, for example, treat an extra status code as a backoff trigger, or stop treating one of the defaults as a trigger, independently of every other scope. -.. _rampup: - -Rampup -====== - -When using APIs that charge per request, like web scraping APIs, you often want -to maximize throughput while staying within rate limits. To do that, set -``"rampup"`` to ``True`` in :setting:`THROTTLING_SCOPES`: - -.. code-block:: python - :caption: ``settings.py`` - - THROTTLING_SCOPES = { - "api.toscrape.com": { - "rampup": True, - }, - } - -Rampup increases concurrency or lowers delay as needed based on the following -setting: - -- .. setting:: RAMPUP_BACKOFF_TARGET - - :setting:`RAMPUP_BACKOFF_TARGET` (default: ``1``) - - Target number of backoff responses per :setting:`BACKOFF_WINDOW` that - :ref:`rampup ` aims for when probing the rate limit of a scope. - -For every :setting:`BACKOFF_WINDOW` that stays **below** -:setting:`RAMPUP_BACKOFF_TARGET` backoff triggers, rampup increases throughput -one step: it first lowers the delay, and once the delay reaches its minimum it -raises the concurrency limit of the scope. Windows that reach or exceed the -target do not ramp up; the rate is -reduced by normal :ref:`backoff ` (which grows the delay) instead. -Rampup only ever probes upward, so the rate settles around the most throughput -a scope allows while triggering fewer than :setting:`RAMPUP_BACKOFF_TARGET` -rate-limit responses per window. - -Rampup behavior can be fine-tuned per scope by giving ``"rampup"`` a dict -instead of ``True``: - -.. code-block:: python - :caption: ``settings.py`` - - THROTTLING_SCOPES = { - "api.toscrape.com": { - "rampup": { - "backoff_target": 1, # overrides RAMPUP_BACKOFF_TARGET - "delay_factor": 0.5, # multiply the delay by this on each ramp-up step - "min_delay": 0.05, # do not ramp the delay below this - }, - }, - } - - .. _retry-after: .. _rate-limiting-headers: @@ -485,9 +435,6 @@ following keys: ``window`` (:class:`float`) Quota window in seconds. Defaults to :setting:`THROTTLING_WINDOW`. -``rampup`` (:class:`bool` or :class:`dict`) - Enables :ref:`rampup ` for the scope. - ``backoff`` (:class:`~scrapy.throttling.BackoffConfig`) Per-scope :ref:`backoff overrides `. @@ -572,11 +519,10 @@ as a string): THROTTLING_SCOPE_MANAGER = "myproject.throttling.MyThrottlingScopeManager" For each throttling scope, an instance of this class is created to manage any -gradual :ref:`backoff ` or :ref:`rampup ` required at run -time. +gradual :ref:`backoff ` required at run time. You can implement your own throttling scope manager if you wish to change the -backoff or rampup behavior beyond what settings allow. +backoff behavior beyond what settings allow. You can also define a custom throttling scope manager for a specific throttling scope by setting the ``"manager"`` key in the :setting:`THROTTLING_SCOPES` @@ -984,9 +930,9 @@ Additional settings :setting:`BACKOFF_WINDOW` (default: ``60.0``) - Time window, in seconds, used by :ref:`backoff ` and - :ref:`rampup `. During backoff, a :ref:`throttling scope - ` must go this many seconds without a new backoff + Time window, in seconds, used by :ref:`backoff `. A + :ref:`throttling scope ` must go this many seconds + without a new backoff trigger (an HTTP error code from :setting:`BACKOFF_HTTP_CODES` or an exception from :setting:`BACKOFF_EXCEPTIONS`) before its delay decreases by one :setting:`BACKOFF_DELAY_FACTOR` step toward the configured value. @@ -1062,8 +1008,6 @@ API .. autoclass:: scrapy.throttling.BackoffConfig -.. autoclass:: scrapy.throttling.RampupConfig - .. autofunction:: scrapy.throttling.scope_cache .. autofunction:: scrapy.throttling.add_scope .. autofunction:: scrapy.throttling.iter_scopes diff --git a/scrapy/core/downloader/handlers/_base_http.py b/scrapy/core/downloader/handlers/_base_http.py index 2ca8ff353..66308389a 100644 --- a/scrapy/core/downloader/handlers/_base_http.py +++ b/scrapy/core/downloader/handlers/_base_http.py @@ -19,22 +19,20 @@ class BaseHttpDownloadHandler(BaseDownloadHandler, ABC): limit, the default ``other``-scope limit, and any explicit :setting:`THROTTLING_SCOPES` concurrency. - A scope with :ref:`rampup ` enabled has no configured - concurrency ceiling; it grows toward :setting:`CONCURRENT_REQUESTS`, so - it counts as that. And since :setting:`CONCURRENT_REQUESTS` caps the - total number of requests in flight, no host can ever exceed it, so it is - also the upper bound of the result. + Since :setting:`CONCURRENT_REQUESTS` caps the total number of requests in + flight, no host can ever exceed it, so it is also the upper bound of the + result. """ global_concurrency = settings.getint("CONCURRENT_REQUESTS") candidates = [ settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN"), settings.getint("THROTTLING_SCOPE_CONCURRENCY"), ] - for scope in settings.getdict("THROTTLING_SCOPES").values(): - if scope.get("rampup"): - candidates.append(global_concurrency) - elif "concurrency" in scope: - candidates.append(int(scope["concurrency"])) + candidates += [ + int(scope["concurrency"]) + for scope in settings.getdict("THROTTLING_SCOPES").values() + if "concurrency" in scope + ] return min(max(candidates), global_concurrency) def __init__(self, crawler: Crawler): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index cc145d241..20859ae30 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -174,7 +174,6 @@ __all__ = [ "PERIODIC_LOG_DELTA", "PERIODIC_LOG_STATS", "PERIODIC_LOG_TIMING_ENABLED", - "RAMPUP_BACKOFF_TARGET", "RANDOMIZE_DOWNLOAD_DELAY", "REACTOR_THREADPOOL_MAXSIZE", "REDIRECT_ENABLED", @@ -521,8 +520,6 @@ PERIODIC_LOG_DELTA = None PERIODIC_LOG_STATS = None PERIODIC_LOG_TIMING_ENABLED = False -RAMPUP_BACKOFF_TARGET = 1 - RANDOMIZE_DOWNLOAD_DELAY = True REACTOR_THREADPOOL_MAXSIZE = 10 diff --git a/scrapy/throttling.py b/scrapy/throttling.py index 646506f47..e460a5163 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -46,20 +46,6 @@ class BackoffConfig(TypedDict, total=False): jitter: float | list[float] -class RampupConfig(TypedDict, total=False): - """Per-scope override of the rampup settings. - - Used as the value of the ``"rampup"`` key of :class:`ThrottlingScopeConfig` - entries when fine-tuning rampup beyond a plain ``True``. Any key left out - falls back to its default (or, for ``backoff_target``, to - :setting:`RAMPUP_BACKOFF_TARGET`). - """ - - backoff_target: float - delay_factor: float - min_delay: float - - class ThrottlingScopeConfig(TypedDict, total=False): """Accepted keys of :setting:`THROTTLING_SCOPES` entries. @@ -78,7 +64,6 @@ class ThrottlingScopeConfig(TypedDict, total=False): quota: float window: float - rampup: bool | RampupConfig manager: str | type """Import path or class of a custom :setting:`THROTTLING_SCOPE_MANAGER` for @@ -214,8 +199,7 @@ def _warn_on_unachievable_concurrency(settings: BaseSettings) -> None: :setting:`CONCURRENT_REQUESTS` caps the total number of requests in flight, so a per-scope (or per-domain) concurrency limit above it can never be - reached. Rampup is not flagged: it has no configured ceiling and simply - grows toward :setting:`CONCURRENT_REQUESTS`. + reached. """ global_concurrency = settings.getint("CONCURRENT_REQUESTS") offenders: list[str] = [ @@ -991,11 +975,6 @@ class ThrottlingScopeManagerProtocol(Protocol): "min_delay": 5.0, "jitter": [0.01, 0.33], }, - "rampup": { - "backoff_target": 1, - "delay_factor": 0.5, - "min_delay": 0.05, - }, } """ @@ -1128,8 +1107,8 @@ class ThrottlingScopeManager: """The default :setting:`THROTTLING_SCOPE_MANAGER` class. It implements a per-scope state machine covering delay, exponential - :ref:`backoff `, :ref:`rampup `, concurrency and - :ref:`quotas `: + :ref:`backoff `, concurrency and :ref:`quotas + `: - A base delay (the scope ``"delay"`` config, defaulting to :setting:`DOWNLOAD_DELAY`) is enforced between consecutive requests for @@ -1147,14 +1126,8 @@ class ThrottlingScopeManager: - After :setting:`BACKOFF_WINDOW` seconds without a new trigger, the delay recovers one step at a time back towards the base delay. - - When the scope is configured with a ``"concurrency"`` limit (or with - ``"rampup"``), no more than that many requests are allowed in flight at - once. - - - When the scope sets ``"rampup": True``, throughput is increased every - :setting:`BACKOFF_WINDOW` that stays under :setting:`RAMPUP_BACKOFF_TARGET` - backoff triggers, first by lowering the delay and then by raising the - concurrency limit. + - When the scope is configured with a ``"concurrency"`` limit, no more + than that many requests are allowed in flight at once. - When the scope is configured with a ``"quota"``, no more than that much quota is consumed per ``"window"`` (default: :setting:`THROTTLING_WINDOW`). @@ -1208,27 +1181,12 @@ class ThrottlingScopeManager: ) self._window: float = settings.getfloat("BACKOFF_WINDOW") - # Rampup. - rampup = config.get("rampup") - self._rampup_enabled: bool = bool(rampup) - rampup_config: dict[str, Any] = rampup if isinstance(rampup, dict) else {} - self._rampup_target: float = float( - rampup_config.get( - "backoff_target", settings.getfloat("RAMPUP_BACKOFF_TARGET") - ) - ) - self._rampup_delay_factor: float = float(rampup_config.get("delay_factor", 0.5)) - self._rampup_min_delay: float = float(rampup_config.get("min_delay", 0.0)) - # Concurrency. ``None`` means no scope-level limit (the downloader slots # enforce concurrency instead); a limit is only set when configured - # explicitly or implied by rampup. + # explicitly. configured_concurrency = config.get("concurrency") if configured_concurrency is not None: self._concurrency: int | None = int(configured_concurrency) - elif self._rampup_enabled: - # Rampup starts conservative at a single slot and probes upward. - self._concurrency = 1 else: self._concurrency = _default_scope_concurrency(settings) or None # Used as the load denominator when the scope enforces no explicit @@ -1253,8 +1211,6 @@ class ThrottlingScopeManager: self._slot_waiters: list[Deferred[None]] = [] self._consumed: float = 0.0 self._quota_window_start: float | None = None - self._rampup_window_start: float | None = None - self._rampup_backoffs: int = 0 @staticmethod def _now(now: float | None) -> float: @@ -1316,45 +1272,6 @@ class ThrottlingScopeManager: break self._delay = max(self._base_delay, self._delay / self._delay_factor) - def _maybe_rampup(self, now: float) -> None: - """Increase throughput once per :setting:`BACKOFF_WINDOW` that stays - under :setting:`RAMPUP_BACKOFF_TARGET` backoff triggers.""" - if not self._rampup_enabled: - return - if self._window <= 0: - # No window: no rampup cadence to step (would spin). - return - if self._rampup_window_start is None: - self._rampup_window_start = now - return - if now - self._rampup_window_start < self._window: - return - # Catch up with elapsed time but apply at most one ramp step per call: a - # scope that stayed idle for several windows must not ramp up - # cumulatively once it becomes active again (that would collapse the - # delay or jump the concurrency limit by several steps at once). - elapsed_windows = int((now - self._rampup_window_start) // self._window) - self._rampup_window_start += elapsed_windows * self._window - if self._rampup_backoffs < self._rampup_target: - self._rampup_step() - self._rampup_backoffs = 0 - - def _rampup_step(self) -> None: - # Backoff in progress: let it recover before probing again. - if self._backoff_level > 0: - return - if self._delay > self._rampup_min_delay: - # Lower only the effective delay while probing for headroom; the - # configured base delay is left untouched so it stays the recovery - # target on backoff and the value reported by get_base_delay(). - self._delay = max( - self._rampup_min_delay, self._delay * self._rampup_delay_factor - ) - else: - # Rampup is only enabled with a concurrency limit set. - assert self._concurrency is not None - self._concurrency += 1 - def _maybe_reset_quota(self, now: float) -> None: if self._quota is None: return @@ -1372,10 +1289,7 @@ class ThrottlingScopeManager: def can_send(self, now: float | None = None, amount: float | None = None) -> float: # can_send() only refreshes passive, time-based state (backoff recovery - # and the quota window) to reflect the current time; it performs no - # active throughput probing. That way a readiness check (is_ready() / - # get_time_until_ready()) has no side effect on the send rate: rampup - # only advances on an actual send, from record_sent(). + # and the quota window) to reflect the current time. now = self._now(now) self._recover(now) self._maybe_reset_quota(now) @@ -1402,9 +1316,6 @@ class ThrottlingScopeManager: self._last_seen = now if self._in_backoff_until is not None and now >= self._in_backoff_until: self._in_backoff_until = None - # An actual send is the cue to probe for more throughput (rampup), - # rather than a mere readiness check; see can_send(). - self._maybe_rampup(now) self._next_allowed_time = now + self._effective_delay() self._active += 1 if self._quota is not None and amount is not None: @@ -1457,7 +1368,6 @@ class ThrottlingScopeManager: self._last_seen = now self._last_backoff_time = now self._backoff_level += 1 - self._rampup_backoffs += 1 if delay is not None: # A hard delay (e.g. a Retry-After header) is a one-time gate: hold # the scope back for at least this long *once*, matching the HTTP diff --git a/tests/test_throttling.py b/tests/test_throttling.py index 3dacbe0ef..b05181a29 100644 --- a/tests/test_throttling.py +++ b/tests/test_throttling.py @@ -638,16 +638,6 @@ class TestThrottlingScopeManager: assert scope.can_send(now=1.0, amount=5.0) == 0.0 assert scope._consumed == 0.0 - def test_zero_window_disables_rampup(self): - # A non-positive window must not make _maybe_rampup spin forever; rampup - # is simply disabled. - scope = _scope_manager( - settings={"BACKOFF_WINDOW": 0}, config={"id": "x", "rampup": True} - ) - scope.can_send(now=0.0) - scope.can_send(now=10_000.0) - assert scope._concurrency == scope._min_concurrency - def test_set_concurrency_fires_slot_event(self): scope = _scope_manager(config={"id": "x", "concurrency": 1}) scope.record_sent(now=0.0) @@ -711,42 +701,6 @@ class TestThrottlingScopeManager: scope.reconcile_quota(remaining=3.0, now=0.0) assert scope._consumed == pytest.approx(7.0) - def test_rampup_lowers_delay_when_quiet(self): - scope = _scope_manager( - {"BACKOFF_WINDOW": 10.0, "RANDOMIZE_DOWNLOAD_DELAY": False}, - { - "id": "x", - "delay": 4.0, - "rampup": {"delay_factor": 0.5, "min_delay": 0.5}, - }, - ) - scope.can_send(now=0.0) # start the rampup window - # A quiet window (no backoff) lowers the delay. - scope.can_send(now=10.0) - assert scope._delay == pytest.approx(2.0) - scope.can_send(now=20.0) - assert scope._delay == pytest.approx(1.0) - - def test_rampup_raises_concurrency_at_min_delay(self): - scope = _scope_manager( - {"BACKOFF_WINDOW": 10.0}, - {"id": "x", "delay": 0.0, "rampup": True, "min_concurrency": 1}, - ) - assert scope._concurrency == 1 - scope.can_send(now=0.0) - scope.can_send(now=10.0) - assert scope._concurrency == 2 - - def test_rampup_holds_when_target_met(self): - scope = _scope_manager( - {"BACKOFF_WINDOW": 10.0, "RAMPUP_BACKOFF_TARGET": 1}, - {"id": "x", "delay": 0.0, "rampup": True, "min_concurrency": 1}, - ) - scope.can_send(now=0.0) - scope.record_backoff(now=1.0) # one trigger == target -> hold, do not probe - scope.can_send(now=10.0) - assert scope._concurrency == 1 - class TestThrottlingManagerReadiness: """The synchronous readiness API used by a throttling-aware scheduler.""" @@ -1129,12 +1083,6 @@ class TestThrottlingScopeManagerEdges: # A randomized base delay lands within [0.5, 1.5] * delay. assert 1.0 <= scope.can_send(now=0.0) <= 3.0 - def test_rampup_target_as_range(self): - scope = _scope_manager( - config={"id": "x", "rampup": {"backoff_target": [1, 3]}}, - ) - assert scope._rampup_target == (1.0, 3.0) - def test_record_done_without_active(self): scope = _scope_manager(config={"id": "x"}) # Calling record_done() with nothing in flight is a harmless no-op. @@ -1160,14 +1108,6 @@ class TestThrottlingScopeManagerEdges: assert scope._base_delay == 0.5 assert scope._delay == backoff_delay - def test_rampup_step_held_during_backoff(self): - scope = _scope_manager(config={"id": "x", "delay": 4.0, "rampup": True}) - scope._backoff_level = 1 - before = scope._delay - scope._rampup_step() - # A rampup probe is skipped while a backoff is in progress. - assert scope._delay == before - def test_record_sent_clears_expired_backoff(self): scope = _scope_manager(config={"id": "x"}) scope.record_backoff(delay=5.0, now=0.0)