diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index 8d3ae004e..565bbd09f 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -63,24 +63,30 @@ The main throttling :ref:`settings ` are: The wait time is measured from when the previous request was sent. -For example, with ``CONCURRENT_REQUESTS_PER_DOMAIN = 2``, ``DOWNLOAD_DELAY = 0.3``, -and ``DOWNLOAD_DELAY_PER_SLOT = 1.0``, sending 3 requests to the same domain -would result in: +For example, with ``DOWNLOAD_DELAY = 1.0`` (and, by default, a single download +slot per domain), requests to the same domain are sent at most once per second: .. code-block:: text - T=0.0s: Request 1 sent (slot 1) - T=0.3s: Request 2 sent (slot 2, respects same-domain delay) - T=0.6s: Request 3 must wait (same-domain delay satisfied, but slot 1 needs 1.0s) - T=1.0s: Request 3 sent (slot 1 can now be reused) + T=0.0s: Request 1 sent + T=1.0s: Request 2 sent + T=2.0s: Request 3 sent + +:setting:`DOWNLOAD_DELAY` (per :ref:`throttling scope `) and +:setting:`DOWNLOAD_DELAY_PER_SLOT` (per download slot) are enforced +independently. By default each domain is both its own scope and its own +download slot, so both apply to the same requests and the effective minimum +spacing is the larger of the two; they only differ when requests are grouped +into custom :ref:`scopes ` or download slots (via the +``download_slot`` request meta key). When configuring these settings, note that: - :setting:`CONCURRENT_REQUESTS` caps ``CONCURRENT_REQUESTS_PER_DOMAIN``. -- If ``DOWNLOAD_DELAY`` ≥ response time, concurrency is effectively ``1``. - This happens because all slots must wait for the delay between requests, - preventing them from sending requests simultaneously. +- If ``DOWNLOAD_DELAY`` ≥ response time, concurrency is effectively ``1``, + because the next request to the domain is not sent until the delay elapses, + by which time the previous response has already arrived. .. [1] You can :ref:`customize ` how requests are grouped for throttling, but domain-based throttling works well in most cases. For @@ -1061,6 +1067,23 @@ Additional settings Whether to log :ref:`throttling ` decisions (per-scope delays, backoff steps and recoveries) for debugging. +- .. setting:: THROTTLING_SCOPE_LIMIT + + :setting:`THROTTLING_SCOPE_LIMIT` (default: ``100000``) + + Maximum number of :ref:`throttling scope ` states kept + in memory at once, to bound memory usage on broad crawls that touch a large + number of scopes (e.g. domains). + + When the limit is exceeded, the least-recently-used idle scopes are evicted + (an evicted scope is recreated from its configuration the next time it is + needed). Scopes with in-flight requests or in active backoff are never + evicted, so the limit may be temporarily exceeded if that many scopes are + busy at once. Set to ``0`` to disable the limit. + + This complements :setting:`THROTTLING_SCOPE_MAX_IDLE`, which evicts scopes + by inactivity time rather than by count. + - .. setting:: THROTTLING_SCOPE_MAX_IDLE :setting:`THROTTLING_SCOPE_MAX_IDLE` (default: ``3600.0``) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 7b761d80b..042468cd7 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -380,11 +380,16 @@ class ExecutionEngine: if delay_fn is None or not scheduler.has_pending_requests(): return delay = delay_fn() - if delay is None: + # ``delay`` is ``None`` when no pending request is time-blocked, and + # ``0`` when some request is ready right now but could not be sent (e.g. + # the downloader is at capacity). Neither case needs a timer: a freed + # slot already re-runs the loop from :meth:`_download`'s ``finally``, + # and arming a ``0``-second timer here would busy-loop the engine while + # the downloader stays full. Only a positive delay, i.e. a time-based + # gate that nothing else would wake us for, needs one. + if delay is None or delay <= 0: return - self._throttling_wakeup = call_later( - max(0.0, delay), self._slot.nextcall.schedule - ) + self._throttling_wakeup = call_later(delay, self._slot.nextcall.schedule) def needs_backout(self) -> bool: """Returns ``True`` if no more requests can be sent at the moment, or diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index f19894658..586c2ecfd 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -598,6 +598,7 @@ THROTTLING_SCOPES = {} THROTTLING_WINDOW = 60.0 THROTTLING_ROBOTSTXT_OBEY = True THROTTLING_ROBOTSTXT_MAX_DELAY = 60.0 +THROTTLING_SCOPE_LIMIT = 100000 THROTTLING_SCOPE_MAX_IDLE = 3600.0 THROTTLING_DEBUG = False diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index e1ebdf707..64b27db68 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -9,5 +9,12 @@ NEWSPIDER_MODULE = "$project_name.spiders" # User-Agent header: #USER_AGENT = "$project_name (+https://your-domain.example)" +# Obey robots.txt rules: +ROBOTSTXT_OBEY = True + +# Throttle crawls to be polite to websites: +CONCURRENT_REQUESTS_PER_DOMAIN = 1 +DOWNLOAD_DELAY = 1 + # Set settings whose default value is deprecated to a future-proof value: FEED_EXPORT_ENCODING = "utf-8" diff --git a/scrapy/throttling.py b/scrapy/throttling.py index bdddd3d3b..ddb540f53 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -5,6 +5,7 @@ import datetime as dt import logging import random import time +from collections import OrderedDict from collections.abc import Awaitable, Callable, Iterable from email.utils import parsedate_to_datetime from functools import wraps @@ -45,7 +46,9 @@ def _parse_retry_after(response: Response) -> float | None: date = date.replace(tzinfo=dt.timezone.utc) now = dt.datetime.now(dt.timezone.utc) seconds_to_wait = (date - now).total_seconds() - return max(0, int(seconds_to_wait)) or None + # Keep sub-second precision (a date less than a second away must not be + # truncated to 0 and dropped); a past or present date yields no delay. + return max(0.0, seconds_to_wait) or None def _parse_ratelimit_reset(response: Response) -> float | None: @@ -485,7 +488,12 @@ class ThrottlingManager: self._scopes_config: dict[str, dict[str, Any]] = crawler.settings.getdict( "THROTTLING_SCOPES" ) - self._scope_managers: dict[ScopeID, ThrottlingScopeManagerProtocol] = {} + # Ordered by least-recently-used first (see _get_scope_manager), so the + # scope limit can evict the coldest idle scopes (see THROTTLING_SCOPE_LIMIT). + self._scope_managers: OrderedDict[ScopeID, ThrottlingScopeManagerProtocol] = ( + OrderedDict() + ) + self._scope_limit: int = crawler.settings.getint("THROTTLING_SCOPE_LIMIT") self._last_eviction: float | None = None # Concurrency slots reserved by acquire(), to be released once the # request finishes downloading. @@ -585,18 +593,46 @@ class ThrottlingManager: def _get_scope_manager(self, scope_id: ScopeID) -> ThrottlingScopeManagerProtocol: manager = self._scope_managers.get(scope_id) - if manager is None: - config: dict[str, Any] = dict(self._scopes_config.get(scope_id, {})) - config.setdefault("id", scope_id) - manager_cls = ( - load_object(config["manager"]) - if "manager" in config - else self._default_scope_manager_cls - ) - manager = build_from_crawler(manager_cls, self.crawler, config) - self._scope_managers[scope_id] = manager + if manager is not None: + # Mark as most-recently-used for the LRU scope limit. + self._scope_managers.move_to_end(scope_id) + return manager + config: dict[str, Any] = dict(self._scopes_config.get(scope_id, {})) + config.setdefault("id", scope_id) + manager_cls = ( + load_object(config["manager"]) + if "manager" in config + else self._default_scope_manager_cls + ) + manager = cast( + "ThrottlingScopeManagerProtocol", + build_from_crawler(manager_cls, self.crawler, config), + ) + self._scope_managers[scope_id] = manager + self._enforce_scope_limit(scope_id) return manager + def _enforce_scope_limit(self, keep: ScopeID) -> None: + """Evict least-recently-used idle scopes while the number of live scope + managers exceeds :setting:`THROTTLING_SCOPE_LIMIT` (``0`` disables the + limit). + + LRU order is kept by :meth:`_get_scope_manager` moving each accessed + scope to the end, so the coldest scopes are at the front. Only scopes + that are idle (no in-flight requests and no active backoff) are evicted; + the just-created *keep* scope is never evicted. A scope evicted while + still throttling is simply recreated from its configuration the next + time it is needed. + """ + if self._scope_limit <= 0 or len(self._scope_managers) <= self._scope_limit: + return + now = time.monotonic() + for scope_id in list(self._scope_managers): + if len(self._scope_managers) <= self._scope_limit: + break + if scope_id != keep and self._scope_managers[scope_id].is_idle(now, 0): + del self._scope_managers[scope_id] + async def acquire(self, request: Request) -> None: # A throttling-aware scheduler reserves the request before handing it # to the engine, so there is nothing left to wait for or record here. @@ -1008,9 +1044,9 @@ class ThrottlingScopeManager: :ref:`backoff `, :ref:`rampup `, concurrency and :ref:`quotas `: - - A base :setting:`DOWNLOAD_DELAY`-style delay (``0`` by default, taken - from the scope ``"delay"`` config) is enforced between consecutive - requests for the scope. + - A base delay (the scope ``"delay"`` config, defaulting to + :setting:`DOWNLOAD_DELAY`) is enforced between consecutive requests for + the scope. - On a backoff trigger (a :setting:`BACKOFF_HTTP_CODES` response or a :setting:`BACKOFF_EXCEPTIONS` exception) the delay grows exponentially @@ -1043,7 +1079,11 @@ class ThrottlingScopeManager: settings = crawler.settings backoff: dict[str, Any] = config.get("backoff", {}) self._id: ScopeID = config.get("id", "") - self._base_delay: float = float(config.get("delay", 0.0)) + # The per-scope delay defaults to DOWNLOAD_DELAY; a scope can override + # it with its own "delay" config (see THROTTLING_SCOPES). + self._base_delay: float = float( + config.get("delay", settings.getfloat("DOWNLOAD_DELAY")) + ) self._randomize: bool = bool( config.get("randomize_delay", settings.getbool("RANDOMIZE_DOWNLOAD_DELAY")) ) diff --git a/tests/test_engine.py b/tests/test_engine.py index 60d911009..032e5df2b 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -694,6 +694,18 @@ class TestEngineThrottling: engine._maybe_arm_throttling_wakeup() assert engine._throttling_wakeup is None + def test_maybe_arm_throttling_wakeup_zero_delay(self, engine): + # A 0 delay means a request is ready but could not be sent (e.g. the + # downloader is at capacity); arming a 0-second timer would busy-loop + # the engine, so no timer must be armed. + scheduler = Mock() + scheduler.has_pending_requests.return_value = True + scheduler.next_request_delay.return_value = 0.0 + engine._slot = Mock() + engine._slot.scheduler = scheduler + engine._maybe_arm_throttling_wakeup() + assert engine._throttling_wakeup is None + def test_warn_delayed_requests(self, engine): engine._delayed_requests_warn_threshold = 1 engine._throttling_waiting = {Request("http://a.example")} diff --git a/tests/test_throttling.py b/tests/test_throttling.py index 9ba6d51eb..d088d6c7e 100644 --- a/tests/test_throttling.py +++ b/tests/test_throttling.py @@ -415,6 +415,32 @@ class TestThrottlingManager: assert "idle.example" not in manager._scope_managers assert "active.example" in manager._scope_managers + def test_scope_limit_evicts_least_recently_used(self): + manager = _manager({"THROTTLING_SCOPE_LIMIT": 2}) + # Use three scopes in order; each send/done leaves them idle. + for scope_id in ("a.example", "b.example", "c.example"): + scope = manager._get_scope_manager(scope_id) + scope.record_sent(now=0.0) + scope.record_done(now=0.0) + # The limit caps live managers at 2, dropping the least-recently-used. + assert set(manager._scope_managers) == {"b.example", "c.example"} + + def test_scope_limit_keeps_active_scopes(self): + manager = _manager({"THROTTLING_SCOPE_LIMIT": 1}) + # Two scopes with in-flight requests cannot be evicted, so the limit is + # exceeded rather than dropping a scope that still tracks a live send. + for scope_id in ("a.example", "b.example"): + manager._get_scope_manager(scope_id).record_sent(now=0.0) + assert set(manager._scope_managers) == {"a.example", "b.example"} + + def test_scope_limit_disabled(self): + manager = _manager({"THROTTLING_SCOPE_LIMIT": 0}) + for i in range(5): + scope = manager._get_scope_manager(f"{i}.example") + scope.record_sent(now=0.0) + scope.record_done(now=0.0) + assert len(manager._scope_managers) == 5 + class TestThrottlingScopeManager: def test_no_delay_by_default(self): @@ -431,6 +457,21 @@ class TestThrottlingScopeManager: assert scope.can_send(now=11.0) == pytest.approx(1.0) assert scope.can_send(now=12.0) == 0 + def test_base_delay_defaults_to_download_delay(self): + # With no explicit scope "delay", the base delay is DOWNLOAD_DELAY. + scope = _scope_manager( + {"DOWNLOAD_DELAY": 2.0, "RANDOMIZE_DOWNLOAD_DELAY": False}, {"id": "x"} + ) + assert scope._base_delay == pytest.approx(2.0) + + def test_scope_delay_overrides_download_delay(self): + # An explicit scope "delay" overrides DOWNLOAD_DELAY. + scope = _scope_manager( + {"DOWNLOAD_DELAY": 2.0, "RANDOMIZE_DOWNLOAD_DELAY": False}, + {"id": "x", "delay": 0.0}, + ) + assert scope._base_delay == pytest.approx(0.0) + def test_exponential_backoff(self): scope = _scope_manager( {