WIP

docs/topics/throttling.rst

@@ -63,24 +63,30 @@ The main throttling :ref:`settings <topics-settings>` are:
 
     The wait time is measured from when the previous request was sent.
 
-For example, with ``CONCURRENT_REQUESTS_PER_DOMAIN = 2``, ``DOWNLOAD_DELAY = 0.3``,
-and ``DOWNLOAD_DELAY_PER_SLOT = 1.0``, sending 3 requests to the same domain
-would result in:
+For example, with ``DOWNLOAD_DELAY = 1.0`` (and, by default, a single download
+slot per domain), requests to the same domain are sent at most once per second:
 
 .. code-block:: text
 
-    T=0.0s: Request 1 sent (slot 1)
-    T=0.3s: Request 2 sent (slot 2, respects same-domain delay)
-    T=0.6s: Request 3 must wait (same-domain delay satisfied, but slot 1 needs 1.0s)
-    T=1.0s: Request 3 sent (slot 1 can now be reused)
+    T=0.0s: Request 1 sent
+    T=1.0s: Request 2 sent
+    T=2.0s: Request 3 sent
+
+:setting:`DOWNLOAD_DELAY` (per :ref:`throttling scope <throttling-scopes>`) and
+:setting:`DOWNLOAD_DELAY_PER_SLOT` (per download slot) are enforced
+independently. By default each domain is both its own scope and its own
+download slot, so both apply to the same requests and the effective minimum
+spacing is the larger of the two; they only differ when requests are grouped
+into custom :ref:`scopes <throttling-scopes>` or download slots (via the
+``download_slot`` request meta key).
 
 When configuring these settings, note that:
 
 -   :setting:`CONCURRENT_REQUESTS` caps ``CONCURRENT_REQUESTS_PER_DOMAIN``.
 
--   If ``DOWNLOAD_DELAY`` ≥ response time, concurrency is effectively ``1``.
-    This happens because all slots must wait for the delay between requests,
-    preventing them from sending requests simultaneously.
+-   If ``DOWNLOAD_DELAY`` ≥ response time, concurrency is effectively ``1``,
+    because the next request to the domain is not sent until the delay elapses,
+    by which time the previous response has already arrived.
 
 .. [1] You can :ref:`customize <throttling-scopes>` how requests are grouped
     for throttling, but domain-based throttling works well in most cases. For
@@ -1061,6 +1067,23 @@ Additional settings
     Whether to log :ref:`throttling <throttling>` decisions (per-scope delays,
     backoff steps and recoveries) for debugging.
 
+-   .. setting:: THROTTLING_SCOPE_LIMIT
+
+    :setting:`THROTTLING_SCOPE_LIMIT` (default: ``100000``)
+
+    Maximum number of :ref:`throttling scope <throttling-scopes>` states kept
+    in memory at once, to bound memory usage on broad crawls that touch a large
+    number of scopes (e.g. domains).
+
+    When the limit is exceeded, the least-recently-used idle scopes are evicted
+    (an evicted scope is recreated from its configuration the next time it is
+    needed). Scopes with in-flight requests or in active backoff are never
+    evicted, so the limit may be temporarily exceeded if that many scopes are
+    busy at once. Set to ``0`` to disable the limit.
+
+    This complements :setting:`THROTTLING_SCOPE_MAX_IDLE`, which evicts scopes
+    by inactivity time rather than by count.
+
 -   .. setting:: THROTTLING_SCOPE_MAX_IDLE
 
     :setting:`THROTTLING_SCOPE_MAX_IDLE` (default: ``3600.0``)

scrapy/core/engine.py

@@ -380,11 +380,16 @@ class ExecutionEngine:
         if delay_fn is None or not scheduler.has_pending_requests():
             return
         delay = delay_fn()
-        if delay is None:
+        # ``delay`` is ``None`` when no pending request is time-blocked, and
+        # ``0`` when some request is ready right now but could not be sent (e.g.
+        # the downloader is at capacity). Neither case needs a timer: a freed
+        # slot already re-runs the loop from :meth:`_download`'s ``finally``,
+        # and arming a ``0``-second timer here would busy-loop the engine while
+        # the downloader stays full. Only a positive delay, i.e. a time-based
+        # gate that nothing else would wake us for, needs one.
+        if delay is None or delay <= 0:
             return
-        self._throttling_wakeup = call_later(
-            max(0.0, delay), self._slot.nextcall.schedule
-        )
+        self._throttling_wakeup = call_later(delay, self._slot.nextcall.schedule)
 
     def needs_backout(self) -> bool:
         """Returns ``True`` if no more requests can be sent at the moment, or

scrapy/settings/default_settings.py

@@ -598,6 +598,7 @@ THROTTLING_SCOPES = {}
 THROTTLING_WINDOW = 60.0
 THROTTLING_ROBOTSTXT_OBEY = True
 THROTTLING_ROBOTSTXT_MAX_DELAY = 60.0
+THROTTLING_SCOPE_LIMIT = 100000
 THROTTLING_SCOPE_MAX_IDLE = 3600.0
 THROTTLING_DEBUG = False
 

scrapy/templates/project/module/settings.py.tmpl

@@ -9,5 +9,12 @@ NEWSPIDER_MODULE = "$project_name.spiders"
 # User-Agent header:
 #USER_AGENT = "$project_name (+https://your-domain.example)"
 
+# Obey robots.txt rules:
+ROBOTSTXT_OBEY = True
+
+# Throttle crawls to be polite to websites:
+CONCURRENT_REQUESTS_PER_DOMAIN = 1
+DOWNLOAD_DELAY = 1
+
 # Set settings whose default value is deprecated to a future-proof value:
 FEED_EXPORT_ENCODING = "utf-8"

scrapy/throttling.py

@@ -5,6 +5,7 @@ import datetime as dt
 import logging
 import random
 import time
+from collections import OrderedDict
 from collections.abc import Awaitable, Callable, Iterable
 from email.utils import parsedate_to_datetime
 from functools import wraps
@@ -45,7 +46,9 @@ def _parse_retry_after(response: Response) -> float | None:
         date = date.replace(tzinfo=dt.timezone.utc)
     now = dt.datetime.now(dt.timezone.utc)
     seconds_to_wait = (date - now).total_seconds()
-    return max(0, int(seconds_to_wait)) or None
+    # Keep sub-second precision (a date less than a second away must not be
+    # truncated to 0 and dropped); a past or present date yields no delay.
+    return max(0.0, seconds_to_wait) or None
 
 
 def _parse_ratelimit_reset(response: Response) -> float | None:
@@ -485,7 +488,12 @@ class ThrottlingManager:
         self._scopes_config: dict[str, dict[str, Any]] = crawler.settings.getdict(
             "THROTTLING_SCOPES"
         )
-        self._scope_managers: dict[ScopeID, ThrottlingScopeManagerProtocol] = {}
+        # Ordered by least-recently-used first (see _get_scope_manager), so the
+        # scope limit can evict the coldest idle scopes (see THROTTLING_SCOPE_LIMIT).
+        self._scope_managers: OrderedDict[ScopeID, ThrottlingScopeManagerProtocol] = (
+            OrderedDict()
+        )
+        self._scope_limit: int = crawler.settings.getint("THROTTLING_SCOPE_LIMIT")
         self._last_eviction: float | None = None
         # Concurrency slots reserved by acquire(), to be released once the
         # request finishes downloading.
@@ -585,18 +593,46 @@ class ThrottlingManager:
 
     def _get_scope_manager(self, scope_id: ScopeID) -> ThrottlingScopeManagerProtocol:
         manager = self._scope_managers.get(scope_id)
-        if manager is None:
-            config: dict[str, Any] = dict(self._scopes_config.get(scope_id, {}))
-            config.setdefault("id", scope_id)
-            manager_cls = (
-                load_object(config["manager"])
-                if "manager" in config
-                else self._default_scope_manager_cls
-            )
-            manager = build_from_crawler(manager_cls, self.crawler, config)
-            self._scope_managers[scope_id] = manager
+        if manager is not None:
+            # Mark as most-recently-used for the LRU scope limit.
+            self._scope_managers.move_to_end(scope_id)
+            return manager
+        config: dict[str, Any] = dict(self._scopes_config.get(scope_id, {}))
+        config.setdefault("id", scope_id)
+        manager_cls = (
+            load_object(config["manager"])
+            if "manager" in config
+            else self._default_scope_manager_cls
+        )
+        manager = cast(
+            "ThrottlingScopeManagerProtocol",
+            build_from_crawler(manager_cls, self.crawler, config),
+        )
+        self._scope_managers[scope_id] = manager
+        self._enforce_scope_limit(scope_id)
         return manager
 
+    def _enforce_scope_limit(self, keep: ScopeID) -> None:
+        """Evict least-recently-used idle scopes while the number of live scope
+        managers exceeds :setting:`THROTTLING_SCOPE_LIMIT` (``0`` disables the
+        limit).
+
+        LRU order is kept by :meth:`_get_scope_manager` moving each accessed
+        scope to the end, so the coldest scopes are at the front. Only scopes
+        that are idle (no in-flight requests and no active backoff) are evicted;
+        the just-created *keep* scope is never evicted. A scope evicted while
+        still throttling is simply recreated from its configuration the next
+        time it is needed.
+        """
+        if self._scope_limit <= 0 or len(self._scope_managers) <= self._scope_limit:
+            return
+        now = time.monotonic()
+        for scope_id in list(self._scope_managers):
+            if len(self._scope_managers) <= self._scope_limit:
+                break
+            if scope_id != keep and self._scope_managers[scope_id].is_idle(now, 0):
+                del self._scope_managers[scope_id]
+
     async def acquire(self, request: Request) -> None:
         # A throttling-aware scheduler reserves the request before handing it
         # to the engine, so there is nothing left to wait for or record here.
@@ -1008,9 +1044,9 @@ class ThrottlingScopeManager:
     :ref:`backoff <backoff>`, :ref:`rampup <rampup>`, concurrency and
     :ref:`quotas <throttling-quotas>`:
 
-    -   A base :setting:`DOWNLOAD_DELAY`-style delay (``0`` by default, taken
-        from the scope ``"delay"`` config) is enforced between consecutive
-        requests for the scope.
+    -   A base delay (the scope ``"delay"`` config, defaulting to
+        :setting:`DOWNLOAD_DELAY`) is enforced between consecutive requests for
+        the scope.
 
     -   On a backoff trigger (a :setting:`BACKOFF_HTTP_CODES` response or a
         :setting:`BACKOFF_EXCEPTIONS` exception) the delay grows exponentially
@@ -1043,7 +1079,11 @@ class ThrottlingScopeManager:
         settings = crawler.settings
         backoff: dict[str, Any] = config.get("backoff", {})
         self._id: ScopeID = config.get("id", "")
-        self._base_delay: float = float(config.get("delay", 0.0))
+        # The per-scope delay defaults to DOWNLOAD_DELAY; a scope can override
+        # it with its own "delay" config (see THROTTLING_SCOPES).
+        self._base_delay: float = float(
+            config.get("delay", settings.getfloat("DOWNLOAD_DELAY"))
+        )
         self._randomize: bool = bool(
             config.get("randomize_delay", settings.getbool("RANDOMIZE_DOWNLOAD_DELAY"))
         )

tests/test_engine.py

@@ -694,6 +694,18 @@ class TestEngineThrottling:
         engine._maybe_arm_throttling_wakeup()
         assert engine._throttling_wakeup is None
 
+    def test_maybe_arm_throttling_wakeup_zero_delay(self, engine):
+        # A 0 delay means a request is ready but could not be sent (e.g. the
+        # downloader is at capacity); arming a 0-second timer would busy-loop
+        # the engine, so no timer must be armed.
+        scheduler = Mock()
+        scheduler.has_pending_requests.return_value = True
+        scheduler.next_request_delay.return_value = 0.0
+        engine._slot = Mock()
+        engine._slot.scheduler = scheduler
+        engine._maybe_arm_throttling_wakeup()
+        assert engine._throttling_wakeup is None
+
     def test_warn_delayed_requests(self, engine):
         engine._delayed_requests_warn_threshold = 1
         engine._throttling_waiting = {Request("http://a.example")}

tests/test_throttling.py

@@ -415,6 +415,32 @@ class TestThrottlingManager:
         assert "idle.example" not in manager._scope_managers
         assert "active.example" in manager._scope_managers
 
+    def test_scope_limit_evicts_least_recently_used(self):
+        manager = _manager({"THROTTLING_SCOPE_LIMIT": 2})
+        # Use three scopes in order; each send/done leaves them idle.
+        for scope_id in ("a.example", "b.example", "c.example"):
+            scope = manager._get_scope_manager(scope_id)
+            scope.record_sent(now=0.0)
+            scope.record_done(now=0.0)
+        # The limit caps live managers at 2, dropping the least-recently-used.
+        assert set(manager._scope_managers) == {"b.example", "c.example"}
+
+    def test_scope_limit_keeps_active_scopes(self):
+        manager = _manager({"THROTTLING_SCOPE_LIMIT": 1})
+        # Two scopes with in-flight requests cannot be evicted, so the limit is
+        # exceeded rather than dropping a scope that still tracks a live send.
+        for scope_id in ("a.example", "b.example"):
+            manager._get_scope_manager(scope_id).record_sent(now=0.0)
+        assert set(manager._scope_managers) == {"a.example", "b.example"}
+
+    def test_scope_limit_disabled(self):
+        manager = _manager({"THROTTLING_SCOPE_LIMIT": 0})
+        for i in range(5):
+            scope = manager._get_scope_manager(f"{i}.example")
+            scope.record_sent(now=0.0)
+            scope.record_done(now=0.0)
+        assert len(manager._scope_managers) == 5
+
 
 class TestThrottlingScopeManager:
     def test_no_delay_by_default(self):
@@ -431,6 +457,21 @@ class TestThrottlingScopeManager:
         assert scope.can_send(now=11.0) == pytest.approx(1.0)
         assert scope.can_send(now=12.0) == 0
 
+    def test_base_delay_defaults_to_download_delay(self):
+        # With no explicit scope "delay", the base delay is DOWNLOAD_DELAY.
+        scope = _scope_manager(
+            {"DOWNLOAD_DELAY": 2.0, "RANDOMIZE_DOWNLOAD_DELAY": False}, {"id": "x"}
+        )
+        assert scope._base_delay == pytest.approx(2.0)
+
+    def test_scope_delay_overrides_download_delay(self):
+        # An explicit scope "delay" overrides DOWNLOAD_DELAY.
+        scope = _scope_manager(
+            {"DOWNLOAD_DELAY": 2.0, "RANDOMIZE_DOWNLOAD_DELAY": False},
+            {"id": "x", "delay": 0.0},
+        )
+        assert scope._base_delay == pytest.approx(0.0)
+
     def test_exponential_backoff(self):
         scope = _scope_manager(
             {