WIP

docs/topics/throttling.rst

@@ -72,11 +72,11 @@ The :setting:`THROTTLING_SCOPES` setting allows you to customize throttling
 behavior for specific domains [1]_.
 
 It is a dict that maps scope names to
-:class:`~scrapy.throttling.ThrottlingScopeConfig` dicts.
+:class:`~scrapy.throttling.ThrottlingScopeConfig` dicts. It is empty by default.
 
-Its default value allows faster crawling of the testing websites used during
-the :ref:`tutorial <intro-tutorial>` while maintaining conservative defaults
-for other domains:
+:command:`startproject` scaffolds an entry for the testing websites used during
+the :ref:`tutorial <intro-tutorial>`, so that they are crawled faster while the
+:ref:`conservative defaults <basic-throttling>` still apply to other domains:
 
 .. code-block:: python
 

scrapy/core/downloader/__init__.py

@@ -86,18 +86,18 @@ class _DeprecatedSlotView:
     @property
     def delay(self) -> float:
         if self._scope is not None:
-            return self._scope._delay  # type: ignore[union-attr]
+            return self._scope.get_delay()
         return 0.0
 
     @delay.setter
     def delay(self, value: float) -> None:
         if self._scope is not None:
-            self._scope._delay = value  # type: ignore[union-attr]
+            self._scope.set_base_delay(value, only_increase=False)
 
     @property
     def randomize_delay(self) -> bool:
         if self._scope is not None:
-            return bool(self._scope._jitter)  # type: ignore[union-attr]
+            return bool(self._scope.get_jitter())
         return False
 
     @property
@@ -109,11 +109,13 @@ class _DeprecatedSlotView:
             stacklevel=2,
         )
         if self._scope is not None:
-            return self._scope._concurrency or 0  # type: ignore[union-attr]
+            return self._scope.get_concurrency() or 0
         return 0
 
     def free_transfer_slots(self) -> int:
-        concurrency = self._scope._concurrency if self._scope is not None else 0  # type: ignore[union-attr]
+        concurrency = (
+            self._scope.get_concurrency() or 0 if self._scope is not None else 0
+        )
         return concurrency - len(self.transferring)
 
     def download_delay(self) -> float:
@@ -132,7 +134,7 @@ class _DeprecatedSlotView:
         return f"_DeprecatedSlotView({self._key!r})"
 
 
-class _DeprecatedSlotsView(Mapping):
+class _DeprecatedSlotsView(Mapping[str, _DeprecatedSlotView]):
     """Deprecated mapping view of active downloads, keyed by slot name."""
 
     __slots__ = ("_downloader", "_throttler")
@@ -152,7 +154,7 @@ class _DeprecatedSlotsView(Mapping):
         if key not in self._active_keys():
             raise KeyError(key)
         scope = (
-            self._throttler._get_scope_manager(key)
+            self._throttler.get_scope_manager(key)
             if self._throttler is not None
             else None
         )
@@ -238,7 +240,7 @@ class Downloader:
     ) -> tuple[str, _DeprecatedSlotView]:
         key = self._get_slot_key(request)
         scope = (
-            self.crawler.throttler._get_scope_manager(key)
+            self.crawler.throttler.get_scope_manager(key)
             if self.crawler.throttler is not None
             else None
         )

scrapy/settings/default_settings.py

@@ -613,10 +613,7 @@ TEMPLATES_DIR = str((Path(__file__).parent / ".." / "templates").resolve())
 
 THROTTLING_MANAGER = "scrapy.throttling.ThrottlingManager"
 THROTTLING_SCOPE_MANAGER = "scrapy.throttling.ThrottlingScopeManager"
-THROTTLING_SCOPES = {
-    "books.toscrape.com": {"concurrency": 16, "delay": 0.1},
-    "quotes.toscrape.com": {"concurrency": 16, "delay": 0.1},
-}
+THROTTLING_SCOPES = {}
 THROTTLING_WINDOW = 60.0
 THROTTLING_ROBOTSTXT_OBEY = True
 THROTTLING_ROBOTSTXT_MAX_DELAY = 60.0

scrapy/templates/project/module/settings.py.tmpl

@@ -16,5 +16,12 @@ ROBOTSTXT_OBEY = True
 CONCURRENT_REQUESTS_PER_DOMAIN = 1
 DOWNLOAD_DELAY = 1
 
+# Crawl the tutorial websites faster, overriding the polite defaults above only
+# for these domains (remove this once you crawl your own sites):
+THROTTLING_SCOPES = {
+    "books.toscrape.com": {"concurrency": 16, "delay": 0.1},
+    "quotes.toscrape.com": {"concurrency": 16, "delay": 0.1},
+}
+
 # Set settings whose default value is deprecated to a future-proof value:
 FEED_EXPORT_ENCODING = "utf-8"

scrapy/throttling.py

@@ -28,6 +28,7 @@ from scrapy.utils.misc import build_from_crawler, load_object
 if TYPE_CHECKING:
     from scrapy.crawler import Crawler
     from scrapy.http import Request, Response
+    from scrapy.settings import BaseSettings
 
 
 logger = logging.getLogger(__name__)
@@ -208,7 +209,7 @@ def _classify_scope(scope_id: ScopeID) -> str:
     return "other"
 
 
-def _default_scope_concurrency(settings: Any, scope_id: ScopeID) -> int:
+def _default_scope_concurrency(settings: BaseSettings, scope_id: ScopeID) -> int:
     """Return the default concurrency for *scope_id* based on its
     :func:`kind <_classify_scope>`.
 
@@ -348,16 +349,18 @@ class ThrottlingManagerProtocol(Protocol):
             those scopes are currently exhausted.
 
         -   A dict with scope names as keys and dict values. Dict values
-            support the following keys:
+            support the following keys (matching :class:`BackoffScopeData`):
 
             -   ``"delay"``: a float indicating how many seconds to wait before
                 sending another request for the scope.
 
-            -   ``"quota"``: a float indicating the remaining :ref:`throttling
-                quota <throttling-quotas>`.
+            -   ``"remaining"``: a float indicating the remaining
+                :ref:`throttling quota <throttling-quotas>` of the scope.
 
-            If ``"quota"`` is not specified, the resource is considered
-            exhausted.
+            -   ``"consumed"``: a float indicating how much of the scope's
+                quota has already been consumed.
+
+            An empty dict marks the scope as currently exhausted.
 
         For example:
 
@@ -366,7 +369,7 @@ class ThrottlingManagerProtocol(Protocol):
             return {
                 "scope1": {"delay": 5.0},
                 "scope2": {},
-                "scope3": {"quota": 42.0},
+                "scope3": {"remaining": 42.0},
             }
         """
 
@@ -488,6 +491,10 @@ class ThrottlingManagerProtocol(Protocol):
         directly (e.g. an adaptive-delay extension).
         """
 
+    def get_scope_manager(self, scope_id: str) -> ThrottlingScopeManagerProtocol:
+        """Return the :class:`ThrottlingScopeManagerProtocol` instance handling
+        the scope identified by *scope_id*, creating it if necessary."""
+
     async def process_response(self, response: Response) -> None:
         """Update the throttling state based on *response*."""
 
@@ -592,7 +599,28 @@ class ThrottlingManager:
         self._scopes_config: dict[str, dict[str, Any]] = crawler.settings.getdict(
             "THROTTLING_SCOPES"
         )
-        # Ordered by least-recently-used first (see _get_scope_manager), so the
+        # Cheap pre-filter for the backoff methods: the union of the global
+        # triggers and every per-scope override. A response whose status (or an
+        # exception type) is not in the union cannot trigger backoff for any
+        # scope, so its scopes need not be resolved (see get_response_backoff
+        # and get_exception_backoff). Each scope still makes the final decision
+        # via its own triggers_backoff_* methods.
+        self._any_backoff_http_codes: set[int] = set(self._backoff_http_codes)
+        self._any_backoff_exceptions: tuple[type[BaseException], ...] = (
+            self._backoff_exceptions
+        )
+        for scope_config in self._scopes_config.values():
+            backoff = scope_config.get("backoff") or {}
+            if "http_codes" in backoff:
+                self._any_backoff_http_codes.update(
+                    int(code) for code in backoff["http_codes"]
+                )
+            if "exceptions" in backoff:
+                self._any_backoff_exceptions += tuple(
+                    load_object(exc) if isinstance(exc, str) else exc
+                    for exc in backoff["exceptions"]
+                )
+        # Ordered by least-recently-used first (see get_scope_manager), so the
         # scope limit can evict the coldest idle scopes (see THROTTLING_SCOPE_LIMIT).
         self._scope_managers: OrderedDict[ScopeID, ThrottlingScopeManagerProtocol] = (
             OrderedDict()
@@ -631,7 +659,7 @@ class ThrottlingManager:
                 category=ScrapyDeprecationWarning,
                 stacklevel=2,
             )
-            return download_slot
+            return cast("RequestScopes", download_slot)
         netloc = urlparse_cached(request).netloc
         if self._per_ip:
             # Best-effort, mirroring the legacy per-IP downloader slots: use the
@@ -686,12 +714,21 @@ class ThrottlingManager:
         assert response.request is not None
         if response.request.meta.get("throttling_dont_track"):
             return None
-        if response.status not in self._backoff_http_codes:
+        if response.status not in self._any_backoff_http_codes:
             return None
         scopes = await self._scopes(response.request)
+        matched = [
+            scope
+            for scope in iter_scopes(scopes)
+            if self.get_scope_manager(scope).triggers_backoff_for_status(
+                response.status
+            )
+        ]
+        if not matched:
+            return None
         if delay := self.get_response_delay(response):
-            scopes = {scope: {"delay": delay} for scope in iter_scopes(scopes)}
-        return scopes
+            return {scope: {"delay": delay} for scope in matched}
+        return matched
 
     def get_response_delay(self, response: Response) -> float | None:
         """Return the throttling delay requested by the response."""
@@ -711,13 +748,19 @@ class ThrottlingManager:
     ) -> BackoffData:
         if request.meta.get("throttling_dont_track"):
             return None
-        if isinstance(exception, self._backoff_exceptions):
-            return await self._scopes(request)
-        return None
+        if not isinstance(exception, self._any_backoff_exceptions):
+            return None
+        scopes = await self._scopes(request)
+        matched = [
+            scope
+            for scope in iter_scopes(scopes)
+            if self.get_scope_manager(scope).triggers_backoff_for_exception(exception)
+        ]
+        return matched or None
 
     # -- Scope-state coordination (called from the request lifecycle) --------
 
-    def _get_scope_manager(self, scope_id: ScopeID) -> ThrottlingScopeManagerProtocol:
+    def get_scope_manager(self, scope_id: ScopeID) -> ThrottlingScopeManagerProtocol:
         manager = self._scope_managers.get(scope_id)
         if manager is not None:
             # Mark as most-recently-used for the LRU scope limit.
@@ -743,7 +786,7 @@ class ThrottlingManager:
         managers exceeds :setting:`THROTTLING_SCOPE_LIMIT` (``0`` disables the
         limit).
 
-        LRU order is kept by :meth:`_get_scope_manager` moving each accessed
+        LRU order is kept by :meth:`get_scope_manager` moving each accessed
         scope to the end, so the coldest scopes are at the front. Only scopes
         that are idle (no in-flight requests and no active backoff) are evicted;
         the just-created *keep* scope is never evicted. A scope evicted while
@@ -771,7 +814,7 @@ class ThrottlingManager:
         if not scope_values:
             return
         managers = [
-            (self._get_scope_manager(scope_id), value)
+            (self.get_scope_manager(scope_id), value)
             for scope_id, value in scope_values
         ]
         while True:
@@ -817,7 +860,7 @@ class ThrottlingManager:
         if self._request_delay_deadline(request, now) > now:
             return False
         for scope_id, value in self._cached_scope_values(request):
-            manager = self._get_scope_manager(scope_id)
+            manager = self.get_scope_manager(scope_id)
             if manager.can_send(now=now, amount=value) > 0:
                 return False
             if manager.concurrency_blocked():
@@ -831,7 +874,7 @@ class ThrottlingManager:
         # up on broad crawls.
         self._maybe_evict(time.monotonic())
         managers = [
-            (self._get_scope_manager(scope_id), value)
+            (self.get_scope_manager(scope_id), value)
             for scope_id, value in self._cached_scope_values(request)
         ]
         for manager, value in managers:
@@ -842,12 +885,12 @@ class ThrottlingManager:
         now = time.monotonic()
         wait = max(0.0, self._request_delay_deadline(request, now) - now)
         for scope_id, value in self._cached_scope_values(request):
-            manager = self._get_scope_manager(scope_id)
+            manager = self.get_scope_manager(scope_id)
             wait = max(wait, manager.can_send(now=now, amount=value))
         return wait if wait > 0 else None
 
     def get_scope_load(self, scope_id: ScopeID) -> float:
-        return self._get_scope_manager(scope_id).get_load()
+        return self.get_scope_manager(scope_id).get_load()
 
     def get_request_delay(self, request: Request, now: float | None = None) -> float:
         now = time.monotonic() if now is None else now
@@ -921,7 +964,7 @@ class ThrottlingManager:
         else:
             items = ((scope_id, None) for scope_id in iter_scopes(data))
         for scope_id, entry in items:
-            manager = self._get_scope_manager(scope_id)
+            manager = self.get_scope_manager(scope_id)
             delay = consumed = remaining = None
             if isinstance(entry, dict):
                 delay = entry.get("delay")
@@ -993,7 +1036,7 @@ class ThrottlingManager:
             return
         if self._debug:
             logger.debug(f"robots.txt Crawl-delay for scope {scope_id}: {capped}s")
-        manager = self._get_scope_manager(scope_id)
+        manager = self.get_scope_manager(scope_id)
         manager.set_base_delay(capped)
         manager.set_concurrency(1)
 
@@ -1002,13 +1045,13 @@ class ThrottlingManager:
             logger.debug(f"Delaying scope {scope_id} for {delay:.2f}s")
         # Like a Retry-After / RateLimit-Reset header, this is a hard minimum
         # delay; unlike those, it is trusted, so it bypasses BACKOFF_MAX_DELAY.
-        self._get_scope_manager(scope_id).record_backoff(delay=float(delay), cap=False)
+        self.get_scope_manager(scope_id).record_backoff(delay=float(delay), cap=False)
 
     def get_scope_delay(self, scope_id: ScopeID) -> float:
-        return self._get_scope_manager(scope_id).get_base_delay()
+        return self.get_scope_manager(scope_id).get_base_delay()
 
     def set_scope_delay(self, scope_id: ScopeID, delay: float) -> None:
-        self._get_scope_manager(scope_id).set_base_delay(
+        self.get_scope_manager(scope_id).set_base_delay(
             float(delay), only_increase=False
         )
 
@@ -1111,6 +1154,26 @@ class ThrottlingScopeManagerProtocol(Protocol):
         reported for a request, correcting the estimate used by
         :meth:`record_sent`."""
 
+    def triggers_backoff_for_status(self, status: int) -> bool:
+        """Return whether a response with the given *status* triggers backoff
+        for this scope (defaults to :setting:`BACKOFF_HTTP_CODES`)."""
+
+    def triggers_backoff_for_exception(self, exception: BaseException) -> bool:
+        """Return whether *exception* triggers backoff for this scope (defaults
+        to :setting:`BACKOFF_EXCEPTIONS`)."""
+
+    def get_delay(self) -> float:
+        """Return the current effective delay of this scope, in seconds,
+        including any active backoff (unlike :meth:`get_base_delay`)."""
+
+    def get_jitter(self) -> float | list[float]:
+        """Return the magnitude of the random variation applied to the delay
+        (the per-scope override of :setting:`RANDOMIZE_DOWNLOAD_DELAY`)."""
+
+    def get_concurrency(self) -> int | None:
+        """Return the maximum number of concurrent requests allowed for this
+        scope, or ``None`` when the scope enforces no explicit limit."""
+
     def get_base_delay(self) -> float:
         """Return the base (non-backoff) delay of this scope, in seconds."""
 
@@ -1240,6 +1303,19 @@ class ThrottlingScopeManager:
         self._backoff_jitter: float | list[float] = backoff.get(
             "jitter", settings.getfloat("BACKOFF_JITTER")
         )
+        # Which responses/exceptions trigger backoff for this scope. Each
+        # defaults to the matching global BACKOFF_* setting (see
+        # triggers_backoff_for_status / triggers_backoff_for_exception).
+        self._backoff_http_codes: set[int] = {
+            int(code)
+            for code in backoff.get(
+                "http_codes", settings.getlist("BACKOFF_HTTP_CODES")
+            )
+        }
+        self._backoff_exceptions: tuple[type[BaseException], ...] = tuple(
+            load_object(exc) if isinstance(exc, str) else exc
+            for exc in backoff.get("exceptions", settings.getlist("BACKOFF_EXCEPTIONS"))
+        )
         self._window: float = settings.getfloat("BACKOFF_WINDOW")
         self._min_concurrency: int = int(config.get("min_concurrency", 1))
 
@@ -1493,6 +1569,21 @@ class ThrottlingScopeManager:
         elif consumed is not None:
             self._consumed = max(0.0, self._consumed + float(consumed))
 
+    def triggers_backoff_for_status(self, status: int) -> bool:
+        return status in self._backoff_http_codes
+
+    def triggers_backoff_for_exception(self, exception: BaseException) -> bool:
+        return isinstance(exception, self._backoff_exceptions)
+
+    def get_delay(self) -> float:
+        return self._delay
+
+    def get_jitter(self) -> float | list[float]:
+        return self._jitter
+
+    def get_concurrency(self) -> int | None:
+        return self._concurrency
+
     def get_base_delay(self) -> float:
         return self._base_delay