KISS

docs/topics/throttling.rst

@@ -75,15 +75,15 @@ behavior for specific domains [1]_.
 It is a dict that maps scope names to
 :class:`~scrapy.throttling.ThrottlingScopeConfig` dicts. It is empty by default.
 
-:command:`startproject` scaffolds an entry for the testing websites used during
-the :ref:`tutorial <intro-tutorial>`, so that they are crawled faster while the
-:ref:`conservative defaults <basic-throttling>` still apply to other domains:
+:command:`startproject` scaffolds a commented-out example entry, so that you can
+uncomment and edit it to crawl domains you own (or that are meant for scraping)
+faster, while the :ref:`conservative defaults <basic-throttling>` still apply to
+other domains:
 
 .. code-block:: python
 
     THROTTLING_SCOPES = {
-        "books.toscrape.com": {"concurrency": 16, "delay": 0.1},
-        "quotes.toscrape.com": {"concurrency": 16, "delay": 0.1},
+        "example.com": {"concurrency": 16, "delay": 0.1},
     }
 
 Additional keys like ``"jitter"`` and ``"backoff"`` can be used here and are

scrapy/core/downloader/__init__.py

@@ -162,9 +162,6 @@ class _DeprecatedSlotView:
     def __repr__(self) -> str:
         return f"_DeprecatedSlotView({self._key!r})"
 
-    def __str__(self) -> str:
-        return f"_DeprecatedSlotView({self._key!r})"
-
 
 class _DeprecatedSlotsView(Mapping[str, _DeprecatedSlotView]):
     """Deprecated mapping view of active downloads, keyed by slot name."""

scrapy/downloadermiddlewares/backoff.py

@@ -1,12 +1,11 @@
 from __future__ import annotations
 
-import datetime as dt
 import logging
-from email.utils import parsedate_to_datetime
 from typing import TYPE_CHECKING
 
 from scrapy.exceptions import NotConfigured
 from scrapy.throttling import _load_exceptions, iter_scopes
+from scrapy.utils._headers import _parse_ratelimit_reset, _parse_retry_after
 from scrapy.utils.decorators import _warn_spider_arg
 
 if TYPE_CHECKING:
@@ -22,47 +21,6 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
-def _decoded_header(response: Response, name: str) -> str | None:
-    """Return the stripped UTF-8 value of the *name* header of *response*, or
-    ``None`` if it is absent or not valid UTF-8."""
-    raw = response.headers.get(name)
-    if not raw:
-        return None
-    try:
-        return raw.decode("utf-8").strip()
-    except UnicodeDecodeError:
-        return None
-
-
-def _parse_retry_after(response: Response) -> float | None:
-    value = _decoded_header(response, "Retry-After")
-    if value is None:
-        return None
-    if value.isdigit():
-        return float(value)  # seconds
-    try:
-        date = parsedate_to_datetime(value)
-    except (TypeError, ValueError, OverflowError):
-        return None
-    if date.tzinfo is None:
-        date = date.replace(tzinfo=dt.timezone.utc)
-    now = dt.datetime.now(dt.timezone.utc)
-    seconds_to_wait = (date - now).total_seconds()
-    # Keep sub-second precision (a date less than a second away must not be
-    # truncated to 0 and dropped); a past or present date yields no delay.
-    return max(0.0, seconds_to_wait) or None
-
-
-def _parse_ratelimit_reset(response: Response) -> float | None:
-    value = _decoded_header(response, "RateLimit-Reset")
-    if value is None:
-        return None
-    try:
-        return float(value)
-    except ValueError:
-        return None
-
-
 class BackoffMiddleware:
     """Downloader middleware that drives :ref:`backoff <backoff>` from download
     outcomes.

scrapy/downloadermiddlewares/redirect.py

@@ -7,10 +7,10 @@ from urllib.parse import urljoin, urlparse
 from w3lib.url import safe_url_string
 
 from scrapy import signals
-from scrapy.downloadermiddlewares.backoff import _parse_retry_after
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import HtmlResponse, Response
 from scrapy.spidermiddlewares.referer import RefererMiddleware
+from scrapy.utils._headers import _parse_retry_after
 from scrapy.utils.decorators import _warn_spider_arg
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import global_object_name

scrapy/throttling.py

@@ -1277,8 +1277,7 @@ class ThrottlingScopeManager:
         if self._backoff_level == 0 or self._last_backoff_time is None:
             return
         if self._window <= 0:
-            # A non-positive window has no recovery cadence to step through (and
-            # would spin forever on a zero-length step), so recover at once.
+            # No window: no recovery cadence to step (would spin); recover at once.
             self._backoff_level = 0
             self._delay = self._base_delay
             self._in_backoff_until = None
@@ -1300,8 +1299,7 @@ class ThrottlingScopeManager:
         if not self._rampup_enabled:
             return
         if self._window <= 0:
-            # No window means no cadence to ramp up on (and a zero-length step
-            # would spin forever).
+            # No window: no rampup cadence to step (would spin).
             return
         if self._rampup_window_start is None:
             self._rampup_window_start = now
@@ -1338,8 +1336,7 @@ class ThrottlingScopeManager:
         if self._quota is None:
             return
         if self._quota_window <= 0:
-            # A non-positive window has no reset cadence to step through (and
-            # would spin forever on a zero-length step), so keep it reset.
+            # No window: no reset cadence to step (would spin); keep it reset.
             self._consumed = 0.0
             self._quota_window_start = now
             return