KISS

docs/intro/tutorial.rst

@@ -130,6 +130,29 @@ and defines some attributes and methods:
   the scraped data as dicts and also finding new URLs to
   follow and creating new requests (:class:`~scrapy.Request`) from them.
 
+Crawling faster for this tutorial
+---------------------------------
+
+By default, Scrapy :ref:`throttles <throttling>` crawls to be polite: it sends
+at most one request at a time to each website, waiting one second between
+requests. That keeps you from overwhelming websites, but it also makes crawling
+slow, and it means the requests above are sent one after another rather than in
+parallel.
+
+``quotes.toscrape.com`` is a sandbox meant for scraping practice, so we can
+safely crawl it faster. Open ``tutorial/settings.py`` and add a
+:setting:`THROTTLING_SCOPES` entry that raises the concurrency and lowers the
+delay for that domain only:
+
+.. code-block:: python
+
+    THROTTLING_SCOPES = {
+        "quotes.toscrape.com": {"concurrency": 16, "delay": 0.1},
+    }
+
+The polite defaults still apply to every other website. Remove this entry once
+you move on to crawling your own sites.
+
 How to run our spider
 ---------------------
 

scrapy/core/downloader/__init__.py

@@ -114,13 +114,19 @@ class _DeprecatedSlotView:
             if r.meta.get(Downloader.DOWNLOAD_SLOT) == self._key
         }
 
+    # This deprecated view reads throttling scope state from private attributes
+    # of the default scope manager rather than through the scope manager
+    # protocol: these are read-only compatibility accessors, so keeping them off
+    # the protocol avoids forcing custom THROTTLING_SCOPE_MANAGER implementations
+    # to provide members that only exist to feed this shim. A custom manager that
+    # lacks the attribute simply falls back to the historical default.
     @property
     def lastseen(self) -> float:
-        return 0.0
+        return getattr(self._scope, "_last_seen", None) or 0.0
 
     @property
     def delay(self) -> float:
-        return self._scope.get_delay()
+        return getattr(self._scope, "_delay", 0.0)
 
     @delay.setter
     def delay(self, value: float) -> None:
@@ -128,7 +134,7 @@ class _DeprecatedSlotView:
 
     @property
     def randomize_delay(self) -> bool:
-        return bool(self._scope.get_jitter())
+        return bool(getattr(self._scope, "_jitter", None))
 
     @property
     def concurrency(self) -> int:
@@ -138,10 +144,10 @@ class _DeprecatedSlotView:
             category=ScrapyDeprecationWarning,
             stacklevel=2,
         )
-        return self._scope.get_concurrency() or 0
+        return getattr(self._scope, "_concurrency", None) or 0
 
     def free_transfer_slots(self) -> int:
-        concurrency = self._scope.get_concurrency() or 0
+        concurrency = getattr(self._scope, "_concurrency", None) or 0
         return concurrency - len(self.transferring)
 
     def download_delay(self) -> float:

scrapy/templates/project/module/settings.py.tmpl

@@ -16,12 +16,11 @@ ROBOTSTXT_OBEY = True
 THROTTLING_SCOPE_CONCURRENCY = 1
 DOWNLOAD_DELAY = 1
 
-# Crawl the tutorial websites faster, overriding the polite defaults above only
-# for these domains (remove this once you crawl your own sites):
-THROTTLING_SCOPES = {
-    "books.toscrape.com": {"concurrency": 16, "delay": 0.1},
-    "quotes.toscrape.com": {"concurrency": 16, "delay": 0.1},
-}
+# Override the polite defaults above for specific domains, e.g. to crawl a site
+# you own (or one meant for scraping) faster:
+#THROTTLING_SCOPES = {
+#    "example.com": {"concurrency": 16, "delay": 0.1},
+#}
 
 # Set settings whose default value is deprecated to a future-proof value:
 FEED_EXPORT_ENCODING = "utf-8"

scrapy/throttling.py

@@ -104,13 +104,7 @@ def iter_scopes(scopes: RequestScopes) -> Iterable[ScopeID]:
     this helper normalizes any of those into an iterable of scope IDs, e.g. to
     react to a request's scopes in a custom middleware.
     """
-    if scopes is None:
-        return ()
-    if isinstance(scopes, str):
-        return (scopes,)
-    if isinstance(scopes, dict):
-        return scopes.keys()
-    return iter(scopes)
+    return (scope for scope, _ in iter_scope_values(scopes))
 
 
 def iter_scope_values(scopes: RequestScopes) -> Iterable[tuple[ScopeID, float | None]]:
@@ -197,17 +191,12 @@ def _warn_on_deprecated_concurrency(settings: BaseSettings) -> None:
             stacklevel=2,
         )
     elif not scope_set:
+        # This warn-then-flip message only makes sense while the two defaults
+        # differ (otherwise it reads "will drop from 1 to 1"). That invariant is
+        # guarded by test_deprecated_concurrency_defaults_differ rather than at
+        # run time, so a crawl is never aborted over it.
         current = settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN")
         future = settings.getint("THROTTLING_SCOPE_CONCURRENCY")
-        # This warning only makes sense while the two defaults differ. If the
-        # default of the deprecated CONCURRENT_REQUESTS_PER_DOMAIN is lowered to
-        # match THROTTLING_SCOPE_CONCURRENCY before this branch is merged, the
-        # message becomes nonsensical ("will change from 1 to 1"); fail loudly
-        # here so it is revisited rather than shipping a bogus warning.
-        assert current != future, (
-            "CONCURRENT_REQUESTS_PER_DOMAIN and THROTTLING_SCOPE_CONCURRENCY now "
-            "share the same default; drop this warn-then-flip warning."
-        )
         warnings.warn(
             f"The effective per-scope (per-domain) concurrency is {current}, "
             f"the default of the deprecated CONCURRENT_REQUESTS_PER_DOMAIN "
@@ -727,9 +716,7 @@ class ThrottlingManager:
                 manager for manager, _ in managers if manager.concurrency_blocked()
             ]
             if not blocked:
-                for manager, value in managers:
-                    manager.record_sent(amount=value)
-                self._reserved[request] = managers
+                self._record_reservation(request, managers)
                 return
             if self._debug:
                 logger.debug(
@@ -738,6 +725,18 @@ class ThrottlingManager:
                 )
             await self._wait_for_slot(blocked)
 
+    def _record_reservation(
+        self,
+        request: Request,
+        managers: list[tuple[ThrottlingScopeManagerProtocol, float | None]],
+    ) -> None:
+        """Record a send on each of *request*'s scope *managers* and mark
+        *request* as reserved, so :meth:`release` can later free the slots. This
+        is the shared tail of :meth:`acquire` and :meth:`reserve`."""
+        for manager, value in managers:
+            manager.record_sent(amount=value)
+        self._reserved[request] = managers
+
     def release(self, request: Request) -> None:
         managers = self._reserved.pop(request, None)
         if not managers:
@@ -769,9 +768,7 @@ class ThrottlingManager:
             (self.get_scope_manager(scope_id), value)
             for scope_id, value in self._cached_scope_values(request)
         ]
-        for manager, value in managers:
-            manager.record_sent(amount=value)
-        self._reserved[request] = managers
+        self._record_reservation(request, managers)
 
     def get_time_until_ready(self, request: Request) -> float | None:
         now = time.monotonic()
@@ -806,14 +803,19 @@ class ThrottlingManager:
 
     async def _delay_request(self, request: Request) -> None:
         """Honor the :reqmeta:`throttling_delay` meta key by holding *request*
-        for the requested number of seconds the first time it is processed."""
-        delay = request.meta.get("throttling_delay")
-        if not delay or request.meta.get("_throttling_delayed"):
+        for the requested number of seconds the first time it is processed.
+
+        This is the blocking (:meth:`acquire`) counterpart of
+        :meth:`_request_delay_deadline`, which the readiness API polls instead;
+        both share the deadline bookkeeping and the one-time debug log. Here the
+        deadline is honored by sleeping until it, then marking the delay as
+        consumed so the request is never held again."""
+        now = time.monotonic()
+        wait = self._request_delay_deadline(request, now) - now
+        if wait <= 0:
             return
+        await sleep(wait)
         request.meta["_throttling_delayed"] = True
-        if self._debug:
-            logger.debug(f"Holding {request} for {delay:.2f}s (throttling_delay)")
-        await sleep(float(delay))
 
     def _request_delay_deadline(self, request: Request, now: float) -> float:
         """Return the monotonic time before which *request* must not be sent due
@@ -1042,18 +1044,6 @@ class ThrottlingScopeManagerProtocol(Protocol):
         """Return whether *exception* triggers backoff for this scope (defaults
         to :setting:`BACKOFF_EXCEPTIONS`)."""
 
-    def get_delay(self) -> float:
-        """Return the current effective delay of this scope, in seconds,
-        including any active backoff (unlike :meth:`get_base_delay`)."""
-
-    def get_jitter(self) -> float | list[float]:
-        """Return the magnitude of the random variation applied to the delay
-        (the per-scope override of :setting:`RANDOMIZE_DOWNLOAD_DELAY`)."""
-
-    def get_concurrency(self) -> int | None:
-        """Return the maximum number of concurrent requests allowed for this
-        scope, or ``None`` when the scope enforces no explicit limit."""
-
     def get_base_delay(self) -> float:
         """Return the base (non-backoff) delay of this scope, in seconds."""
 
@@ -1167,11 +1157,14 @@ class ThrottlingScopeManager:
         self._base_delay: float = float(
             config.get("delay", settings.getfloat("DOWNLOAD_DELAY"))
         )
-        # Magnitude of the random variation applied to the (non-backoff) delay.
+        # Magnitude of the random variation applied to the (non-backoff) delay,
+        # normalized to a (low, high) multiplier range (or None for no jitter).
         # Defaults to RANDOMIZE_DOWNLOAD_DELAY's historical ±50% when delay
         # randomization is on, or to no variation when it is off.
-        self._jitter: float | list[float] = config.get(
-            "jitter", 0.5 if settings.getbool("RANDOMIZE_DOWNLOAD_DELAY") else 0.0
+        self._jitter: tuple[float, float] | None = self._normalize_jitter(
+            config.get(
+                "jitter", 0.5 if settings.getbool("RANDOMIZE_DOWNLOAD_DELAY") else 0.0
+            )
         )
         self._delay_factor: float = float(
             backoff.get("delay_factor", settings.getfloat("BACKOFF_DELAY_FACTOR"))
@@ -1182,8 +1175,8 @@ class ThrottlingScopeManager:
         self._min_delay: float = float(
             backoff.get("min_delay", settings.getfloat("BACKOFF_MIN_DELAY"))
         )
-        self._backoff_jitter: float | list[float] = backoff.get(
-            "jitter", settings.getfloat("BACKOFF_JITTER")
+        self._backoff_jitter: tuple[float, float] | None = self._normalize_jitter(
+            backoff.get("jitter", settings.getfloat("BACKOFF_JITTER"))
         )
         # Which responses/exceptions trigger backoff for this scope. Each
         # defaults to the matching global BACKOFF_* setting (see
@@ -1252,13 +1245,28 @@ class ThrottlingScopeManager:
         return time.monotonic() if now is None else now
 
     @staticmethod
-    def _apply_jitter(value: float, jitter: float | list[float]) -> float:
+    def _normalize_jitter(
+        jitter: float | list[float],
+    ) -> tuple[float, float] | None:
+        """Normalize a ``jitter`` config value to a ``(low, high)`` multiplier
+        range, or ``None`` when no jitter applies.
+
+        A scalar ``j`` means the symmetric range ``(-j, +j)``, so that
+        ``value * (1 + uniform(-j, +j))`` matches the historical
+        ``value * uniform(1 - j, 1 + j)``; a list/tuple is taken as an explicit
+        ``[low, high]`` range.
+        """
         if isinstance(jitter, (list, tuple)):
-            low, high = jitter[0], jitter[1]
-            return value * (1 + random.uniform(low, high))  # noqa: S311
+            return (float(jitter[0]), float(jitter[1]))
         if not jitter:
+            return None
+        return (-float(jitter), float(jitter))
+
+    @staticmethod
+    def _apply_jitter(value: float, jitter: tuple[float, float] | None) -> float:
+        if jitter is None:
             return value
-        return value * random.uniform(1 - jitter, 1 + jitter)  # noqa: S311
+        return value * (1 + random.uniform(*jitter))  # noqa: S311
 
     def _effective_delay(self) -> float:
         # ``self._delay`` is the deterministic delay (the base delay, or the
@@ -1474,15 +1482,6 @@ class ThrottlingScopeManager:
     def triggers_backoff_for_exception(self, exception: BaseException) -> bool:
         return isinstance(exception, self._backoff_exceptions)
 
-    def get_delay(self) -> float:
-        return self._delay
-
-    def get_jitter(self) -> float | list[float]:
-        return self._jitter
-
-    def get_concurrency(self) -> int | None:
-        return self._concurrency
-
     def get_base_delay(self) -> float:
         return self._base_delay
 

tests/test_throttling.py

@@ -7,6 +7,7 @@ import pytest
 from scrapy import signals
 from scrapy.exceptions import DownloadTimeoutError
 from scrapy.http import Request, Response
+from scrapy.settings import default_settings
 from scrapy.throttling import (
     ThrottlingManager,
     ThrottlingScopeManager,
@@ -58,6 +59,18 @@ def _response(status=200, headers=None, url="http://example.com", meta=None):
     return Response(url, status=status, headers=headers or {}, request=request)
 
 
+def test_deprecated_concurrency_defaults_differ():
+    """``_warn_on_deprecated_concurrency`` emits a warn-then-flip message that
+    only makes sense while the two concurrency defaults differ (otherwise it
+    reads "will drop from N to N"). Guard that invariant here so that lowering
+    the deprecated default to match is caught by the test suite instead of
+    shipping a bogus warning or aborting a crawl."""
+    assert (
+        default_settings.CONCURRENT_REQUESTS_PER_DOMAIN
+        != default_settings.THROTTLING_SCOPE_CONCURRENCY
+    )
+
+
 class TestThrottlingManager:
     @coroutine_test
     async def test_get_scopes_returns_netloc(self):