Human review

scrapy/core/scheduler.py

@@ -516,7 +516,7 @@ def _queue_supports_peek(queue_cls: type) -> bool:
 
 
 class ThrottlingAwareScheduler(Scheduler):
-    """A :class:`Scheduler` that only ever hands the engine requests whose
+    """A :setting:`SCHEDULER` that only ever hands the engine requests whose
     :ref:`throttling scopes <throttling-scopes>` allow them to be sent **right
     now**.
 

scrapy/crawler.py

@@ -105,10 +105,6 @@ class Crawler:
 
         self.addons.load_settings(self.settings)
         self._apply_deprecated_spider_attr("download_delay", "DOWNLOAD_DELAY")
-        # 'max_concurrent_requests' historically overrode the per-domain slot
-        # concurrency, which is now THROTTLING_SCOPE_CONCURRENCY (see
-        # scrapy.throttling._default_scope_concurrency); the old deprecation
-        # message pointed at CONCURRENT_REQUESTS, which never matched that.
         self._apply_deprecated_spider_attr(
             "max_concurrent_requests", "THROTTLING_SCOPE_CONCURRENCY"
         )
@@ -189,7 +185,7 @@ class Crawler:
             return
         warnings.warn(
             f"The {attr!r} spider attribute is deprecated. Use the {setting} "
-            f"setting or per-domain THROTTLING_SCOPES instead.",
+            f"setting or THROTTLING_SCOPES instead.",
             category=ScrapyDeprecationWarning,
             stacklevel=3,
         )

scrapy/downloadermiddlewares/backoff.py

@@ -4,9 +4,10 @@ import logging
 from typing import TYPE_CHECKING
 
 from scrapy.exceptions import NotConfigured
-from scrapy.throttling import _load_exceptions, iter_scopes
+from scrapy.throttling import iter_scopes
 from scrapy.utils._headers import _parse_ratelimit_reset, _parse_retry_after
 from scrapy.utils.decorators import _warn_spider_arg
+from scrapy.utils.misc import _load_objects
 
 if TYPE_CHECKING:
     # typing.Self requires Python 3.11
@@ -37,25 +38,20 @@ class BackoffMiddleware:
     See :ref:`throttling` for details.
     """
 
+    @classmethod
+    def from_crawler(cls, crawler: Crawler) -> Self:
+        return cls(crawler)
+
     def __init__(self, crawler: Crawler):
         if not crawler.settings.getbool("BACKOFF_ENABLED"):
             raise NotConfigured
-        # Throttling is a core, always-on subsystem: THROTTLING_MANAGER has a
-        # non-None default and is instantiated before the downloader is built,
-        # so crawler.throttler is always set here (the engine likewise asserts
-        # it in its download path).
         assert crawler.throttler is not None
         self._throttler: ThrottlingManagerProtocol = crawler.throttler
         settings = crawler.settings
-        # Union of the global backoff triggers and every per-scope override: a
-        # response status (or exception type) outside it cannot trigger backoff
-        # for any scope, so the scopes of such a request need not be resolved.
-        # Each scope still makes the final decision via its scope manager's
-        # triggers_backoff_* methods (which read the per-scope overrides).
         self._http_codes: set[int] = {
             int(code) for code in settings.getlist("BACKOFF_HTTP_CODES")
         }
-        self._exceptions: tuple[type[BaseException], ...] = _load_exceptions(
+        self._exceptions: tuple[type[BaseException], ...] = _load_objects(
             settings.getlist("BACKOFF_EXCEPTIONS")
         )
         for scope_config in settings.getdict("THROTTLING_SCOPES").values():
@@ -63,11 +59,7 @@ class BackoffMiddleware:
             if "http_codes" in backoff:
                 self._http_codes.update(int(code) for code in backoff["http_codes"])
             if "exceptions" in backoff:
-                self._exceptions += _load_exceptions(backoff["exceptions"])
-
-    @classmethod
-    def from_crawler(cls, crawler: Crawler) -> Self:
-        return cls(crawler)
+                self._exceptions += _load_objects(backoff["exceptions"])
 
     @_warn_spider_arg
     def process_response(

scrapy/extensions/throttle.py

@@ -44,7 +44,6 @@ class AutoThrottle:
                 f"AUTOTHROTTLE_TARGET_CONCURRENCY "
                 f"({self.target_concurrency!r}) must be higher than 0."
             )
-        # Scopes whose start delay has already been applied (see _scope_delay).
         self._started_scopes: set[str] = set()
         crawler.signals.connect(self._spider_opened, signal=signals.spider_opened)
         crawler.signals.connect(
@@ -81,11 +80,6 @@ class AutoThrottle:
         ):
             return
 
-        # AutoThrottle predates throttling scopes, so it adjusts the delay of
-        # the request's domain scope, matching its historical per-domain slots.
-        # Key by hostname (not netloc) to match the default ThrottlingManager
-        # scope, so the adjusted delay applies to the scope actually enforced
-        # even for non-default ports.
         scope_id = urlparse_cached(request).hostname or ""
         olddelay = self._scope_delay(throttler, scope_id)
         newdelay = self._adjust_delay(olddelay, latency, response)

scrapy/pqueues.py

@@ -237,10 +237,6 @@ class ScrapyPriorityQueue:
         """
         if self.curprio is None:
             return None
-        # Mirror pop(): at a given priority it drains the regular queue before
-        # the start queue, so peek() must report the regular head first for the
-        # two to agree on "the next request" (a throttling-aware queue relies on
-        # this to check readiness of the exact request pop() will return).
         try:
             queue = self.queues[self.curprio]
         except KeyError:
@@ -509,18 +505,18 @@ class ThrottlingAwarePriorityQueue:
         self.key: str = key
         self.crawler: Crawler = crawler
 
-        # scope set -> priority queue
         self.pqueues: dict[frozenset[ScopeID], ScrapyPriorityQueue] = {}
-        # Min-heap of (deadline, seq, scope_set, request) for requests held back
-        # by a per-request throttling_delay; seq keeps ordering stable and
-        # avoids comparing requests when deadlines tie.
-        self._delayed: list[tuple[float, int, frozenset[ScopeID], Request]] = []
-        self._delayed_seq: int = 0
         if slot_startprios:
             for set_key, startprios in slot_startprios.items():
                 scope_set = _scope_set_from_key(set_key)
                 self.pqueues[scope_set] = self.pqfactory(scope_set, startprios)
 
+        # Min-heap of (deadline, seq, scope_set, request) for requests held
+        # back by a per-request throttling_delay; seq keeps ordering stable and
+        # avoids comparing requests when deadlines tie.
+        self._delayed: list[tuple[float, int, frozenset[ScopeID], Request]] = []
+        self._delayed_seq: int = 0
+
     def pqfactory(
         self, scope_set: frozenset[ScopeID], startprios: Iterable[int] = ()
     ) -> ScrapyPriorityQueue:

scrapy/throttling.py

@@ -19,7 +19,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.settings import SETTINGS_PRIORITIES
 from scrapy.utils.asyncio import sleep, wait_for_first
 from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.misc import build_from_crawler, load_object
+from scrapy.utils.misc import _load_objects, build_from_crawler, load_object
 
 if TYPE_CHECKING:
     from scrapy.crawler import Crawler
@@ -126,14 +126,6 @@ def iter_scope_values(scopes: RequestScopes) -> Iterable[tuple[ScopeID, float |
         yield scope, None
 
 
-def _load_exceptions(exceptions: Iterable[Any]) -> tuple[type[BaseException], ...]:
-    """Resolve *exceptions* (exception classes or their import paths) to a tuple
-    of exception classes."""
-    return tuple(
-        load_object(exc) if isinstance(exc, str) else exc for exc in exceptions
-    )
-
-
 def _effective_priority(settings: BaseSettings, name: str) -> int:
     """Return the priority of setting *name*, treating an unset setting (no
     priority, ``None``) as just below ``"default"`` so it never wins over one
@@ -1180,7 +1172,7 @@ class ThrottlingScopeManager:
                 "http_codes", settings.getlist("BACKOFF_HTTP_CODES")
             )
         }
-        self._backoff_exceptions: tuple[type[BaseException], ...] = _load_exceptions(
+        self._backoff_exceptions: tuple[type[BaseException], ...] = _load_objects(
             backoff.get("exceptions", settings.getlist("BACKOFF_EXCEPTIONS"))
         )
         self._window: float = settings.getfloat("BACKOFF_WINDOW")

scrapy/utils/misc.py

@@ -90,6 +90,11 @@ def load_object(path: str | Callable[..., Any]) -> Any:
     return obj
 
 
+def _load_objects(objects: Iterable[str | Callable[..., Any]]) -> tuple[Any, ...]:
+    """Resolve *objects* (objects or import paths) to a tuple of objects."""
+    return tuple(load_object(obj) if isinstance(obj, str) else obj for obj in objects)
+
+
 def walk_modules_iter(path: str) -> Iterable[ModuleType]:
     """Loads a module and all its submodules from the given module path and
     returns them. If *any* module throws an exception while importing, that