From 9fe2a229b63fb4d94fae6659d11a8a2f529d46af Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 3 Jul 2026 10:24:00 +0200 Subject: [PATCH] Human review --- scrapy/core/scheduler.py | 2 +- scrapy/crawler.py | 6 +----- scrapy/downloadermiddlewares/backoff.py | 24 ++++++++---------------- scrapy/extensions/throttle.py | 6 ------ scrapy/pqueues.py | 16 ++++++---------- scrapy/throttling.py | 12 ++---------- scrapy/utils/misc.py | 5 +++++ 7 files changed, 23 insertions(+), 48 deletions(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index d863ab20c..78d13bcb4 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -516,7 +516,7 @@ def _queue_supports_peek(queue_cls: type) -> bool: class ThrottlingAwareScheduler(Scheduler): - """A :class:`Scheduler` that only ever hands the engine requests whose + """A :setting:`SCHEDULER` that only ever hands the engine requests whose :ref:`throttling scopes ` allow them to be sent **right now**. diff --git a/scrapy/crawler.py b/scrapy/crawler.py index e5df6f15f..0198ef93e 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -105,10 +105,6 @@ class Crawler: self.addons.load_settings(self.settings) self._apply_deprecated_spider_attr("download_delay", "DOWNLOAD_DELAY") - # 'max_concurrent_requests' historically overrode the per-domain slot - # concurrency, which is now THROTTLING_SCOPE_CONCURRENCY (see - # scrapy.throttling._default_scope_concurrency); the old deprecation - # message pointed at CONCURRENT_REQUESTS, which never matched that. self._apply_deprecated_spider_attr( "max_concurrent_requests", "THROTTLING_SCOPE_CONCURRENCY" ) @@ -189,7 +185,7 @@ class Crawler: return warnings.warn( f"The {attr!r} spider attribute is deprecated. Use the {setting} " - f"setting or per-domain THROTTLING_SCOPES instead.", + f"setting or THROTTLING_SCOPES instead.", category=ScrapyDeprecationWarning, stacklevel=3, ) diff --git a/scrapy/downloadermiddlewares/backoff.py b/scrapy/downloadermiddlewares/backoff.py index c2d57b750..4925973ed 100644 --- a/scrapy/downloadermiddlewares/backoff.py +++ b/scrapy/downloadermiddlewares/backoff.py @@ -4,9 +4,10 @@ import logging from typing import TYPE_CHECKING from scrapy.exceptions import NotConfigured -from scrapy.throttling import _load_exceptions, iter_scopes +from scrapy.throttling import iter_scopes from scrapy.utils._headers import _parse_ratelimit_reset, _parse_retry_after from scrapy.utils.decorators import _warn_spider_arg +from scrapy.utils.misc import _load_objects if TYPE_CHECKING: # typing.Self requires Python 3.11 @@ -37,25 +38,20 @@ class BackoffMiddleware: See :ref:`throttling` for details. """ + @classmethod + def from_crawler(cls, crawler: Crawler) -> Self: + return cls(crawler) + def __init__(self, crawler: Crawler): if not crawler.settings.getbool("BACKOFF_ENABLED"): raise NotConfigured - # Throttling is a core, always-on subsystem: THROTTLING_MANAGER has a - # non-None default and is instantiated before the downloader is built, - # so crawler.throttler is always set here (the engine likewise asserts - # it in its download path). assert crawler.throttler is not None self._throttler: ThrottlingManagerProtocol = crawler.throttler settings = crawler.settings - # Union of the global backoff triggers and every per-scope override: a - # response status (or exception type) outside it cannot trigger backoff - # for any scope, so the scopes of such a request need not be resolved. - # Each scope still makes the final decision via its scope manager's - # triggers_backoff_* methods (which read the per-scope overrides). self._http_codes: set[int] = { int(code) for code in settings.getlist("BACKOFF_HTTP_CODES") } - self._exceptions: tuple[type[BaseException], ...] = _load_exceptions( + self._exceptions: tuple[type[BaseException], ...] = _load_objects( settings.getlist("BACKOFF_EXCEPTIONS") ) for scope_config in settings.getdict("THROTTLING_SCOPES").values(): @@ -63,11 +59,7 @@ class BackoffMiddleware: if "http_codes" in backoff: self._http_codes.update(int(code) for code in backoff["http_codes"]) if "exceptions" in backoff: - self._exceptions += _load_exceptions(backoff["exceptions"]) - - @classmethod - def from_crawler(cls, crawler: Crawler) -> Self: - return cls(crawler) + self._exceptions += _load_objects(backoff["exceptions"]) @_warn_spider_arg def process_response( diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 5a5bd540f..93f82268f 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -44,7 +44,6 @@ class AutoThrottle: f"AUTOTHROTTLE_TARGET_CONCURRENCY " f"({self.target_concurrency!r}) must be higher than 0." ) - # Scopes whose start delay has already been applied (see _scope_delay). self._started_scopes: set[str] = set() crawler.signals.connect(self._spider_opened, signal=signals.spider_opened) crawler.signals.connect( @@ -81,11 +80,6 @@ class AutoThrottle: ): return - # AutoThrottle predates throttling scopes, so it adjusts the delay of - # the request's domain scope, matching its historical per-domain slots. - # Key by hostname (not netloc) to match the default ThrottlingManager - # scope, so the adjusted delay applies to the scope actually enforced - # even for non-default ports. scope_id = urlparse_cached(request).hostname or "" olddelay = self._scope_delay(throttler, scope_id) newdelay = self._adjust_delay(olddelay, latency, response) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 69a0e7ea9..25008cc17 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -237,10 +237,6 @@ class ScrapyPriorityQueue: """ if self.curprio is None: return None - # Mirror pop(): at a given priority it drains the regular queue before - # the start queue, so peek() must report the regular head first for the - # two to agree on "the next request" (a throttling-aware queue relies on - # this to check readiness of the exact request pop() will return). try: queue = self.queues[self.curprio] except KeyError: @@ -509,18 +505,18 @@ class ThrottlingAwarePriorityQueue: self.key: str = key self.crawler: Crawler = crawler - # scope set -> priority queue self.pqueues: dict[frozenset[ScopeID], ScrapyPriorityQueue] = {} - # Min-heap of (deadline, seq, scope_set, request) for requests held back - # by a per-request throttling_delay; seq keeps ordering stable and - # avoids comparing requests when deadlines tie. - self._delayed: list[tuple[float, int, frozenset[ScopeID], Request]] = [] - self._delayed_seq: int = 0 if slot_startprios: for set_key, startprios in slot_startprios.items(): scope_set = _scope_set_from_key(set_key) self.pqueues[scope_set] = self.pqfactory(scope_set, startprios) + # Min-heap of (deadline, seq, scope_set, request) for requests held + # back by a per-request throttling_delay; seq keeps ordering stable and + # avoids comparing requests when deadlines tie. + self._delayed: list[tuple[float, int, frozenset[ScopeID], Request]] = [] + self._delayed_seq: int = 0 + def pqfactory( self, scope_set: frozenset[ScopeID], startprios: Iterable[int] = () ) -> ScrapyPriorityQueue: diff --git a/scrapy/throttling.py b/scrapy/throttling.py index 2681c3983..5c77dad6c 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -19,7 +19,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import SETTINGS_PRIORITIES from scrapy.utils.asyncio import sleep, wait_for_first from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.misc import build_from_crawler, load_object +from scrapy.utils.misc import _load_objects, build_from_crawler, load_object if TYPE_CHECKING: from scrapy.crawler import Crawler @@ -126,14 +126,6 @@ def iter_scope_values(scopes: RequestScopes) -> Iterable[tuple[ScopeID, float | yield scope, None -def _load_exceptions(exceptions: Iterable[Any]) -> tuple[type[BaseException], ...]: - """Resolve *exceptions* (exception classes or their import paths) to a tuple - of exception classes.""" - return tuple( - load_object(exc) if isinstance(exc, str) else exc for exc in exceptions - ) - - def _effective_priority(settings: BaseSettings, name: str) -> int: """Return the priority of setting *name*, treating an unset setting (no priority, ``None``) as just below ``"default"`` so it never wins over one @@ -1180,7 +1172,7 @@ class ThrottlingScopeManager: "http_codes", settings.getlist("BACKOFF_HTTP_CODES") ) } - self._backoff_exceptions: tuple[type[BaseException], ...] = _load_exceptions( + self._backoff_exceptions: tuple[type[BaseException], ...] = _load_objects( backoff.get("exceptions", settings.getlist("BACKOFF_EXCEPTIONS")) ) self._window: float = settings.getfloat("BACKOFF_WINDOW") diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 20e7cb381..57b526be6 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -90,6 +90,11 @@ def load_object(path: str | Callable[..., Any]) -> Any: return obj +def _load_objects(objects: Iterable[str | Callable[..., Any]]) -> tuple[Any, ...]: + """Resolve *objects* (objects or import paths) to a tuple of objects.""" + return tuple(load_object(obj) if isinstance(obj, str) else obj for obj in objects) + + def walk_modules_iter(path: str) -> Iterable[ModuleType]: """Loads a module and all its submodules from the given module path and returns them. If *any* module throws an exception while importing, that