diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index 61d7198a4..f6da9f615 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -667,9 +667,7 @@ Alternative approaches include: @scope_cache async def get_scopes(self, request): extracted = tldextract.extract(request.url) - if extracted.domain and extracted.suffix: - return f"{extracted.domain}.{extracted.suffix}" - return urlparse_cached(request).netloc + return extracted.registered_domain or urlparse_cached(request).netloc THROTTLING_MANAGER = MyThrottlingManager @@ -698,18 +696,10 @@ Alternative approaches include: @scope_cache async def get_scopes(self, request): extracted = tldextract.extract(request.url) - if not (extracted.domain and extracted.suffix): + if not extracted.registered_domain: return urlparse_cached(request).netloc - scopes = set() - registrable_domain = f"{extracted.domain}.{extracted.suffix}" - scopes.add(registrable_domain) - if extracted.subdomain: - subdomain_parts = extracted.subdomain.split(".") - for i in range(len(subdomain_parts)): - subdomain = ".".join(subdomain_parts[i:]) - full_domain = f"{subdomain}.{registrable_domain}" - scopes.add(full_domain) - return scopes + # The registrable domain, plus the full host for a subdomain. + return {extracted.registered_domain, extracted.fqdn} THROTTLING_MANAGER = MyThrottlingManager diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 84f6774d5..e5df6f15f 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -104,8 +104,14 @@ class Crawler: return self.addons.load_settings(self.settings) - self._apply_spider_download_delay() - self._apply_spider_max_concurrent_requests() + self._apply_deprecated_spider_attr("download_delay", "DOWNLOAD_DELAY") + # 'max_concurrent_requests' historically overrode the per-domain slot + # concurrency, which is now THROTTLING_SCOPE_CONCURRENCY (see + # scrapy.throttling._default_scope_concurrency); the old deprecation + # message pointed at CONCURRENT_REQUESTS, which never matched that. + self._apply_deprecated_spider_attr( + "max_concurrent_requests", "THROTTLING_SCOPE_CONCURRENCY" + ) self.stats = load_object(self.settings["STATS_CLASS"])(self) lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) @@ -165,65 +171,29 @@ class Crawler: "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)} ) - def _apply_spider_download_delay(self) -> None: + def _apply_deprecated_spider_attr(self, attr: str, setting: str) -> None: + """Bridge a deprecated spider attribute onto *setting*, warning about + the deprecation (and about being ignored when *setting* is already set + at spider or higher priority).""" spider = self.spider if self.spider is not None else self.spidercls - if not hasattr(spider, "download_delay"): + if not hasattr(spider, attr): return - delay_prio = self.settings.getpriority("DOWNLOAD_DELAY") or 0 - if delay_prio >= SETTINGS_PRIORITIES["spider"]: + if (self.settings.getpriority(setting) or 0) >= SETTINGS_PRIORITIES["spider"]: warnings.warn( - "The 'download_delay' spider attribute is deprecated. " - "It is also being ignored because DOWNLOAD_DELAY is already set " - "at spider or higher priority. Remove the 'download_delay' " - "attribute from your spider.", + f"The {attr!r} spider attribute is deprecated. It is also being " + f"ignored because {setting} is already set at spider or higher " + f"priority. Remove the {attr!r} attribute from your spider.", category=ScrapyDeprecationWarning, - stacklevel=2, + stacklevel=3, ) - else: - warnings.warn( - "The 'download_delay' spider attribute is deprecated. Use the " - "DOWNLOAD_DELAY setting or per-domain THROTTLING_SCOPES instead.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - self.settings.set( - "DOWNLOAD_DELAY", spider.download_delay, priority="spider" - ) - - def _apply_spider_max_concurrent_requests(self) -> None: - spider = self.spider if self.spider is not None else self.spidercls - if not hasattr(spider, "max_concurrent_requests"): return - # Historically this attribute overrode the per-domain slot concurrency, - # which is now THROTTLING_SCOPE_CONCURRENCY (see - # scrapy.throttling._default_scope_concurrency). The old deprecation - # message pointed at CONCURRENT_REQUESTS, but that never matched its - # actual per-domain effect. - concurrency_prio = ( - self.settings.getpriority("THROTTLING_SCOPE_CONCURRENCY") or 0 + warnings.warn( + f"The {attr!r} spider attribute is deprecated. Use the {setting} " + f"setting or per-domain THROTTLING_SCOPES instead.", + category=ScrapyDeprecationWarning, + stacklevel=3, ) - if concurrency_prio >= SETTINGS_PRIORITIES["spider"]: - warnings.warn( - "The 'max_concurrent_requests' spider attribute is deprecated. " - "It is also being ignored because THROTTLING_SCOPE_CONCURRENCY is " - "already set at spider or higher priority. Remove the " - "'max_concurrent_requests' attribute from your spider.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - else: - warnings.warn( - "The 'max_concurrent_requests' spider attribute is deprecated. Use " - "the THROTTLING_SCOPE_CONCURRENCY setting or per-domain " - "THROTTLING_SCOPES instead.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - self.settings.set( - "THROTTLING_SCOPE_CONCURRENCY", - spider.max_concurrent_requests, - priority="spider", - ) + self.settings.set(setting, getattr(spider, attr), priority="spider") def _apply_reactorless_default_settings(self) -> None: """Change some setting defaults when not using a Twisted reactor. diff --git a/scrapy/downloadermiddlewares/backoff.py b/scrapy/downloadermiddlewares/backoff.py index 0bc45ed7c..803143d34 100644 --- a/scrapy/downloadermiddlewares/backoff.py +++ b/scrapy/downloadermiddlewares/backoff.py @@ -6,9 +6,8 @@ from email.utils import parsedate_to_datetime from typing import TYPE_CHECKING from scrapy.exceptions import NotConfigured -from scrapy.throttling import iter_scopes +from scrapy.throttling import _load_exceptions, iter_scopes from scrapy.utils.decorators import _warn_spider_arg -from scrapy.utils.misc import load_object if TYPE_CHECKING: # typing.Self requires Python 3.11 @@ -94,19 +93,15 @@ class BackoffMiddleware: self._http_codes: set[int] = { int(code) for code in settings.getlist("BACKOFF_HTTP_CODES") } - self._exceptions: tuple[type[BaseException], ...] = tuple( - load_object(exc) if isinstance(exc, str) else exc - for exc in settings.getlist("BACKOFF_EXCEPTIONS") + self._exceptions: tuple[type[BaseException], ...] = _load_exceptions( + settings.getlist("BACKOFF_EXCEPTIONS") ) for scope_config in settings.getdict("THROTTLING_SCOPES").values(): backoff = scope_config.get("backoff") or {} if "http_codes" in backoff: self._http_codes.update(int(code) for code in backoff["http_codes"]) if "exceptions" in backoff: - self._exceptions += tuple( - load_object(exc) if isinstance(exc, str) else exc - for exc in backoff["exceptions"] - ) + self._exceptions += _load_exceptions(backoff["exceptions"]) @classmethod def from_crawler(cls, crawler: Crawler) -> Self: diff --git a/scrapy/throttling.py b/scrapy/throttling.py index b27bd9319..19fc66a61 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -132,6 +132,14 @@ def iter_scope_values(scopes: RequestScopes) -> Iterable[tuple[ScopeID, float | yield scope, None +def _load_exceptions(exceptions: Iterable[Any]) -> tuple[type[BaseException], ...]: + """Resolve *exceptions* (exception classes or their import paths) to a tuple + of exception classes.""" + return tuple( + load_object(exc) if isinstance(exc, str) else exc for exc in exceptions + ) + + def _default_scope_concurrency(settings: BaseSettings) -> int: """Return the default concurrency of a throttling scope that does not set its own ``concurrency``. @@ -1186,9 +1194,8 @@ class ThrottlingScopeManager: "http_codes", settings.getlist("BACKOFF_HTTP_CODES") ) } - self._backoff_exceptions: tuple[type[BaseException], ...] = tuple( - load_object(exc) if isinstance(exc, str) else exc - for exc in backoff.get("exceptions", settings.getlist("BACKOFF_EXCEPTIONS")) + self._backoff_exceptions: tuple[type[BaseException], ...] = _load_exceptions( + backoff.get("exceptions", settings.getlist("BACKOFF_EXCEPTIONS")) ) self._window: float = settings.getfloat("BACKOFF_WINDOW")