RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

KISS

This commit is contained in:
Adrian Chaves 2026-07-03 05:32:31 +02:00
parent 584462815d
commit 5b276a3da1
4 changed files with 42 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
docs/topics/throttling.rst
View File

@ -667,9 +667,7 @@ Alternative approaches include:
@scope_cache
async def get_scopes(self, request):
extracted = tldextract.extract(request.url)
if extracted.domain and extracted.suffix:
return f"{extracted.domain}.{extracted.suffix}"
return urlparse_cached(request).netloc
return extracted.registered_domain or urlparse_cached(request).netloc
THROTTLING_MANAGER = MyThrottlingManager
@ -698,18 +696,10 @@ Alternative approaches include:
@scope_cache
async def get_scopes(self, request):
extracted = tldextract.extract(request.url)
if not (extracted.domain and extracted.suffix):
if not extracted.registered_domain:
return urlparse_cached(request).netloc
scopes = set()
registrable_domain = f"{extracted.domain}.{extracted.suffix}"
scopes.add(registrable_domain)
if extracted.subdomain:
subdomain_parts = extracted.subdomain.split(".")
for i in range(len(subdomain_parts)):
subdomain = ".".join(subdomain_parts[i:])
full_domain = f"{subdomain}.{registrable_domain}"
scopes.add(full_domain)
return scopes
# The registrable domain, plus the full host for a subdomain.
return {extracted.registered_domain, extracted.fqdn}
THROTTLING_MANAGER = MyThrottlingManager

78
scrapy/crawler.py
View File

@ -104,8 +104,14 @@ class Crawler:
return
self.addons.load_settings(self.settings)
self._apply_spider_download_delay()
self._apply_spider_max_concurrent_requests()
self._apply_deprecated_spider_attr("download_delay", "DOWNLOAD_DELAY")
# 'max_concurrent_requests' historically overrode the per-domain slot
# concurrency, which is now THROTTLING_SCOPE_CONCURRENCY (see
# scrapy.throttling._default_scope_concurrency); the old deprecation
# message pointed at CONCURRENT_REQUESTS, which never matched that.
self._apply_deprecated_spider_attr(
"max_concurrent_requests", "THROTTLING_SCOPE_CONCURRENCY"
)
self.stats = load_object(self.settings["STATS_CLASS"])(self)
lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"])
@ -165,65 +171,29 @@ class Crawler:
"Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)}
)
def _apply_spider_download_delay(self) -> None:
def _apply_deprecated_spider_attr(self, attr: str, setting: str) -> None:
"""Bridge a deprecated spider attribute onto *setting*, warning about
the deprecation (and about being ignored when *setting* is already set
at spider or higher priority)."""
spider = self.spider if self.spider is not None else self.spidercls
if not hasattr(spider, "download_delay"):
if not hasattr(spider, attr):
return
delay_prio = self.settings.getpriority("DOWNLOAD_DELAY") or 0
if delay_prio >= SETTINGS_PRIORITIES["spider"]:
if (self.settings.getpriority(setting) or 0) >= SETTINGS_PRIORITIES["spider"]:
warnings.warn(
"The 'download_delay' spider attribute is deprecated. "
"It is also being ignored because DOWNLOAD_DELAY is already set "
"at spider or higher priority. Remove the 'download_delay' "
"attribute from your spider.",
f"The {attr!r} spider attribute is deprecated. It is also being "
f"ignored because {setting} is already set at spider or higher "
f"priority. Remove the {attr!r} attribute from your spider.",
category=ScrapyDeprecationWarning,
stacklevel=2,
stacklevel=3,
)
else:
warnings.warn(
"The 'download_delay' spider attribute is deprecated. Use the "
"DOWNLOAD_DELAY setting or per-domain THROTTLING_SCOPES instead.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
self.settings.set(
"DOWNLOAD_DELAY", spider.download_delay, priority="spider"
)
def _apply_spider_max_concurrent_requests(self) -> None:
spider = self.spider if self.spider is not None else self.spidercls
if not hasattr(spider, "max_concurrent_requests"):
return
# Historically this attribute overrode the per-domain slot concurrency,
# which is now THROTTLING_SCOPE_CONCURRENCY (see
# scrapy.throttling._default_scope_concurrency). The old deprecation
# message pointed at CONCURRENT_REQUESTS, but that never matched its
# actual per-domain effect.
concurrency_prio = (
self.settings.getpriority("THROTTLING_SCOPE_CONCURRENCY") or 0
warnings.warn(
f"The {attr!r} spider attribute is deprecated. Use the {setting} "
f"setting or per-domain THROTTLING_SCOPES instead.",
category=ScrapyDeprecationWarning,
stacklevel=3,
)
if concurrency_prio >= SETTINGS_PRIORITIES["spider"]:
warnings.warn(
"The 'max_concurrent_requests' spider attribute is deprecated. "
"It is also being ignored because THROTTLING_SCOPE_CONCURRENCY is "
"already set at spider or higher priority. Remove the "
"'max_concurrent_requests' attribute from your spider.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
else:
warnings.warn(
"The 'max_concurrent_requests' spider attribute is deprecated. Use "
"the THROTTLING_SCOPE_CONCURRENCY setting or per-domain "
"THROTTLING_SCOPES instead.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
self.settings.set(
"THROTTLING_SCOPE_CONCURRENCY",
spider.max_concurrent_requests,
priority="spider",
)
self.settings.set(setting, getattr(spider, attr), priority="spider")
def _apply_reactorless_default_settings(self) -> None:
"""Change some setting defaults when not using a Twisted reactor.

13
scrapy/downloadermiddlewares/backoff.py
View File

@ -6,9 +6,8 @@ from email.utils import parsedate_to_datetime
from typing import TYPE_CHECKING
from scrapy.exceptions import NotConfigured
from scrapy.throttling import iter_scopes
from scrapy.throttling import _load_exceptions, iter_scopes
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.misc import load_object
if TYPE_CHECKING:
# typing.Self requires Python 3.11
@ -94,19 +93,15 @@ class BackoffMiddleware:
self._http_codes: set[int] = {
int(code) for code in settings.getlist("BACKOFF_HTTP_CODES")
}
self._exceptions: tuple[type[BaseException], ...] = tuple(
load_object(exc) if isinstance(exc, str) else exc
for exc in settings.getlist("BACKOFF_EXCEPTIONS")
self._exceptions: tuple[type[BaseException], ...] = _load_exceptions(
settings.getlist("BACKOFF_EXCEPTIONS")
)
for scope_config in settings.getdict("THROTTLING_SCOPES").values():
backoff = scope_config.get("backoff") or {}
if "http_codes" in backoff:
self._http_codes.update(int(code) for code in backoff["http_codes"])
if "exceptions" in backoff:
self._exceptions += tuple(
load_object(exc) if isinstance(exc, str) else exc
for exc in backoff["exceptions"]
)
self._exceptions += _load_exceptions(backoff["exceptions"])
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:

13
scrapy/throttling.py
View File

@ -132,6 +132,14 @@ def iter_scope_values(scopes: RequestScopes) -> Iterable[tuple[ScopeID, float |
yield scope, None
def _load_exceptions(exceptions: Iterable[Any]) -> tuple[type[BaseException], ...]:
"""Resolve *exceptions* (exception classes or their import paths) to a tuple
of exception classes."""
return tuple(
load_object(exc) if isinstance(exc, str) else exc for exc in exceptions
)
def _default_scope_concurrency(settings: BaseSettings) -> int:
"""Return the default concurrency of a throttling scope that does not set
its own ``concurrency``.
@ -1186,9 +1194,8 @@ class ThrottlingScopeManager:
"http_codes", settings.getlist("BACKOFF_HTTP_CODES")
)
}
self._backoff_exceptions: tuple[type[BaseException], ...] = tuple(
load_object(exc) if isinstance(exc, str) else exc
for exc in backoff.get("exceptions", settings.getlist("BACKOFF_EXCEPTIONS"))
self._backoff_exceptions: tuple[type[BaseException], ...] = _load_exceptions(
backoff.get("exceptions", settings.getlist("BACKOFF_EXCEPTIONS"))
)
self._window: float = settings.getfloat("BACKOFF_WINDOW")