From 6dfbe1e30bd18d5305d95a0402d115e414e706c2 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 3 Jul 2026 06:16:55 +0200 Subject: [PATCH] KISS --- docs/intro/tutorial.rst | 23 ++++ scrapy/core/downloader/__init__.py | 16 ++- .../templates/project/module/settings.py.tmpl | 11 +- scrapy/throttling.py | 115 +++++++++--------- tests/test_throttling.py | 13 ++ 5 files changed, 109 insertions(+), 69 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index c4e04364b..d61b373ba 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -130,6 +130,29 @@ and defines some attributes and methods: the scraped data as dicts and also finding new URLs to follow and creating new requests (:class:`~scrapy.Request`) from them. +Crawling faster for this tutorial +--------------------------------- + +By default, Scrapy :ref:`throttles ` crawls to be polite: it sends +at most one request at a time to each website, waiting one second between +requests. That keeps you from overwhelming websites, but it also makes crawling +slow, and it means the requests above are sent one after another rather than in +parallel. + +``quotes.toscrape.com`` is a sandbox meant for scraping practice, so we can +safely crawl it faster. Open ``tutorial/settings.py`` and add a +:setting:`THROTTLING_SCOPES` entry that raises the concurrency and lowers the +delay for that domain only: + +.. code-block:: python + + THROTTLING_SCOPES = { + "quotes.toscrape.com": {"concurrency": 16, "delay": 0.1}, + } + +The polite defaults still apply to every other website. Remove this entry once +you move on to crawling your own sites. + How to run our spider --------------------- diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 7e570ace5..5394926f1 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -114,13 +114,19 @@ class _DeprecatedSlotView: if r.meta.get(Downloader.DOWNLOAD_SLOT) == self._key } + # This deprecated view reads throttling scope state from private attributes + # of the default scope manager rather than through the scope manager + # protocol: these are read-only compatibility accessors, so keeping them off + # the protocol avoids forcing custom THROTTLING_SCOPE_MANAGER implementations + # to provide members that only exist to feed this shim. A custom manager that + # lacks the attribute simply falls back to the historical default. @property def lastseen(self) -> float: - return 0.0 + return getattr(self._scope, "_last_seen", None) or 0.0 @property def delay(self) -> float: - return self._scope.get_delay() + return getattr(self._scope, "_delay", 0.0) @delay.setter def delay(self, value: float) -> None: @@ -128,7 +134,7 @@ class _DeprecatedSlotView: @property def randomize_delay(self) -> bool: - return bool(self._scope.get_jitter()) + return bool(getattr(self._scope, "_jitter", None)) @property def concurrency(self) -> int: @@ -138,10 +144,10 @@ class _DeprecatedSlotView: category=ScrapyDeprecationWarning, stacklevel=2, ) - return self._scope.get_concurrency() or 0 + return getattr(self._scope, "_concurrency", None) or 0 def free_transfer_slots(self) -> int: - concurrency = self._scope.get_concurrency() or 0 + concurrency = getattr(self._scope, "_concurrency", None) or 0 return concurrency - len(self.transferring) def download_delay(self) -> float: diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index a035cc9ed..b9faac9a5 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -16,12 +16,11 @@ ROBOTSTXT_OBEY = True THROTTLING_SCOPE_CONCURRENCY = 1 DOWNLOAD_DELAY = 1 -# Crawl the tutorial websites faster, overriding the polite defaults above only -# for these domains (remove this once you crawl your own sites): -THROTTLING_SCOPES = { - "books.toscrape.com": {"concurrency": 16, "delay": 0.1}, - "quotes.toscrape.com": {"concurrency": 16, "delay": 0.1}, -} +# Override the polite defaults above for specific domains, e.g. to crawl a site +# you own (or one meant for scraping) faster: +#THROTTLING_SCOPES = { +# "example.com": {"concurrency": 16, "delay": 0.1}, +#} # Set settings whose default value is deprecated to a future-proof value: FEED_EXPORT_ENCODING = "utf-8" diff --git a/scrapy/throttling.py b/scrapy/throttling.py index 19fc66a61..f782bae42 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -104,13 +104,7 @@ def iter_scopes(scopes: RequestScopes) -> Iterable[ScopeID]: this helper normalizes any of those into an iterable of scope IDs, e.g. to react to a request's scopes in a custom middleware. """ - if scopes is None: - return () - if isinstance(scopes, str): - return (scopes,) - if isinstance(scopes, dict): - return scopes.keys() - return iter(scopes) + return (scope for scope, _ in iter_scope_values(scopes)) def iter_scope_values(scopes: RequestScopes) -> Iterable[tuple[ScopeID, float | None]]: @@ -197,17 +191,12 @@ def _warn_on_deprecated_concurrency(settings: BaseSettings) -> None: stacklevel=2, ) elif not scope_set: + # This warn-then-flip message only makes sense while the two defaults + # differ (otherwise it reads "will drop from 1 to 1"). That invariant is + # guarded by test_deprecated_concurrency_defaults_differ rather than at + # run time, so a crawl is never aborted over it. current = settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN") future = settings.getint("THROTTLING_SCOPE_CONCURRENCY") - # This warning only makes sense while the two defaults differ. If the - # default of the deprecated CONCURRENT_REQUESTS_PER_DOMAIN is lowered to - # match THROTTLING_SCOPE_CONCURRENCY before this branch is merged, the - # message becomes nonsensical ("will change from 1 to 1"); fail loudly - # here so it is revisited rather than shipping a bogus warning. - assert current != future, ( - "CONCURRENT_REQUESTS_PER_DOMAIN and THROTTLING_SCOPE_CONCURRENCY now " - "share the same default; drop this warn-then-flip warning." - ) warnings.warn( f"The effective per-scope (per-domain) concurrency is {current}, " f"the default of the deprecated CONCURRENT_REQUESTS_PER_DOMAIN " @@ -727,9 +716,7 @@ class ThrottlingManager: manager for manager, _ in managers if manager.concurrency_blocked() ] if not blocked: - for manager, value in managers: - manager.record_sent(amount=value) - self._reserved[request] = managers + self._record_reservation(request, managers) return if self._debug: logger.debug( @@ -738,6 +725,18 @@ class ThrottlingManager: ) await self._wait_for_slot(blocked) + def _record_reservation( + self, + request: Request, + managers: list[tuple[ThrottlingScopeManagerProtocol, float | None]], + ) -> None: + """Record a send on each of *request*'s scope *managers* and mark + *request* as reserved, so :meth:`release` can later free the slots. This + is the shared tail of :meth:`acquire` and :meth:`reserve`.""" + for manager, value in managers: + manager.record_sent(amount=value) + self._reserved[request] = managers + def release(self, request: Request) -> None: managers = self._reserved.pop(request, None) if not managers: @@ -769,9 +768,7 @@ class ThrottlingManager: (self.get_scope_manager(scope_id), value) for scope_id, value in self._cached_scope_values(request) ] - for manager, value in managers: - manager.record_sent(amount=value) - self._reserved[request] = managers + self._record_reservation(request, managers) def get_time_until_ready(self, request: Request) -> float | None: now = time.monotonic() @@ -806,14 +803,19 @@ class ThrottlingManager: async def _delay_request(self, request: Request) -> None: """Honor the :reqmeta:`throttling_delay` meta key by holding *request* - for the requested number of seconds the first time it is processed.""" - delay = request.meta.get("throttling_delay") - if not delay or request.meta.get("_throttling_delayed"): + for the requested number of seconds the first time it is processed. + + This is the blocking (:meth:`acquire`) counterpart of + :meth:`_request_delay_deadline`, which the readiness API polls instead; + both share the deadline bookkeeping and the one-time debug log. Here the + deadline is honored by sleeping until it, then marking the delay as + consumed so the request is never held again.""" + now = time.monotonic() + wait = self._request_delay_deadline(request, now) - now + if wait <= 0: return + await sleep(wait) request.meta["_throttling_delayed"] = True - if self._debug: - logger.debug(f"Holding {request} for {delay:.2f}s (throttling_delay)") - await sleep(float(delay)) def _request_delay_deadline(self, request: Request, now: float) -> float: """Return the monotonic time before which *request* must not be sent due @@ -1042,18 +1044,6 @@ class ThrottlingScopeManagerProtocol(Protocol): """Return whether *exception* triggers backoff for this scope (defaults to :setting:`BACKOFF_EXCEPTIONS`).""" - def get_delay(self) -> float: - """Return the current effective delay of this scope, in seconds, - including any active backoff (unlike :meth:`get_base_delay`).""" - - def get_jitter(self) -> float | list[float]: - """Return the magnitude of the random variation applied to the delay - (the per-scope override of :setting:`RANDOMIZE_DOWNLOAD_DELAY`).""" - - def get_concurrency(self) -> int | None: - """Return the maximum number of concurrent requests allowed for this - scope, or ``None`` when the scope enforces no explicit limit.""" - def get_base_delay(self) -> float: """Return the base (non-backoff) delay of this scope, in seconds.""" @@ -1167,11 +1157,14 @@ class ThrottlingScopeManager: self._base_delay: float = float( config.get("delay", settings.getfloat("DOWNLOAD_DELAY")) ) - # Magnitude of the random variation applied to the (non-backoff) delay. + # Magnitude of the random variation applied to the (non-backoff) delay, + # normalized to a (low, high) multiplier range (or None for no jitter). # Defaults to RANDOMIZE_DOWNLOAD_DELAY's historical ±50% when delay # randomization is on, or to no variation when it is off. - self._jitter: float | list[float] = config.get( - "jitter", 0.5 if settings.getbool("RANDOMIZE_DOWNLOAD_DELAY") else 0.0 + self._jitter: tuple[float, float] | None = self._normalize_jitter( + config.get( + "jitter", 0.5 if settings.getbool("RANDOMIZE_DOWNLOAD_DELAY") else 0.0 + ) ) self._delay_factor: float = float( backoff.get("delay_factor", settings.getfloat("BACKOFF_DELAY_FACTOR")) @@ -1182,8 +1175,8 @@ class ThrottlingScopeManager: self._min_delay: float = float( backoff.get("min_delay", settings.getfloat("BACKOFF_MIN_DELAY")) ) - self._backoff_jitter: float | list[float] = backoff.get( - "jitter", settings.getfloat("BACKOFF_JITTER") + self._backoff_jitter: tuple[float, float] | None = self._normalize_jitter( + backoff.get("jitter", settings.getfloat("BACKOFF_JITTER")) ) # Which responses/exceptions trigger backoff for this scope. Each # defaults to the matching global BACKOFF_* setting (see @@ -1252,13 +1245,28 @@ class ThrottlingScopeManager: return time.monotonic() if now is None else now @staticmethod - def _apply_jitter(value: float, jitter: float | list[float]) -> float: + def _normalize_jitter( + jitter: float | list[float], + ) -> tuple[float, float] | None: + """Normalize a ``jitter`` config value to a ``(low, high)`` multiplier + range, or ``None`` when no jitter applies. + + A scalar ``j`` means the symmetric range ``(-j, +j)``, so that + ``value * (1 + uniform(-j, +j))`` matches the historical + ``value * uniform(1 - j, 1 + j)``; a list/tuple is taken as an explicit + ``[low, high]`` range. + """ if isinstance(jitter, (list, tuple)): - low, high = jitter[0], jitter[1] - return value * (1 + random.uniform(low, high)) # noqa: S311 + return (float(jitter[0]), float(jitter[1])) if not jitter: + return None + return (-float(jitter), float(jitter)) + + @staticmethod + def _apply_jitter(value: float, jitter: tuple[float, float] | None) -> float: + if jitter is None: return value - return value * random.uniform(1 - jitter, 1 + jitter) # noqa: S311 + return value * (1 + random.uniform(*jitter)) # noqa: S311 def _effective_delay(self) -> float: # ``self._delay`` is the deterministic delay (the base delay, or the @@ -1474,15 +1482,6 @@ class ThrottlingScopeManager: def triggers_backoff_for_exception(self, exception: BaseException) -> bool: return isinstance(exception, self._backoff_exceptions) - def get_delay(self) -> float: - return self._delay - - def get_jitter(self) -> float | list[float]: - return self._jitter - - def get_concurrency(self) -> int | None: - return self._concurrency - def get_base_delay(self) -> float: return self._base_delay diff --git a/tests/test_throttling.py b/tests/test_throttling.py index 72cabaa0c..3dacbe0ef 100644 --- a/tests/test_throttling.py +++ b/tests/test_throttling.py @@ -7,6 +7,7 @@ import pytest from scrapy import signals from scrapy.exceptions import DownloadTimeoutError from scrapy.http import Request, Response +from scrapy.settings import default_settings from scrapy.throttling import ( ThrottlingManager, ThrottlingScopeManager, @@ -58,6 +59,18 @@ def _response(status=200, headers=None, url="http://example.com", meta=None): return Response(url, status=status, headers=headers or {}, request=request) +def test_deprecated_concurrency_defaults_differ(): + """``_warn_on_deprecated_concurrency`` emits a warn-then-flip message that + only makes sense while the two concurrency defaults differ (otherwise it + reads "will drop from N to N"). Guard that invariant here so that lowering + the deprecated default to match is caught by the test suite instead of + shipping a bogus warning or aborting a crawl.""" + assert ( + default_settings.CONCURRENT_REQUESTS_PER_DOMAIN + != default_settings.THROTTLING_SCOPE_CONCURRENCY + ) + + class TestThrottlingManager: @coroutine_test async def test_get_scopes_returns_netloc(self):