diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index f6da9f615..bfa126fc8 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -75,15 +75,15 @@ behavior for specific domains [1]_. It is a dict that maps scope names to :class:`~scrapy.throttling.ThrottlingScopeConfig` dicts. It is empty by default. -:command:`startproject` scaffolds an entry for the testing websites used during -the :ref:`tutorial `, so that they are crawled faster while the -:ref:`conservative defaults ` still apply to other domains: +:command:`startproject` scaffolds a commented-out example entry, so that you can +uncomment and edit it to crawl domains you own (or that are meant for scraping) +faster, while the :ref:`conservative defaults ` still apply to +other domains: .. code-block:: python THROTTLING_SCOPES = { - "books.toscrape.com": {"concurrency": 16, "delay": 0.1}, - "quotes.toscrape.com": {"concurrency": 16, "delay": 0.1}, + "example.com": {"concurrency": 16, "delay": 0.1}, } Additional keys like ``"jitter"`` and ``"backoff"`` can be used here and are diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index a28333c90..6a3c155d5 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -162,9 +162,6 @@ class _DeprecatedSlotView: def __repr__(self) -> str: return f"_DeprecatedSlotView({self._key!r})" - def __str__(self) -> str: - return f"_DeprecatedSlotView({self._key!r})" - class _DeprecatedSlotsView(Mapping[str, _DeprecatedSlotView]): """Deprecated mapping view of active downloads, keyed by slot name.""" diff --git a/scrapy/downloadermiddlewares/backoff.py b/scrapy/downloadermiddlewares/backoff.py index 54b429fb7..c2d57b750 100644 --- a/scrapy/downloadermiddlewares/backoff.py +++ b/scrapy/downloadermiddlewares/backoff.py @@ -1,12 +1,11 @@ from __future__ import annotations -import datetime as dt import logging -from email.utils import parsedate_to_datetime from typing import TYPE_CHECKING from scrapy.exceptions import NotConfigured from scrapy.throttling import _load_exceptions, iter_scopes +from scrapy.utils._headers import _parse_ratelimit_reset, _parse_retry_after from scrapy.utils.decorators import _warn_spider_arg if TYPE_CHECKING: @@ -22,47 +21,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -def _decoded_header(response: Response, name: str) -> str | None: - """Return the stripped UTF-8 value of the *name* header of *response*, or - ``None`` if it is absent or not valid UTF-8.""" - raw = response.headers.get(name) - if not raw: - return None - try: - return raw.decode("utf-8").strip() - except UnicodeDecodeError: - return None - - -def _parse_retry_after(response: Response) -> float | None: - value = _decoded_header(response, "Retry-After") - if value is None: - return None - if value.isdigit(): - return float(value) # seconds - try: - date = parsedate_to_datetime(value) - except (TypeError, ValueError, OverflowError): - return None - if date.tzinfo is None: - date = date.replace(tzinfo=dt.timezone.utc) - now = dt.datetime.now(dt.timezone.utc) - seconds_to_wait = (date - now).total_seconds() - # Keep sub-second precision (a date less than a second away must not be - # truncated to 0 and dropped); a past or present date yields no delay. - return max(0.0, seconds_to_wait) or None - - -def _parse_ratelimit_reset(response: Response) -> float | None: - value = _decoded_header(response, "RateLimit-Reset") - if value is None: - return None - try: - return float(value) - except ValueError: - return None - - class BackoffMiddleware: """Downloader middleware that drives :ref:`backoff ` from download outcomes. diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 40fa5511c..35b347d08 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -7,10 +7,10 @@ from urllib.parse import urljoin, urlparse from w3lib.url import safe_url_string from scrapy import signals -from scrapy.downloadermiddlewares.backoff import _parse_retry_after from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import HtmlResponse, Response from scrapy.spidermiddlewares.referer import RefererMiddleware +from scrapy.utils._headers import _parse_retry_after from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import global_object_name diff --git a/scrapy/throttling.py b/scrapy/throttling.py index 480bb3a8f..2681c3983 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -1277,8 +1277,7 @@ class ThrottlingScopeManager: if self._backoff_level == 0 or self._last_backoff_time is None: return if self._window <= 0: - # A non-positive window has no recovery cadence to step through (and - # would spin forever on a zero-length step), so recover at once. + # No window: no recovery cadence to step (would spin); recover at once. self._backoff_level = 0 self._delay = self._base_delay self._in_backoff_until = None @@ -1300,8 +1299,7 @@ class ThrottlingScopeManager: if not self._rampup_enabled: return if self._window <= 0: - # No window means no cadence to ramp up on (and a zero-length step - # would spin forever). + # No window: no rampup cadence to step (would spin). return if self._rampup_window_start is None: self._rampup_window_start = now @@ -1338,8 +1336,7 @@ class ThrottlingScopeManager: if self._quota is None: return if self._quota_window <= 0: - # A non-positive window has no reset cadence to step through (and - # would spin forever on a zero-length step), so keep it reset. + # No window: no reset cadence to step (would spin); keep it reset. self._consumed = 0.0 self._quota_window_start = now return