diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 5394926f1..a28333c90 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -274,15 +274,6 @@ class Downloader: assert self.crawler.throttler is not None return _DeprecatedSlotsView(self, self.crawler.throttler) - @_warn_spider_arg - def _get_slot( - self, request: Request, spider: Spider | None = None - ) -> tuple[str, _DeprecatedSlotView]: - key = self._get_slot_key(request) - assert self.crawler.throttler is not None - scope = self.crawler.throttler.get_scope_manager(key) - return key, _DeprecatedSlotView(self, key, scope) - def _get_slot_key(self, request: Request) -> str: assert self.crawler.throttler is not None return self.crawler.throttler.get_slot_key(request) diff --git a/scrapy/downloadermiddlewares/backoff.py b/scrapy/downloadermiddlewares/backoff.py index 803143d34..54b429fb7 100644 --- a/scrapy/downloadermiddlewares/backoff.py +++ b/scrapy/downloadermiddlewares/backoff.py @@ -22,14 +22,22 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -def _parse_retry_after(response: Response) -> float | None: - raw = response.headers.get("Retry-After") +def _decoded_header(response: Response, name: str) -> str | None: + """Return the stripped UTF-8 value of the *name* header of *response*, or + ``None`` if it is absent or not valid UTF-8.""" + raw = response.headers.get(name) if not raw: return None try: - value = raw.decode("utf-8").strip() + return raw.decode("utf-8").strip() except UnicodeDecodeError: return None + + +def _parse_retry_after(response: Response) -> float | None: + value = _decoded_header(response, "Retry-After") + if value is None: + return None if value.isdigit(): return float(value) # seconds try: @@ -46,12 +54,8 @@ def _parse_retry_after(response: Response) -> float | None: def _parse_ratelimit_reset(response: Response) -> float | None: - raw = response.headers.get("RateLimit-Reset") - if not raw: - return None - try: - value = raw.decode("utf-8").strip() - except UnicodeDecodeError: + value = _decoded_header(response, "RateLimit-Reset") + if value is None: return None try: return float(value) @@ -151,7 +155,6 @@ class BackoffMiddleware: ] if matched: self._throttler.back_off(matched) - return @staticmethod def _response_delay(response: Response) -> float | None: diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 23ff40b9c..b9b9d19f3 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -268,7 +268,6 @@ BACKOFF_MAX_DELAY = 300.0 BACKOFF_MIN_DELAY = 1.0 BACKOFF_WINDOW = 60.0 - BOT_NAME = "scrapybot" CLOSESPIDER_ERRORCOUNT = 0