KISS

scrapy/core/downloader/__init__.py

@@ -274,15 +274,6 @@ class Downloader:
         assert self.crawler.throttler is not None
         return _DeprecatedSlotsView(self, self.crawler.throttler)
 
-    @_warn_spider_arg
-    def _get_slot(
-        self, request: Request, spider: Spider | None = None
-    ) -> tuple[str, _DeprecatedSlotView]:
-        key = self._get_slot_key(request)
-        assert self.crawler.throttler is not None
-        scope = self.crawler.throttler.get_scope_manager(key)
-        return key, _DeprecatedSlotView(self, key, scope)
-
     def _get_slot_key(self, request: Request) -> str:
         assert self.crawler.throttler is not None
         return self.crawler.throttler.get_slot_key(request)

scrapy/downloadermiddlewares/backoff.py

@@ -22,14 +22,22 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
-def _parse_retry_after(response: Response) -> float | None:
-    raw = response.headers.get("Retry-After")
+def _decoded_header(response: Response, name: str) -> str | None:
+    """Return the stripped UTF-8 value of the *name* header of *response*, or
+    ``None`` if it is absent or not valid UTF-8."""
+    raw = response.headers.get(name)
     if not raw:
         return None
     try:
-        value = raw.decode("utf-8").strip()
+        return raw.decode("utf-8").strip()
     except UnicodeDecodeError:
         return None
+
+
+def _parse_retry_after(response: Response) -> float | None:
+    value = _decoded_header(response, "Retry-After")
+    if value is None:
+        return None
     if value.isdigit():
         return float(value)  # seconds
     try:
@@ -46,12 +54,8 @@ def _parse_retry_after(response: Response) -> float | None:
 
 
 def _parse_ratelimit_reset(response: Response) -> float | None:
-    raw = response.headers.get("RateLimit-Reset")
-    if not raw:
-        return None
-    try:
-        value = raw.decode("utf-8").strip()
-    except UnicodeDecodeError:
+    value = _decoded_header(response, "RateLimit-Reset")
+    if value is None:
         return None
     try:
         return float(value)
@@ -151,7 +155,6 @@ class BackoffMiddleware:
         ]
         if matched:
             self._throttler.back_off(matched)
-        return
 
     @staticmethod
     def _response_delay(response: Response) -> float | None:

scrapy/settings/default_settings.py

@@ -268,7 +268,6 @@ BACKOFF_MAX_DELAY = 300.0
 BACKOFF_MIN_DELAY = 1.0
 BACKOFF_WINDOW = 60.0
 
-
 BOT_NAME = "scrapybot"
 
 CLOSESPIDER_ERRORCOUNT = 0