WIP

2026-07-02 10:05:47 +02:00 · 2026-07-02 10:05:47 +02:00 · 27f239074c
parent 8f1f66e1eb
commit 27f239074c
5 changed files with 31 additions and 60 deletions
--- a/docs/news.rst
+++ b/docs/news.rst
@ -1173,7 +1173,7 @@ Deprecations
    (:issue:`7005`, :issue:`7043`)

 -   The ``CONCURRENT_REQUESTS_PER_IP`` setting is deprecated, use
-    :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` instead.
+    ``CONCURRENT_REQUESTS_PER_DOMAIN`` instead.
    (:issue:`6917`, :issue:`6921`)

 -   The ``scrapy.core.downloader.handlers.http`` module is deprecated. You
@ -1505,7 +1505,7 @@ Scrapy 2.13.3 (2025-07-02)
 --------------------------

 -   Changed the values for :setting:`DOWNLOAD_DELAY` (from ``0`` to ``1``) and
-    :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` (from ``8`` to ``1``) in the
+    ``CONCURRENT_REQUESTS_PER_DOMAIN`` (from ``8`` to ``1``) in the
    default project template.
    (:issue:`6597`, :issue:`6918`, :issue:`6923`)

@ -3259,7 +3259,7 @@ New features
 ~~~~~~~~~~~~

 -   Settings corresponding to :setting:`DOWNLOAD_DELAY`,
-    :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and
+    ``CONCURRENT_REQUESTS_PER_DOMAIN`` and
    :setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per-domain basis
    via the new ``DOWNLOAD_SLOTS`` setting. (:issue:`5328`)

@ -8753,7 +8753,7 @@ New features and settings
 - In request errbacks, offending requests are now received in ``failure.request`` attribute (:rev:`2738`)
 - Big downloader refactoring to support per domain/ip concurrency limits (:rev:`2732`)
   - ``CONCURRENT_REQUESTS_PER_SPIDER`` setting has been deprecated and replaced by:
-      - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, ``CONCURRENT_REQUESTS_PER_IP``
+      - :setting:`CONCURRENT_REQUESTS`, ``CONCURRENT_REQUESTS_PER_DOMAIN``, ``CONCURRENT_REQUESTS_PER_IP``
   - check the documentation for more details
 - Added builtin caching DNS resolver (:rev:`2728`)
 - Moved Amazon AWS-related components/extensions (SQS spider queue, SimpleDB stats collector) to a separate project: [scaws](https://github.com/scrapinghub/scaws) (:rev:`2706`, :rev:`2714`)
--- a/scrapy/core/downloader/init.py
+++ b/scrapy/core/downloader/init.py
@ -26,7 +26,10 @@ if TYPE_CHECKING:
    from scrapy.http import Response
    from scrapy.settings import BaseSettings
    from scrapy.signalmanager import SignalManager
-    from scrapy.throttling import ThrottlingScopeManagerProtocol
+    from scrapy.throttling import (
+        ThrottlingManagerProtocol,
+        ThrottlingScopeManagerProtocol,
+    )


@dataclass(slots=True, eq=False)
@ -56,7 +59,7 @@ class _DeprecatedSlotView:
        self,
        downloader: Downloader,
        key: str,
-        scope: ThrottlingScopeManagerProtocol | None,
+        scope: ThrottlingScopeManagerProtocol,
    ) -> None:
        self._downloader = downloader
        self._key = key
@ -84,20 +87,15 @@ class _DeprecatedSlotView:

    @property
    def delay(self) -> float:
-        if self._scope is not None:
-            return self._scope.get_delay()
-        return 0.0
+        return self._scope.get_delay()

    @delay.setter
    def delay(self, value: float) -> None:
-        if self._scope is not None:
-            self._scope.set_base_delay(value, only_increase=False)
+        self._scope.set_base_delay(value, only_increase=False)

    @property
    def randomize_delay(self) -> bool:
-        if self._scope is not None:
-            return bool(self._scope.get_jitter())
-        return False
+        return bool(self._scope.get_jitter())

    @property
    def concurrency(self) -> int:
@ -107,14 +105,10 @@ class _DeprecatedSlotView:
            category=ScrapyDeprecationWarning,
            stacklevel=2,
        )
-        if self._scope is not None:
-            return self._scope.get_concurrency() or 0
-        return 0
+        return self._scope.get_concurrency() or 0

    def free_transfer_slots(self) -> int:
-        concurrency = (
-            self._scope.get_concurrency() or 0 if self._scope is not None else 0
-        )
+        concurrency = self._scope.get_concurrency() or 0
        return concurrency - len(self.transferring)

    def download_delay(self) -> float:
@ -138,7 +132,9 @@ class _DeprecatedSlotsView(Mapping[str, _DeprecatedSlotView]):

    __slots__ = ("_downloader", "_throttler")

-    def __init__(self, downloader: Downloader, throttler: Any) -> None:
+    def __init__(
+        self, downloader: Downloader, throttler: ThrottlingManagerProtocol
+    ) -> None:
        self._downloader = downloader
        self._throttler = throttler

@ -152,11 +148,7 @@ class _DeprecatedSlotsView(Mapping[str, _DeprecatedSlotView]):
    def __getitem__(self, key: str) -> _DeprecatedSlotView:
        if key not in self._active_keys():
            raise KeyError(key)
-        scope = (
-            self._throttler.get_scope_manager(key)
-            if self._throttler is not None
-            else None
-        )
+        scope = self._throttler.get_scope_manager(key)
        return _DeprecatedSlotView(self._downloader, key, scope)

    def __iter__(self) -> Iterator[str]:
@ -193,16 +185,6 @@ class Downloader:
                category=ScrapyDeprecationWarning,
                stacklevel=2,
            )
-            for slot_settings in self.per_slot_settings.values():
-                for deprecated_key in ("concurrency", "delay", "randomize_delay"):
-                    if deprecated_key in slot_settings:
-                        warnings.warn(
-                            f"The '{deprecated_key}' key in DOWNLOAD_SLOTS is deprecated."
-                            " Use THROTTLING_SCOPES to configure per-domain settings"
-                            " instead.",
-                            category=ScrapyDeprecationWarning,
-                            stacklevel=2,
-                        )

    @inlineCallbacks
    @_warn_spider_arg
@ -230,6 +212,7 @@ class Downloader:
            category=ScrapyDeprecationWarning,
            stacklevel=2,
        )
+        assert self.crawler.throttler is not None
        return _DeprecatedSlotsView(self, self.crawler.throttler)

    @_warn_spider_arg
@ -237,22 +220,18 @@ class Downloader:
        self, request: Request, spider: Spider | None = None
    ) -> tuple[str, _DeprecatedSlotView]:
        key = self._get_slot_key(request)
-        scope = (
-            self.crawler.throttler.get_scope_manager(key)
-            if self.crawler.throttler is not None
-            else None
-        )
+        assert self.crawler.throttler is not None
+        scope = self.crawler.throttler.get_scope_manager(key)
        return key, _DeprecatedSlotView(self, key, scope)

    def _get_slot_key(self, request: Request) -> str:
-        throttler = self.crawler.throttler
-        if throttler is not None:
-            return throttler.get_slot_key(request)
-        return self.get_slot_key(request)
+        assert self.crawler.throttler is not None
+        return self.crawler.throttler.get_slot_key(request)

    def get_slot_key(self, request: Request) -> str:
-        # This fallback is used only when no throttler is set; it mirrors the
-        # historical keying (an explicit download_slot wins, else the domain).
+        # Retained as public backward-compatible API. It mirrors the historical
+        # keying (an explicit download_slot wins, else the domain); the slot key
+        # used at run time comes from the throttler (see _get_slot_key()).
        meta_slot: str | None = request.meta.get(self.DOWNLOAD_SLOT)
        if meta_slot is not None:
            return meta_slot
--- a/scrapy/extensions/throttle.py
+++ b/scrapy/extensions/throttle.py
@ -73,10 +73,10 @@ class AutoThrottle:
        self, response: Response, request: Request, spider: Spider
    ) -> None:
        throttler = self.crawler.throttler
+        assert throttler is not None
        latency = request.meta.get("download_latency")
        if (
            latency is None
-            or throttler is None
            or request.meta.get("autothrottle_dont_adjust_delay", False) is True
        ):
            return
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@ -16,7 +16,6 @@ if TYPE_CHECKING:
    from typing_extensions import Self

    from scrapy import Request
-    from scrapy.core.downloader import Downloader
    from scrapy.crawler import Crawler
    from scrapy.throttling import ScopeID, ThrottlingManagerProtocol

@ -271,22 +270,17 @@ class ScrapyPriorityQueue:

 class DownloaderInterface:
    def __init__(self, crawler: Crawler):
-        assert crawler.engine
-        self.downloader: Downloader = crawler.engine.downloader
-        self._throttler: ThrottlingManagerProtocol | None = crawler.throttler
+        assert crawler.throttler is not None
+        self._throttler: ThrottlingManagerProtocol = crawler.throttler

    def stats(self, possible_slots: Iterable[str]) -> list[tuple[float, str]]:
        return [(self._slot_load(slot), slot) for slot in possible_slots]

    def get_slot_key(self, request: Request) -> str:
-        if self._throttler is not None:
-            return self._throttler.get_slot_key(request)
-        return self.downloader.get_slot_key(request)
+        return self._throttler.get_slot_key(request)

    def _slot_load(self, slot: str) -> float:
-        if self._throttler is not None:
-            return self._throttler.get_scope_load(slot)
-        return 0.0
+        return self._throttler.get_scope_load(slot)


 class DownloaderAwarePriorityQueue:
--- a/tests/test_downloaderslotssettings.py
+++ b/tests/test_downloaderslotssettings.py
@ -56,7 +56,6 @@ async def test_concurrency_key_deprecated():
        downloader = Downloader(crawler)
    messages = [str(w.message) for w in warns]
    assert any("DOWNLOAD_SLOTS setting is deprecated" in m for m in messages)
-    assert any("'concurrency' key in DOWNLOAD_SLOTS" in m for m in messages)
    downloader._get_slot(Request("https://example.com"))
    downloader.close()

@ -115,7 +114,6 @@ async def test_delay_deprecated():
        downloader = Downloader(crawler)
    messages = [str(w.message) for w in warns]
    assert any("DOWNLOAD_SLOTS setting is deprecated" in m for m in messages)
-    assert any("'delay' key in DOWNLOAD_SLOTS" in m for m in messages)
    downloader._get_slot(Request("https://example.com"))
    downloader.close()