From 27f239074c7c98718198f8bb38e7e66e95f00f87 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Thu, 2 Jul 2026 10:05:47 +0200 Subject: [PATCH] WIP --- docs/news.rst | 8 ++-- scrapy/core/downloader/__init__.py | 65 +++++++++------------------ scrapy/extensions/throttle.py | 2 +- scrapy/pqueues.py | 14 ++---- tests/test_downloaderslotssettings.py | 2 - 5 files changed, 31 insertions(+), 60 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 37d0d8c8a..7a87e31b3 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1173,7 +1173,7 @@ Deprecations (:issue:`7005`, :issue:`7043`) - The ``CONCURRENT_REQUESTS_PER_IP`` setting is deprecated, use - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` instead. + ``CONCURRENT_REQUESTS_PER_DOMAIN`` instead. (:issue:`6917`, :issue:`6921`) - The ``scrapy.core.downloader.handlers.http`` module is deprecated. You @@ -1505,7 +1505,7 @@ Scrapy 2.13.3 (2025-07-02) -------------------------- - Changed the values for :setting:`DOWNLOAD_DELAY` (from ``0`` to ``1``) and - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` (from ``8`` to ``1``) in the + ``CONCURRENT_REQUESTS_PER_DOMAIN`` (from ``8`` to ``1``) in the default project template. (:issue:`6597`, :issue:`6918`, :issue:`6923`) @@ -3259,7 +3259,7 @@ New features ~~~~~~~~~~~~ - Settings corresponding to :setting:`DOWNLOAD_DELAY`, - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and + ``CONCURRENT_REQUESTS_PER_DOMAIN`` and :setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per-domain basis via the new ``DOWNLOAD_SLOTS`` setting. (:issue:`5328`) @@ -8753,7 +8753,7 @@ New features and settings - In request errbacks, offending requests are now received in ``failure.request`` attribute (:rev:`2738`) - Big downloader refactoring to support per domain/ip concurrency limits (:rev:`2732`) - ``CONCURRENT_REQUESTS_PER_SPIDER`` setting has been deprecated and replaced by: - - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, ``CONCURRENT_REQUESTS_PER_IP`` + - :setting:`CONCURRENT_REQUESTS`, ``CONCURRENT_REQUESTS_PER_DOMAIN``, ``CONCURRENT_REQUESTS_PER_IP`` - check the documentation for more details - Added builtin caching DNS resolver (:rev:`2728`) - Moved Amazon AWS-related components/extensions (SQS spider queue, SimpleDB stats collector) to a separate project: [scaws](https://github.com/scrapinghub/scaws) (:rev:`2706`, :rev:`2714`) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 6c5598335..8e4053a92 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -26,7 +26,10 @@ if TYPE_CHECKING: from scrapy.http import Response from scrapy.settings import BaseSettings from scrapy.signalmanager import SignalManager - from scrapy.throttling import ThrottlingScopeManagerProtocol + from scrapy.throttling import ( + ThrottlingManagerProtocol, + ThrottlingScopeManagerProtocol, + ) @dataclass(slots=True, eq=False) @@ -56,7 +59,7 @@ class _DeprecatedSlotView: self, downloader: Downloader, key: str, - scope: ThrottlingScopeManagerProtocol | None, + scope: ThrottlingScopeManagerProtocol, ) -> None: self._downloader = downloader self._key = key @@ -84,20 +87,15 @@ class _DeprecatedSlotView: @property def delay(self) -> float: - if self._scope is not None: - return self._scope.get_delay() - return 0.0 + return self._scope.get_delay() @delay.setter def delay(self, value: float) -> None: - if self._scope is not None: - self._scope.set_base_delay(value, only_increase=False) + self._scope.set_base_delay(value, only_increase=False) @property def randomize_delay(self) -> bool: - if self._scope is not None: - return bool(self._scope.get_jitter()) - return False + return bool(self._scope.get_jitter()) @property def concurrency(self) -> int: @@ -107,14 +105,10 @@ class _DeprecatedSlotView: category=ScrapyDeprecationWarning, stacklevel=2, ) - if self._scope is not None: - return self._scope.get_concurrency() or 0 - return 0 + return self._scope.get_concurrency() or 0 def free_transfer_slots(self) -> int: - concurrency = ( - self._scope.get_concurrency() or 0 if self._scope is not None else 0 - ) + concurrency = self._scope.get_concurrency() or 0 return concurrency - len(self.transferring) def download_delay(self) -> float: @@ -138,7 +132,9 @@ class _DeprecatedSlotsView(Mapping[str, _DeprecatedSlotView]): __slots__ = ("_downloader", "_throttler") - def __init__(self, downloader: Downloader, throttler: Any) -> None: + def __init__( + self, downloader: Downloader, throttler: ThrottlingManagerProtocol + ) -> None: self._downloader = downloader self._throttler = throttler @@ -152,11 +148,7 @@ class _DeprecatedSlotsView(Mapping[str, _DeprecatedSlotView]): def __getitem__(self, key: str) -> _DeprecatedSlotView: if key not in self._active_keys(): raise KeyError(key) - scope = ( - self._throttler.get_scope_manager(key) - if self._throttler is not None - else None - ) + scope = self._throttler.get_scope_manager(key) return _DeprecatedSlotView(self._downloader, key, scope) def __iter__(self) -> Iterator[str]: @@ -193,16 +185,6 @@ class Downloader: category=ScrapyDeprecationWarning, stacklevel=2, ) - for slot_settings in self.per_slot_settings.values(): - for deprecated_key in ("concurrency", "delay", "randomize_delay"): - if deprecated_key in slot_settings: - warnings.warn( - f"The '{deprecated_key}' key in DOWNLOAD_SLOTS is deprecated." - " Use THROTTLING_SCOPES to configure per-domain settings" - " instead.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) @inlineCallbacks @_warn_spider_arg @@ -230,6 +212,7 @@ class Downloader: category=ScrapyDeprecationWarning, stacklevel=2, ) + assert self.crawler.throttler is not None return _DeprecatedSlotsView(self, self.crawler.throttler) @_warn_spider_arg @@ -237,22 +220,18 @@ class Downloader: self, request: Request, spider: Spider | None = None ) -> tuple[str, _DeprecatedSlotView]: key = self._get_slot_key(request) - scope = ( - self.crawler.throttler.get_scope_manager(key) - if self.crawler.throttler is not None - else None - ) + assert self.crawler.throttler is not None + scope = self.crawler.throttler.get_scope_manager(key) return key, _DeprecatedSlotView(self, key, scope) def _get_slot_key(self, request: Request) -> str: - throttler = self.crawler.throttler - if throttler is not None: - return throttler.get_slot_key(request) - return self.get_slot_key(request) + assert self.crawler.throttler is not None + return self.crawler.throttler.get_slot_key(request) def get_slot_key(self, request: Request) -> str: - # This fallback is used only when no throttler is set; it mirrors the - # historical keying (an explicit download_slot wins, else the domain). + # Retained as public backward-compatible API. It mirrors the historical + # keying (an explicit download_slot wins, else the domain); the slot key + # used at run time comes from the throttler (see _get_slot_key()). meta_slot: str | None = request.meta.get(self.DOWNLOAD_SLOT) if meta_slot is not None: return meta_slot diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index ee76d3786..db3d97444 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -73,10 +73,10 @@ class AutoThrottle: self, response: Response, request: Request, spider: Spider ) -> None: throttler = self.crawler.throttler + assert throttler is not None latency = request.meta.get("download_latency") if ( latency is None - or throttler is None or request.meta.get("autothrottle_dont_adjust_delay", False) is True ): return diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index a476602e9..8aac29b7b 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -16,7 +16,6 @@ if TYPE_CHECKING: from typing_extensions import Self from scrapy import Request - from scrapy.core.downloader import Downloader from scrapy.crawler import Crawler from scrapy.throttling import ScopeID, ThrottlingManagerProtocol @@ -271,22 +270,17 @@ class ScrapyPriorityQueue: class DownloaderInterface: def __init__(self, crawler: Crawler): - assert crawler.engine - self.downloader: Downloader = crawler.engine.downloader - self._throttler: ThrottlingManagerProtocol | None = crawler.throttler + assert crawler.throttler is not None + self._throttler: ThrottlingManagerProtocol = crawler.throttler def stats(self, possible_slots: Iterable[str]) -> list[tuple[float, str]]: return [(self._slot_load(slot), slot) for slot in possible_slots] def get_slot_key(self, request: Request) -> str: - if self._throttler is not None: - return self._throttler.get_slot_key(request) - return self.downloader.get_slot_key(request) + return self._throttler.get_slot_key(request) def _slot_load(self, slot: str) -> float: - if self._throttler is not None: - return self._throttler.get_scope_load(slot) - return 0.0 + return self._throttler.get_scope_load(slot) class DownloaderAwarePriorityQueue: diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 5540c2e98..80b204e4d 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -56,7 +56,6 @@ async def test_concurrency_key_deprecated(): downloader = Downloader(crawler) messages = [str(w.message) for w in warns] assert any("DOWNLOAD_SLOTS setting is deprecated" in m for m in messages) - assert any("'concurrency' key in DOWNLOAD_SLOTS" in m for m in messages) downloader._get_slot(Request("https://example.com")) downloader.close() @@ -115,7 +114,6 @@ async def test_delay_deprecated(): downloader = Downloader(crawler) messages = [str(w.message) for w in warns] assert any("DOWNLOAD_SLOTS setting is deprecated" in m for m in messages) - assert any("'delay' key in DOWNLOAD_SLOTS" in m for m in messages) downloader._get_slot(Request("https://example.com")) downloader.close()