diff --git a/docs/news.rst b/docs/news.rst index b923f8135..37d0d8c8a 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -6275,7 +6275,7 @@ New features ``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be :ref:`enabled ` for a significant scheduling improvement on crawls targeting multiple web domains, at the - cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`) + cost of no ``CONCURRENT_REQUESTS_PER_IP`` support (:issue:`3520`) * A new :attr:`.Request.cb_kwargs` attribute provides a cleaner way to pass keyword arguments to callback methods @@ -8753,7 +8753,7 @@ New features and settings - In request errbacks, offending requests are now received in ``failure.request`` attribute (:rev:`2738`) - Big downloader refactoring to support per domain/ip concurrency limits (:rev:`2732`) - ``CONCURRENT_REQUESTS_PER_SPIDER`` setting has been deprecated and replaced by: - - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, :setting:`CONCURRENT_REQUESTS_PER_IP` + - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, ``CONCURRENT_REQUESTS_PER_IP`` - check the documentation for more details - Added builtin caching DNS resolver (:rev:`2728`) - Moved Amazon AWS-related components/extensions (SQS spider queue, SimpleDB stats collector) to a separate project: [scaws](https://github.com/scrapinghub/scaws) (:rev:`2706`, :rev:`2714`) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 517be0fcd..6c5598335 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -251,7 +251,11 @@ class Downloader: return self.get_slot_key(request) def get_slot_key(self, request: Request) -> str: - # This fallback (used only when no throttler is set) keys by domain. + # This fallback is used only when no throttler is set; it mirrors the + # historical keying (an explicit download_slot wins, else the domain). + meta_slot: str | None = request.meta.get(self.DOWNLOAD_SLOT) + if meta_slot is not None: + return meta_slot return urlparse_cached(request).netloc or "" async def _enqueue_request(self, request: Request) -> Response: diff --git a/scrapy/crawler.py b/scrapy/crawler.py index cd457c22a..84f6774d5 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -105,6 +105,7 @@ class Crawler: self.addons.load_settings(self.settings) self._apply_spider_download_delay() + self._apply_spider_max_concurrent_requests() self.stats = load_object(self.settings["STATS_CLASS"])(self) lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) @@ -189,6 +190,41 @@ class Crawler: "DOWNLOAD_DELAY", spider.download_delay, priority="spider" ) + def _apply_spider_max_concurrent_requests(self) -> None: + spider = self.spider if self.spider is not None else self.spidercls + if not hasattr(spider, "max_concurrent_requests"): + return + # Historically this attribute overrode the per-domain slot concurrency, + # which is now THROTTLING_SCOPE_CONCURRENCY (see + # scrapy.throttling._default_scope_concurrency). The old deprecation + # message pointed at CONCURRENT_REQUESTS, but that never matched its + # actual per-domain effect. + concurrency_prio = ( + self.settings.getpriority("THROTTLING_SCOPE_CONCURRENCY") or 0 + ) + if concurrency_prio >= SETTINGS_PRIORITIES["spider"]: + warnings.warn( + "The 'max_concurrent_requests' spider attribute is deprecated. " + "It is also being ignored because THROTTLING_SCOPE_CONCURRENCY is " + "already set at spider or higher priority. Remove the " + "'max_concurrent_requests' attribute from your spider.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + else: + warnings.warn( + "The 'max_concurrent_requests' spider attribute is deprecated. Use " + "the THROTTLING_SCOPE_CONCURRENCY setting or per-domain " + "THROTTLING_SCOPES instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + self.settings.set( + "THROTTLING_SCOPE_CONCURRENCY", + spider.max_concurrent_requests, + priority="spider", + ) + def _apply_reactorless_default_settings(self) -> None: """Change some setting defaults when not using a Twisted reactor. diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 75bec1a46..a476602e9 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -342,11 +342,6 @@ class DownloaderAwarePriorityQueue: *, start_queue_cls: type[QueueProtocol] | None = None, ): - if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0: - raise ValueError( - f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP' - ) - if slot_startprios and not isinstance(slot_startprios, dict): raise ValueError( "DownloaderAwarePriorityQueue accepts " diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 932463ba9..cdf8b2f1b 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -149,15 +149,6 @@ class BaseSettings(MutableMapping[str, Any]): :param default: the value to return if no setting is found :type default: object """ - if name == "CONCURRENT_REQUESTS_PER_IP" and ( - isinstance(self[name], int) and self[name] != 0 - ): - warnings.warn( - "The CONCURRENT_REQUESTS_PER_IP setting is deprecated, use CONCURRENT_REQUESTS_PER_DOMAIN instead.", - ScrapyDeprecationWarning, - stacklevel=2, - ) - if name == "DNS_RESOLVER": warnings.warn( "The DNS_RESOLVER setting is deprecated, please use " diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 4dba40fe0..e4f9788d4 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -16,7 +16,6 @@ Scrapy developers, if you add a setting here remember to: import sys from importlib import import_module from pathlib import Path -from typing import Any __all__ = [ "ADDONS", @@ -635,19 +634,3 @@ URLLENGTH_LIMIT = 2083 USER_AGENT = f"Scrapy/{import_module('scrapy').__version__} (+https://scrapy.org)" WARN_ON_GENERATOR_RETURN_VALUE = True - - -def __getattr__(name: str) -> Any: - if name == "CONCURRENT_REQUESTS_PER_IP": - import warnings # noqa: PLC0415 - - from scrapy.exceptions import ScrapyDeprecationWarning # noqa: PLC0415 - - warnings.warn( - "The scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_IP attribute is deprecated, use scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_DOMAIN instead.", - ScrapyDeprecationWarning, - stacklevel=2, - ) - return 0 - - raise AttributeError(f"module {__name__!r} has no attribute {name!r}")