Address backward compatibility issues, remove CONCURRENT_REQUESTS_PER_IP

docs/news.rst

@@ -6275,7 +6275,7 @@ New features
     ``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be
     :ref:`enabled <broad-crawls-scheduler-priority-queue>` for a significant
     scheduling improvement on crawls targeting multiple web domains, at the
-    cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`)
+    cost of no ``CONCURRENT_REQUESTS_PER_IP`` support (:issue:`3520`)
 
 *   A new :attr:`.Request.cb_kwargs` attribute
     provides a cleaner way to pass keyword arguments to callback methods
@@ -8753,7 +8753,7 @@ New features and settings
 - In request errbacks, offending requests are now received in ``failure.request`` attribute (:rev:`2738`)
 - Big downloader refactoring to support per domain/ip concurrency limits (:rev:`2732`)
    - ``CONCURRENT_REQUESTS_PER_SPIDER`` setting has been deprecated and replaced by:
-      - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, :setting:`CONCURRENT_REQUESTS_PER_IP`
+      - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, ``CONCURRENT_REQUESTS_PER_IP``
    - check the documentation for more details
 - Added builtin caching DNS resolver (:rev:`2728`)
 - Moved Amazon AWS-related components/extensions (SQS spider queue, SimpleDB stats collector) to a separate project: [scaws](https://github.com/scrapinghub/scaws) (:rev:`2706`, :rev:`2714`)

scrapy/core/downloader/__init__.py

@@ -251,7 +251,11 @@ class Downloader:
         return self.get_slot_key(request)
 
     def get_slot_key(self, request: Request) -> str:
-        # This fallback (used only when no throttler is set) keys by domain.
+        # This fallback is used only when no throttler is set; it mirrors the
+        # historical keying (an explicit download_slot wins, else the domain).
+        meta_slot: str | None = request.meta.get(self.DOWNLOAD_SLOT)
+        if meta_slot is not None:
+            return meta_slot
         return urlparse_cached(request).netloc or ""
 
     async def _enqueue_request(self, request: Request) -> Response:

scrapy/crawler.py

@@ -105,6 +105,7 @@ class Crawler:
 
         self.addons.load_settings(self.settings)
         self._apply_spider_download_delay()
+        self._apply_spider_max_concurrent_requests()
         self.stats = load_object(self.settings["STATS_CLASS"])(self)
 
         lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"])
@@ -189,6 +190,41 @@ class Crawler:
                 "DOWNLOAD_DELAY", spider.download_delay, priority="spider"
             )
 
+    def _apply_spider_max_concurrent_requests(self) -> None:
+        spider = self.spider if self.spider is not None else self.spidercls
+        if not hasattr(spider, "max_concurrent_requests"):
+            return
+        # Historically this attribute overrode the per-domain slot concurrency,
+        # which is now THROTTLING_SCOPE_CONCURRENCY (see
+        # scrapy.throttling._default_scope_concurrency). The old deprecation
+        # message pointed at CONCURRENT_REQUESTS, but that never matched its
+        # actual per-domain effect.
+        concurrency_prio = (
+            self.settings.getpriority("THROTTLING_SCOPE_CONCURRENCY") or 0
+        )
+        if concurrency_prio >= SETTINGS_PRIORITIES["spider"]:
+            warnings.warn(
+                "The 'max_concurrent_requests' spider attribute is deprecated. "
+                "It is also being ignored because THROTTLING_SCOPE_CONCURRENCY is "
+                "already set at spider or higher priority. Remove the "
+                "'max_concurrent_requests' attribute from your spider.",
+                category=ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
+        else:
+            warnings.warn(
+                "The 'max_concurrent_requests' spider attribute is deprecated. Use "
+                "the THROTTLING_SCOPE_CONCURRENCY setting or per-domain "
+                "THROTTLING_SCOPES instead.",
+                category=ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
+            self.settings.set(
+                "THROTTLING_SCOPE_CONCURRENCY",
+                spider.max_concurrent_requests,
+                priority="spider",
+            )
+
     def _apply_reactorless_default_settings(self) -> None:
         """Change some setting defaults when not using a Twisted reactor.
 

scrapy/pqueues.py

@@ -342,11 +342,6 @@ class DownloaderAwarePriorityQueue:
         *,
         start_queue_cls: type[QueueProtocol] | None = None,
     ):
-        if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0:
-            raise ValueError(
-                f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP'
-            )
-
         if slot_startprios and not isinstance(slot_startprios, dict):
             raise ValueError(
                 "DownloaderAwarePriorityQueue accepts "

scrapy/settings/__init__.py

@@ -149,15 +149,6 @@ class BaseSettings(MutableMapping[str, Any]):
         :param default: the value to return if no setting is found
         :type default: object
         """
-        if name == "CONCURRENT_REQUESTS_PER_IP" and (
-            isinstance(self[name], int) and self[name] != 0
-        ):
-            warnings.warn(
-                "The CONCURRENT_REQUESTS_PER_IP setting is deprecated, use CONCURRENT_REQUESTS_PER_DOMAIN instead.",
-                ScrapyDeprecationWarning,
-                stacklevel=2,
-            )
-
         if name == "DNS_RESOLVER":
             warnings.warn(
                 "The DNS_RESOLVER setting is deprecated, please use "

scrapy/settings/default_settings.py

@@ -16,7 +16,6 @@ Scrapy developers, if you add a setting here remember to:
 import sys
 from importlib import import_module
 from pathlib import Path
-from typing import Any
 
 __all__ = [
     "ADDONS",
@@ -635,19 +634,3 @@ URLLENGTH_LIMIT = 2083
 USER_AGENT = f"Scrapy/{import_module('scrapy').__version__} (+https://scrapy.org)"
 
 WARN_ON_GENERATOR_RETURN_VALUE = True
-
-
-def __getattr__(name: str) -> Any:
-    if name == "CONCURRENT_REQUESTS_PER_IP":
-        import warnings  # noqa: PLC0415
-
-        from scrapy.exceptions import ScrapyDeprecationWarning  # noqa: PLC0415
-
-        warnings.warn(
-            "The scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_IP attribute is deprecated, use scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_DOMAIN instead.",
-            ScrapyDeprecationWarning,
-            stacklevel=2,
-        )
-        return 0
-
-    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")