WIP

docs/topics/throttling.rst

@@ -278,7 +278,8 @@ to wait between requests.
 If :setting:`ROBOTSTXT_OBEY` and :setting:`THROTTLING_ROBOTSTXT_OBEY` are
 ``True`` (default), valid ``Crawl-Delay`` directives override
 :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`DOWNLOAD_DELAY`.
-Concurrency is set to ``1`` and delay is set to the value of ``Crawl-Delay``, capped at
+Concurrency is set to ``1`` and the delay is raised to at least the
+``Crawl-Delay`` value (a larger configured delay is kept), capped at
 :setting:`THROTTLING_ROBOTSTXT_MAX_DELAY` (default: ``60.0``).
 
 If :setting:`THROTTLING_SCOPES` defines a different concurrency or delay, it
@@ -897,6 +898,13 @@ request, resolved from the DNS cache, as a second scope, limited to that many
 concurrent requests. IP scopes whose ``concurrency`` is left unset default to
 :setting:`CONCURRENT_REQUESTS_PER_IP`.
 
+.. note:: Enabling :setting:`CONCURRENT_REQUESTS_PER_IP` requires the
+    :ref:`throttling-aware scheduler <throttling-aware-scheduler>`. The default
+    :class:`~scrapy.pqueues.DownloaderAwarePriorityQueue` does not support it
+    and raises an error at start up if it is set, so switch
+    :setting:`SCHEDULER` and :setting:`SCHEDULER_PRIORITY_QUEUE` as shown
+    there.
+
 For finer control (e.g. resolving IPs that are not in the DNS cache yet, or
 grouping several hosts under one address), implement a :ref:`throttling manager
 <custom-throttling-scopes>` that adds the request's IP as a second scope

scrapy/core/downloader/__init__.py

@@ -12,7 +12,6 @@ from scrapy import Request, Spider, signals
 from scrapy.core.downloader.handlers import DownloadHandlers
 from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
 from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.resolver import dnscache
 from scrapy.utils.decorators import _warn_spider_arg
 from scrapy.utils.defer import _defer_sleep_async, deferred_from_coro
 from scrapy.utils.deprecate import create_deprecated_class
@@ -181,7 +180,6 @@ class Downloader:
         self._transferring: set[Request] = set()
         self.handlers: DownloadHandlers = DownloadHandlers(crawler)
         self.total_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS")
-        self.ip_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS_PER_IP")
         self.middleware: DownloaderMiddlewareManager = (
             DownloaderMiddlewareManager.from_crawler(crawler)
         )
@@ -253,10 +251,10 @@ class Downloader:
         return self.get_slot_key(request)
 
     def get_slot_key(self, request: Request) -> str:
-        key = urlparse_cached(request).netloc or ""
-        if self.ip_concurrency:
-            key = dnscache.get(key, key)
-        return key
+        # Per-IP grouping (CONCURRENT_REQUESTS_PER_IP) is now a throttling scope
+        # handled by the throttler, not a downloader slot key; this fallback (used
+        # only when no throttler is set) keys by domain alone.
+        return urlparse_cached(request).netloc or ""
 
     async def _enqueue_request(self, request: Request) -> Response:
         key = self._get_slot_key(request)

scrapy/pqueues.py

@@ -339,7 +339,10 @@ class DownloaderAwarePriorityQueue:
     ):
         if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0:
             raise ValueError(
-                f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP'
+                f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP. '
+                "Set SCHEDULER_PRIORITY_QUEUE to "
+                "'scrapy.pqueues.ThrottlingAwarePriorityQueue' (along with the "
+                "matching SCHEDULER) to use per-IP concurrency limiting."
             )
 
         if slot_startprios and not isinstance(slot_startprios, dict):

scrapy/throttling.py

@@ -1095,7 +1095,7 @@ class ThrottlingScopeManagerProtocol(Protocol):
             },
             "rampup": {
                 "backoff_target": 1,
-                "delay_factor": 0.8,
+                "delay_factor": 0.5,
                 "min_delay": 0.05,
             },
         }