diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index 1f358f789..ecb7f7a40 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -278,7 +278,8 @@ to wait between requests. If :setting:`ROBOTSTXT_OBEY` and :setting:`THROTTLING_ROBOTSTXT_OBEY` are ``True`` (default), valid ``Crawl-Delay`` directives override :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`DOWNLOAD_DELAY`. -Concurrency is set to ``1`` and delay is set to the value of ``Crawl-Delay``, capped at +Concurrency is set to ``1`` and the delay is raised to at least the +``Crawl-Delay`` value (a larger configured delay is kept), capped at :setting:`THROTTLING_ROBOTSTXT_MAX_DELAY` (default: ``60.0``). If :setting:`THROTTLING_SCOPES` defines a different concurrency or delay, it @@ -897,6 +898,13 @@ request, resolved from the DNS cache, as a second scope, limited to that many concurrent requests. IP scopes whose ``concurrency`` is left unset default to :setting:`CONCURRENT_REQUESTS_PER_IP`. +.. note:: Enabling :setting:`CONCURRENT_REQUESTS_PER_IP` requires the + :ref:`throttling-aware scheduler `. The default + :class:`~scrapy.pqueues.DownloaderAwarePriorityQueue` does not support it + and raises an error at start up if it is set, so switch + :setting:`SCHEDULER` and :setting:`SCHEDULER_PRIORITY_QUEUE` as shown + there. + For finer control (e.g. resolving IPs that are not in the DNS cache yet, or grouping several hosts under one address), implement a :ref:`throttling manager ` that adds the request's IP as a second scope diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 50c2165eb..6941e35cc 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -12,7 +12,6 @@ from scrapy import Request, Spider, signals from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.resolver import dnscache from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.defer import _defer_sleep_async, deferred_from_coro from scrapy.utils.deprecate import create_deprecated_class @@ -181,7 +180,6 @@ class Downloader: self._transferring: set[Request] = set() self.handlers: DownloadHandlers = DownloadHandlers(crawler) self.total_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS") - self.ip_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS_PER_IP") self.middleware: DownloaderMiddlewareManager = ( DownloaderMiddlewareManager.from_crawler(crawler) ) @@ -253,10 +251,10 @@ class Downloader: return self.get_slot_key(request) def get_slot_key(self, request: Request) -> str: - key = urlparse_cached(request).netloc or "" - if self.ip_concurrency: - key = dnscache.get(key, key) - return key + # Per-IP grouping (CONCURRENT_REQUESTS_PER_IP) is now a throttling scope + # handled by the throttler, not a downloader slot key; this fallback (used + # only when no throttler is set) keys by domain alone. + return urlparse_cached(request).netloc or "" async def _enqueue_request(self, request: Request) -> Response: key = self._get_slot_key(request) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 4394f7407..211b0b256 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -339,7 +339,10 @@ class DownloaderAwarePriorityQueue: ): if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0: raise ValueError( - f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP' + f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP. ' + "Set SCHEDULER_PRIORITY_QUEUE to " + "'scrapy.pqueues.ThrottlingAwarePriorityQueue' (along with the " + "matching SCHEDULER) to use per-IP concurrency limiting." ) if slot_startprios and not isinstance(slot_startprios, dict): diff --git a/scrapy/throttling.py b/scrapy/throttling.py index 737343b35..0cfea5d6d 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -1095,7 +1095,7 @@ class ThrottlingScopeManagerProtocol(Protocol): }, "rampup": { "backoff_target": 1, - "delay_factor": 0.8, + "delay_factor": 0.5, "min_delay": 0.05, }, }