From dabdc7550e06f4547249104721817835f553d064 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Mon, 30 Jun 2025 08:51:05 -0300 Subject: [PATCH] Deprecate CONCURRENT_REQUESTS_PER_IP setting (#6921) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Deprecate CONCURRENT_REQUESTS_PER_IP setting * Typing fix * PR review * tests * oops * Update scrapy/settings/default_settings.py Co-authored-by: Adrián Chaves * update logic --------- Co-authored-by: Adrián Chaves --- docs/topics/autothrottle.rst | 11 +- docs/topics/broad-crawls.rst | 7 +- docs/topics/settings.rst | 23 +--- scrapy/settings/__init__.py | 11 ++ scrapy/settings/default_settings.py | 191 +++++++++++++++++++++++++++- tests/test_scheduler.py | 11 +- tests/test_scrapy__getattr__.py | 14 ++ tests/test_settings/__init__.py | 12 ++ 8 files changed, 239 insertions(+), 41 deletions(-) diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index 5bd72fa15..d0321c906 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -37,8 +37,7 @@ processed in parallel. Instead of adjusting the delays one can just set a small fixed download delay and impose hard limits on concurrency using -:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or -:setting:`CONCURRENT_REQUESTS_PER_IP` options. It will provide a similar +:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. It will provide a similar effect, but there are some important differences: * because the download delay is small there will be occasional bursts @@ -71,7 +70,6 @@ AutoThrottle algorithm adjusts download delays based on the following rules: .. note:: The AutoThrottle extension honours the standard Scrapy settings for concurrency and delay. This means that it will respect :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and - :setting:`CONCURRENT_REQUESTS_PER_IP` options and never set a download delay lower than :setting:`DOWNLOAD_DELAY`. .. _download-latency: @@ -123,7 +121,6 @@ The settings used to control the AutoThrottle extension are: * :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` * :setting:`AUTOTHROTTLE_DEBUG` * :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` -* :setting:`CONCURRENT_REQUESTS_PER_IP` * :setting:`DOWNLOAD_DELAY` For more information see :ref:`autothrottle-algorithm`. @@ -171,12 +168,10 @@ a higher value (e.g. ``2.0``) to increase the throughput and the load on remote servers. A lower ``AUTOTHROTTLE_TARGET_CONCURRENCY`` value (e.g. ``0.5``) makes the crawler more conservative and polite. -Note that :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` -and :setting:`CONCURRENT_REQUESTS_PER_IP` options are still respected +Note that :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` is still respected when AutoThrottle extension is enabled. This means that if ``AUTOTHROTTLE_TARGET_CONCURRENCY`` is set to a value higher than -:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or -:setting:`CONCURRENT_REQUESTS_PER_IP`, the crawler won't reach this number +:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, the crawler won't reach this number of concurrent requests. At every given time point Scrapy can be sending more or less concurrent diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 248e38b61..ecde3da43 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -61,12 +61,7 @@ Increase concurrency Concurrency is the number of requests that are processed in parallel. There is a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional limit that -can be set either per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`) or per -IP (:setting:`CONCURRENT_REQUESTS_PER_IP`). - -.. note:: The scheduler priority queue :ref:`recommended for broad crawls - ` does not support - :setting:`CONCURRENT_REQUESTS_PER_IP`. +can be set per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`). The default global concurrency limit in Scrapy is not suitable for crawling many different domains in parallel, so you will want to increase it. How much diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 01921e922..db65fb993 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -526,23 +526,6 @@ See also: :ref:`topics-autothrottle` and its :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` option. -.. setting:: CONCURRENT_REQUESTS_PER_IP - -CONCURRENT_REQUESTS_PER_IP --------------------------- - -Default: ``0`` - -The maximum number of concurrent (i.e. simultaneous) requests that will be -performed to any single IP. If non-zero, the -:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` setting is ignored, and this one is -used instead. In other words, concurrency limits will be applied per IP, not -per domain. - -This setting also affects :setting:`DOWNLOAD_DELAY` and -:ref:`topics-autothrottle`: if :setting:`CONCURRENT_REQUESTS_PER_IP` -is non-zero, download delay is enforced per IP, not per domain. - .. setting:: DEFAULT_DROPITEM_LOG_LEVEL DEFAULT_DROPITEM_LOG_LEVEL @@ -884,9 +867,6 @@ every 10 seconds:: This setting is also affected by the :setting:`RANDOMIZE_DOWNLOAD_DELAY` setting, which is enabled by default. -When :setting:`CONCURRENT_REQUESTS_PER_IP` is non-zero, delays are enforced -per IP address instead of per domain. - Note that :setting:`DOWNLOAD_DELAY` can lower the effective per-domain concurrency below :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. If the response time of a domain is lower than :setting:`DOWNLOAD_DELAY`, the effective @@ -1765,8 +1745,7 @@ Type of priority queue used by the scheduler. Another available type is ``scrapy.pqueues.DownloaderAwarePriorityQueue``. ``scrapy.pqueues.DownloaderAwarePriorityQueue`` works better than ``scrapy.pqueues.ScrapyPriorityQueue`` when you crawl many different -domains in parallel. But currently ``scrapy.pqueues.DownloaderAwarePriorityQueue`` -does not work together with :setting:`CONCURRENT_REQUESTS_PER_IP`. +domains in parallel. .. setting:: SCHEDULER_START_DISK_QUEUE diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index cc4853c8f..9363da492 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -2,11 +2,13 @@ from __future__ import annotations import copy import json +import warnings from collections.abc import Iterable, Iterator, Mapping, MutableMapping from importlib import import_module from pprint import pformat from typing import TYPE_CHECKING, Any, Union, cast +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import default_settings from scrapy.utils.misc import load_object @@ -147,6 +149,15 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): :param default: the value to return if no setting is found :type default: object """ + if name == "CONCURRENT_REQUESTS_PER_IP" and ( + isinstance(self[name], int) and self[name] != 0 + ): + warnings.warn( + "The CONCURRENT_REQUESTS_PER_IP setting is deprecated, use CONCURRENT_REQUESTS_PER_DOMAIN instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return self[name] if self[name] is not None else default def getbool(self, name: _SettingsKeyT, default: bool = False) -> bool: diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index b6f47f1c3..50c0088e7 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -17,6 +17,180 @@ import sys from importlib import import_module from pathlib import Path +__all__ = [ + "ADDONS", + "AJAXCRAWL_ENABLED", + "AJAXCRAWL_MAXSIZE", + "ASYNCIO_EVENT_LOOP", + "AUTOTHROTTLE_DEBUG", + "AUTOTHROTTLE_ENABLED", + "AUTOTHROTTLE_MAX_DELAY", + "AUTOTHROTTLE_START_DELAY", + "AUTOTHROTTLE_TARGET_CONCURRENCY", + "BOT_NAME", + "CLOSESPIDER_ERRORCOUNT", + "CLOSESPIDER_ITEMCOUNT", + "CLOSESPIDER_PAGECOUNT", + "CLOSESPIDER_TIMEOUT", + "COMMANDS_MODULE", + "COMPRESSION_ENABLED", + "CONCURRENT_ITEMS", + "CONCURRENT_REQUESTS", + "CONCURRENT_REQUESTS_PER_DOMAIN", + "COOKIES_DEBUG", + "COOKIES_ENABLED", + "CRAWLSPIDER_FOLLOW_LINKS", + "DEFAULT_DROPITEM_LOG_LEVEL", + "DEFAULT_ITEM_CLASS", + "DEFAULT_REQUEST_HEADERS", + "DEPTH_LIMIT", + "DEPTH_PRIORITY", + "DEPTH_STATS_VERBOSE", + "DNSCACHE_ENABLED", + "DNSCACHE_SIZE", + "DNS_RESOLVER", + "DNS_TIMEOUT", + "DOWNLOADER", + "DOWNLOADER_CLIENTCONTEXTFACTORY", + "DOWNLOADER_CLIENT_TLS_CIPHERS", + "DOWNLOADER_CLIENT_TLS_METHOD", + "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING", + "DOWNLOADER_HTTPCLIENTFACTORY", + "DOWNLOADER_MIDDLEWARES", + "DOWNLOADER_MIDDLEWARES_BASE", + "DOWNLOADER_STATS", + "DOWNLOAD_DELAY", + "DOWNLOAD_FAIL_ON_DATALOSS", + "DOWNLOAD_HANDLERS", + "DOWNLOAD_HANDLERS_BASE", + "DOWNLOAD_MAXSIZE", + "DOWNLOAD_TIMEOUT", + "DOWNLOAD_WARNSIZE", + "DUPEFILTER_CLASS", + "EDITOR", + "EXTENSIONS", + "EXTENSIONS_BASE", + "FEEDS", + "FEED_EXPORTERS", + "FEED_EXPORTERS_BASE", + "FEED_EXPORT_BATCH_ITEM_COUNT", + "FEED_EXPORT_ENCODING", + "FEED_EXPORT_FIELDS", + "FEED_EXPORT_INDENT", + "FEED_FORMAT", + "FEED_STORAGES", + "FEED_STORAGES_BASE", + "FEED_STORAGE_FTP_ACTIVE", + "FEED_STORAGE_GCS_ACL", + "FEED_STORAGE_S3_ACL", + "FEED_STORE_EMPTY", + "FEED_TEMPDIR", + "FEED_URI_PARAMS", + "FILES_STORE_GCS_ACL", + "FILES_STORE_S3_ACL", + "FORCE_CRAWLER_PROCESS", + "FTP_PASSIVE_MODE", + "FTP_PASSWORD", + "FTP_USER", + "GCS_PROJECT_ID", + "HTTPCACHE_ALWAYS_STORE", + "HTTPCACHE_DBM_MODULE", + "HTTPCACHE_DIR", + "HTTPCACHE_ENABLED", + "HTTPCACHE_EXPIRATION_SECS", + "HTTPCACHE_GZIP", + "HTTPCACHE_IGNORE_HTTP_CODES", + "HTTPCACHE_IGNORE_MISSING", + "HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS", + "HTTPCACHE_IGNORE_SCHEMES", + "HTTPCACHE_POLICY", + "HTTPCACHE_STORAGE", + "HTTPPROXY_AUTH_ENCODING", + "HTTPPROXY_ENABLED", + "IMAGES_STORE_GCS_ACL", + "IMAGES_STORE_S3_ACL", + "ITEM_PIPELINES", + "ITEM_PIPELINES_BASE", + "ITEM_PROCESSOR", + "JOBDIR", + "LOGSTATS_INTERVAL", + "LOG_DATEFORMAT", + "LOG_ENABLED", + "LOG_ENCODING", + "LOG_FILE", + "LOG_FILE_APPEND", + "LOG_FORMAT", + "LOG_FORMATTER", + "LOG_LEVEL", + "LOG_SHORT_NAMES", + "LOG_STDOUT", + "LOG_VERSIONS", + "MAIL_FROM", + "MAIL_HOST", + "MAIL_PASS", + "MAIL_PORT", + "MAIL_USER", + "MEMDEBUG_ENABLED", + "MEMDEBUG_NOTIFY", + "MEMUSAGE_CHECK_INTERVAL_SECONDS", + "MEMUSAGE_ENABLED", + "MEMUSAGE_LIMIT_MB", + "MEMUSAGE_NOTIFY_MAIL", + "MEMUSAGE_WARNING_MB", + "METAREFRESH_ENABLED", + "METAREFRESH_IGNORE_TAGS", + "METAREFRESH_MAXDELAY", + "NEWSPIDER_MODULE", + "PERIODIC_LOG_DELTA", + "PERIODIC_LOG_STATS", + "PERIODIC_LOG_TIMING_ENABLED", + "RANDOMIZE_DOWNLOAD_DELAY", + "REACTOR_THREADPOOL_MAXSIZE", + "REDIRECT_ENABLED", + "REDIRECT_MAX_TIMES", + "REDIRECT_PRIORITY_ADJUST", + "REFERER_ENABLED", + "REFERRER_POLICY", + "REQUEST_FINGERPRINTER_CLASS", + "REQUEST_FINGERPRINTER_IMPLEMENTATION", + "RETRY_ENABLED", + "RETRY_EXCEPTIONS", + "RETRY_HTTP_CODES", + "RETRY_PRIORITY_ADJUST", + "RETRY_TIMES", + "ROBOTSTXT_OBEY", + "ROBOTSTXT_PARSER", + "ROBOTSTXT_USER_AGENT", + "SCHEDULER", + "SCHEDULER_DEBUG", + "SCHEDULER_DISK_QUEUE", + "SCHEDULER_MEMORY_QUEUE", + "SCHEDULER_PRIORITY_QUEUE", + "SCHEDULER_START_DISK_QUEUE", + "SCHEDULER_START_MEMORY_QUEUE", + "SCRAPER_SLOT_MAX_ACTIVE_SIZE", + "SPIDER_CONTRACTS", + "SPIDER_CONTRACTS_BASE", + "SPIDER_LOADER_CLASS", + "SPIDER_LOADER_WARN_ONLY", + "SPIDER_MIDDLEWARES", + "SPIDER_MIDDLEWARES_BASE", + "SPIDER_MODULES", + "STATSMAILER_RCPTS", + "STATS_CLASS", + "STATS_DUMP", + "TELNETCONSOLE_ENABLED", + "TELNETCONSOLE_HOST", + "TELNETCONSOLE_PASSWORD", + "TELNETCONSOLE_PORT", + "TELNETCONSOLE_USERNAME", + "TEMPLATES_DIR", + "TWISTED_REACTOR", + "URLLENGTH_LIMIT", + "USER_AGENT", + "WARN_ON_GENERATOR_RETURN_VALUE", +] + ADDONS = {} AJAXCRAWL_ENABLED = False @@ -45,7 +219,6 @@ CONCURRENT_ITEMS = 100 CONCURRENT_REQUESTS = 16 CONCURRENT_REQUESTS_PER_DOMAIN = 8 -CONCURRENT_REQUESTS_PER_IP = 0 COOKIES_ENABLED = True COOKIES_DEBUG = False @@ -358,3 +531,19 @@ URLLENGTH_LIMIT = 2083 USER_AGENT = f"Scrapy/{import_module('scrapy').__version__} (+https://scrapy.org)" WARN_ON_GENERATOR_RETURN_VALUE = True + + +def __getattr__(name: str): + if name == "CONCURRENT_REQUESTS_PER_IP": + import warnings + + from scrapy.exceptions import ScrapyDeprecationWarning + + warnings.warn( + "The scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_IP attribute is deprecated, use scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_DOMAIN instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return 0 + + raise AttributeError diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 456c8537f..d31ba48e5 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -2,6 +2,7 @@ from __future__ import annotations import shutil import tempfile +import warnings from abc import ABC, abstractmethod from collections import deque from typing import Any, NamedTuple @@ -389,7 +390,9 @@ class TestIncompatibility: scheduler.open(spider) def test_incompatibility(self): - with pytest.raises( - ValueError, match="does not support CONCURRENT_REQUESTS_PER_IP" - ): - self._incompatible() + with warnings.catch_warnings(): + warnings.filterwarnings("ignore") + with pytest.raises( + ValueError, match="does not support CONCURRENT_REQUESTS_PER_IP" + ): + self._incompatible() diff --git a/tests/test_scrapy__getattr__.py b/tests/test_scrapy__getattr__.py index 443e26a3c..b1956ee5c 100644 --- a/tests/test_scrapy__getattr__.py +++ b/tests/test_scrapy__getattr__.py @@ -11,3 +11,17 @@ def test_deprecated_twisted_version(): "The scrapy.twisted_version attribute is deprecated, use twisted.version instead" in warns[0].message.args ) + + +def test_deprecated_concurrent_requests_per_ip_attribute(): + with warnings.catch_warnings(record=True) as warns: + from scrapy.settings.default_settings import ( + CONCURRENT_REQUESTS_PER_IP, + ) + + assert CONCURRENT_REQUESTS_PER_IP is not None + assert isinstance(CONCURRENT_REQUESTS_PER_IP, int) + assert ( + "The scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_IP attribute is deprecated, use scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_DOMAIN instead." + in warns[0].message.args + ) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index d7d900546..85dc0679e 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -1,6 +1,7 @@ # pylint: disable=unsubscriptable-object,unsupported-membership-test,use-implicit-booleaness-not-comparison # (too many false positives) +import warnings from unittest import mock import pytest @@ -559,6 +560,17 @@ def test_remove_from_list(before, name, item, after): assert settings.getpriority(name) == expected_settings.getpriority(name) +def test_deprecated_concurrent_requests_per_ip_setting(): + with warnings.catch_warnings(record=True) as warns: + settings = Settings({"CONCURRENT_REQUESTS_PER_IP": 1}) + settings.get("CONCURRENT_REQUESTS_PER_IP") + + assert ( + str(warns[0].message) + == "The CONCURRENT_REQUESTS_PER_IP setting is deprecated, use CONCURRENT_REQUESTS_PER_DOMAIN instead." + ) + + class Component1: pass