diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index cace1f883..21096bdfb 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -47,8 +47,8 @@ Increase concurrency ==================== Concurrency is the number of requests that are processed in parallel. There is -a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional limit that -can be set per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`). +a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional per-scope +(per-domain by default) limit (:setting:`THROTTLING_SCOPE_CONCURRENCY`). The default global concurrency limit in Scrapy is not suitable for crawling many different domains in parallel, so you will want to increase it. How much diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index f6e4a2100..accf6a590 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -26,15 +26,16 @@ throttling limits, as do ``toscrape.com`` and ``books.toscrape.com``. The main throttling :ref:`settings ` are: -- .. setting:: CONCURRENT_REQUESTS_PER_DOMAIN +- .. setting:: THROTTLING_SCOPE_CONCURRENCY - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` (default: ``1`` - (:ref:`fallback `: ``8``)) + :setting:`THROTTLING_SCOPE_CONCURRENCY` (default: ``1``) - Maximum number of simultaneous requests per domain. + Default maximum number of simultaneous requests per :ref:`throttling scope + `. Requests are grouped by domain by default, so this is + the maximum number of simultaneous requests per domain. - It defines a number of “slots” per domain. Each slot can send 1 request at - a time: it sends a request, waits for the response, then sends the next + It defines a number of “slots” per scope. Each slot can send 1 request at a + time: it sends a request, waits for the response, then sends the next request, and so on. - .. setting:: DOWNLOAD_DELAY @@ -50,7 +51,7 @@ The main throttling :ref:`settings ` are: When configuring these settings, note that: -- :setting:`CONCURRENT_REQUESTS` caps :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. +- :setting:`CONCURRENT_REQUESTS` caps :setting:`THROTTLING_SCOPE_CONCURRENCY`. - If ``DOWNLOAD_DELAY`` ≥ response time, concurrency is effectively ``1``, because the next request to the domain is not sent until the delay elapses, @@ -296,7 +297,7 @@ to wait between requests. If :setting:`ROBOTSTXT_OBEY` and :setting:`THROTTLING_ROBOTSTXT_OBEY` are ``True`` (default), valid ``Crawl-Delay`` directives override -:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`DOWNLOAD_DELAY`. +:setting:`THROTTLING_SCOPE_CONCURRENCY` and :setting:`DOWNLOAD_DELAY`. Concurrency is set to ``1`` and the delay is raised to at least the ``Crawl-Delay`` value (a larger configured delay is kept), capped at :setting:`THROTTLING_ROBOTSTXT_MAX_DELAY` (default: ``60.0``). @@ -465,10 +466,8 @@ Its keys are scope names and its values are following keys: ``concurrency`` (:class:`int`) - Maximum number of concurrent requests for the scope. When unset, the - default depends on the kind of scope: :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` - for domain scopes, :setting:`CONCURRENT_REQUESTS_PER_IP` for IP scopes, and - :setting:`THROTTLING_SCOPE_CONCURRENCY` for any other scope. + Maximum number of concurrent requests for the scope. Defaults to + :setting:`THROTTLING_SCOPE_CONCURRENCY`. ``delay`` (:class:`float`) Minimum seconds between requests for the scope. Defaults to @@ -921,46 +920,30 @@ window (:setting:`THROTTLING_WINDOW`). You can use :ref:`throttling quotas Per-IP concurrency limiting --------------------------- -.. setting:: CONCURRENT_REQUESTS_PER_IP +A concurrency limit keyed by IP is just a throttling scope whose id is the +request's IP, with a ``concurrency`` limit. A request then carries two scopes, +its domain and its IP, and is only sent when **both** allow it (see +:ref:`multiple-throttling-scopes`). -A concurrency limit keyed by IP is a throttling scope whose id is the request's -IP. A request then carries two scopes, its domain and its IP, and is only sent -when **both** allow it (see :ref:`multiple-throttling-scopes`). +- Implement a :ref:`throttling manager ` that adds + the request's IP as a second scope: -Set :setting:`CONCURRENT_REQUESTS_PER_IP` (default: ``0``, disabled) to a -positive number to enable this built in: the throttler adds the IP of each -request, resolved from the DNS cache, as a second scope, limited to that many -concurrent requests. IP scopes whose ``concurrency`` is left unset default to -:setting:`CONCURRENT_REQUESTS_PER_IP`. + .. code-block:: python -.. note:: Enabling :setting:`CONCURRENT_REQUESTS_PER_IP` requires the - :ref:`throttling-aware scheduler `. The default - :class:`~scrapy.pqueues.DownloaderAwarePriorityQueue` does not support it - and raises an error at start up if it is set, so switch - :setting:`SCHEDULER` and :setting:`SCHEDULER_PRIORITY_QUEUE` as shown - there. + import socket -For finer control (e.g. resolving IPs that are not in the DNS cache yet, or -grouping several hosts under one address), implement a :ref:`throttling manager -` that adds the request's IP as a second scope -instead: - -.. code-block:: python - - import socket - - from scrapy.throttling import ThrottlingManager, add_scope, scope_cache - from scrapy.utils.asyncio import run_in_thread - from scrapy.utils.httpobj import urlparse_cached + from scrapy.throttling import ThrottlingManager, add_scope, scope_cache + from scrapy.utils.asyncio import run_in_thread + from scrapy.utils.httpobj import urlparse_cached - class IPThrottlingManager(ThrottlingManager): - @scope_cache - async def get_scopes(self, request): - scopes = await super().get_scopes(request) - host = urlparse_cached(request).hostname - address = await run_in_thread(socket.gethostbyname, host) - return add_scope(scopes, address) + class IPThrottlingManager(ThrottlingManager): + @scope_cache + async def get_scopes(self, request): + scopes = await super().get_scopes(request) + host = urlparse_cached(request).hostname + address = await run_in_thread(socket.gethostbyname, host) + return add_scope(scopes, address) .. _throttling-settings: @@ -1048,26 +1031,6 @@ Additional settings Whether to log :ref:`throttling ` decisions (per-scope delays, backoff steps and recoveries) for debugging. -- .. setting:: THROTTLING_SCOPE_CONCURRENCY - - :setting:`THROTTLING_SCOPE_CONCURRENCY` (default: ``1``) - - Default maximum number of concurrent requests for :ref:`throttling scopes - ` that are neither domains nor IPs, e.g. custom scopes - added by a :ref:`custom throttling manager `. - - A scope counts as a domain only if it looks like a hostname with at least - one dot, so single-label hosts such as ``localhost`` (or intranet host - names) fall in this category and default to - :setting:`THROTTLING_SCOPE_CONCURRENCY` rather than - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. Raise it, or give the host an - explicit ``concurrency`` in :setting:`THROTTLING_SCOPES`, if you need more - concurrency against such hosts (e.g. a local development server). - - Domain and IP scopes use :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and - :setting:`CONCURRENT_REQUESTS_PER_IP` instead. A scope ``concurrency`` set - in :setting:`THROTTLING_SCOPES` overrides this. - - .. setting:: THROTTLING_SCOPE_LIMIT :setting:`THROTTLING_SCOPE_LIMIT` (default: ``100000``) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 6941e35cc..517be0fcd 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -251,9 +251,7 @@ class Downloader: return self.get_slot_key(request) def get_slot_key(self, request: Request) -> str: - # Per-IP grouping (CONCURRENT_REQUESTS_PER_IP) is now a throttling scope - # handled by the throttler, not a downloader slot key; this fallback (used - # only when no throttler is set) keys by domain alone. + # This fallback (used only when no throttler is set) keys by domain. return urlparse_cached(request).netloc or "" async def _enqueue_request(self, request: Request) -> Response: diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 30b8d73e0..49422b0bf 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -93,11 +93,10 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler): self._pool: HTTPConnectionPool = HTTPConnectionPool(reactor, persistent=True) # Keep enough persistent connections per host to match the highest - # per-host concurrency the throttler may admit: the per-domain and - # per-IP limits, the default "other"-scope limit, and any explicit - # THROTTLING_SCOPES concurrency. Per-scope concurrency can grow beyond - # this at runtime (rampup); the excess simply uses non-persistent - # connections. + # per-host concurrency the throttler may admit: the per-domain limit, + # the default "other"-scope limit, and any explicit THROTTLING_SCOPES + # concurrency. Per-scope concurrency can grow beyond this at runtime + # (rampup); the excess simply uses non-persistent connections. scope_concurrencies = [ scope["concurrency"] for scope in crawler.settings.getdict("THROTTLING_SCOPES").values() @@ -106,7 +105,6 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler): self._pool.maxPersistentPerHost = max( [ crawler.settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN"), - crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP"), crawler.settings.getint("THROTTLING_SCOPE_CONCURRENCY"), *scope_concurrencies, ] diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 8c11db2f4..75bec1a46 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -344,10 +344,7 @@ class DownloaderAwarePriorityQueue: ): if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0: raise ValueError( - f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP. ' - "Set SCHEDULER_PRIORITY_QUEUE to " - "'scrapy.pqueues.ThrottlingAwarePriorityQueue' (along with the " - "matching SCHEDULER) to use per-IP concurrency limiting." + f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP' ) if slot_startprios and not isinstance(slot_startprios, dict): diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index cdf8b2f1b..932463ba9 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -149,6 +149,15 @@ class BaseSettings(MutableMapping[str, Any]): :param default: the value to return if no setting is found :type default: object """ + if name == "CONCURRENT_REQUESTS_PER_IP" and ( + isinstance(self[name], int) and self[name] != 0 + ): + warnings.warn( + "The CONCURRENT_REQUESTS_PER_IP setting is deprecated, use CONCURRENT_REQUESTS_PER_DOMAIN instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + if name == "DNS_RESOLVER": warnings.warn( "The DNS_RESOLVER setting is deprecated, please use " diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index a363f2ec0..4dba40fe0 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -16,6 +16,7 @@ Scrapy developers, if you add a setting here remember to: import sys from importlib import import_module from pathlib import Path +from typing import Any __all__ = [ "ADDONS", @@ -51,7 +52,6 @@ __all__ = [ "CONCURRENT_ITEMS", "CONCURRENT_REQUESTS", "CONCURRENT_REQUESTS_PER_DOMAIN", - "CONCURRENT_REQUESTS_PER_IP", "COOKIES_DEBUG", "COOKIES_ENABLED", "CRAWLSPIDER_FOLLOW_LINKS", @@ -285,7 +285,6 @@ CONCURRENT_ITEMS = 100 CONCURRENT_REQUESTS = 16 CONCURRENT_REQUESTS_PER_DOMAIN = 8 -CONCURRENT_REQUESTS_PER_IP = 0 COOKIES_ENABLED = True COOKIES_DEBUG = False @@ -636,3 +635,19 @@ URLLENGTH_LIMIT = 2083 USER_AGENT = f"Scrapy/{import_module('scrapy').__version__} (+https://scrapy.org)" WARN_ON_GENERATOR_RETURN_VALUE = True + + +def __getattr__(name: str) -> Any: + if name == "CONCURRENT_REQUESTS_PER_IP": + import warnings # noqa: PLC0415 + + from scrapy.exceptions import ScrapyDeprecationWarning # noqa: PLC0415 + + warnings.warn( + "The scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_IP attribute is deprecated, use scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_DOMAIN instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return 0 + + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 592fa5619..a035cc9ed 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -13,7 +13,7 @@ NEWSPIDER_MODULE = "$project_name.spiders" ROBOTSTXT_OBEY = True # Throttle crawls to be polite to websites: -CONCURRENT_REQUESTS_PER_DOMAIN = 1 +THROTTLING_SCOPE_CONCURRENCY = 1 DOWNLOAD_DELAY = 1 # Crawl the tutorial websites faster, overriding the polite defaults above only diff --git a/scrapy/throttling.py b/scrapy/throttling.py index 356571a05..cc38a0d47 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -1,10 +1,8 @@ from __future__ import annotations import contextlib -import ipaddress import logging import random -import re import time import warnings from collections import OrderedDict @@ -18,7 +16,7 @@ from typing_extensions import Self from scrapy import signals from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.resolver import dnscache +from scrapy.settings import SETTINGS_PRIORITIES from scrapy.utils.asyncio import sleep, wait_for_first from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import build_from_crawler, load_object @@ -134,59 +132,67 @@ def iter_scope_values(scopes: RequestScopes) -> Iterable[tuple[ScopeID, float | yield scope, None -# A DNS hostname with at least one dot, e.g. "books.toscrape.com"; labels are -# alphanumeric (plus hyphens, not at the edges) and an optional trailing dot -# marks a fully-qualified name. -_HOSTNAME_RE = re.compile( - r"(?i)^(?!-)[a-z0-9-]{1,63}(? int: + """Return the default concurrency of a throttling scope that does not set + its own ``concurrency``. - -def _classify_scope(scope_id: ScopeID) -> str: - """Classify *scope_id* as ``"ip"``, ``"domain"`` or ``"other"`` to pick its - default concurrency setting (see :func:`_default_scope_concurrency`). - - A scope that parses as an IP address is ``"ip"``; one that *looks like* a - DNS hostname with at least one dot is ``"domain"``; anything else (custom - group names, single labels like ``"localhost"``) is ``"other"``. + This is :setting:`THROTTLING_SCOPE_CONCURRENCY`, except that the deprecated + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` setting is bridged in when set at a + higher :ref:`priority `. When neither is set + explicitly, the historical :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` value is + kept for backward compatibility (its default flips to + :setting:`THROTTLING_SCOPE_CONCURRENCY` in a future version; see + :func:`_warn_on_deprecated_concurrency`). """ - host = scope_id - if host.startswith("[") and "]" in host: # bracketed IPv6, e.g. "[::1]:80" - host = host[1 : host.index("]")] - with contextlib.suppress(ValueError): - ipaddress.ip_address(host) - return "ip" - # Drop a trailing ":port" for the hostname check (an IPv4 "host:port" is - # re-tested as an IP below). - if host.count(":") == 1: - host = host.rsplit(":", 1)[0] - with contextlib.suppress(ValueError): - ipaddress.ip_address(host) - return "ip" - if "." in host and _HOSTNAME_RE.match(host): - return "domain" - return "other" - - -def _default_scope_concurrency(settings: BaseSettings, scope_id: ScopeID) -> int: - """Return the default concurrency for *scope_id* based on its - :func:`kind <_classify_scope>`. - - Domain scopes default to :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, IP - scopes to :setting:`CONCURRENT_REQUESTS_PER_IP` (falling back to the - per-domain value when per-IP limiting is off, so IP-literal crawls are not - left unbounded), and any other scope to :setting:`THROTTLING_SCOPE_CONCURRENCY`. - """ - kind = _classify_scope(scope_id) - if kind == "ip": - return settings.getint("CONCURRENT_REQUESTS_PER_IP") or settings.getint( - "CONCURRENT_REQUESTS_PER_DOMAIN" - ) - if kind == "domain": + default_priority = SETTINGS_PRIORITIES["default"] + # An unset setting has no priority (``None``); treat it as below "default" + # so it never wins over one that is at least at its default value. + domain_priority = settings.getpriority("CONCURRENT_REQUESTS_PER_DOMAIN") + domain_priority = ( + default_priority - 1 if domain_priority is None else domain_priority + ) + scope_priority = settings.getpriority("THROTTLING_SCOPE_CONCURRENCY") + scope_priority = default_priority - 1 if scope_priority is None else scope_priority + if domain_priority > scope_priority: + return settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN") + if scope_priority > domain_priority: + return settings.getint("THROTTLING_SCOPE_CONCURRENCY") + # Equal priority: on an explicit (higher-than-default) tie the new setting + # wins; when neither is set (both at "default") keep the historical + # per-domain value so existing behavior is preserved. + if domain_priority <= default_priority: return settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN") return settings.getint("THROTTLING_SCOPE_CONCURRENCY") +def _warn_on_deprecated_concurrency(settings: BaseSettings) -> None: + """Warn about the concurrency settings bridged by + :func:`_default_scope_concurrency`. Call once per crawl (see + :meth:`ThrottlingManager.__init__`).""" + default_priority = SETTINGS_PRIORITIES["default"] + domain_priority = settings.getpriority("CONCURRENT_REQUESTS_PER_DOMAIN") + scope_priority = settings.getpriority("THROTTLING_SCOPE_CONCURRENCY") + domain_set = domain_priority is not None and domain_priority > default_priority + scope_set = scope_priority is not None and scope_priority > default_priority + if domain_set: + warnings.warn( + "The CONCURRENT_REQUESTS_PER_DOMAIN setting is deprecated, use " + "THROTTLING_SCOPE_CONCURRENCY instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + elif not scope_set: + current = settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN") + future = settings.getint("THROTTLING_SCOPE_CONCURRENCY") + warnings.warn( + f"The default value of THROTTLING_SCOPE_CONCURRENCY will change " + f"from {current} to {future} in a future Scrapy version. Set " + f"THROTTLING_SCOPE_CONCURRENCY explicitly to silence this warning.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + + def _to_scope_dict(collection: Any, default: Callable[[], Any]) -> dict[ScopeID, Any]: """Normalize *collection* (``None``, str, iterable or dict) into a dict mapping scope names to values produced by *default*.""" @@ -496,10 +502,8 @@ class ThrottlingManager: def __init__(self, crawler: Crawler) -> None: self.crawler = crawler + _warn_on_deprecated_concurrency(crawler.settings) self._debug = crawler.settings.getbool("THROTTLING_DEBUG") - # When set, each request also gets an IP scope (its resolved address), - # enforced alongside its domain scope (see _resolve_scopes_sync). - self._per_ip: int = crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") self._max_idle = crawler.settings.getfloat("THROTTLING_SCOPE_MAX_IDLE") self._robotstxt_obey = crawler.settings.getbool( "ROBOTSTXT_OBEY" @@ -588,15 +592,7 @@ class ThrottlingManager: stacklevel=2, ) return cast("RequestScopes", download_slot) - netloc = urlparse_cached(request).netloc - if self._per_ip: - # Best-effort, mirroring the legacy per-IP downloader slots: use the - # cached resolved address if available, else fall back to the domain - # scope alone (the IP is not known until the host is resolved). - ip = dnscache.get(netloc) - if isinstance(ip, str) and ip != netloc: - return (netloc, ip) - return netloc + return urlparse_cached(request).netloc def get_slot_key(self, request: Request) -> str: scopes = self._resolve_scopes_sync(request) @@ -1196,7 +1192,7 @@ class ThrottlingScopeManager: # Rampup starts conservative at a single slot and probes upward. self._concurrency = 1 else: - self._concurrency = _default_scope_concurrency(settings, self._id) or None + self._concurrency = _default_scope_concurrency(settings) or None # Used as the load denominator when the scope enforces no explicit # concurrency limit (see get_load()). self._global_concurrency: int = settings.getint("CONCURRENT_REQUESTS") diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 38bd8c124..ee289b4e9 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -66,13 +66,16 @@ def get_crawler( will be used to populate the crawler settings with a project level priority. """ - # When needed, useful settings can be added here, e.g. ones that prevent - # deprecation warnings (see prevent_warnings). settings: dict[str, Any] = { "TELNETCONSOLE_ENABLED": False, **get_reactor_settings(), **(settings_dict or {}), } + if prevent_warnings: + # Pin the pre-deprecation default per-scope concurrency so the suite + # keeps its historical behavior and does not emit the warn-then-flip + # deprecation warning (see throttling._warn_on_deprecated_concurrency). + settings.setdefault("THROTTLING_SCOPE_CONCURRENCY", 8) runner: CrawlerRunnerBase if is_reactor_installed(): runner = CrawlerRunner(settings) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index b249c8916..d68d63392 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -5,6 +5,7 @@ import logging import re import signal import threading +import warnings from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar from unittest.mock import MagicMock @@ -52,6 +53,77 @@ def get_raw_crawler( return Crawler(spidercls or DefaultSpider, settings) +class TestScopeConcurrencyBridge: + """CONCURRENT_REQUESTS_PER_DOMAIN is deprecated and bridged into + THROTTLING_SCOPE_CONCURRENCY by setting priority (see + scrapy.throttling._default_scope_concurrency).""" + + @staticmethod + def _resolve(settings: Settings) -> int: + from scrapy.throttling import _default_scope_concurrency # noqa: PLC0415 + + return _default_scope_concurrency(settings) + + def test_neither_set_preserves_per_domain_default(self) -> None: + # Neither setting overridden: keep the historical per-domain default. + assert self._resolve(Settings()) == 8 + + def test_per_domain_overrides(self) -> None: + settings = Settings() + settings.set("CONCURRENT_REQUESTS_PER_DOMAIN", 4, priority="project") + assert self._resolve(settings) == 4 + + def test_scope_concurrency_overrides(self) -> None: + settings = Settings() + settings.set("THROTTLING_SCOPE_CONCURRENCY", 4, priority="project") + assert self._resolve(settings) == 4 + + def test_new_setting_wins_on_non_default_tie(self) -> None: + settings = Settings() + settings.set("CONCURRENT_REQUESTS_PER_DOMAIN", 4, priority="project") + settings.set("THROTTLING_SCOPE_CONCURRENCY", 9, priority="project") + assert self._resolve(settings) == 9 + + def test_higher_priority_deprecated_setting_wins(self) -> None: + settings = Settings() + settings.set("THROTTLING_SCOPE_CONCURRENCY", 9, priority="project") + settings.set("CONCURRENT_REQUESTS_PER_DOMAIN", 4, priority="cmdline") + assert self._resolve(settings) == 4 + + def test_warns_flip_when_neither_set(self) -> None: + with pytest.warns( + ScrapyDeprecationWarning, + match="THROTTLING_SCOPE_CONCURRENCY will change", + ): + get_crawler(prevent_warnings=False) + + def test_warns_deprecated_when_per_domain_set(self) -> None: + with pytest.warns( + ScrapyDeprecationWarning, + match="CONCURRENT_REQUESTS_PER_DOMAIN setting is deprecated", + ): + get_crawler( + settings_dict={"CONCURRENT_REQUESTS_PER_DOMAIN": 4}, + prevent_warnings=False, + ) + + def test_no_concurrency_warning_when_scope_set(self) -> None: + with warnings.catch_warnings(record=True) as recorded: + warnings.simplefilter("always") + get_crawler( + settings_dict={"THROTTLING_SCOPE_CONCURRENCY": 4}, + prevent_warnings=False, + ) + messages = [str(w.message) for w in recorded] + assert not any( + "THROTTLING_SCOPE_CONCURRENCY will change" in m for m in messages + ) + assert not any( + "CONCURRENT_REQUESTS_PER_DOMAIN setting is deprecated" in m + for m in messages + ) + + class TestBaseCrawler: @staticmethod def assertOptionIsDefault(settings: Settings, key: str) -> None: diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index ab8a20c14..3282b0591 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -769,6 +769,15 @@ def test_deprecated_dns_resolver_setting(): settings.get("DNS_RESOLVER") +def test_deprecated_concurrent_requests_per_ip_setting(): + settings = Settings({"CONCURRENT_REQUESTS_PER_IP": 1}) + with pytest.warns( + ScrapyDeprecationWarning, + match="The CONCURRENT_REQUESTS_PER_IP setting is deprecated", + ): + settings.get("CONCURRENT_REQUESTS_PER_IP") + + class Component1: pass