From e0cc7ed7db3c93bb49c431a539824f83f75c59c3 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Tue, 30 Jun 2026 11:33:37 +0200 Subject: [PATCH] WIP --- docs/topics/broad-crawls.rst | 2 +- docs/topics/throttling.rst | 13 +- scrapy/core/scheduler.py | 5 +- scrapy/crawler.py | 16 --- scrapy/extensions/throttle.py | 73 ++++++------ scrapy/settings/__init__.py | 25 ---- scrapy/settings/default_settings.py | 21 +--- .../templates/project/module/settings.py.tmpl | 2 +- scrapy/throttling.py | 111 ++++++++++++++++-- tests/test_crawler.py | 35 +----- tests/test_settings/__init__.py | 9 -- tests/test_throttling.py | 6 +- 12 files changed, 162 insertions(+), 156 deletions(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index cace1f883..81a4679ba 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -48,7 +48,7 @@ Increase concurrency Concurrency is the number of requests that are processed in parallel. There is a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional limit that -can be set per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`). +can be set per domain (:setting:`THROTTLING_SCOPE_CONCURRENCY`). The default global concurrency limit in Scrapy is not suitable for crawling many different domains in parallel, so you will want to increase it. How much diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index 07b0fa0a5..fec88d265 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -26,9 +26,9 @@ throttling limits, as do ``toscrape.com`` and ``books.toscrape.com``. The main throttling :ref:`settings ` are: -- .. setting:: CONCURRENT_REQUESTS_PER_DOMAIN +- .. setting:: THROTTLING_SCOPE_CONCURRENCY - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` (default: ``1`` (:ref:`fallback `: ``8``)) + :setting:`THROTTLING_SCOPE_CONCURRENCY` (default: ``1`` (:ref:`fallback `: ``8``)) Maximum number of simultaneous requests per domain. @@ -49,7 +49,7 @@ The main throttling :ref:`settings ` are: When configuring these settings, note that: -- :setting:`CONCURRENT_REQUESTS` caps ``CONCURRENT_REQUESTS_PER_DOMAIN``. +- :setting:`CONCURRENT_REQUESTS` caps :setting:`THROTTLING_SCOPE_CONCURRENCY`. - If ``DOWNLOAD_DELAY`` ≥ response time, concurrency is effectively ``1``, because the next request to the domain is not sent until the delay elapses, @@ -277,7 +277,7 @@ to wait between requests. If :setting:`ROBOTSTXT_OBEY` and :setting:`THROTTLING_ROBOTSTXT_OBEY` are ``True`` (default), valid ``Crawl-Delay`` directives override -:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`DOWNLOAD_DELAY`. Concurrency +:setting:`THROTTLING_SCOPE_CONCURRENCY` and :setting:`DOWNLOAD_DELAY`. Concurrency is set to ``1`` and delay is set to the value of ``Crawl-Delay``, capped at :setting:`THROTTLING_ROBOTSTXT_MAX_DELAY` (default: ``60.0``). @@ -440,9 +440,8 @@ Its keys are scope names and its values are following keys: ``concurrency`` (:class:`int`) - Maximum number of concurrent requests for the scope. When unset, the - per-domain concurrency (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`) applies - instead. + Maximum number of concurrent requests for the scope. When unset, + :setting:`THROTTLING_SCOPE_CONCURRENCY` applies instead. ``min_concurrency`` (:class:`int`) Concurrency floor that :ref:`backoff ` and :ref:`rampup ` diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 90a892dc3..f233f8c84 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -211,9 +211,8 @@ class Scheduler(BaseScheduler): ------------------------- While pending requests are below the configured values of - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` - or :setting:`CONCURRENT_REQUESTS_PER_IP`, those requests are sent - concurrently. + :setting:`CONCURRENT_REQUESTS` or :setting:`THROTTLING_SCOPE_CONCURRENCY`, + those requests are sent concurrently. As a result, the first few requests of a crawl may not follow the desired order. Lowering those settings to ``1`` enforces the desired order except diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 492877e7a..28bb2bf83 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -98,7 +98,6 @@ class Crawler: return self.addons.load_settings(self.settings) - self._warn_on_deprecated_default_settings() self._apply_spider_download_delay() self.stats = load_object(self.settings["STATS_CLASS"])(self) @@ -184,21 +183,6 @@ class Crawler: "DOWNLOAD_DELAY", spider.download_delay, priority="spider" ) - def _warn_on_deprecated_default_settings(self) -> None: - default_priority = SETTINGS_PRIORITIES["default"] - for setting_name, current_default, future_default in ( - ("THROTTLING_SCOPE_CONCURRENCY", 8, 1), - ): - if self.settings.getpriority(setting_name) == default_priority: - warnings.warn( - f"The default value of {setting_name} will change from " - f"{current_default!r} to {future_default!r} in a future " - f"Scrapy version. Explicitly set {setting_name} in your " - f"settings to silence this warning.", - category=ScrapyDeprecationWarning, - stacklevel=3, - ) - def _apply_reactorless_default_settings(self) -> None: """Change some setting defaults when not using a Twisted reactor. diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 42e58cbc9..d3203f335 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -6,14 +6,15 @@ from warnings import warn from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self - from scrapy.core.downloader import Slot from scrapy.crawler import Crawler from scrapy.http import Response + from scrapy.throttling import ThrottlingManagerProtocol logger = logging.getLogger(__name__) @@ -43,6 +44,8 @@ class AutoThrottle: f"AUTOTHROTTLE_TARGET_CONCURRENCY " f"({self.target_concurrency!r}) must be higher than 0." ) + # Scopes whose start delay has already been applied (see _scope_delay). + self._started_scopes: set[str] = set() crawler.signals.connect(self._spider_opened, signal=signals.spider_opened) crawler.signals.connect( self._response_downloaded, signal=signals.response_downloaded @@ -55,7 +58,9 @@ class AutoThrottle: def _spider_opened(self, spider: Spider) -> None: self.mindelay = self._min_delay(spider) self.maxdelay = self._max_delay(spider) - spider.download_delay = self._start_delay(spider) # type: ignore[attr-defined] + self.startdelay = max( + self.mindelay, self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY") + ) def _min_delay(self, spider: Spider) -> float: s = self.crawler.settings @@ -64,55 +69,55 @@ class AutoThrottle: def _max_delay(self, spider: Spider) -> float: return self.crawler.settings.getfloat("AUTOTHROTTLE_MAX_DELAY") - def _start_delay(self, spider: Spider) -> float: - return max( - self.mindelay, self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY") - ) - def _response_downloaded( self, response: Response, request: Request, spider: Spider ) -> None: - key, slot = self._get_slot(request, spider) + throttler = self.crawler.throttler latency = request.meta.get("download_latency") if ( latency is None - or slot is None + or throttler is None or request.meta.get("autothrottle_dont_adjust_delay", False) is True ): return - olddelay = slot.delay - self._adjust_delay(slot, latency, response) + # AutoThrottle predates throttling scopes, so it adjusts the delay of + # the request's domain scope, matching its historical per-domain slots. + scope_id = urlparse_cached(request).netloc + olddelay = self._scope_delay(throttler, scope_id) + newdelay = self._adjust_delay(olddelay, latency, response) + throttler.set_scope_delay(scope_id, newdelay) if self.debug: - diff = slot.delay - olddelay - size = len(response.body) - conc = len(slot.transferring) logger.info( - "slot: %(slot)s | conc:%(concurrency)2d | " + "slot: %(slot)s | " "delay:%(delay)5d ms (%(delaydiff)+d) | " "latency:%(latency)5d ms | size:%(size)6d bytes", { - "slot": key, - "concurrency": conc, - "delay": slot.delay * 1000, - "delaydiff": diff * 1000, + "slot": scope_id, + "delay": newdelay * 1000, + "delaydiff": (newdelay - olddelay) * 1000, "latency": latency * 1000, - "size": size, + "size": len(response.body), }, extra={"spider": spider}, ) - def _get_slot( - self, request: Request, spider: Spider - ) -> tuple[str | None, Slot | None]: - key: str | None = request.meta.get("download_slot") - if key is None: - return None, None - assert self.crawler.engine - return key, self.crawler.engine.downloader.slots.get(key) + def _scope_delay( + self, throttler: ThrottlingManagerProtocol, scope_id: str + ) -> float: + """Return the current delay of *scope_id*, applying AUTOTHROTTLE_START_DELAY + the first time the scope is seen.""" + delay = throttler.get_scope_delay(scope_id) + if scope_id not in self._started_scopes: + self._started_scopes.add(scope_id) + delay = max(delay, self.startdelay) + return delay - def _adjust_delay(self, slot: Slot, latency: float, response: Response) -> None: - """Define delay adjustment policy""" + def _adjust_delay( + self, olddelay: float, latency: float, response: Response + ) -> float: + """Return the new delay given the current *olddelay* and the observed + *latency*.""" # If a server needs `latency` seconds to respond then # we should send a request each `latency/N` seconds @@ -120,7 +125,7 @@ class AutoThrottle: target_delay = latency / self.target_concurrency # Adjust the delay to make it closer to target_delay - new_delay = (slot.delay + target_delay) / 2.0 + new_delay = (olddelay + target_delay) / 2.0 # If target delay is bigger than old delay, then use it instead of mean. # It works better with problematic sites. @@ -133,7 +138,7 @@ class AutoThrottle: # than old one, as error pages (and redirections) are usually small and # so tend to reduce latency, thus provoking a positive feedback by # reducing delay instead of increase. - if response.status != 200 and new_delay <= slot.delay: - return + if response.status != 200 and new_delay <= olddelay: + return olddelay - slot.delay = new_delay + return new_delay diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 833c71f44..0ee6c83dd 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -149,15 +149,6 @@ class BaseSettings(MutableMapping[str, Any]): :param default: the value to return if no setting is found :type default: object """ - if name == "CONCURRENT_REQUESTS_PER_IP" and ( - isinstance(self[name], int) and self[name] != 0 - ): - warnings.warn( - "The CONCURRENT_REQUESTS_PER_IP setting is deprecated, use CONCURRENT_REQUESTS_PER_DOMAIN instead.", - ScrapyDeprecationWarning, - stacklevel=2, - ) - if name == "DNS_RESOLVER": warnings.warn( "The DNS_RESOLVER setting is deprecated, please use " @@ -166,22 +157,6 @@ class BaseSettings(MutableMapping[str, Any]): stacklevel=2, ) - if name == "THROTTLING_SCOPE_CONCURRENCY": - per_domain_prio = self.getpriority("CONCURRENT_REQUESTS_PER_DOMAIN") or 0 - new_prio = self.getpriority(name) or 0 - if ( - per_domain_prio > SETTINGS_PRIORITIES["default"] - and new_prio <= SETTINGS_PRIORITIES["default"] - ): - warnings.warn( - "The CONCURRENT_REQUESTS_PER_DOMAIN setting is deprecated, use " - "THROTTLING_SCOPE_CONCURRENCY instead.", - ScrapyDeprecationWarning, - stacklevel=2, - ) - per_domain_val = self["CONCURRENT_REQUESTS_PER_DOMAIN"] - return per_domain_val if per_domain_val is not None else default - return self[name] if self[name] is not None else default def getbool(self, name: str, default: bool = False) -> bool: diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 6bfd73777..6d8cdf154 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -16,7 +16,6 @@ Scrapy developers, if you add a setting here remember to: import sys from importlib import import_module from pathlib import Path -from typing import Any __all__ = [ "ADDONS", @@ -45,6 +44,7 @@ __all__ = [ "CONCURRENT_ITEMS", "CONCURRENT_REQUESTS", "CONCURRENT_REQUESTS_PER_DOMAIN", + "CONCURRENT_REQUESTS_PER_IP", "COOKIES_DEBUG", "COOKIES_ENABLED", "CRAWLSPIDER_FOLLOW_LINKS", @@ -265,6 +265,7 @@ CONCURRENT_ITEMS = 100 CONCURRENT_REQUESTS = 16 CONCURRENT_REQUESTS_PER_DOMAIN = 8 +CONCURRENT_REQUESTS_PER_IP = 0 COOKIES_ENABLED = True COOKIES_DEBUG = False @@ -597,7 +598,7 @@ THROTTLING_SCOPES = {} THROTTLING_WINDOW = 60.0 THROTTLING_ROBOTSTXT_OBEY = True THROTTLING_ROBOTSTXT_MAX_DELAY = 60.0 -THROTTLING_SCOPE_CONCURRENCY = 8 +THROTTLING_SCOPE_CONCURRENCY = 1 THROTTLING_SCOPE_LIMIT = 100000 THROTTLING_SCOPE_MAX_IDLE = 3600.0 THROTTLING_DEBUG = False @@ -613,19 +614,3 @@ URLLENGTH_LIMIT = 2083 USER_AGENT = f"Scrapy/{import_module('scrapy').__version__} (+https://scrapy.org)" WARN_ON_GENERATOR_RETURN_VALUE = True - - -def __getattr__(name: str) -> Any: - if name == "CONCURRENT_REQUESTS_PER_IP": - import warnings # noqa: PLC0415 - - from scrapy.exceptions import ScrapyDeprecationWarning # noqa: PLC0415 - - warnings.warn( - "The scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_IP attribute is deprecated, use scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_DOMAIN instead.", - ScrapyDeprecationWarning, - stacklevel=2, - ) - return 0 - - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 64b27db68..71c7596f5 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -13,7 +13,7 @@ NEWSPIDER_MODULE = "$project_name.spiders" ROBOTSTXT_OBEY = True # Throttle crawls to be polite to websites: -CONCURRENT_REQUESTS_PER_DOMAIN = 1 +THROTTLING_SCOPE_CONCURRENCY = 1 DOWNLOAD_DELAY = 1 # Set settings whose default value is deprecated to a future-proof value: diff --git a/scrapy/throttling.py b/scrapy/throttling.py index 040e15a28..fe69fe6b5 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -2,8 +2,10 @@ from __future__ import annotations import contextlib import datetime as dt +import ipaddress import logging import random +import re import time import warnings from collections import OrderedDict @@ -18,6 +20,7 @@ from typing_extensions import NotRequired, Self from scrapy import signals from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.resolver import dnscache from scrapy.utils.asyncio import sleep, wait_for_first from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import build_from_crawler, load_object @@ -148,6 +151,59 @@ def iter_scope_values(scopes: RequestScopes) -> Iterable[tuple[ScopeID, float | yield scope, None +# A DNS hostname with at least one dot, e.g. "books.toscrape.com"; labels are +# alphanumeric (plus hyphens, not at the edges) and an optional trailing dot +# marks a fully-qualified name. +_HOSTNAME_RE = re.compile( + r"(?i)^(?!-)[a-z0-9-]{1,63}(? str: + """Classify *scope_id* as ``"ip"``, ``"domain"`` or ``"other"`` to pick its + default concurrency setting (see :func:`_default_scope_concurrency`). + + A scope that parses as an IP address is ``"ip"``; one that *looks like* a + DNS hostname with at least one dot is ``"domain"``; anything else (custom + group names, single labels like ``"localhost"``) is ``"other"``. + """ + host = scope_id + if host.startswith("[") and "]" in host: # bracketed IPv6, e.g. "[::1]:80" + host = host[1 : host.index("]")] + with contextlib.suppress(ValueError): + ipaddress.ip_address(host) + return "ip" + # Drop a trailing ":port" for the hostname check (an IPv4 "host:port" is + # re-tested as an IP below). + if host.count(":") == 1: + host = host.rsplit(":", 1)[0] + with contextlib.suppress(ValueError): + ipaddress.ip_address(host) + return "ip" + if "." in host and _HOSTNAME_RE.match(host): + return "domain" + return "other" + + +def _default_scope_concurrency(settings: Any, scope_id: ScopeID) -> int: + """Return the default concurrency for *scope_id* based on its + :func:`kind <_classify_scope>`. + + Domain scopes default to :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, IP + scopes to :setting:`CONCURRENT_REQUESTS_PER_IP` (falling back to the + per-domain value when per-IP limiting is off, so IP-literal crawls are not + left unbounded), and any other scope to :setting:`THROTTLING_SCOPE_CONCURRENCY`. + """ + kind = _classify_scope(scope_id) + if kind == "ip": + return settings.getint("CONCURRENT_REQUESTS_PER_IP") or settings.getint( + "CONCURRENT_REQUESTS_PER_DOMAIN" + ) + if kind == "domain": + return settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN") + return settings.getint("THROTTLING_SCOPE_CONCURRENCY") + + def _to_scope_dict(collection: Any, default: Callable[[], Any]) -> dict[ScopeID, Any]: """Normalize *collection* (``None``, str, iterable or dict) into a dict mapping scope names to values produced by *default*.""" @@ -398,6 +454,17 @@ class ThrottlingManagerProtocol(Protocol): whereas a ``delay_scope`` call is trusted. """ + def get_scope_delay(self, scope_id: str) -> float: + """Return the current base (non-backoff) delay of *scope_id*, in seconds.""" + + def set_scope_delay(self, scope_id: str, delay: float) -> None: + """Set the base (non-backoff) delay of *scope_id* to *delay* seconds. + + Unlike :meth:`delay_scope`, this both raises and lowers the delay and is + not counted as backoff; it lets a component drive the scope delay + directly (e.g. an adaptive-delay extension). + """ + async def process_response(self, response: Response) -> None: """Update the throttling state based on *response*.""" @@ -480,6 +547,9 @@ class ThrottlingManager: load_object(cls) for cls in crawler.settings.getlist("BACKOFF_EXCEPTIONS") ) self._debug = crawler.settings.getbool("THROTTLING_DEBUG") + # When set, each request also gets an IP scope (its resolved address), + # enforced alongside its domain scope (see _resolve_scopes_sync). + self._per_ip: int = crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") self._max_idle = crawler.settings.getfloat("THROTTLING_SCOPE_MAX_IDLE") self._robotstxt_obey = crawler.settings.getbool( "ROBOTSTXT_OBEY" @@ -539,7 +609,15 @@ class ThrottlingManager: stacklevel=2, ) return download_slot - return urlparse_cached(request).netloc + netloc = urlparse_cached(request).netloc + if self._per_ip: + # Best-effort, mirroring the legacy per-IP downloader slots: use the + # cached resolved address if available, else fall back to the domain + # scope alone (the IP is not known until the host is resolved). + ip = dnscache.get(netloc) + if isinstance(ip, str) and ip != netloc: + return (netloc, ip) + return netloc def get_slot_key(self, request: Request) -> str: scopes = self._resolve_scopes_sync(request) @@ -903,6 +981,14 @@ class ThrottlingManager: # delay; unlike those, it is trusted, so it bypasses BACKOFF_MAX_DELAY. self._get_scope_manager(scope_id).record_backoff(delay=float(delay), cap=False) + def get_scope_delay(self, scope_id: ScopeID) -> float: + return self._get_scope_manager(scope_id).get_base_delay() + + def set_scope_delay(self, scope_id: ScopeID, delay: float) -> None: + self._get_scope_manager(scope_id).set_base_delay( + float(delay), only_increase=False + ) + def _maybe_evict(self, now: float) -> None: if self._max_idle <= 0: return @@ -1002,11 +1088,15 @@ class ThrottlingScopeManagerProtocol(Protocol): reported for a request, correcting the estimate used by :meth:`record_sent`.""" - def set_base_delay(self, delay: float) -> None: - """Raise the base (non-backoff) delay of this scope to *delay* seconds. + def get_base_delay(self) -> float: + """Return the base (non-backoff) delay of this scope, in seconds.""" - It never lowers the configured base delay; it is used to honor external - hints such as a robots.txt ``Crawl-delay`` directive. + def set_base_delay(self, delay: float, *, only_increase: bool = True) -> None: + """Set the base (non-backoff) delay of this scope to *delay* seconds. + + By default it only raises the delay, to honor external hints such as a + robots.txt ``Crawl-delay`` directive. Pass ``only_increase=False`` to + also allow lowering it. """ def set_concurrency(self, concurrency: int) -> None: @@ -1146,7 +1236,7 @@ class ThrottlingScopeManager: elif self._rampup_enabled: self._concurrency = self._min_concurrency else: - self._concurrency = settings.getint("THROTTLING_SCOPE_CONCURRENCY") or None + self._concurrency = _default_scope_concurrency(settings, self._id) or None # Used as the load denominator when the scope enforces no explicit # concurrency limit (see get_load()). self._global_concurrency: int = settings.getint("CONCURRENT_REQUESTS") @@ -1374,10 +1464,15 @@ class ThrottlingScopeManager: elif consumed is not None: self._consumed = max(0.0, self._consumed + float(consumed)) - def set_base_delay(self, delay: float) -> None: - if delay <= self._base_delay: + def get_base_delay(self) -> float: + return self._base_delay + + def set_base_delay(self, delay: float, *, only_increase: bool = True) -> None: + if only_increase and delay <= self._base_delay: return self._base_delay = delay + # Reflect the change in the effective delay unless a backoff is raising + # it above the base right now. if self._backoff_level == 0: self._delay = delay diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 5588775cf..ada7e53f1 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -3,7 +3,6 @@ from __future__ import annotations import asyncio import logging import re -import warnings from pathlib import Path from typing import Any, ClassVar @@ -82,41 +81,11 @@ class TestCrawler(TestBaseCrawler): assert not settings.frozen assert crawler.settings.frozen - @pytest.mark.parametrize( - ("setting_name", "current_default", "future_default"), - [ - ("THROTTLING_SCOPE_CONCURRENCY", 8, 1), - ], - ) - def test_deprecated_default_settings_warn( - self, setting_name: str, current_default: Any, future_default: Any - ) -> None: - crawler = Crawler(DefaultSpider) - with pytest.warns( - ScrapyDeprecationWarning, - match=rf"The default value of {setting_name} will change from {current_default!r} to {future_default!r}", - ): - crawler._apply_settings() - - @pytest.mark.parametrize( - ("setting_name", "current_default"), - [ - ("THROTTLING_SCOPE_CONCURRENCY", 8), - ], - ) - def test_deprecated_default_settings_no_warn_when_set( - self, setting_name: str, current_default: int - ) -> None: - crawler = Crawler(DefaultSpider, {setting_name: current_default}) - with warnings.catch_warnings(): - warnings.simplefilter("error", ScrapyDeprecationWarning) - crawler._apply_settings() - def test_spider_download_delay_deprecated(self) -> None: class DelaySpider(DefaultSpider): download_delay = 2.5 - crawler = Crawler(DelaySpider, {"THROTTLING_SCOPE_CONCURRENCY": 8}) + crawler = Crawler(DelaySpider) with pytest.warns( ScrapyDeprecationWarning, match="'download_delay' spider attribute" ): @@ -127,7 +96,7 @@ class TestCrawler(TestBaseCrawler): class DelaySpider(DefaultSpider): download_delay = 2.5 - crawler = Crawler(DelaySpider, {"THROTTLING_SCOPE_CONCURRENCY": 8}) + crawler = Crawler(DelaySpider) crawler.settings.set("DOWNLOAD_DELAY", 5.0, priority="spider") with pytest.warns( ScrapyDeprecationWarning, diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 3282b0591..ab8a20c14 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -769,15 +769,6 @@ def test_deprecated_dns_resolver_setting(): settings.get("DNS_RESOLVER") -def test_deprecated_concurrent_requests_per_ip_setting(): - settings = Settings({"CONCURRENT_REQUESTS_PER_IP": 1}) - with pytest.warns( - ScrapyDeprecationWarning, - match="The CONCURRENT_REQUESTS_PER_IP setting is deprecated", - ): - settings.get("CONCURRENT_REQUESTS_PER_IP") - - class Component1: pass diff --git a/tests/test_throttling.py b/tests/test_throttling.py index ef07bc381..c9b28067f 100644 --- a/tests/test_throttling.py +++ b/tests/test_throttling.py @@ -575,7 +575,11 @@ class TestThrottlingScopeManager: assert scope._concurrency == 8 def test_no_scope_concurrency_limit_when_zero(self): - scope = _scope_manager(settings={"THROTTLING_SCOPE_CONCURRENCY": 0}) + # THROTTLING_SCOPE_CONCURRENCY governs scopes that are neither a domain + # nor an IP (here a bare "custom" group name). + scope = _scope_manager( + settings={"THROTTLING_SCOPE_CONCURRENCY": 0}, config={"id": "custom"} + ) assert scope._concurrency is None for _ in range(100): scope.record_sent(now=0.0)