RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

WIP

This commit is contained in:
Adrian Chaves 2026-06-30 11:33:37 +02:00
parent 80f161ef3e
commit e0cc7ed7db
12 changed files with 162 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/topics/broad-crawls.rst
View File

@ -48,7 +48,7 @@ Increase concurrency
Concurrency is the number of requests that are processed in parallel. There is
a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional limit that
can be set per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`).
can be set per domain (:setting:`THROTTLING_SCOPE_CONCURRENCY`).
The default global concurrency limit in Scrapy is not suitable for crawling
many different domains in parallel, so you will want to increase it. How much

13
docs/topics/throttling.rst
View File

@ -26,9 +26,9 @@ throttling limits, as do ``toscrape.com`` and ``books.toscrape.com``.
The main throttling :ref:`settings <topics-settings>` are:
- .. setting:: CONCURRENT_REQUESTS_PER_DOMAIN
- .. setting:: THROTTLING_SCOPE_CONCURRENCY
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` (default: ``1`` (:ref:`fallback <default-settings>`: ``8``))
:setting:`THROTTLING_SCOPE_CONCURRENCY` (default: ``1`` (:ref:`fallback <default-settings>`: ``8``))
Maximum number of simultaneous requests per domain.
@ -49,7 +49,7 @@ The main throttling :ref:`settings <topics-settings>` are:
When configuring these settings, note that:
- :setting:`CONCURRENT_REQUESTS` caps ``CONCURRENT_REQUESTS_PER_DOMAIN``.
- :setting:`CONCURRENT_REQUESTS` caps :setting:`THROTTLING_SCOPE_CONCURRENCY`.
- If ``DOWNLOAD_DELAY`` ≥ response time, concurrency is effectively ``1``,
because the next request to the domain is not sent until the delay elapses,
@ -277,7 +277,7 @@ to wait between requests.
If :setting:`ROBOTSTXT_OBEY` and :setting:`THROTTLING_ROBOTSTXT_OBEY` are
``True`` (default), valid ``Crawl-Delay`` directives override
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`DOWNLOAD_DELAY`. Concurrency
:setting:`THROTTLING_SCOPE_CONCURRENCY` and :setting:`DOWNLOAD_DELAY`. Concurrency
is set to ``1`` and delay is set to the value of ``Crawl-Delay``, capped at
:setting:`THROTTLING_ROBOTSTXT_MAX_DELAY` (default: ``60.0``).
@ -440,9 +440,8 @@ Its keys are scope names and its values are
following keys:
``concurrency`` (:class:`int`)
Maximum number of concurrent requests for the scope. When unset, the
per-domain concurrency (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`) applies
instead.
Maximum number of concurrent requests for the scope. When unset,
:setting:`THROTTLING_SCOPE_CONCURRENCY` applies instead.
``min_concurrency`` (:class:`int`)
Concurrency floor that :ref:`backoff <backoff>` and :ref:`rampup <rampup>`

5
scrapy/core/scheduler.py
View File

@ -211,9 +211,8 @@ class Scheduler(BaseScheduler):
-------------------------
While pending requests are below the configured values of
:setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`
or :setting:`CONCURRENT_REQUESTS_PER_IP`, those requests are sent
concurrently.
:setting:`CONCURRENT_REQUESTS` or :setting:`THROTTLING_SCOPE_CONCURRENCY`,
those requests are sent concurrently.
As a result, the first few requests of a crawl may not follow the desired
order. Lowering those settings to ``1`` enforces the desired order except

16
scrapy/crawler.py
View File

@ -98,7 +98,6 @@ class Crawler:
return
self.addons.load_settings(self.settings)
self._warn_on_deprecated_default_settings()
self._apply_spider_download_delay()
self.stats = load_object(self.settings["STATS_CLASS"])(self)
@ -184,21 +183,6 @@ class Crawler:
"DOWNLOAD_DELAY", spider.download_delay, priority="spider"
)
def _warn_on_deprecated_default_settings(self) -> None:
default_priority = SETTINGS_PRIORITIES["default"]
for setting_name, current_default, future_default in (
("THROTTLING_SCOPE_CONCURRENCY", 8, 1),
):
if self.settings.getpriority(setting_name) == default_priority:
warnings.warn(
f"The default value of {setting_name} will change from "
f"{current_default!r} to {future_default!r} in a future "
f"Scrapy version. Explicitly set {setting_name} in your "
f"settings to silence this warning.",
category=ScrapyDeprecationWarning,
stacklevel=3,
)
def _apply_reactorless_default_settings(self) -> None:
"""Change some setting defaults when not using a Twisted reactor.

73
scrapy/extensions/throttle.py
View File

@ -6,14 +6,15 @@ from warnings import warn
from scrapy import Request, Spider, signals
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.utils.httpobj import urlparse_cached
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.core.downloader import Slot
from scrapy.crawler import Crawler
from scrapy.http import Response
from scrapy.throttling import ThrottlingManagerProtocol
logger = logging.getLogger(__name__)
@ -43,6 +44,8 @@ class AutoThrottle:
f"AUTOTHROTTLE_TARGET_CONCURRENCY "
f"({self.target_concurrency!r}) must be higher than 0."
)
# Scopes whose start delay has already been applied (see _scope_delay).
self._started_scopes: set[str] = set()
crawler.signals.connect(self._spider_opened, signal=signals.spider_opened)
crawler.signals.connect(
self._response_downloaded, signal=signals.response_downloaded
@ -55,7 +58,9 @@ class AutoThrottle:
def _spider_opened(self, spider: Spider) -> None:
self.mindelay = self._min_delay(spider)
self.maxdelay = self._max_delay(spider)
spider.download_delay = self._start_delay(spider) # type: ignore[attr-defined]
self.startdelay = max(
self.mindelay, self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY")
)
def _min_delay(self, spider: Spider) -> float:
s = self.crawler.settings
@ -64,55 +69,55 @@ class AutoThrottle:
def _max_delay(self, spider: Spider) -> float:
return self.crawler.settings.getfloat("AUTOTHROTTLE_MAX_DELAY")
def _start_delay(self, spider: Spider) -> float:
return max(
self.mindelay, self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY")
)
def _response_downloaded(
self, response: Response, request: Request, spider: Spider
) -> None:
key, slot = self._get_slot(request, spider)
throttler = self.crawler.throttler
latency = request.meta.get("download_latency")
if (
latency is None
or slot is None
or throttler is None
or request.meta.get("autothrottle_dont_adjust_delay", False) is True
):
return
olddelay = slot.delay
self._adjust_delay(slot, latency, response)
# AutoThrottle predates throttling scopes, so it adjusts the delay of
# the request's domain scope, matching its historical per-domain slots.
scope_id = urlparse_cached(request).netloc
olddelay = self._scope_delay(throttler, scope_id)
newdelay = self._adjust_delay(olddelay, latency, response)
throttler.set_scope_delay(scope_id, newdelay)
if self.debug:
diff = slot.delay - olddelay
size = len(response.body)
conc = len(slot.transferring)
logger.info(
"slot: %(slot)s | conc:%(concurrency)2d | "
"slot: %(slot)s | "
"delay:%(delay)5d ms (%(delaydiff)+d) | "
"latency:%(latency)5d ms | size:%(size)6d bytes",
{
"slot": key,
"concurrency": conc,
"delay": slot.delay * 1000,
"delaydiff": diff * 1000,
"slot": scope_id,
"delay": newdelay * 1000,
"delaydiff": (newdelay - olddelay) * 1000,
"latency": latency * 1000,
"size": size,
"size": len(response.body),
},
extra={"spider": spider},
)
def _get_slot(
self, request: Request, spider: Spider
) -> tuple[str | None, Slot | None]:
key: str | None = request.meta.get("download_slot")
if key is None:
return None, None
assert self.crawler.engine
return key, self.crawler.engine.downloader.slots.get(key)
def _scope_delay(
self, throttler: ThrottlingManagerProtocol, scope_id: str
) -> float:
"""Return the current delay of *scope_id*, applying AUTOTHROTTLE_START_DELAY
the first time the scope is seen."""
delay = throttler.get_scope_delay(scope_id)
if scope_id not in self._started_scopes:
self._started_scopes.add(scope_id)
delay = max(delay, self.startdelay)
return delay
def _adjust_delay(self, slot: Slot, latency: float, response: Response) -> None:
"""Define delay adjustment policy"""
def _adjust_delay(
self, olddelay: float, latency: float, response: Response
) -> float:
"""Return the new delay given the current *olddelay* and the observed
*latency*."""
# If a server needs `latency` seconds to respond then
# we should send a request each `latency/N` seconds
@ -120,7 +125,7 @@ class AutoThrottle:
target_delay = latency / self.target_concurrency
# Adjust the delay to make it closer to target_delay
new_delay = (slot.delay + target_delay) / 2.0
new_delay = (olddelay + target_delay) / 2.0
# If target delay is bigger than old delay, then use it instead of mean.
# It works better with problematic sites.
@ -133,7 +138,7 @@ class AutoThrottle:
# than old one, as error pages (and redirections) are usually small and
# so tend to reduce latency, thus provoking a positive feedback by
# reducing delay instead of increase.
if response.status != 200 and new_delay <= slot.delay:
return
if response.status != 200 and new_delay <= olddelay:
return olddelay
slot.delay = new_delay
return new_delay

25
scrapy/settings/__init__.py
View File

@ -149,15 +149,6 @@ class BaseSettings(MutableMapping[str, Any]):
:param default: the value to return if no setting is found
:type default: object
"""
if name == "CONCURRENT_REQUESTS_PER_IP" and (
isinstance(self[name], int) and self[name] != 0
):
warnings.warn(
"The CONCURRENT_REQUESTS_PER_IP setting is deprecated, use CONCURRENT_REQUESTS_PER_DOMAIN instead.",
ScrapyDeprecationWarning,
stacklevel=2,
)
if name == "DNS_RESOLVER":
warnings.warn(
"The DNS_RESOLVER setting is deprecated, please use "
@ -166,22 +157,6 @@ class BaseSettings(MutableMapping[str, Any]):
stacklevel=2,
)
if name == "THROTTLING_SCOPE_CONCURRENCY":
per_domain_prio = self.getpriority("CONCURRENT_REQUESTS_PER_DOMAIN") or 0
new_prio = self.getpriority(name) or 0
if (
per_domain_prio > SETTINGS_PRIORITIES["default"]
and new_prio <= SETTINGS_PRIORITIES["default"]
):
warnings.warn(
"The CONCURRENT_REQUESTS_PER_DOMAIN setting is deprecated, use "
"THROTTLING_SCOPE_CONCURRENCY instead.",
ScrapyDeprecationWarning,
stacklevel=2,
)
per_domain_val = self["CONCURRENT_REQUESTS_PER_DOMAIN"]
return per_domain_val if per_domain_val is not None else default
return self[name] if self[name] is not None else default
def getbool(self, name: str, default: bool = False) -> bool:

21
scrapy/settings/default_settings.py
View File

@ -16,7 +16,6 @@ Scrapy developers, if you add a setting here remember to:
import sys
from importlib import import_module
from pathlib import Path
from typing import Any
__all__ = [
"ADDONS",
@ -45,6 +44,7 @@ __all__ = [
"CONCURRENT_ITEMS",
"CONCURRENT_REQUESTS",
"CONCURRENT_REQUESTS_PER_DOMAIN",
"CONCURRENT_REQUESTS_PER_IP",
"COOKIES_DEBUG",
"COOKIES_ENABLED",
"CRAWLSPIDER_FOLLOW_LINKS",
@ -265,6 +265,7 @@ CONCURRENT_ITEMS = 100
CONCURRENT_REQUESTS = 16
CONCURRENT_REQUESTS_PER_DOMAIN = 8
CONCURRENT_REQUESTS_PER_IP = 0
COOKIES_ENABLED = True
COOKIES_DEBUG = False
@ -597,7 +598,7 @@ THROTTLING_SCOPES = {}
THROTTLING_WINDOW = 60.0
THROTTLING_ROBOTSTXT_OBEY = True
THROTTLING_ROBOTSTXT_MAX_DELAY = 60.0
THROTTLING_SCOPE_CONCURRENCY = 8
THROTTLING_SCOPE_CONCURRENCY = 1
THROTTLING_SCOPE_LIMIT = 100000
THROTTLING_SCOPE_MAX_IDLE = 3600.0
THROTTLING_DEBUG = False
@ -613,19 +614,3 @@ URLLENGTH_LIMIT = 2083
USER_AGENT = f"Scrapy/{import_module('scrapy').__version__} (+https://scrapy.org)"
WARN_ON_GENERATOR_RETURN_VALUE = True
def __getattr__(name: str) -> Any:
if name == "CONCURRENT_REQUESTS_PER_IP":
import warnings # noqa: PLC0415
from scrapy.exceptions import ScrapyDeprecationWarning # noqa: PLC0415
warnings.warn(
"The scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_IP attribute is deprecated, use scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_DOMAIN instead.",
ScrapyDeprecationWarning,
stacklevel=2,
)
return 0
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

2
scrapy/templates/project/module/settings.py.tmpl
View File

@ -13,7 +13,7 @@ NEWSPIDER_MODULE = "$project_name.spiders"
ROBOTSTXT_OBEY = True
# Throttle crawls to be polite to websites:
CONCURRENT_REQUESTS_PER_DOMAIN = 1
THROTTLING_SCOPE_CONCURRENCY = 1
DOWNLOAD_DELAY = 1
# Set settings whose default value is deprecated to a future-proof value:

111
scrapy/throttling.py
View File

@ -2,8 +2,10 @@ from __future__ import annotations
import contextlib
import datetime as dt
import ipaddress
import logging
import random
import re
import time
import warnings
from collections import OrderedDict
@ -18,6 +20,7 @@ from typing_extensions import NotRequired, Self
from scrapy import signals
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.resolver import dnscache
from scrapy.utils.asyncio import sleep, wait_for_first
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import build_from_crawler, load_object
@ -148,6 +151,59 @@ def iter_scope_values(scopes: RequestScopes) -> Iterable[tuple[ScopeID, float |
yield scope, None
# A DNS hostname with at least one dot, e.g. "books.toscrape.com"; labels are
# alphanumeric (plus hyphens, not at the edges) and an optional trailing dot
# marks a fully-qualified name.
_HOSTNAME_RE = re.compile(
r"(?i)^(?!-)[a-z0-9-]{1,63}(?<!-)(\.(?!-)[a-z0-9-]{1,63}(?<!-))+\.?$"
)
def _classify_scope(scope_id: ScopeID) -> str:
"""Classify *scope_id* as ``"ip"``, ``"domain"`` or ``"other"`` to pick its
default concurrency setting (see :func:`_default_scope_concurrency`).
A scope that parses as an IP address is ``"ip"``; one that *looks like* a
DNS hostname with at least one dot is ``"domain"``; anything else (custom
group names, single labels like ``"localhost"``) is ``"other"``.
"""
host = scope_id
if host.startswith("[") and "]" in host: # bracketed IPv6, e.g. "[::1]:80"
host = host[1 : host.index("]")]
with contextlib.suppress(ValueError):
ipaddress.ip_address(host)
return "ip"
# Drop a trailing ":port" for the hostname check (an IPv4 "host:port" is
# re-tested as an IP below).
if host.count(":") == 1:
host = host.rsplit(":", 1)[0]
with contextlib.suppress(ValueError):
ipaddress.ip_address(host)
return "ip"
if "." in host and _HOSTNAME_RE.match(host):
return "domain"
return "other"
def _default_scope_concurrency(settings: Any, scope_id: ScopeID) -> int:
"""Return the default concurrency for *scope_id* based on its
:func:`kind <_classify_scope>`.
Domain scopes default to :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, IP
scopes to :setting:`CONCURRENT_REQUESTS_PER_IP` (falling back to the
per-domain value when per-IP limiting is off, so IP-literal crawls are not
left unbounded), and any other scope to :setting:`THROTTLING_SCOPE_CONCURRENCY`.
"""
kind = _classify_scope(scope_id)
if kind == "ip":
return settings.getint("CONCURRENT_REQUESTS_PER_IP") or settings.getint(
"CONCURRENT_REQUESTS_PER_DOMAIN"
)
if kind == "domain":
return settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN")
return settings.getint("THROTTLING_SCOPE_CONCURRENCY")
def _to_scope_dict(collection: Any, default: Callable[[], Any]) -> dict[ScopeID, Any]:
"""Normalize *collection* (``None``, str, iterable or dict) into a dict
mapping scope names to values produced by *default*."""
@ -398,6 +454,17 @@ class ThrottlingManagerProtocol(Protocol):
whereas a ``delay_scope`` call is trusted.
"""
def get_scope_delay(self, scope_id: str) -> float:
"""Return the current base (non-backoff) delay of *scope_id*, in seconds."""
def set_scope_delay(self, scope_id: str, delay: float) -> None:
"""Set the base (non-backoff) delay of *scope_id* to *delay* seconds.
Unlike :meth:`delay_scope`, this both raises and lowers the delay and is
not counted as backoff; it lets a component drive the scope delay
directly (e.g. an adaptive-delay extension).
"""
async def process_response(self, response: Response) -> None:
"""Update the throttling state based on *response*."""
@ -480,6 +547,9 @@ class ThrottlingManager:
load_object(cls) for cls in crawler.settings.getlist("BACKOFF_EXCEPTIONS")
)
self._debug = crawler.settings.getbool("THROTTLING_DEBUG")
# When set, each request also gets an IP scope (its resolved address),
# enforced alongside its domain scope (see _resolve_scopes_sync).
self._per_ip: int = crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP")
self._max_idle = crawler.settings.getfloat("THROTTLING_SCOPE_MAX_IDLE")
self._robotstxt_obey = crawler.settings.getbool(
"ROBOTSTXT_OBEY"
@ -539,7 +609,15 @@ class ThrottlingManager:
stacklevel=2,
)
return download_slot
return urlparse_cached(request).netloc
netloc = urlparse_cached(request).netloc
if self._per_ip:
# Best-effort, mirroring the legacy per-IP downloader slots: use the
# cached resolved address if available, else fall back to the domain
# scope alone (the IP is not known until the host is resolved).
ip = dnscache.get(netloc)
if isinstance(ip, str) and ip != netloc:
return (netloc, ip)
return netloc
def get_slot_key(self, request: Request) -> str:
scopes = self._resolve_scopes_sync(request)
@ -903,6 +981,14 @@ class ThrottlingManager:
# delay; unlike those, it is trusted, so it bypasses BACKOFF_MAX_DELAY.
self._get_scope_manager(scope_id).record_backoff(delay=float(delay), cap=False)
def get_scope_delay(self, scope_id: ScopeID) -> float:
return self._get_scope_manager(scope_id).get_base_delay()
def set_scope_delay(self, scope_id: ScopeID, delay: float) -> None:
self._get_scope_manager(scope_id).set_base_delay(
float(delay), only_increase=False
)
def _maybe_evict(self, now: float) -> None:
if self._max_idle <= 0:
return
@ -1002,11 +1088,15 @@ class ThrottlingScopeManagerProtocol(Protocol):
reported for a request, correcting the estimate used by
:meth:`record_sent`."""
def set_base_delay(self, delay: float) -> None:
"""Raise the base (non-backoff) delay of this scope to *delay* seconds.
def get_base_delay(self) -> float:
"""Return the base (non-backoff) delay of this scope, in seconds."""
It never lowers the configured base delay; it is used to honor external
hints such as a robots.txt ``Crawl-delay`` directive.
def set_base_delay(self, delay: float, *, only_increase: bool = True) -> None:
"""Set the base (non-backoff) delay of this scope to *delay* seconds.
By default it only raises the delay, to honor external hints such as a
robots.txt ``Crawl-delay`` directive. Pass ``only_increase=False`` to
also allow lowering it.
"""
def set_concurrency(self, concurrency: int) -> None:
@ -1146,7 +1236,7 @@ class ThrottlingScopeManager:
elif self._rampup_enabled:
self._concurrency = self._min_concurrency
else:
self._concurrency = settings.getint("THROTTLING_SCOPE_CONCURRENCY") or None
self._concurrency = _default_scope_concurrency(settings, self._id) or None
# Used as the load denominator when the scope enforces no explicit
# concurrency limit (see get_load()).
self._global_concurrency: int = settings.getint("CONCURRENT_REQUESTS")
@ -1374,10 +1464,15 @@ class ThrottlingScopeManager:
elif consumed is not None:
self._consumed = max(0.0, self._consumed + float(consumed))
def set_base_delay(self, delay: float) -> None:
if delay <= self._base_delay:
def get_base_delay(self) -> float:
return self._base_delay
def set_base_delay(self, delay: float, *, only_increase: bool = True) -> None:
if only_increase and delay <= self._base_delay:
return
self._base_delay = delay
# Reflect the change in the effective delay unless a backoff is raising
# it above the base right now.
if self._backoff_level == 0:
self._delay = delay

35
tests/test_crawler.py
View File

@ -3,7 +3,6 @@ from __future__ import annotations
import asyncio
import logging
import re
import warnings
from pathlib import Path
from typing import Any, ClassVar
@ -82,41 +81,11 @@ class TestCrawler(TestBaseCrawler):
assert not settings.frozen
assert crawler.settings.frozen
@pytest.mark.parametrize(
("setting_name", "current_default", "future_default"),
[
("THROTTLING_SCOPE_CONCURRENCY", 8, 1),
],
)
def test_deprecated_default_settings_warn(
self, setting_name: str, current_default: Any, future_default: Any
) -> None:
crawler = Crawler(DefaultSpider)
with pytest.warns(
ScrapyDeprecationWarning,
match=rf"The default value of {setting_name} will change from {current_default!r} to {future_default!r}",
):
crawler._apply_settings()
@pytest.mark.parametrize(
("setting_name", "current_default"),
[
("THROTTLING_SCOPE_CONCURRENCY", 8),
],
)
def test_deprecated_default_settings_no_warn_when_set(
self, setting_name: str, current_default: int
) -> None:
crawler = Crawler(DefaultSpider, {setting_name: current_default})
with warnings.catch_warnings():
warnings.simplefilter("error", ScrapyDeprecationWarning)
crawler._apply_settings()
def test_spider_download_delay_deprecated(self) -> None:
class DelaySpider(DefaultSpider):
download_delay = 2.5
crawler = Crawler(DelaySpider, {"THROTTLING_SCOPE_CONCURRENCY": 8})
crawler = Crawler(DelaySpider)
with pytest.warns(
ScrapyDeprecationWarning, match="'download_delay' spider attribute"
):
@ -127,7 +96,7 @@ class TestCrawler(TestBaseCrawler):
class DelaySpider(DefaultSpider):
download_delay = 2.5
crawler = Crawler(DelaySpider, {"THROTTLING_SCOPE_CONCURRENCY": 8})
crawler = Crawler(DelaySpider)
crawler.settings.set("DOWNLOAD_DELAY", 5.0, priority="spider")
with pytest.warns(
ScrapyDeprecationWarning,

9
tests/test_settings/__init__.py
View File

@ -769,15 +769,6 @@ def test_deprecated_dns_resolver_setting():
settings.get("DNS_RESOLVER")
def test_deprecated_concurrent_requests_per_ip_setting():
settings = Settings({"CONCURRENT_REQUESTS_PER_IP": 1})
with pytest.warns(
ScrapyDeprecationWarning,
match="The CONCURRENT_REQUESTS_PER_IP setting is deprecated",
):
settings.get("CONCURRENT_REQUESTS_PER_IP")
class Component1:
pass

6
tests/test_throttling.py
View File

@ -575,7 +575,11 @@ class TestThrottlingScopeManager:
assert scope._concurrency == 8
def test_no_scope_concurrency_limit_when_zero(self):
scope = _scope_manager(settings={"THROTTLING_SCOPE_CONCURRENCY": 0})
# THROTTLING_SCOPE_CONCURRENCY governs scopes that are neither a domain
# nor an IP (here a bare "custom" group name).
scope = _scope_manager(
settings={"THROTTLING_SCOPE_CONCURRENCY": 0}, config={"id": "custom"}
)
assert scope._concurrency is None
for _ in range(100):
scope.record_sent(now=0.0)