This commit is contained in:
Adrian Chaves 2026-07-02 06:48:46 +02:00
parent c21f8dcfc5
commit e203c326c2
2 changed files with 17 additions and 0 deletions

View File

@ -1044,6 +1044,14 @@ Additional settings
<throttling-scopes>` that are neither domains nor IPs, e.g. custom scopes
added by a :ref:`custom throttling manager <custom-throttling-scopes>`.
A scope counts as a domain only if it looks like a hostname with at least
one dot, so single-label hosts such as ``localhost`` (or intranet host
names) fall in this category and default to
:setting:`THROTTLING_SCOPE_CONCURRENCY` rather than
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. Raise it, or give the host an
explicit ``concurrency`` in :setting:`THROTTLING_SCOPES`, if you need more
concurrency against such hosts (e.g. a local development server).
Domain and IP scopes use :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and
:setting:`CONCURRENT_REQUESTS_PER_IP` instead. A scope ``concurrency`` set
in :setting:`THROTTLING_SCOPES` overrides this.
@ -1101,3 +1109,4 @@ API
.. autofunction:: scrapy.throttling.scope_cache
.. autofunction:: scrapy.throttling.add_scope
.. autofunction:: scrapy.throttling.iter_scopes

View File

@ -98,6 +98,14 @@ RequestScopes = None | ScopeID | Iterable[ScopeID] | dict[ScopeID, float | None]
def iter_scopes(scopes: RequestScopes) -> Iterable[ScopeID]:
"""Iterate over the scope IDs of *scopes*, whatever its form.
:class:`~ThrottlingManagerProtocol.get_scopes` (and
:meth:`~ThrottlingManagerProtocol.get_resolved_scopes`) may return a single
scope ID, an iterable of them, a ``{scope_id: quota}`` mapping, or ``None``;
this helper normalizes any of those into an iterable of scope IDs, e.g. to
react to a request's scopes in a custom middleware.
"""
if scopes is None:
return ()
if isinstance(scopes, str):