diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index 99a5fe0d7..2de8cea8d 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -1044,6 +1044,14 @@ Additional settings ` that are neither domains nor IPs, e.g. custom scopes added by a :ref:`custom throttling manager `. + A scope counts as a domain only if it looks like a hostname with at least + one dot, so single-label hosts such as ``localhost`` (or intranet host + names) fall in this category and default to + :setting:`THROTTLING_SCOPE_CONCURRENCY` rather than + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. Raise it, or give the host an + explicit ``concurrency`` in :setting:`THROTTLING_SCOPES`, if you need more + concurrency against such hosts (e.g. a local development server). + Domain and IP scopes use :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`CONCURRENT_REQUESTS_PER_IP` instead. A scope ``concurrency`` set in :setting:`THROTTLING_SCOPES` overrides this. @@ -1101,3 +1109,4 @@ API .. autofunction:: scrapy.throttling.scope_cache .. autofunction:: scrapy.throttling.add_scope +.. autofunction:: scrapy.throttling.iter_scopes diff --git a/scrapy/throttling.py b/scrapy/throttling.py index eec7c9ddd..356571a05 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -98,6 +98,14 @@ RequestScopes = None | ScopeID | Iterable[ScopeID] | dict[ScopeID, float | None] def iter_scopes(scopes: RequestScopes) -> Iterable[ScopeID]: + """Iterate over the scope IDs of *scopes*, whatever its form. + + :class:`~ThrottlingManagerProtocol.get_scopes` (and + :meth:`~ThrottlingManagerProtocol.get_resolved_scopes`) may return a single + scope ID, an iterable of them, a ``{scope_id: quota}`` mapping, or ``None``; + this helper normalizes any of those into an iterable of scope IDs, e.g. to + react to a request's scopes in a custom middleware. + """ if scopes is None: return () if isinstance(scopes, str):