mirror of https://github.com/scrapy/scrapy.git
Rederepcate PER_IP, deprecate PER_DOMAIN
This commit is contained in:
Adrian Chaves
parent
8e488d837c
commit
7784594f7a
|
|
@ -47,8 +47,8 @@ Increase concurrency
|
|||
====================
|
||||
|
||||
Concurrency is the number of requests that are processed in parallel. There is
|
||||
a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional limit that
|
||||
can be set per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`).
|
||||
a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional per-scope
|
||||
(per-domain by default) limit (:setting:`THROTTLING_SCOPE_CONCURRENCY`).
|
||||
|
||||
The default global concurrency limit in Scrapy is not suitable for crawling
|
||||
many different domains in parallel, so you will want to increase it. How much
|
||||
|
|
|
|||
|
|
@ -26,15 +26,16 @@ throttling limits, as do ``toscrape.com`` and ``books.toscrape.com``.
|
|||
|
||||
The main throttling :ref:`settings <topics-settings>` are:
|
||||
|
||||
- .. setting:: CONCURRENT_REQUESTS_PER_DOMAIN
|
||||
- .. setting:: THROTTLING_SCOPE_CONCURRENCY
|
||||
|
||||
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` (default: ``1``
|
||||
(:ref:`fallback <default-settings>`: ``8``))
|
||||
:setting:`THROTTLING_SCOPE_CONCURRENCY` (default: ``1``)
|
||||
|
||||
Maximum number of simultaneous requests per domain.
|
||||
Default maximum number of simultaneous requests per :ref:`throttling scope
|
||||
<throttling-scopes>`. Requests are grouped by domain by default, so this is
|
||||
the maximum number of simultaneous requests per domain.
|
||||
|
||||
It defines a number of “slots” per domain. Each slot can send 1 request at
|
||||
a time: it sends a request, waits for the response, then sends the next
|
||||
It defines a number of “slots” per scope. Each slot can send 1 request at a
|
||||
time: it sends a request, waits for the response, then sends the next
|
||||
request, and so on.
|
||||
|
||||
- .. setting:: DOWNLOAD_DELAY
|
||||
|
|
@ -50,7 +51,7 @@ The main throttling :ref:`settings <topics-settings>` are:
|
|||
|
||||
When configuring these settings, note that:
|
||||
|
||||
- :setting:`CONCURRENT_REQUESTS` caps :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`.
|
||||
- :setting:`CONCURRENT_REQUESTS` caps :setting:`THROTTLING_SCOPE_CONCURRENCY`.
|
||||
|
||||
- If ``DOWNLOAD_DELAY`` ≥ response time, concurrency is effectively ``1``,
|
||||
because the next request to the domain is not sent until the delay elapses,
|
||||
|
|
@ -296,7 +297,7 @@ to wait between requests.
|
|||
|
||||
If :setting:`ROBOTSTXT_OBEY` and :setting:`THROTTLING_ROBOTSTXT_OBEY` are
|
||||
``True`` (default), valid ``Crawl-Delay`` directives override
|
||||
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`DOWNLOAD_DELAY`.
|
||||
:setting:`THROTTLING_SCOPE_CONCURRENCY` and :setting:`DOWNLOAD_DELAY`.
|
||||
Concurrency is set to ``1`` and the delay is raised to at least the
|
||||
``Crawl-Delay`` value (a larger configured delay is kept), capped at
|
||||
:setting:`THROTTLING_ROBOTSTXT_MAX_DELAY` (default: ``60.0``).
|
||||
|
|
@ -465,10 +466,8 @@ Its keys are scope names and its values are
|
|||
following keys:
|
||||
|
||||
``concurrency`` (:class:`int`)
|
||||
Maximum number of concurrent requests for the scope. When unset, the
|
||||
default depends on the kind of scope: :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`
|
||||
for domain scopes, :setting:`CONCURRENT_REQUESTS_PER_IP` for IP scopes, and
|
||||
:setting:`THROTTLING_SCOPE_CONCURRENCY` for any other scope.
|
||||
Maximum number of concurrent requests for the scope. Defaults to
|
||||
:setting:`THROTTLING_SCOPE_CONCURRENCY`.
|
||||
|
||||
``delay`` (:class:`float`)
|
||||
Minimum seconds between requests for the scope. Defaults to
|
||||
|
|
@ -921,46 +920,30 @@ window (:setting:`THROTTLING_WINDOW`). You can use :ref:`throttling quotas
|
|||
Per-IP concurrency limiting
|
||||
---------------------------
|
||||
|
||||
.. setting:: CONCURRENT_REQUESTS_PER_IP
|
||||
A concurrency limit keyed by IP is just a throttling scope whose id is the
|
||||
request's IP, with a ``concurrency`` limit. A request then carries two scopes,
|
||||
its domain and its IP, and is only sent when **both** allow it (see
|
||||
:ref:`multiple-throttling-scopes`).
|
||||
|
||||
A concurrency limit keyed by IP is a throttling scope whose id is the request's
|
||||
IP. A request then carries two scopes, its domain and its IP, and is only sent
|
||||
when **both** allow it (see :ref:`multiple-throttling-scopes`).
|
||||
- Implement a :ref:`throttling manager <custom-throttling-scopes>` that adds
|
||||
the request's IP as a second scope:
|
||||
|
||||
Set :setting:`CONCURRENT_REQUESTS_PER_IP` (default: ``0``, disabled) to a
|
||||
positive number to enable this built in: the throttler adds the IP of each
|
||||
request, resolved from the DNS cache, as a second scope, limited to that many
|
||||
concurrent requests. IP scopes whose ``concurrency`` is left unset default to
|
||||
:setting:`CONCURRENT_REQUESTS_PER_IP`.
|
||||
.. code-block:: python
|
||||
|
||||
.. note:: Enabling :setting:`CONCURRENT_REQUESTS_PER_IP` requires the
|
||||
:ref:`throttling-aware scheduler <throttling-aware-scheduler>`. The default
|
||||
:class:`~scrapy.pqueues.DownloaderAwarePriorityQueue` does not support it
|
||||
and raises an error at start up if it is set, so switch
|
||||
:setting:`SCHEDULER` and :setting:`SCHEDULER_PRIORITY_QUEUE` as shown
|
||||
there.
|
||||
import socket
|
||||
|
||||
For finer control (e.g. resolving IPs that are not in the DNS cache yet, or
|
||||
grouping several hosts under one address), implement a :ref:`throttling manager
|
||||
<custom-throttling-scopes>` that adds the request's IP as a second scope
|
||||
instead:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import socket
|
||||
|
||||
from scrapy.throttling import ThrottlingManager, add_scope, scope_cache
|
||||
from scrapy.utils.asyncio import run_in_thread
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.throttling import ThrottlingManager, add_scope, scope_cache
|
||||
from scrapy.utils.asyncio import run_in_thread
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
|
||||
|
||||
class IPThrottlingManager(ThrottlingManager):
|
||||
@scope_cache
|
||||
async def get_scopes(self, request):
|
||||
scopes = await super().get_scopes(request)
|
||||
host = urlparse_cached(request).hostname
|
||||
address = await run_in_thread(socket.gethostbyname, host)
|
||||
return add_scope(scopes, address)
|
||||
class IPThrottlingManager(ThrottlingManager):
|
||||
@scope_cache
|
||||
async def get_scopes(self, request):
|
||||
scopes = await super().get_scopes(request)
|
||||
host = urlparse_cached(request).hostname
|
||||
address = await run_in_thread(socket.gethostbyname, host)
|
||||
return add_scope(scopes, address)
|
||||
|
||||
.. _throttling-settings:
|
||||
|
||||
|
|
@ -1048,26 +1031,6 @@ Additional settings
|
|||
Whether to log :ref:`throttling <throttling>` decisions (per-scope delays,
|
||||
backoff steps and recoveries) for debugging.
|
||||
|
||||
- .. setting:: THROTTLING_SCOPE_CONCURRENCY
|
||||
|
||||
:setting:`THROTTLING_SCOPE_CONCURRENCY` (default: ``1``)
|
||||
|
||||
Default maximum number of concurrent requests for :ref:`throttling scopes
|
||||
<throttling-scopes>` that are neither domains nor IPs, e.g. custom scopes
|
||||
added by a :ref:`custom throttling manager <custom-throttling-scopes>`.
|
||||
|
||||
A scope counts as a domain only if it looks like a hostname with at least
|
||||
one dot, so single-label hosts such as ``localhost`` (or intranet host
|
||||
names) fall in this category and default to
|
||||
:setting:`THROTTLING_SCOPE_CONCURRENCY` rather than
|
||||
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. Raise it, or give the host an
|
||||
explicit ``concurrency`` in :setting:`THROTTLING_SCOPES`, if you need more
|
||||
concurrency against such hosts (e.g. a local development server).
|
||||
|
||||
Domain and IP scopes use :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and
|
||||
:setting:`CONCURRENT_REQUESTS_PER_IP` instead. A scope ``concurrency`` set
|
||||
in :setting:`THROTTLING_SCOPES` overrides this.
|
||||
|
||||
- .. setting:: THROTTLING_SCOPE_LIMIT
|
||||
|
||||
:setting:`THROTTLING_SCOPE_LIMIT` (default: ``100000``)
|
||||
|
|
|
|||
|
|
@ -251,9 +251,7 @@ class Downloader:
|
|||
return self.get_slot_key(request)
|
||||
|
||||
def get_slot_key(self, request: Request) -> str:
|
||||
# Per-IP grouping (CONCURRENT_REQUESTS_PER_IP) is now a throttling scope
|
||||
# handled by the throttler, not a downloader slot key; this fallback (used
|
||||
# only when no throttler is set) keys by domain alone.
|
||||
# This fallback (used only when no throttler is set) keys by domain.
|
||||
return urlparse_cached(request).netloc or ""
|
||||
|
||||
async def _enqueue_request(self, request: Request) -> Response:
|
||||
|
|
|
|||
|
|
@ -93,11 +93,10 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler):
|
|||
|
||||
self._pool: HTTPConnectionPool = HTTPConnectionPool(reactor, persistent=True)
|
||||
# Keep enough persistent connections per host to match the highest
|
||||
# per-host concurrency the throttler may admit: the per-domain and
|
||||
# per-IP limits, the default "other"-scope limit, and any explicit
|
||||
# THROTTLING_SCOPES concurrency. Per-scope concurrency can grow beyond
|
||||
# this at runtime (rampup); the excess simply uses non-persistent
|
||||
# connections.
|
||||
# per-host concurrency the throttler may admit: the per-domain limit,
|
||||
# the default "other"-scope limit, and any explicit THROTTLING_SCOPES
|
||||
# concurrency. Per-scope concurrency can grow beyond this at runtime
|
||||
# (rampup); the excess simply uses non-persistent connections.
|
||||
scope_concurrencies = [
|
||||
scope["concurrency"]
|
||||
for scope in crawler.settings.getdict("THROTTLING_SCOPES").values()
|
||||
|
|
@ -106,7 +105,6 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler):
|
|||
self._pool.maxPersistentPerHost = max(
|
||||
[
|
||||
crawler.settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN"),
|
||||
crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP"),
|
||||
crawler.settings.getint("THROTTLING_SCOPE_CONCURRENCY"),
|
||||
*scope_concurrencies,
|
||||
]
|
||||
|
|
|
|||
|
|
@ -344,10 +344,7 @@ class DownloaderAwarePriorityQueue:
|
|||
):
|
||||
if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0:
|
||||
raise ValueError(
|
||||
f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP. '
|
||||
"Set SCHEDULER_PRIORITY_QUEUE to "
|
||||
"'scrapy.pqueues.ThrottlingAwarePriorityQueue' (along with the "
|
||||
"matching SCHEDULER) to use per-IP concurrency limiting."
|
||||
f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP'
|
||||
)
|
||||
|
||||
if slot_startprios and not isinstance(slot_startprios, dict):
|
||||
|
|
|
|||
|
|
@ -149,6 +149,15 @@ class BaseSettings(MutableMapping[str, Any]):
|
|||
:param default: the value to return if no setting is found
|
||||
:type default: object
|
||||
"""
|
||||
if name == "CONCURRENT_REQUESTS_PER_IP" and (
|
||||
isinstance(self[name], int) and self[name] != 0
|
||||
):
|
||||
warnings.warn(
|
||||
"The CONCURRENT_REQUESTS_PER_IP setting is deprecated, use CONCURRENT_REQUESTS_PER_DOMAIN instead.",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
if name == "DNS_RESOLVER":
|
||||
warnings.warn(
|
||||
"The DNS_RESOLVER setting is deprecated, please use "
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ Scrapy developers, if you add a setting here remember to:
|
|||
import sys
|
||||
from importlib import import_module
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
__all__ = [
|
||||
"ADDONS",
|
||||
|
|
@ -51,7 +52,6 @@ __all__ = [
|
|||
"CONCURRENT_ITEMS",
|
||||
"CONCURRENT_REQUESTS",
|
||||
"CONCURRENT_REQUESTS_PER_DOMAIN",
|
||||
"CONCURRENT_REQUESTS_PER_IP",
|
||||
"COOKIES_DEBUG",
|
||||
"COOKIES_ENABLED",
|
||||
"CRAWLSPIDER_FOLLOW_LINKS",
|
||||
|
|
@ -285,7 +285,6 @@ CONCURRENT_ITEMS = 100
|
|||
|
||||
CONCURRENT_REQUESTS = 16
|
||||
CONCURRENT_REQUESTS_PER_DOMAIN = 8
|
||||
CONCURRENT_REQUESTS_PER_IP = 0
|
||||
|
||||
COOKIES_ENABLED = True
|
||||
COOKIES_DEBUG = False
|
||||
|
|
@ -636,3 +635,19 @@ URLLENGTH_LIMIT = 2083
|
|||
USER_AGENT = f"Scrapy/{import_module('scrapy').__version__} (+https://scrapy.org)"
|
||||
|
||||
WARN_ON_GENERATOR_RETURN_VALUE = True
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
if name == "CONCURRENT_REQUESTS_PER_IP":
|
||||
import warnings # noqa: PLC0415
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning # noqa: PLC0415
|
||||
|
||||
warnings.warn(
|
||||
"The scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_IP attribute is deprecated, use scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_DOMAIN instead.",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return 0
|
||||
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ NEWSPIDER_MODULE = "$project_name.spiders"
|
|||
ROBOTSTXT_OBEY = True
|
||||
|
||||
# Throttle crawls to be polite to websites:
|
||||
CONCURRENT_REQUESTS_PER_DOMAIN = 1
|
||||
THROTTLING_SCOPE_CONCURRENCY = 1
|
||||
DOWNLOAD_DELAY = 1
|
||||
|
||||
# Crawl the tutorial websites faster, overriding the polite defaults above only
|
||||
|
|
|
|||
|
|
@ -1,10 +1,8 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import ipaddress
|
||||
import logging
|
||||
import random
|
||||
import re
|
||||
import time
|
||||
import warnings
|
||||
from collections import OrderedDict
|
||||
|
|
@ -18,7 +16,7 @@ from typing_extensions import Self
|
|||
|
||||
from scrapy import signals
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.resolver import dnscache
|
||||
from scrapy.settings import SETTINGS_PRIORITIES
|
||||
from scrapy.utils.asyncio import sleep, wait_for_first
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.misc import build_from_crawler, load_object
|
||||
|
|
@ -134,59 +132,67 @@ def iter_scope_values(scopes: RequestScopes) -> Iterable[tuple[ScopeID, float |
|
|||
yield scope, None
|
||||
|
||||
|
||||
# A DNS hostname with at least one dot, e.g. "books.toscrape.com"; labels are
|
||||
# alphanumeric (plus hyphens, not at the edges) and an optional trailing dot
|
||||
# marks a fully-qualified name.
|
||||
_HOSTNAME_RE = re.compile(
|
||||
r"(?i)^(?!-)[a-z0-9-]{1,63}(?<!-)(\.(?!-)[a-z0-9-]{1,63}(?<!-))+\.?$"
|
||||
)
|
||||
def _default_scope_concurrency(settings: BaseSettings) -> int:
|
||||
"""Return the default concurrency of a throttling scope that does not set
|
||||
its own ``concurrency``.
|
||||
|
||||
|
||||
def _classify_scope(scope_id: ScopeID) -> str:
|
||||
"""Classify *scope_id* as ``"ip"``, ``"domain"`` or ``"other"`` to pick its
|
||||
default concurrency setting (see :func:`_default_scope_concurrency`).
|
||||
|
||||
A scope that parses as an IP address is ``"ip"``; one that *looks like* a
|
||||
DNS hostname with at least one dot is ``"domain"``; anything else (custom
|
||||
group names, single labels like ``"localhost"``) is ``"other"``.
|
||||
This is :setting:`THROTTLING_SCOPE_CONCURRENCY`, except that the deprecated
|
||||
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` setting is bridged in when set at a
|
||||
higher :ref:`priority <populating-settings>`. When neither is set
|
||||
explicitly, the historical :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` value is
|
||||
kept for backward compatibility (its default flips to
|
||||
:setting:`THROTTLING_SCOPE_CONCURRENCY` in a future version; see
|
||||
:func:`_warn_on_deprecated_concurrency`).
|
||||
"""
|
||||
host = scope_id
|
||||
if host.startswith("[") and "]" in host: # bracketed IPv6, e.g. "[::1]:80"
|
||||
host = host[1 : host.index("]")]
|
||||
with contextlib.suppress(ValueError):
|
||||
ipaddress.ip_address(host)
|
||||
return "ip"
|
||||
# Drop a trailing ":port" for the hostname check (an IPv4 "host:port" is
|
||||
# re-tested as an IP below).
|
||||
if host.count(":") == 1:
|
||||
host = host.rsplit(":", 1)[0]
|
||||
with contextlib.suppress(ValueError):
|
||||
ipaddress.ip_address(host)
|
||||
return "ip"
|
||||
if "." in host and _HOSTNAME_RE.match(host):
|
||||
return "domain"
|
||||
return "other"
|
||||
|
||||
|
||||
def _default_scope_concurrency(settings: BaseSettings, scope_id: ScopeID) -> int:
|
||||
"""Return the default concurrency for *scope_id* based on its
|
||||
:func:`kind <_classify_scope>`.
|
||||
|
||||
Domain scopes default to :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, IP
|
||||
scopes to :setting:`CONCURRENT_REQUESTS_PER_IP` (falling back to the
|
||||
per-domain value when per-IP limiting is off, so IP-literal crawls are not
|
||||
left unbounded), and any other scope to :setting:`THROTTLING_SCOPE_CONCURRENCY`.
|
||||
"""
|
||||
kind = _classify_scope(scope_id)
|
||||
if kind == "ip":
|
||||
return settings.getint("CONCURRENT_REQUESTS_PER_IP") or settings.getint(
|
||||
"CONCURRENT_REQUESTS_PER_DOMAIN"
|
||||
)
|
||||
if kind == "domain":
|
||||
default_priority = SETTINGS_PRIORITIES["default"]
|
||||
# An unset setting has no priority (``None``); treat it as below "default"
|
||||
# so it never wins over one that is at least at its default value.
|
||||
domain_priority = settings.getpriority("CONCURRENT_REQUESTS_PER_DOMAIN")
|
||||
domain_priority = (
|
||||
default_priority - 1 if domain_priority is None else domain_priority
|
||||
)
|
||||
scope_priority = settings.getpriority("THROTTLING_SCOPE_CONCURRENCY")
|
||||
scope_priority = default_priority - 1 if scope_priority is None else scope_priority
|
||||
if domain_priority > scope_priority:
|
||||
return settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN")
|
||||
if scope_priority > domain_priority:
|
||||
return settings.getint("THROTTLING_SCOPE_CONCURRENCY")
|
||||
# Equal priority: on an explicit (higher-than-default) tie the new setting
|
||||
# wins; when neither is set (both at "default") keep the historical
|
||||
# per-domain value so existing behavior is preserved.
|
||||
if domain_priority <= default_priority:
|
||||
return settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN")
|
||||
return settings.getint("THROTTLING_SCOPE_CONCURRENCY")
|
||||
|
||||
|
||||
def _warn_on_deprecated_concurrency(settings: BaseSettings) -> None:
|
||||
"""Warn about the concurrency settings bridged by
|
||||
:func:`_default_scope_concurrency`. Call once per crawl (see
|
||||
:meth:`ThrottlingManager.__init__`)."""
|
||||
default_priority = SETTINGS_PRIORITIES["default"]
|
||||
domain_priority = settings.getpriority("CONCURRENT_REQUESTS_PER_DOMAIN")
|
||||
scope_priority = settings.getpriority("THROTTLING_SCOPE_CONCURRENCY")
|
||||
domain_set = domain_priority is not None and domain_priority > default_priority
|
||||
scope_set = scope_priority is not None and scope_priority > default_priority
|
||||
if domain_set:
|
||||
warnings.warn(
|
||||
"The CONCURRENT_REQUESTS_PER_DOMAIN setting is deprecated, use "
|
||||
"THROTTLING_SCOPE_CONCURRENCY instead.",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
elif not scope_set:
|
||||
current = settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN")
|
||||
future = settings.getint("THROTTLING_SCOPE_CONCURRENCY")
|
||||
warnings.warn(
|
||||
f"The default value of THROTTLING_SCOPE_CONCURRENCY will change "
|
||||
f"from {current} to {future} in a future Scrapy version. Set "
|
||||
f"THROTTLING_SCOPE_CONCURRENCY explicitly to silence this warning.",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
|
||||
def _to_scope_dict(collection: Any, default: Callable[[], Any]) -> dict[ScopeID, Any]:
|
||||
"""Normalize *collection* (``None``, str, iterable or dict) into a dict
|
||||
mapping scope names to values produced by *default*."""
|
||||
|
|
@ -496,10 +502,8 @@ class ThrottlingManager:
|
|||
|
||||
def __init__(self, crawler: Crawler) -> None:
|
||||
self.crawler = crawler
|
||||
_warn_on_deprecated_concurrency(crawler.settings)
|
||||
self._debug = crawler.settings.getbool("THROTTLING_DEBUG")
|
||||
# When set, each request also gets an IP scope (its resolved address),
|
||||
# enforced alongside its domain scope (see _resolve_scopes_sync).
|
||||
self._per_ip: int = crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP")
|
||||
self._max_idle = crawler.settings.getfloat("THROTTLING_SCOPE_MAX_IDLE")
|
||||
self._robotstxt_obey = crawler.settings.getbool(
|
||||
"ROBOTSTXT_OBEY"
|
||||
|
|
@ -588,15 +592,7 @@ class ThrottlingManager:
|
|||
stacklevel=2,
|
||||
)
|
||||
return cast("RequestScopes", download_slot)
|
||||
netloc = urlparse_cached(request).netloc
|
||||
if self._per_ip:
|
||||
# Best-effort, mirroring the legacy per-IP downloader slots: use the
|
||||
# cached resolved address if available, else fall back to the domain
|
||||
# scope alone (the IP is not known until the host is resolved).
|
||||
ip = dnscache.get(netloc)
|
||||
if isinstance(ip, str) and ip != netloc:
|
||||
return (netloc, ip)
|
||||
return netloc
|
||||
return urlparse_cached(request).netloc
|
||||
|
||||
def get_slot_key(self, request: Request) -> str:
|
||||
scopes = self._resolve_scopes_sync(request)
|
||||
|
|
@ -1196,7 +1192,7 @@ class ThrottlingScopeManager:
|
|||
# Rampup starts conservative at a single slot and probes upward.
|
||||
self._concurrency = 1
|
||||
else:
|
||||
self._concurrency = _default_scope_concurrency(settings, self._id) or None
|
||||
self._concurrency = _default_scope_concurrency(settings) or None
|
||||
# Used as the load denominator when the scope enforces no explicit
|
||||
# concurrency limit (see get_load()).
|
||||
self._global_concurrency: int = settings.getint("CONCURRENT_REQUESTS")
|
||||
|
|
|
|||
|
|
@ -66,13 +66,16 @@ def get_crawler(
|
|||
will be used to populate the crawler settings with a project level
|
||||
priority.
|
||||
"""
|
||||
# When needed, useful settings can be added here, e.g. ones that prevent
|
||||
# deprecation warnings (see prevent_warnings).
|
||||
settings: dict[str, Any] = {
|
||||
"TELNETCONSOLE_ENABLED": False,
|
||||
**get_reactor_settings(),
|
||||
**(settings_dict or {}),
|
||||
}
|
||||
if prevent_warnings:
|
||||
# Pin the pre-deprecation default per-scope concurrency so the suite
|
||||
# keeps its historical behavior and does not emit the warn-then-flip
|
||||
# deprecation warning (see throttling._warn_on_deprecated_concurrency).
|
||||
settings.setdefault("THROTTLING_SCOPE_CONCURRENCY", 8)
|
||||
runner: CrawlerRunnerBase
|
||||
if is_reactor_installed():
|
||||
runner = CrawlerRunner(settings)
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import logging
|
|||
import re
|
||||
import signal
|
||||
import threading
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
from unittest.mock import MagicMock
|
||||
|
|
@ -52,6 +53,77 @@ def get_raw_crawler(
|
|||
return Crawler(spidercls or DefaultSpider, settings)
|
||||
|
||||
|
||||
class TestScopeConcurrencyBridge:
|
||||
"""CONCURRENT_REQUESTS_PER_DOMAIN is deprecated and bridged into
|
||||
THROTTLING_SCOPE_CONCURRENCY by setting priority (see
|
||||
scrapy.throttling._default_scope_concurrency)."""
|
||||
|
||||
@staticmethod
|
||||
def _resolve(settings: Settings) -> int:
|
||||
from scrapy.throttling import _default_scope_concurrency # noqa: PLC0415
|
||||
|
||||
return _default_scope_concurrency(settings)
|
||||
|
||||
def test_neither_set_preserves_per_domain_default(self) -> None:
|
||||
# Neither setting overridden: keep the historical per-domain default.
|
||||
assert self._resolve(Settings()) == 8
|
||||
|
||||
def test_per_domain_overrides(self) -> None:
|
||||
settings = Settings()
|
||||
settings.set("CONCURRENT_REQUESTS_PER_DOMAIN", 4, priority="project")
|
||||
assert self._resolve(settings) == 4
|
||||
|
||||
def test_scope_concurrency_overrides(self) -> None:
|
||||
settings = Settings()
|
||||
settings.set("THROTTLING_SCOPE_CONCURRENCY", 4, priority="project")
|
||||
assert self._resolve(settings) == 4
|
||||
|
||||
def test_new_setting_wins_on_non_default_tie(self) -> None:
|
||||
settings = Settings()
|
||||
settings.set("CONCURRENT_REQUESTS_PER_DOMAIN", 4, priority="project")
|
||||
settings.set("THROTTLING_SCOPE_CONCURRENCY", 9, priority="project")
|
||||
assert self._resolve(settings) == 9
|
||||
|
||||
def test_higher_priority_deprecated_setting_wins(self) -> None:
|
||||
settings = Settings()
|
||||
settings.set("THROTTLING_SCOPE_CONCURRENCY", 9, priority="project")
|
||||
settings.set("CONCURRENT_REQUESTS_PER_DOMAIN", 4, priority="cmdline")
|
||||
assert self._resolve(settings) == 4
|
||||
|
||||
def test_warns_flip_when_neither_set(self) -> None:
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning,
|
||||
match="THROTTLING_SCOPE_CONCURRENCY will change",
|
||||
):
|
||||
get_crawler(prevent_warnings=False)
|
||||
|
||||
def test_warns_deprecated_when_per_domain_set(self) -> None:
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning,
|
||||
match="CONCURRENT_REQUESTS_PER_DOMAIN setting is deprecated",
|
||||
):
|
||||
get_crawler(
|
||||
settings_dict={"CONCURRENT_REQUESTS_PER_DOMAIN": 4},
|
||||
prevent_warnings=False,
|
||||
)
|
||||
|
||||
def test_no_concurrency_warning_when_scope_set(self) -> None:
|
||||
with warnings.catch_warnings(record=True) as recorded:
|
||||
warnings.simplefilter("always")
|
||||
get_crawler(
|
||||
settings_dict={"THROTTLING_SCOPE_CONCURRENCY": 4},
|
||||
prevent_warnings=False,
|
||||
)
|
||||
messages = [str(w.message) for w in recorded]
|
||||
assert not any(
|
||||
"THROTTLING_SCOPE_CONCURRENCY will change" in m for m in messages
|
||||
)
|
||||
assert not any(
|
||||
"CONCURRENT_REQUESTS_PER_DOMAIN setting is deprecated" in m
|
||||
for m in messages
|
||||
)
|
||||
|
||||
|
||||
class TestBaseCrawler:
|
||||
@staticmethod
|
||||
def assertOptionIsDefault(settings: Settings, key: str) -> None:
|
||||
|
|
|
|||
|
|
@ -769,6 +769,15 @@ def test_deprecated_dns_resolver_setting():
|
|||
settings.get("DNS_RESOLVER")
|
||||
|
||||
|
||||
def test_deprecated_concurrent_requests_per_ip_setting():
|
||||
settings = Settings({"CONCURRENT_REQUESTS_PER_IP": 1})
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning,
|
||||
match="The CONCURRENT_REQUESTS_PER_IP setting is deprecated",
|
||||
):
|
||||
settings.get("CONCURRENT_REQUESTS_PER_IP")
|
||||
|
||||
|
||||
class Component1:
|
||||
pass
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue