From ba8bb9e4e97ddeefb91fe26ca59c38cdfc63bb07 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Tue, 30 Jun 2026 12:53:18 +0200 Subject: [PATCH] WIP --- docs/news.rst | 6 +-- docs/topics/settings.rst | 40 +++++++++++++++ docs/topics/throttling.rst | 80 ++++++++++++++++++++---------- scrapy/core/downloader/__init__.py | 2 +- scrapy/throttling.py | 53 +++++++++++++++----- scrapy/utils/test.py | 4 +- tests/test_throttling.py | 2 +- 7 files changed, 141 insertions(+), 46 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index e5ccd561e..b923f8135 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3261,7 +3261,7 @@ New features - Settings corresponding to :setting:`DOWNLOAD_DELAY`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per-domain basis - via the new :setting:`DOWNLOAD_SLOTS` setting. (:issue:`5328`) + via the new ``DOWNLOAD_SLOTS`` setting. (:issue:`5328`) - Added :meth:`.TextResponse.jmespath`, a shortcut for JMESPath selectors available since parsel_ 1.8.1. (:issue:`5894`, :issue:`5915`) @@ -7448,7 +7448,7 @@ This 1.1 release brings a lot of interesting features and bug fixes: - Item loaders now support nested loaders (:issue:`1467`). - ``FormRequest.from_response`` improvements (:issue:`1382`, :issue:`1137`). - - Added setting :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` and improved + - Added setting ``AUTOTHROTTLE_TARGET_CONCURRENCY`` and improved AutoThrottle docs (:issue:`1324`). - Added ``response.text`` to get body as unicode (:issue:`1730`). - Anonymous S3 connections (:issue:`1358`). @@ -8651,7 +8651,7 @@ Scrapy changes: - added :ref:`topics-contracts`, a mechanism for testing spiders in a formal/reproducible way - added options ``-o`` and ``-t`` to the :command:`runspider` command -- documented ``scrapy.extensions.throttle.AutoThrottle`` and added to extensions installed by default. You still need to enable it with :setting:`AUTOTHROTTLE_ENABLED` +- documented ``scrapy.extensions.throttle.AutoThrottle`` and added to extensions installed by default. You still need to enable it with ``AUTOTHROTTLE_ENABLED`` - major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backward compatibility is kept on the Stats Collector API and signals. - added a ``process_start_requests()`` method to spider middlewares - dropped Signals singleton. Signals should now be accessed through the Crawler.signals attribute. See the signals documentation for more info. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 6275e2e56..d6b9c2623 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -882,6 +882,46 @@ Default: ``True`` Whether to enable downloader stats collection. +.. setting:: DOWNLOAD_BIND_ADDRESS + +DOWNLOAD_BIND_ADDRESS +--------------------- + +Default: ``None`` + +The default local outgoing address for download-handler connections. + +This setting can be either: + +- a host address as a string (e.g. ``"127.0.0.2"``), in which case the local + port is chosen automatically, or + +- a ``(host, port)`` tuple (e.g. ``("127.0.0.2", 50000)``) to bind to both a + specific local interface and a specific local port. + +For example: + +.. code-block:: python + + # Bind to this local address + DOWNLOAD_BIND_ADDRESS = "127.0.0.2" + +.. code-block:: python + + # Bind to this local address and local port + DOWNLOAD_BIND_ADDRESS = ("127.0.0.2", 5000) + +If set, built-in HTTP download handlers use this value by default. +Set the :reqmeta:`bindaddress` request meta key to override it for a specific +request. + +.. note:: + + Handling of this setting needs to be implemented inside the :ref:`download + handler `, so it's not guaranteed to be supported + by all 3rd-party handlers. Specifying the port is unsupported by + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. + .. setting:: DOWNLOAD_HANDLERS DOWNLOAD_HANDLERS diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index fec88d265..13e9c0e50 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -26,9 +26,9 @@ throttling limits, as do ``toscrape.com`` and ``books.toscrape.com``. The main throttling :ref:`settings ` are: -- .. setting:: THROTTLING_SCOPE_CONCURRENCY +- .. setting:: CONCURRENT_REQUESTS_PER_DOMAIN - :setting:`THROTTLING_SCOPE_CONCURRENCY` (default: ``1`` (:ref:`fallback `: ``8``)) + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` (default: ``8``) Maximum number of simultaneous requests per domain. @@ -49,7 +49,7 @@ The main throttling :ref:`settings ` are: When configuring these settings, note that: -- :setting:`CONCURRENT_REQUESTS` caps :setting:`THROTTLING_SCOPE_CONCURRENCY`. +- :setting:`CONCURRENT_REQUESTS` caps :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. - If ``DOWNLOAD_DELAY`` ≥ response time, concurrency is effectively ``1``, because the next request to the domain is not sent until the delay elapses, @@ -277,8 +277,8 @@ to wait between requests. If :setting:`ROBOTSTXT_OBEY` and :setting:`THROTTLING_ROBOTSTXT_OBEY` are ``True`` (default), valid ``Crawl-Delay`` directives override -:setting:`THROTTLING_SCOPE_CONCURRENCY` and :setting:`DOWNLOAD_DELAY`. Concurrency -is set to ``1`` and delay is set to the value of ``Crawl-Delay``, capped at +:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`DOWNLOAD_DELAY`. +Concurrency is set to ``1`` and delay is set to the value of ``Crawl-Delay``, capped at :setting:`THROTTLING_ROBOTSTXT_MAX_DELAY` (default: ``60.0``). If :setting:`THROTTLING_SCOPES` defines a different concurrency or delay, it @@ -440,8 +440,10 @@ Its keys are scope names and its values are following keys: ``concurrency`` (:class:`int`) - Maximum number of concurrent requests for the scope. When unset, - :setting:`THROTTLING_SCOPE_CONCURRENCY` applies instead. + Maximum number of concurrent requests for the scope. When unset, the + default depends on the kind of scope: :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` + for domain scopes, :setting:`CONCURRENT_REQUESTS_PER_IP` for IP scopes, and + :setting:`THROTTLING_SCOPE_CONCURRENCY` for any other scope. ``min_concurrency`` (:class:`int`) Concurrency floor that :ref:`backoff ` and :ref:`rampup ` @@ -452,7 +454,10 @@ following keys: :setting:`DOWNLOAD_DELAY`. ``jitter`` (:class:`float` or 2-:class:`list`) - Per-scope override of :setting:`RANDOMIZE_DOWNLOAD_DELAY`. + Magnitude of the random variation applied to ``delay``; the per-scope + override of :setting:`RANDOMIZE_DOWNLOAD_DELAY`. ``0`` disables it, ``0.5`` + means ±50% (the default when :setting:`RANDOMIZE_DOWNLOAD_DELAY` is on), and + a ``[low, high]`` pair multiplies the delay by ``1 + uniform(low, high)``. ``quota`` (:class:`float`) Maximum :ref:`quota ` consumed per ``window``. @@ -924,30 +929,39 @@ window (:setting:`BACKOFF_WINDOW`). You can use :ref:`throttling quotas Per-IP concurrency limiting --------------------------- -A concurrency limit keyed by IP is just a throttling scope whose id is the -request's IP, with a ``concurrency`` limit. A request then carries two scopes, -its domain and its IP, and is only sent when **both** allow it (see -:ref:`multiple-throttling-scopes`). +.. setting:: CONCURRENT_REQUESTS_PER_IP -- Implement a :ref:`throttling manager ` that adds - the request's IP as a second scope: +A concurrency limit keyed by IP is a throttling scope whose id is the request's +IP. A request then carries two scopes, its domain and its IP, and is only sent +when **both** allow it (see :ref:`multiple-throttling-scopes`). - .. code-block:: python +Set :setting:`CONCURRENT_REQUESTS_PER_IP` (default: ``0``, disabled) to a +positive number to enable this built in: the throttler adds the IP of each +request, resolved from the DNS cache, as a second scope, limited to that many +concurrent requests. IP scopes whose ``concurrency`` is left unset default to +:setting:`CONCURRENT_REQUESTS_PER_IP`. - import socket +For finer control (e.g. resolving IPs that are not in the DNS cache yet, or +grouping several hosts under one address), implement a :ref:`throttling manager +` that adds the request's IP as a second scope +instead: - from scrapy.throttling import ThrottlingManager, add_scope, scope_cache - from scrapy.utils.asyncio import run_in_thread - from scrapy.utils.httpobj import urlparse_cached +.. code-block:: python + + import socket + + from scrapy.throttling import ThrottlingManager, add_scope, scope_cache + from scrapy.utils.asyncio import run_in_thread + from scrapy.utils.httpobj import urlparse_cached - class IPThrottlingManager(ThrottlingManager): - @scope_cache - async def get_scopes(self, request): - scopes = await super().get_scopes(request) - host = urlparse_cached(request).hostname - address = await run_in_thread(socket.gethostbyname, host) - return add_scope(scopes, address) + class IPThrottlingManager(ThrottlingManager): + @scope_cache + async def get_scopes(self, request): + scopes = await super().get_scopes(request) + host = urlparse_cached(request).hostname + address = await run_in_thread(socket.gethostbyname, host) + return add_scope(scopes, address) .. _throttling-settings: @@ -1033,6 +1047,18 @@ Additional settings Whether to log :ref:`throttling ` decisions (per-scope delays, backoff steps and recoveries) for debugging. +- .. setting:: THROTTLING_SCOPE_CONCURRENCY + + :setting:`THROTTLING_SCOPE_CONCURRENCY` (default: ``1``) + + Default maximum number of concurrent requests for :ref:`throttling scopes + ` that are neither domains nor IPs, e.g. custom scopes + added by a :ref:`custom throttling manager `. + + Domain and IP scopes use :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and + :setting:`CONCURRENT_REQUESTS_PER_IP` instead. A scope ``concurrency`` set + in :setting:`THROTTLING_SCOPES` overrides this. + - .. setting:: THROTTLING_SCOPE_LIMIT :setting:`THROTTLING_SCOPE_LIMIT` (default: ``100000``) @@ -1068,7 +1094,7 @@ The ``AutoThrottle`` extension is deprecated in favor of the throttling and :ref:`backoff ` system described here, which is always active and does not need to be enabled. -Setting :setting:`AUTOTHROTTLE_ENABLED` to ``True`` still works but logs a +Setting ``AUTOTHROTTLE_ENABLED`` to ``True`` still works but logs a deprecation warning. To migrate, drop the ``AUTOTHROTTLE_*`` settings and use the following equivalents: diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index e22b1192d..ec6ba527e 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -97,7 +97,7 @@ class _DeprecatedSlotView: @property def randomize_delay(self) -> bool: if self._scope is not None: - return self._scope._randomize # type: ignore[union-attr] + return bool(self._scope._jitter) # type: ignore[union-attr] return False @property diff --git a/scrapy/throttling.py b/scrapy/throttling.py index fe69fe6b5..2d7d14e15 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -92,6 +92,20 @@ class BackoffConfig(TypedDict, total=False): jitter: float | list[float] +class RampupConfig(TypedDict, total=False): + """Per-scope override of the rampup settings. + + Used as the value of the ``"rampup"`` key of :class:`ThrottlingScopeConfig` + entries when fine-tuning rampup beyond a plain ``True``. Any key left out + falls back to its default (or, for ``backoff_target``, to + :setting:`RAMPUP_BACKOFF_TARGET`). + """ + + backoff_target: float | list[float] + delay_factor: float + min_delay: float + + class ThrottlingScopeConfig(TypedDict, total=False): """Accepted keys of :setting:`THROTTLING_SCOPES` entries. @@ -105,10 +119,15 @@ class ThrottlingScopeConfig(TypedDict, total=False): """Floor. Never drop below this during backoff/rampup.""" delay: float + jitter: float | list[float] + """Magnitude of the random variation applied to ``delay``; the per-scope + override of :setting:`RANDOMIZE_DOWNLOAD_DELAY` (``0`` disables it, ``0.5`` + means ±50%).""" + quota: float window: float - rampup: bool + rampup: bool | RampupConfig manager: str | type """Import path or class of a custom :setting:`THROTTLING_SCOPE_MANAGER` for @@ -116,6 +135,10 @@ class ThrottlingScopeConfig(TypedDict, total=False): backoff: BackoffConfig + ignore_robots_txt: bool + """Silence the warning logged when this configuration is more aggressive + than a robots.txt ``Crawl-delay``.""" + ScopeID = str BackoffData = None | ScopeID | Iterable[ScopeID] | dict[ScopeID, BackoffScopeData] @@ -1199,8 +1222,11 @@ class ThrottlingScopeManager: self._base_delay: float = float( config.get("delay", settings.getfloat("DOWNLOAD_DELAY")) ) - self._randomize: bool = bool( - config.get("randomize_delay", settings.getbool("RANDOMIZE_DOWNLOAD_DELAY")) + # Magnitude of the random variation applied to the (non-backoff) delay. + # Defaults to RANDOMIZE_DOWNLOAD_DELAY's historical ±50% when delay + # randomization is on, or to no variation when it is off. + self._jitter: float | list[float] = config.get( + "jitter", 0.5 if settings.getbool("RANDOMIZE_DOWNLOAD_DELAY") else 0.0 ) self._delay_factor: float = float( backoff.get("delay_factor", settings.getfloat("BACKOFF_DELAY_FACTOR")) @@ -1211,7 +1237,7 @@ class ThrottlingScopeManager: self._min_delay: float = float( backoff.get("min_delay", settings.getfloat("BACKOFF_MIN_DELAY")) ) - self._jitter: float | list[float] = backoff.get( + self._backoff_jitter: float | list[float] = backoff.get( "jitter", settings.getfloat("BACKOFF_JITTER") ) self._window: float = settings.getfloat("BACKOFF_WINDOW") @@ -1272,17 +1298,18 @@ class ThrottlingScopeManager: return float(value[0]), float(value[1]) return float(value), float(value) - def _apply_jitter(self, value: float) -> float: - if isinstance(self._jitter, (list, tuple)): - low, high = self._jitter[0], self._jitter[1] + @staticmethod + def _apply_jitter(value: float, jitter: float | list[float]) -> float: + if isinstance(jitter, (list, tuple)): + low, high = jitter[0], jitter[1] return value * (1 + random.uniform(low, high)) # noqa: S311 - if not self._jitter: + if not jitter: return value - return value * random.uniform(1 - self._jitter, 1 + self._jitter) # noqa: S311 + return value * random.uniform(1 - jitter, 1 + jitter) # noqa: S311 def _effective_delay(self) -> float: - if self._backoff_level == 0 and self._randomize and self._delay > 0: - return random.uniform(0.5 * self._delay, 1.5 * self._delay) # noqa: S311 + if self._backoff_level == 0 and self._delay > 0: + return self._apply_jitter(self._delay, self._jitter) return self._delay def _recover(self, now: float) -> None: @@ -1447,7 +1474,9 @@ class ThrottlingScopeManager: ) grown = max(self._min_delay, grown) grown = min(grown, self._max_delay) - self._delay = min(self._apply_jitter(grown), self._max_delay) + self._delay = min( + self._apply_jitter(grown, self._backoff_jitter), self._max_delay + ) self._next_allowed_time = now + self._delay def reconcile_quota( diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index fc86cbe7b..38bd8c124 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -66,13 +66,13 @@ def get_crawler( will be used to populate the crawler settings with a project level priority. """ + # When needed, useful settings can be added here, e.g. ones that prevent + # deprecation warnings (see prevent_warnings). settings: dict[str, Any] = { "TELNETCONSOLE_ENABLED": False, **get_reactor_settings(), **(settings_dict or {}), } - if prevent_warnings: - settings.setdefault("THROTTLING_SCOPE_CONCURRENCY", 8) runner: CrawlerRunnerBase if is_reactor_installed(): runner = CrawlerRunner(settings) diff --git a/tests/test_throttling.py b/tests/test_throttling.py index c9b28067f..ddc14ff9b 100644 --- a/tests/test_throttling.py +++ b/tests/test_throttling.py @@ -1106,7 +1106,7 @@ class TestThrottlingScopeManagerEdges: config={"id": "x", "backoff": {"jitter": [0.0, 0.0]}, "min_delay": 1.0} ) # A [low, high] jitter range of [0, 0] leaves the value unchanged. - assert scope._apply_jitter(4.0) == pytest.approx(4.0) + assert scope._apply_jitter(4.0, scope._backoff_jitter) == pytest.approx(4.0) def test_effective_delay_randomized(self): scope = _scope_manager(