From 370ff74dc91e949ea7600584b13e93a2d43efe56 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Thu, 2 Jul 2026 15:02:13 +0200 Subject: [PATCH] WIP --- docs/topics/throttling.rst | 9 +++++++++ scrapy/downloadermiddlewares/backoff.py | 6 ++++++ scrapy/extensions/throttle.py | 5 ++++- scrapy/pqueues.py | 4 ++-- scrapy/settings/default_settings.py | 2 ++ 5 files changed, 23 insertions(+), 3 deletions(-) diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index accf6a590..61d7198a4 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -949,6 +949,15 @@ its domain and its IP, and is only sent when **both** allow it (see Additional settings =================== +- .. setting:: BACKOFF_ENABLED + + :setting:`BACKOFF_ENABLED` (default: ``True``) + + Whether to enable the :class:`~scrapy.downloadermiddlewares.backoff.BackoffMiddleware`, + which drives :ref:`backoff ` from download outcomes. Set it to + ``False`` to disable backoff without having to remove the middleware from + :setting:`DOWNLOADER_MIDDLEWARES`. + - .. setting:: BACKOFF_EXCEPTIONS :setting:`BACKOFF_EXCEPTIONS` diff --git a/scrapy/downloadermiddlewares/backoff.py b/scrapy/downloadermiddlewares/backoff.py index da6f2a9f1..0bc45ed7c 100644 --- a/scrapy/downloadermiddlewares/backoff.py +++ b/scrapy/downloadermiddlewares/backoff.py @@ -5,6 +5,7 @@ import logging from email.utils import parsedate_to_datetime from typing import TYPE_CHECKING +from scrapy.exceptions import NotConfigured from scrapy.throttling import iter_scopes from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.misc import load_object @@ -69,10 +70,15 @@ class BackoffMiddleware: ` to back off the request's scopes through its :meth:`~scrapy.throttling.ThrottlingManagerProtocol.back_off` API. + It is enabled by default; set :setting:`BACKOFF_ENABLED` to ``False`` to + disable it without removing it from :setting:`DOWNLOADER_MIDDLEWARES`. + See :ref:`throttling` for details. """ def __init__(self, crawler: Crawler): + if not crawler.settings.getbool("BACKOFF_ENABLED"): + raise NotConfigured # Throttling is a core, always-on subsystem: THROTTLING_MANAGER has a # non-None default and is instantiated before the downloader is built, # so crawler.throttler is always set here (the engine likewise asserts diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index db3d97444..5a5bd540f 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -83,7 +83,10 @@ class AutoThrottle: # AutoThrottle predates throttling scopes, so it adjusts the delay of # the request's domain scope, matching its historical per-domain slots. - scope_id = urlparse_cached(request).netloc + # Key by hostname (not netloc) to match the default ThrottlingManager + # scope, so the adjusted delay applies to the scope actually enforced + # even for non-default ports. + scope_id = urlparse_cached(request).hostname or "" olddelay = self._scope_delay(throttler, scope_id) newdelay = self._adjust_delay(olddelay, latency, response) throttler.set_scope_delay(scope_id, newdelay) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 8aac29b7b..a767c9dbd 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -268,7 +268,7 @@ class ScrapyPriorityQueue: ) -class DownloaderInterface: +class _DownloaderInterface: def __init__(self, crawler: Crawler): assert crawler.throttler is not None self._throttler: ThrottlingManagerProtocol = crawler.throttler @@ -347,7 +347,7 @@ class DownloaderAwarePriorityQueue: "queue class can be resumed." ) - self._downloader_interface: DownloaderInterface = DownloaderInterface(crawler) + self._downloader_interface: _DownloaderInterface = _DownloaderInterface(crawler) self.downstream_queue_cls: type[QueueProtocol] = downstream_queue_cls self._start_queue_cls: type[QueueProtocol] | None = start_queue_cls self.key: str = key diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index e4f9788d4..23ff40b9c 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -33,6 +33,7 @@ __all__ = [ "AWS_USE_SSL", "AWS_VERIFY", "BACKOFF_DELAY_FACTOR", + "BACKOFF_ENABLED", "BACKOFF_EXCEPTIONS", "BACKOFF_HTTP_CODES", "BACKOFF_JITTER", @@ -255,6 +256,7 @@ AWS_USE_SSL = None AWS_VERIFY = None BACKOFF_DELAY_FACTOR = 2.0 +BACKOFF_ENABLED = True BACKOFF_EXCEPTIONS = [ "scrapy.exceptions.DownloadFailedError", "scrapy.exceptions.DownloadTimeoutError",