WIP

docs/topics/throttling.rst

@@ -949,6 +949,15 @@ its domain and its IP, and is only sent when **both** allow it (see
 
 Additional settings
 ===================
+-   .. setting:: BACKOFF_ENABLED
+
+    :setting:`BACKOFF_ENABLED` (default: ``True``)
+
+    Whether to enable the :class:`~scrapy.downloadermiddlewares.backoff.BackoffMiddleware`,
+    which drives :ref:`backoff <backoff>` from download outcomes. Set it to
+    ``False`` to disable backoff without having to remove the middleware from
+    :setting:`DOWNLOADER_MIDDLEWARES`.
+
 -   .. setting:: BACKOFF_EXCEPTIONS
 
     :setting:`BACKOFF_EXCEPTIONS`

scrapy/downloadermiddlewares/backoff.py

@@ -5,6 +5,7 @@ import logging
 from email.utils import parsedate_to_datetime
 from typing import TYPE_CHECKING
 
+from scrapy.exceptions import NotConfigured
 from scrapy.throttling import iter_scopes
 from scrapy.utils.decorators import _warn_spider_arg
 from scrapy.utils.misc import load_object
@@ -69,10 +70,15 @@ class BackoffMiddleware:
     <throttling>` to back off the request's scopes through its
     :meth:`~scrapy.throttling.ThrottlingManagerProtocol.back_off` API.
 
+    It is enabled by default; set :setting:`BACKOFF_ENABLED` to ``False`` to
+    disable it without removing it from :setting:`DOWNLOADER_MIDDLEWARES`.
+
     See :ref:`throttling` for details.
     """
 
     def __init__(self, crawler: Crawler):
+        if not crawler.settings.getbool("BACKOFF_ENABLED"):
+            raise NotConfigured
         # Throttling is a core, always-on subsystem: THROTTLING_MANAGER has a
         # non-None default and is instantiated before the downloader is built,
         # so crawler.throttler is always set here (the engine likewise asserts

scrapy/extensions/throttle.py

@@ -83,7 +83,10 @@ class AutoThrottle:
 
         # AutoThrottle predates throttling scopes, so it adjusts the delay of
         # the request's domain scope, matching its historical per-domain slots.
-        scope_id = urlparse_cached(request).netloc
+        # Key by hostname (not netloc) to match the default ThrottlingManager
+        # scope, so the adjusted delay applies to the scope actually enforced
+        # even for non-default ports.
+        scope_id = urlparse_cached(request).hostname or ""
         olddelay = self._scope_delay(throttler, scope_id)
         newdelay = self._adjust_delay(olddelay, latency, response)
         throttler.set_scope_delay(scope_id, newdelay)

scrapy/pqueues.py

@@ -268,7 +268,7 @@ class ScrapyPriorityQueue:
         )
 
 
-class DownloaderInterface:
+class _DownloaderInterface:
     def __init__(self, crawler: Crawler):
         assert crawler.throttler is not None
         self._throttler: ThrottlingManagerProtocol = crawler.throttler
@@ -347,7 +347,7 @@ class DownloaderAwarePriorityQueue:
                 "queue class can be resumed."
             )
 
-        self._downloader_interface: DownloaderInterface = DownloaderInterface(crawler)
+        self._downloader_interface: _DownloaderInterface = _DownloaderInterface(crawler)
         self.downstream_queue_cls: type[QueueProtocol] = downstream_queue_cls
         self._start_queue_cls: type[QueueProtocol] | None = start_queue_cls
         self.key: str = key

scrapy/settings/default_settings.py

@@ -33,6 +33,7 @@ __all__ = [
     "AWS_USE_SSL",
     "AWS_VERIFY",
     "BACKOFF_DELAY_FACTOR",
+    "BACKOFF_ENABLED",
     "BACKOFF_EXCEPTIONS",
     "BACKOFF_HTTP_CODES",
     "BACKOFF_JITTER",
@@ -255,6 +256,7 @@ AWS_USE_SSL = None
 AWS_VERIFY = None
 
 BACKOFF_DELAY_FACTOR = 2.0
+BACKOFF_ENABLED = True
 BACKOFF_EXCEPTIONS = [
     "scrapy.exceptions.DownloadFailedError",
     "scrapy.exceptions.DownloadTimeoutError",