From 2a4b7fe0f8b2e1ce8c43998aad503f2b0b68495b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 7 Nov 2024 16:17:16 +0100 Subject: [PATCH] =?UTF-8?q?dont=5Fthrottle=20=E2=86=92=20autothrottle=5Fdo?= =?UTF-8?q?nt=5Fadjust=5Fdelay?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/topics/autothrottle.rst | 8 ++++---- docs/topics/request-response.rst | 2 +- scrapy/extensions/throttle.py | 2 +- tests/test_extension_throttle.py | 6 +++++- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index 48d742f63..cfd6440f2 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -47,19 +47,19 @@ effect, but there are some important differences: AutoThrottle doesn't have these issues. -.. reqmeta:: dont_throttle +.. reqmeta:: autothrottle_dont_adjust_delay Disabling the throttling of a request ===================================== -To disable AutoThrottle for a specific request, set the ``dont_throttle`` -request metadata key to ``True``: +To disable AutoThrottle for a specific request, set the +``autothrottle_dont_adjust_delay`` request metadata key to ``True``: .. code-block:: python from scrapy import Request - Request("https://example.com", meta={"dont_throttle": True}) + Request("https://example.com", meta={"autothrottle_dont_adjust_delay": True}) Note, however, that AutoThrottle still determines the starting delay of every slot by setting the ``download_delay`` attribute on the running spider. You diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 18b5cbdd0..7c15b67e8 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -668,6 +668,7 @@ are some special keys recognized by Scrapy and its built-in extensions. Those are: +* :reqmeta:`autothrottle_dont_adjust_delay` * :reqmeta:`bindaddress` * :reqmeta:`cookiejar` * :reqmeta:`dont_cache` @@ -675,7 +676,6 @@ Those are: * :reqmeta:`dont_obey_robotstxt` * :reqmeta:`dont_redirect` * :reqmeta:`dont_retry` -* :reqmeta:`dont_throttle` * :reqmeta:`download_fail_on_dataloss` * :reqmeta:`download_latency` * :reqmeta:`download_maxsize` diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index fbac48b1e..cdb0671ae 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -67,7 +67,7 @@ class AutoThrottle: if ( latency is None or slot is None - or request.meta.get("dont_throttle", False) is True + or request.meta.get("autothrottle_dont_adjust_delay", False) is True ): return diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.py index 602b48e78..f2c9dc063 100644 --- a/tests/test_extension_throttle.py +++ b/tests/test_extension_throttle.py @@ -165,7 +165,11 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected) ({"download_slot": "foo"}, "foo"), ({"download_latency": 1.0, "download_slot": "foo"}, None), ( - {"download_latency": 1.0, "download_slot": "foo", "dont_throttle": True}, + { + "download_latency": 1.0, + "download_slot": "foo", + "autothrottle_dont_adjust_delay": True, + }, "foo", ), ),