From f57fc454beb4d7746002bb69457cf8add6cc3bcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 5 Nov 2024 19:44:30 +0100 Subject: [PATCH 1/5] Replace Slot.throttle with Request.meta['dont_throttle'] --- docs/topics/autothrottle.rst | 15 ++++++++++----- docs/topics/request-response.rst | 1 + docs/topics/settings.rst | 10 +--------- scrapy/core/downloader/__init__.py | 10 ++-------- scrapy/extensions/throttle.py | 6 +++++- tests/test_core_downloader.py | 2 +- tests/test_downloaderslotssettings.py | 1 - tests/test_extension_throttle.py | 23 ++++++++++++----------- 8 files changed, 32 insertions(+), 36 deletions(-) diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index 8a13b8976..fbfdd0647 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -47,12 +47,17 @@ effect, but there are some important differences: AutoThrottle doesn't have these issues. -Disabling throttling on a downloader slot -========================================= +.. reqmeta:: dont_throttle -It is possible to disable AutoThrottle for a specific download slot at run time -by setting its ``throttle`` attribute to ``False``, e.g. using -:setting:`DOWNLOAD_SLOTS`. +Disabling the throttling of a request +===================================== + +To disable AutoThrottle for a specific request, set the ``dont_throttle`` +request metadata key to ``True``: + +.. code-block:: python + + yield Request("https://example.com", meta={"dont_throttle": True}) Note, however, that AutoThrottle still determines the starting delay of every slot by setting the ``download_delay`` attribute on the running spider. You diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 3c2843bc1..18b5cbdd0 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -675,6 +675,7 @@ Those are: * :reqmeta:`dont_obey_robotstxt` * :reqmeta:`dont_redirect` * :reqmeta:`dont_retry` +* :reqmeta:`dont_throttle` * :reqmeta:`download_fail_on_dataloss` * :reqmeta:`download_latency` * :reqmeta:`download_maxsize` diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 116e8226e..cce4a7b3e 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -845,12 +845,7 @@ Allows to define concurrency/delay parameters on per slot (domain) basis: .. code-block:: python DOWNLOAD_SLOTS = { - "quotes.toscrape.com": { - "concurrency": 1, - "delay": 2, - "randomize_delay": False, - "throttle": False, - }, + "quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False}, "books.toscrape.com": {"delay": 3, "randomize_delay": False}, } @@ -862,9 +857,6 @@ Allows to define concurrency/delay parameters on per slot (domain) basis: - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency`` - :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay`` - There is no global setting for ``throttle``, whose default value is - ``None``. - .. setting:: DOWNLOAD_TIMEOUT diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 1cc0422b7..5040741e2 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -36,13 +36,10 @@ class Slot: concurrency: int, delay: float, randomize_delay: bool, - *, - throttle: bool | None = None, ): self.concurrency: int = concurrency self.delay: float = delay self.randomize_delay: bool = randomize_delay - self.throttle = throttle self.active: set[Request] = set() self.queue: deque[tuple[Request, Deferred[Response]]] = deque() @@ -67,15 +64,13 @@ class Slot: return ( f"{cls_name}(concurrency={self.concurrency!r}, " f"delay={self.delay:.2f}, " - f"randomize_delay={self.randomize_delay!r}, " - f"throttle={self.throttle!r})" + f"randomize_delay={self.randomize_delay!r})" ) def __str__(self) -> str: return ( f"" @@ -146,8 +141,7 @@ class Downloader: slot_settings.get("delay", delay), ) randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay) - throttle = slot_settings.get("throttle", None) - new_slot = Slot(conc, delay, randomize_delay, throttle=throttle) + new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot return key, self.slots[key] diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index d4b4f0e9d..fbac48b1e 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -64,7 +64,11 @@ class AutoThrottle: ) -> None: key, slot = self._get_slot(request, spider) latency = request.meta.get("download_latency") - if latency is None or slot is None or slot.throttle is False: + if ( + latency is None + or slot is None + or request.meta.get("dont_throttle", False) is True + ): return olddelay = slot.delay diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index 81cff4947..d929a9369 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -8,5 +8,5 @@ class SlotTest(unittest.TestCase): slot = Slot(concurrency=8, delay=0.1, randomize_delay=True) self.assertEqual( repr(slot), - "Slot(concurrency=8, delay=0.10, randomize_delay=True, throttle=None)", + "Slot(concurrency=8, delay=0.10, randomize_delay=True)", ) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index ea8c5b4f0..55f9ecac9 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -80,7 +80,6 @@ def test_params(): "concurrency": 1, "delay": 2, "randomize_delay": False, - "throttle": False, } settings = { "DOWNLOAD_SLOTS": { diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.py index 722a05c26..602b48e78 100644 --- a/tests/test_extension_throttle.py +++ b/tests/test_extension_throttle.py @@ -157,17 +157,20 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected) @pytest.mark.parametrize( - ("meta", "slot", "throttle"), + ("meta", "slot"), ( - ({}, None, None), - ({"download_latency": 1.0}, None, None), - ({"download_slot": "foo"}, None, None), - ({"download_slot": "foo"}, "foo", None), - ({"download_latency": 1.0, "download_slot": "foo"}, None, None), - ({"download_latency": 1.0, "download_slot": "foo"}, "foo", False), + ({}, None), + ({"download_latency": 1.0}, None), + ({"download_slot": "foo"}, None), + ({"download_slot": "foo"}, "foo"), + ({"download_latency": 1.0, "download_slot": "foo"}, None), + ( + {"download_latency": 1.0, "download_slot": "foo", "dont_throttle": True}, + "foo", + ), ), ) -def test_skipped(meta, slot, throttle): +def test_skipped(meta, slot): crawler = get_crawler() at = build_from_crawler(AutoThrottle, crawler) spider = TestSpider() @@ -178,9 +181,7 @@ def test_skipped(meta, slot, throttle): crawler.engine.downloader = Mock() crawler.engine.downloader.slots = {} if slot is not None: - _slot = Mock() - _slot.throttle = throttle - crawler.engine.downloader.slots[slot] = _slot + crawler.engine.downloader.slots[slot] = object() at._adjust_delay = None # Raise exception if called. at._response_downloaded(None, request, spider) From 5862216bb1c4717b5f4eebe8c410ad8cef60c6d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 5 Nov 2024 19:55:28 +0100 Subject: [PATCH 2/5] Fix docs example --- docs/topics/autothrottle.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index fbfdd0647..9f9114e83 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -57,7 +57,7 @@ request metadata key to ``True``: .. code-block:: python - yield Request("https://example.com", meta={"dont_throttle": True}) + Request("https://example.com", meta={"dont_throttle": True}) Note, however, that AutoThrottle still determines the starting delay of every slot by setting the ``download_delay`` attribute on the running spider. You From b244ea7ac028e2aae69d7014a808d49fa26d7c6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 5 Nov 2024 20:05:58 +0100 Subject: [PATCH 3/5] Add the missing import to the docs example --- docs/topics/autothrottle.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index 9f9114e83..48d742f63 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -57,6 +57,8 @@ request metadata key to ``True``: .. code-block:: python + from scrapy import Request + Request("https://example.com", meta={"dont_throttle": True}) Note, however, that AutoThrottle still determines the starting delay of every From 2a4b7fe0f8b2e1ce8c43998aad503f2b0b68495b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 7 Nov 2024 16:17:16 +0100 Subject: [PATCH 4/5] =?UTF-8?q?dont=5Fthrottle=20=E2=86=92=20autothrottle?= =?UTF-8?q?=5Fdont=5Fadjust=5Fdelay?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/topics/autothrottle.rst | 8 ++++---- docs/topics/request-response.rst | 2 +- scrapy/extensions/throttle.py | 2 +- tests/test_extension_throttle.py | 6 +++++- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index 48d742f63..cfd6440f2 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -47,19 +47,19 @@ effect, but there are some important differences: AutoThrottle doesn't have these issues. -.. reqmeta:: dont_throttle +.. reqmeta:: autothrottle_dont_adjust_delay Disabling the throttling of a request ===================================== -To disable AutoThrottle for a specific request, set the ``dont_throttle`` -request metadata key to ``True``: +To disable AutoThrottle for a specific request, set the +``autothrottle_dont_adjust_delay`` request metadata key to ``True``: .. code-block:: python from scrapy import Request - Request("https://example.com", meta={"dont_throttle": True}) + Request("https://example.com", meta={"autothrottle_dont_adjust_delay": True}) Note, however, that AutoThrottle still determines the starting delay of every slot by setting the ``download_delay`` attribute on the running spider. You diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 18b5cbdd0..7c15b67e8 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -668,6 +668,7 @@ are some special keys recognized by Scrapy and its built-in extensions. Those are: +* :reqmeta:`autothrottle_dont_adjust_delay` * :reqmeta:`bindaddress` * :reqmeta:`cookiejar` * :reqmeta:`dont_cache` @@ -675,7 +676,6 @@ Those are: * :reqmeta:`dont_obey_robotstxt` * :reqmeta:`dont_redirect` * :reqmeta:`dont_retry` -* :reqmeta:`dont_throttle` * :reqmeta:`download_fail_on_dataloss` * :reqmeta:`download_latency` * :reqmeta:`download_maxsize` diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index fbac48b1e..cdb0671ae 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -67,7 +67,7 @@ class AutoThrottle: if ( latency is None or slot is None - or request.meta.get("dont_throttle", False) is True + or request.meta.get("autothrottle_dont_adjust_delay", False) is True ): return diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.py index 602b48e78..f2c9dc063 100644 --- a/tests/test_extension_throttle.py +++ b/tests/test_extension_throttle.py @@ -165,7 +165,11 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected) ({"download_slot": "foo"}, "foo"), ({"download_latency": 1.0, "download_slot": "foo"}, None), ( - {"download_latency": 1.0, "download_slot": "foo", "dont_throttle": True}, + { + "download_latency": 1.0, + "download_slot": "foo", + "autothrottle_dont_adjust_delay": True, + }, "foo", ), ), From dc3ebb6cf76daa1953418af5aae3b83ffc12d02a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 7 Nov 2024 16:38:48 +0100 Subject: [PATCH 5/5] Refactor the docs --- docs/topics/autothrottle.rst | 57 ++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index cfd6440f2..5bd72fa15 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -21,9 +21,14 @@ Design goals How it works ============ -AutoThrottle extension adjusts download delays dynamically to make spider send -:setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` concurrent requests on average -to each remote website. +Scrapy allows defining the concurrency and delay of different download slots, +e.g. through the :setting:`DOWNLOAD_SLOTS` setting. By default requests are +assigned to slots based on their URL domain, although it is possible to +customize the download slot of any request. + +The AutoThrottle extension adjusts the delay of each download slot dynamically, +to make your spider send :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` concurrent +requests on average to each remote website. It uses download latency to compute the delays. The main idea is the following: if a server needs ``latency`` seconds to respond, a client @@ -47,25 +52,6 @@ effect, but there are some important differences: AutoThrottle doesn't have these issues. -.. reqmeta:: autothrottle_dont_adjust_delay - -Disabling the throttling of a request -===================================== - -To disable AutoThrottle for a specific request, set the -``autothrottle_dont_adjust_delay`` request metadata key to ``True``: - -.. code-block:: python - - from scrapy import Request - - Request("https://example.com", meta={"autothrottle_dont_adjust_delay": True}) - -Note, however, that AutoThrottle still determines the starting delay of every -slot by setting the ``download_delay`` attribute on the running spider. You -might want to set a custom value for the ``delay`` attribute of the slot, e.g. -using :setting:`DOWNLOAD_SLOTS`. - Throttling algorithm ==================== @@ -99,6 +85,33 @@ callback, for example, and unable to attend downloads. However, these latencies should still give a reasonable estimate of how busy Scrapy (and ultimately, the server) is, and this extension builds on that premise. +.. reqmeta:: autothrottle_dont_adjust_delay + +Prevent specific requests from triggering slot delay adjustments +================================================================ + +AutoThrottle adjusts the delay of download slots based on the latencies of +responses that belong to that download slot. The only exceptions are non-200 +responses, which are only taken into account to increase that delay, but +ignored if they would decrease that delay. + +You can also set the ``autothrottle_dont_adjust_delay`` request metadata key to +``True`` in any request to prevent its response latency from impacting the +delay of its download slot: + +.. code-block:: python + + from scrapy import Request + + Request("https://example.com", meta={"autothrottle_dont_adjust_delay": True}) + +Note, however, that AutoThrottle still determines the starting delay of every +download slot by setting the ``download_delay`` attribute on the running +spider. If you want AutoThrottle not to impact a download slot at all, in +addition to setting this meta key in all requests that use that download slot, +you might want to set a custom value for the ``delay`` attribute of that +download slot, e.g. using :setting:`DOWNLOAD_SLOTS`. + Settings ========