diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index 5370d77b3..8a13b8976 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -47,6 +47,18 @@ effect, but there are some important differences: AutoThrottle doesn't have these issues. +Disabling throttling on a downloader slot +========================================= + +It is possible to disable AutoThrottle for a specific download slot at run time +by setting its ``throttle`` attribute to ``False``, e.g. using +:setting:`DOWNLOAD_SLOTS`. + +Note, however, that AutoThrottle still determines the starting delay of every +slot by setting the ``download_delay`` attribute on the running spider. You +might want to set a custom value for the ``delay`` attribute of the slot, e.g. +using :setting:`DOWNLOAD_SLOTS`. + Throttling algorithm ==================== diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 439aedc18..2bd9cf1ed 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -835,7 +835,7 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2: .. setting:: DOWNLOAD_SLOTS DOWNLOAD_SLOTS ----------------- +-------------- Default: ``{}`` @@ -844,7 +844,12 @@ Allows to define concurrency/delay parameters on per slot (domain) basis: .. code-block:: python DOWNLOAD_SLOTS = { - "quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False}, + "quotes.toscrape.com": { + "concurrency": 1, + "delay": 2, + "randomize_delay": False, + "throttle": False, + }, "books.toscrape.com": {"delay": 3, "randomize_delay": False}, } @@ -856,6 +861,9 @@ Allows to define concurrency/delay parameters on per slot (domain) basis: - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency`` - :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay`` + There is no global setting for ``throttle``, whose default value is + ``None``. + .. setting:: DOWNLOAD_TIMEOUT diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 666282856..ecd3e8b56 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -2,7 +2,7 @@ import random from collections import deque from datetime import datetime from time import time -from typing import TYPE_CHECKING, Any, Deque, Dict, Set, Tuple, cast +from typing import TYPE_CHECKING, Any, Deque, Dict, Optional, Set, Tuple, cast from twisted.internet import task from twisted.internet.defer import Deferred @@ -24,10 +24,18 @@ if TYPE_CHECKING: class Slot: """Downloader slot""" - def __init__(self, concurrency: int, delay: float, randomize_delay: bool): + def __init__( + self, + concurrency: int, + delay: float, + randomize_delay: bool, + *, + throttle: Optional[bool] = None, + ): self.concurrency: int = concurrency self.delay: float = delay self.randomize_delay: bool = randomize_delay + self.throttle = throttle self.active: Set[Request] = set() self.queue: Deque[Tuple[Request, Deferred]] = deque() @@ -52,13 +60,15 @@ class Slot: return ( f"{cls_name}(concurrency={self.concurrency!r}, " f"delay={self.delay:.2f}, " - f"randomize_delay={self.randomize_delay!r})" + f"randomize_delay={self.randomize_delay!r}, " + f"throttle={self.throttle!r})" ) def __str__(self) -> str: return ( f"" @@ -127,7 +137,8 @@ class Downloader: slot_settings.get("delay", delay), ) randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay) - new_slot = Slot(conc, delay, randomize_delay) + throttle = slot_settings.get("throttle", None) + new_slot = Slot(conc, delay, randomize_delay, throttle=throttle) self.slots[key] = new_slot return key, self.slots[key] diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index bf4e6bb63..217e61a81 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -62,7 +62,7 @@ class AutoThrottle: ) -> None: key, slot = self._get_slot(request, spider) latency = request.meta.get("download_latency") - if latency is None or slot is None: + if latency is None or slot is None or slot.throttle is False: return olddelay = slot.delay diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index 9a6e9e4ff..81cff4947 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -7,5 +7,6 @@ class SlotTest(unittest.TestCase): def test_repr(self): slot = Slot(concurrency=8, delay=0.1, randomize_delay=True) self.assertEqual( - repr(slot), "Slot(concurrency=8, delay=0.10, randomize_delay=True)" + repr(slot), + "Slot(concurrency=8, delay=0.10, randomize_delay=True, throttle=None)", ) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 9d4072d19..ea8c5b4f0 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -3,8 +3,10 @@ import time from twisted.internet import defer from twisted.trial.unittest import TestCase +from scrapy import Request +from scrapy.core.downloader import Downloader, Slot from scrapy.crawler import CrawlerRunner -from scrapy.http import Request +from scrapy.utils.test import get_crawler from tests.mockserver import MockServer from tests.spiders import MetaSpider @@ -20,6 +22,7 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): "concurrency": 1, "delay": 2, "randomize_delay": False, + "throttle": False, }, "books.toscrape.com": {"delay": 3, "randomize_delay": False}, }, @@ -70,3 +73,27 @@ class CrawlTestCase(TestCase): } self.assertTrue(max(list(error_delta.values())) < tolerance) + + +def test_params(): + params = { + "concurrency": 1, + "delay": 2, + "randomize_delay": False, + "throttle": False, + } + settings = { + "DOWNLOAD_SLOTS": { + "example.com": params, + }, + } + crawler = get_crawler(settings_dict=settings) + downloader = Downloader(crawler) + downloader._slot_gc_loop.stop() # Prevent an unclean reactor. + request = Request("https://example.com") + _, actual = downloader._get_slot(request, spider=None) + expected = Slot(**params) + for param in params: + assert getattr(expected, param) == getattr( + actual, param + ), f"Slot.{param}: {getattr(expected, param)!r} != {getattr(actual, param)!r}" diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.py index dae4ea966..722a05c26 100644 --- a/tests/test_extension_throttle.py +++ b/tests/test_extension_throttle.py @@ -157,16 +157,17 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected) @pytest.mark.parametrize( - ("meta", "slot"), + ("meta", "slot", "throttle"), ( - ({}, None), - ({"download_latency": 1.0}, None), - ({"download_slot": "foo"}, None), - ({"download_slot": "foo"}, "foo"), - ({"download_latency": 1.0, "download_slot": "foo"}, None), + ({}, None, None), + ({"download_latency": 1.0}, None, None), + ({"download_slot": "foo"}, None, None), + ({"download_slot": "foo"}, "foo", None), + ({"download_latency": 1.0, "download_slot": "foo"}, None, None), + ({"download_latency": 1.0, "download_slot": "foo"}, "foo", False), ), ) -def test_skipped(meta, slot): +def test_skipped(meta, slot, throttle): crawler = get_crawler() at = build_from_crawler(AutoThrottle, crawler) spider = TestSpider() @@ -177,7 +178,9 @@ def test_skipped(meta, slot): crawler.engine.downloader = Mock() crawler.engine.downloader.slots = {} if slot is not None: - crawler.engine.downloader.slots[slot] = object() + _slot = Mock() + _slot.throttle = throttle + crawler.engine.downloader.slots[slot] = _slot at._adjust_delay = None # Raise exception if called. at._response_downloaded(None, request, spider)