From 10f4cd34e8dc071269e17d131628af240cbab655 Mon Sep 17 00:00:00 2001 From: Hok Kan Cheung Date: Mon, 15 Dec 2025 14:45:24 -0500 Subject: [PATCH] Deprecate download_delay attr in scrapy/extensions/throttle.py, scrapy/core/downloader/__init__.py and tests/test_extension_throttle.py --- scrapy/core/downloader/__init__.py | 2 ++ scrapy/extensions/throttle.py | 7 ++++--- tests/test_extension_throttle.py | 21 +++++++++------------ 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 1c3cbc875..1e92f9d6c 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -97,6 +97,8 @@ def _get_concurrency_delay( if hasattr(spider, "download_delay"): # pragma: no cover warn_on_deprecated_spider_attribute("download_delay", "DOWNLOAD_DELAY") delay = spider.download_delay + if settings.getbool("AUTOTHROTTLE_ENABLED"): + delay = max(delay, settings.getfloat("AUTOTHROTTLE_START_DELAY")) if hasattr(spider, "max_concurrent_requests"): # pragma: no cover warn_on_deprecated_spider_attribute( diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index ae00e66b9..8c3cf424e 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -44,12 +44,13 @@ class AutoThrottle: return cls(crawler) def _spider_opened(self, spider: Spider) -> None: - if hasattr(spider, "download_delay"): # pragma: no cover - warn_on_deprecated_spider_attribute("download_delay", "DOWNLOAD_DELAY") self.mindelay = self._min_delay(spider) self.maxdelay = self._max_delay(spider) def _min_delay(self, spider: Spider) -> float: + if hasattr(spider, "download_delay"): # pragma: no cover + warn_on_deprecated_spider_attribute("download_delay", "DOWNLOAD_DELAY") + return spider.download_delay return self.crawler.settings.getfloat("DOWNLOAD_DELAY") def _max_delay(self, spider: Spider) -> float: @@ -57,7 +58,7 @@ class AutoThrottle: def _start_delay(self, spider: Spider) -> float: return max( - self.crawler.settings.getfloat("DOWNLOAD_DELAY"), + self.mindelay, self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY"), ) diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.py index b374885c7..498e0cd42 100644 --- a/tests/test_extension_throttle.py +++ b/tests/test_extension_throttle.py @@ -63,10 +63,10 @@ def test_target_concurrency_invalid(value): ("spider", "setting", "expected"), [ (UNSET, UNSET, DOWNLOAD_DELAY), - (1.0, UNSET, DOWNLOAD_DELAY), + (1.0, UNSET, 1.0), (UNSET, 1.0, 1.0), - (1.0, 2.0, 2.0), - (3.0, 2.0, 2.0), + (1.0, 2.0, 1.0), + (3.0, 2.0, 3.0), ], ) def test_mindelay_definition(spider, setting, expected): @@ -82,7 +82,8 @@ def test_mindelay_definition(spider, setting, expected): crawler = get_crawler(settings, _TestSpider) at = build_from_crawler(AutoThrottle, crawler) - assert at._min_delay(_TestSpider()) == expected + at._spider_opened(_TestSpider()) + assert at.mindelay == expected @pytest.mark.parametrize( @@ -107,7 +108,7 @@ def test_maxdelay_definition(value, expected): [ (UNSET, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), (AUTOTHROTTLE_START_DELAY - 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), - (AUTOTHROTTLE_START_DELAY + 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), + (AUTOTHROTTLE_START_DELAY + 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY + 1.0), (UNSET, AUTOTHROTTLE_START_DELAY - 1.0, UNSET, AUTOTHROTTLE_START_DELAY), (UNSET, AUTOTHROTTLE_START_DELAY + 1.0, UNSET, AUTOTHROTTLE_START_DELAY + 1.0), (UNSET, UNSET, AUTOTHROTTLE_START_DELAY - 1.0, AUTOTHROTTLE_START_DELAY - 1.0), @@ -116,13 +117,13 @@ def test_maxdelay_definition(value, expected): AUTOTHROTTLE_START_DELAY + 1.0, AUTOTHROTTLE_START_DELAY + 2.0, UNSET, - AUTOTHROTTLE_START_DELAY + 2.0, + AUTOTHROTTLE_START_DELAY + 1.0, ), ( AUTOTHROTTLE_START_DELAY + 2.0, UNSET, AUTOTHROTTLE_START_DELAY + 1.0, - AUTOTHROTTLE_START_DELAY + 1.0, + AUTOTHROTTLE_START_DELAY + 2.0, ), ( AUTOTHROTTLE_START_DELAY + 1.0, @@ -149,11 +150,7 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected) at = build_from_crawler(AutoThrottle, crawler) spider = _TestSpider() at._spider_opened(spider) - download_delay = min_setting if min_setting is not UNSET else DOWNLOAD_DELAY - autothrottle_start = ( - start_setting if start_setting is not UNSET else AUTOTHROTTLE_START_DELAY - ) - assert max(download_delay, autothrottle_start) == expected + assert at._start_delay(spider) == expected @pytest.mark.parametrize(