From 2686f1a37d7a155d34872d94522127fbbabc0682 Mon Sep 17 00:00:00 2001 From: Hok Kan Cheung Date: Fri, 12 Dec 2025 23:50:19 -0500 Subject: [PATCH 1/5] Update throttle.py to deprecate download_delay attr --- scrapy/extensions/throttle.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index cdb0671ae..e98af05b1 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -45,18 +45,17 @@ class AutoThrottle: def _spider_opened(self, spider: Spider) -> None: self.mindelay = self._min_delay(spider) self.maxdelay = self._max_delay(spider) - spider.download_delay = self._start_delay(spider) # type: ignore[attr-defined] def _min_delay(self, spider: Spider) -> float: - s = self.crawler.settings - return getattr(spider, "download_delay", s.getfloat("DOWNLOAD_DELAY")) + return self.crawler.settings.getfloat("DOWNLOAD_DELAY") def _max_delay(self, spider: Spider) -> float: return self.crawler.settings.getfloat("AUTOTHROTTLE_MAX_DELAY") def _start_delay(self, spider: Spider) -> float: return max( - self.mindelay, self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY") + self.crawler.settings.getfloat("DOWNLOAD_DELAY"), + self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY"), ) def _response_downloaded( From 77a0df8c7beaa280a4d16be41c12ce6f26aa3700 Mon Sep 17 00:00:00 2001 From: Hok Kan Cheung Date: Mon, 15 Dec 2025 08:31:20 -0500 Subject: [PATCH 2/5] Add deprecation warning on download_delay to throttle.py --- scrapy/extensions/throttle.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index e98af05b1..ae00e66b9 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured +from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute if TYPE_CHECKING: # typing.Self requires Python 3.11 @@ -43,6 +44,8 @@ class AutoThrottle: return cls(crawler) def _spider_opened(self, spider: Spider) -> None: + if hasattr(spider, "download_delay"): # pragma: no cover + warn_on_deprecated_spider_attribute("download_delay", "DOWNLOAD_DELAY") self.mindelay = self._min_delay(spider) self.maxdelay = self._max_delay(spider) From 9bd7c108e36ebc8531f182df495d7546182c71f8 Mon Sep 17 00:00:00 2001 From: Hok Kan Cheung Date: Mon, 15 Dec 2025 08:34:01 -0500 Subject: [PATCH 3/5] Update test_extension_throttle.py to deprecate download_delay --- tests/test_extension_throttle.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.py index 4874f284a..b374885c7 100644 --- a/tests/test_extension_throttle.py +++ b/tests/test_extension_throttle.py @@ -63,10 +63,10 @@ def test_target_concurrency_invalid(value): ("spider", "setting", "expected"), [ (UNSET, UNSET, DOWNLOAD_DELAY), - (1.0, UNSET, 1.0), + (1.0, UNSET, DOWNLOAD_DELAY), (UNSET, 1.0, 1.0), - (1.0, 2.0, 1.0), - (3.0, 2.0, 3.0), + (1.0, 2.0, 2.0), + (3.0, 2.0, 2.0), ], ) def test_mindelay_definition(spider, setting, expected): @@ -82,8 +82,7 @@ def test_mindelay_definition(spider, setting, expected): crawler = get_crawler(settings, _TestSpider) at = build_from_crawler(AutoThrottle, crawler) - at._spider_opened(_TestSpider()) - assert at.mindelay == expected + assert at._min_delay(_TestSpider()) == expected @pytest.mark.parametrize( @@ -108,7 +107,7 @@ def test_maxdelay_definition(value, expected): [ (UNSET, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), (AUTOTHROTTLE_START_DELAY - 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), - (AUTOTHROTTLE_START_DELAY + 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY + 1.0), + (AUTOTHROTTLE_START_DELAY + 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), (UNSET, AUTOTHROTTLE_START_DELAY - 1.0, UNSET, AUTOTHROTTLE_START_DELAY), (UNSET, AUTOTHROTTLE_START_DELAY + 1.0, UNSET, AUTOTHROTTLE_START_DELAY + 1.0), (UNSET, UNSET, AUTOTHROTTLE_START_DELAY - 1.0, AUTOTHROTTLE_START_DELAY - 1.0), @@ -117,13 +116,13 @@ def test_maxdelay_definition(value, expected): AUTOTHROTTLE_START_DELAY + 1.0, AUTOTHROTTLE_START_DELAY + 2.0, UNSET, - AUTOTHROTTLE_START_DELAY + 1.0, + AUTOTHROTTLE_START_DELAY + 2.0, ), ( AUTOTHROTTLE_START_DELAY + 2.0, UNSET, AUTOTHROTTLE_START_DELAY + 1.0, - AUTOTHROTTLE_START_DELAY + 2.0, + AUTOTHROTTLE_START_DELAY + 1.0, ), ( AUTOTHROTTLE_START_DELAY + 1.0, @@ -150,7 +149,11 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected) at = build_from_crawler(AutoThrottle, crawler) spider = _TestSpider() at._spider_opened(spider) - assert spider.download_delay == expected + download_delay = min_setting if min_setting is not UNSET else DOWNLOAD_DELAY + autothrottle_start = ( + start_setting if start_setting is not UNSET else AUTOTHROTTLE_START_DELAY + ) + assert max(download_delay, autothrottle_start) == expected @pytest.mark.parametrize( From 3d3c84ceb436167397d8b93c1857e63dc04a417e Mon Sep 17 00:00:00 2001 From: Hok Kan Cheung Date: Mon, 15 Dec 2025 09:32:46 -0500 Subject: [PATCH 4/5] Add deprecation warning on download_delay to scrapy/core/downloader/__init__.py --- scrapy/core/downloader/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index be5efba87..1c3cbc875 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -94,7 +94,8 @@ def _get_concurrency_delay( concurrency: int, spider: Spider, settings: BaseSettings ) -> tuple[int, float]: delay: float = settings.getfloat("DOWNLOAD_DELAY") - if hasattr(spider, "download_delay"): + if hasattr(spider, "download_delay"): # pragma: no cover + warn_on_deprecated_spider_attribute("download_delay", "DOWNLOAD_DELAY") delay = spider.download_delay if hasattr(spider, "max_concurrent_requests"): # pragma: no cover From 10f4cd34e8dc071269e17d131628af240cbab655 Mon Sep 17 00:00:00 2001 From: Hok Kan Cheung Date: Mon, 15 Dec 2025 14:45:24 -0500 Subject: [PATCH 5/5] Deprecate download_delay attr in scrapy/extensions/throttle.py, scrapy/core/downloader/__init__.py and tests/test_extension_throttle.py --- scrapy/core/downloader/__init__.py | 2 ++ scrapy/extensions/throttle.py | 7 ++++--- tests/test_extension_throttle.py | 21 +++++++++------------ 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 1c3cbc875..1e92f9d6c 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -97,6 +97,8 @@ def _get_concurrency_delay( if hasattr(spider, "download_delay"): # pragma: no cover warn_on_deprecated_spider_attribute("download_delay", "DOWNLOAD_DELAY") delay = spider.download_delay + if settings.getbool("AUTOTHROTTLE_ENABLED"): + delay = max(delay, settings.getfloat("AUTOTHROTTLE_START_DELAY")) if hasattr(spider, "max_concurrent_requests"): # pragma: no cover warn_on_deprecated_spider_attribute( diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index ae00e66b9..8c3cf424e 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -44,12 +44,13 @@ class AutoThrottle: return cls(crawler) def _spider_opened(self, spider: Spider) -> None: - if hasattr(spider, "download_delay"): # pragma: no cover - warn_on_deprecated_spider_attribute("download_delay", "DOWNLOAD_DELAY") self.mindelay = self._min_delay(spider) self.maxdelay = self._max_delay(spider) def _min_delay(self, spider: Spider) -> float: + if hasattr(spider, "download_delay"): # pragma: no cover + warn_on_deprecated_spider_attribute("download_delay", "DOWNLOAD_DELAY") + return spider.download_delay return self.crawler.settings.getfloat("DOWNLOAD_DELAY") def _max_delay(self, spider: Spider) -> float: @@ -57,7 +58,7 @@ class AutoThrottle: def _start_delay(self, spider: Spider) -> float: return max( - self.crawler.settings.getfloat("DOWNLOAD_DELAY"), + self.mindelay, self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY"), ) diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.py index b374885c7..498e0cd42 100644 --- a/tests/test_extension_throttle.py +++ b/tests/test_extension_throttle.py @@ -63,10 +63,10 @@ def test_target_concurrency_invalid(value): ("spider", "setting", "expected"), [ (UNSET, UNSET, DOWNLOAD_DELAY), - (1.0, UNSET, DOWNLOAD_DELAY), + (1.0, UNSET, 1.0), (UNSET, 1.0, 1.0), - (1.0, 2.0, 2.0), - (3.0, 2.0, 2.0), + (1.0, 2.0, 1.0), + (3.0, 2.0, 3.0), ], ) def test_mindelay_definition(spider, setting, expected): @@ -82,7 +82,8 @@ def test_mindelay_definition(spider, setting, expected): crawler = get_crawler(settings, _TestSpider) at = build_from_crawler(AutoThrottle, crawler) - assert at._min_delay(_TestSpider()) == expected + at._spider_opened(_TestSpider()) + assert at.mindelay == expected @pytest.mark.parametrize( @@ -107,7 +108,7 @@ def test_maxdelay_definition(value, expected): [ (UNSET, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), (AUTOTHROTTLE_START_DELAY - 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), - (AUTOTHROTTLE_START_DELAY + 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), + (AUTOTHROTTLE_START_DELAY + 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY + 1.0), (UNSET, AUTOTHROTTLE_START_DELAY - 1.0, UNSET, AUTOTHROTTLE_START_DELAY), (UNSET, AUTOTHROTTLE_START_DELAY + 1.0, UNSET, AUTOTHROTTLE_START_DELAY + 1.0), (UNSET, UNSET, AUTOTHROTTLE_START_DELAY - 1.0, AUTOTHROTTLE_START_DELAY - 1.0), @@ -116,13 +117,13 @@ def test_maxdelay_definition(value, expected): AUTOTHROTTLE_START_DELAY + 1.0, AUTOTHROTTLE_START_DELAY + 2.0, UNSET, - AUTOTHROTTLE_START_DELAY + 2.0, + AUTOTHROTTLE_START_DELAY + 1.0, ), ( AUTOTHROTTLE_START_DELAY + 2.0, UNSET, AUTOTHROTTLE_START_DELAY + 1.0, - AUTOTHROTTLE_START_DELAY + 1.0, + AUTOTHROTTLE_START_DELAY + 2.0, ), ( AUTOTHROTTLE_START_DELAY + 1.0, @@ -149,11 +150,7 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected) at = build_from_crawler(AutoThrottle, crawler) spider = _TestSpider() at._spider_opened(spider) - download_delay = min_setting if min_setting is not UNSET else DOWNLOAD_DELAY - autothrottle_start = ( - start_setting if start_setting is not UNSET else AUTOTHROTTLE_START_DELAY - ) - assert max(download_delay, autothrottle_start) == expected + assert at._start_delay(spider) == expected @pytest.mark.parametrize(