RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Deprecate download_delay attr in scrapy/extensions/throttle.py, scrapy/core/downloader/__init__.py and tests/test_extension_throttle.py

This commit is contained in:
Hok Kan Cheung 2025-12-15 14:45:24 -05:00
parent 3d3c84ceb4
commit 10f4cd34e8
3 changed files with 15 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/core/downloader/__init__.py
View File

@ -97,6 +97,8 @@ def _get_concurrency_delay(
if hasattr(spider, "download_delay"): # pragma: no cover
warn_on_deprecated_spider_attribute("download_delay", "DOWNLOAD_DELAY")
delay = spider.download_delay
if settings.getbool("AUTOTHROTTLE_ENABLED"):
delay = max(delay, settings.getfloat("AUTOTHROTTLE_START_DELAY"))
if hasattr(spider, "max_concurrent_requests"): # pragma: no cover
warn_on_deprecated_spider_attribute(

7
scrapy/extensions/throttle.py
View File

@ -44,12 +44,13 @@ class AutoThrottle:
return cls(crawler)
def _spider_opened(self, spider: Spider) -> None:
if hasattr(spider, "download_delay"): # pragma: no cover
warn_on_deprecated_spider_attribute("download_delay", "DOWNLOAD_DELAY")
self.mindelay = self._min_delay(spider)
self.maxdelay = self._max_delay(spider)
def _min_delay(self, spider: Spider) -> float:
if hasattr(spider, "download_delay"): # pragma: no cover
warn_on_deprecated_spider_attribute("download_delay", "DOWNLOAD_DELAY")
return spider.download_delay
return self.crawler.settings.getfloat("DOWNLOAD_DELAY")
def _max_delay(self, spider: Spider) -> float:
@ -57,7 +58,7 @@ class AutoThrottle:
def _start_delay(self, spider: Spider) -> float:
return max(
self.crawler.settings.getfloat("DOWNLOAD_DELAY"),
self.mindelay,
self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY"),
)

21
tests/test_extension_throttle.py
View File

@ -63,10 +63,10 @@ def test_target_concurrency_invalid(value):
("spider", "setting", "expected"),
[
(UNSET, UNSET, DOWNLOAD_DELAY),
(1.0, UNSET, DOWNLOAD_DELAY),
(1.0, UNSET, 1.0),
(UNSET, 1.0, 1.0),
(1.0, 2.0, 2.0),
(3.0, 2.0, 2.0),
(1.0, 2.0, 1.0),
(3.0, 2.0, 3.0),
],
)
def test_mindelay_definition(spider, setting, expected):
@ -82,7 +82,8 @@ def test_mindelay_definition(spider, setting, expected):
crawler = get_crawler(settings, _TestSpider)
at = build_from_crawler(AutoThrottle, crawler)
assert at._min_delay(_TestSpider()) == expected
at._spider_opened(_TestSpider())
assert at.mindelay == expected
@pytest.mark.parametrize(
@ -107,7 +108,7 @@ def test_maxdelay_definition(value, expected):
[
(UNSET, UNSET, UNSET, AUTOTHROTTLE_START_DELAY),
(AUTOTHROTTLE_START_DELAY - 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY),
(AUTOTHROTTLE_START_DELAY + 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY),
(AUTOTHROTTLE_START_DELAY + 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY + 1.0),
(UNSET, AUTOTHROTTLE_START_DELAY - 1.0, UNSET, AUTOTHROTTLE_START_DELAY),
(UNSET, AUTOTHROTTLE_START_DELAY + 1.0, UNSET, AUTOTHROTTLE_START_DELAY + 1.0),
(UNSET, UNSET, AUTOTHROTTLE_START_DELAY - 1.0, AUTOTHROTTLE_START_DELAY - 1.0),
@ -116,13 +117,13 @@ def test_maxdelay_definition(value, expected):
AUTOTHROTTLE_START_DELAY + 1.0,
AUTOTHROTTLE_START_DELAY + 2.0,
UNSET,
AUTOTHROTTLE_START_DELAY + 2.0,
AUTOTHROTTLE_START_DELAY + 1.0,
),
(
AUTOTHROTTLE_START_DELAY + 2.0,
UNSET,
AUTOTHROTTLE_START_DELAY + 1.0,
AUTOTHROTTLE_START_DELAY + 1.0,
AUTOTHROTTLE_START_DELAY + 2.0,
),
(
AUTOTHROTTLE_START_DELAY + 1.0,
@ -149,11 +150,7 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected)
at = build_from_crawler(AutoThrottle, crawler)
spider = _TestSpider()
at._spider_opened(spider)
download_delay = min_setting if min_setting is not UNSET else DOWNLOAD_DELAY
autothrottle_start = (
start_setting if start_setting is not UNSET else AUTOTHROTTLE_START_DELAY
)
assert max(download_delay, autothrottle_start) == expected
assert at._start_delay(spider) == expected
@pytest.mark.parametrize(