diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 7c0ee0eec..759a63f5d 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -84,8 +84,11 @@ def _get_concurrency_delay( concurrency: int, spider: Spider, settings: BaseSettings ) -> tuple[int, float]: delay: float = settings.getfloat("DOWNLOAD_DELAY") - if hasattr(spider, "download_delay"): + if hasattr(spider, "download_delay"): # pragma: no cover + warn_on_deprecated_spider_attribute("download_delay", "DOWNLOAD_DELAY") delay = spider.download_delay + if settings.getbool("AUTOTHROTTLE_ENABLED"): + delay = max(delay, settings.getfloat("AUTOTHROTTLE_START_DELAY")) if hasattr(spider, "max_concurrent_requests"): # pragma: no cover warn_on_deprecated_spider_attribute( diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 542ff1cdc..82da07133 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured +from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute if TYPE_CHECKING: # typing.Self requires Python 3.11 @@ -45,18 +46,20 @@ class AutoThrottle: def _spider_opened(self, spider: Spider) -> None: self.mindelay = self._min_delay(spider) self.maxdelay = self._max_delay(spider) - spider.download_delay = self._start_delay(spider) # type: ignore[attr-defined] def _min_delay(self, spider: Spider) -> float: - s = self.crawler.settings - return getattr(spider, "download_delay", s.getfloat("DOWNLOAD_DELAY")) + if hasattr(spider, "download_delay"): # pragma: no cover + warn_on_deprecated_spider_attribute("download_delay", "DOWNLOAD_DELAY") + return spider.download_delay + return self.crawler.settings.getfloat("DOWNLOAD_DELAY") def _max_delay(self, spider: Spider) -> float: return self.crawler.settings.getfloat("AUTOTHROTTLE_MAX_DELAY") def _start_delay(self, spider: Spider) -> float: return max( - self.mindelay, self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY") + self.mindelay, + self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY"), ) def _response_downloaded( diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.py index 4874f284a..498e0cd42 100644 --- a/tests/test_extension_throttle.py +++ b/tests/test_extension_throttle.py @@ -150,7 +150,7 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected) at = build_from_crawler(AutoThrottle, crawler) spider = _TestSpider() at._spider_opened(spider) - assert spider.download_delay == expected + assert at._start_delay(spider) == expected @pytest.mark.parametrize(