From 0292d23d7a0ad89432c17b6920a6c9f7a01aa85a Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 19 Jun 2026 17:46:22 +0200 Subject: [PATCH] Improve deprecation messaging around SCRAPER_SLOT_MAX_ACTIVE_SIZE --- scrapy/core/downloader/__init__.py | 42 ++++++++++++++++++++---------- tests/test_downloader.py | 31 +++++++++++++++------- 2 files changed, 50 insertions(+), 23 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index d0239414d..23a563d41 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -142,24 +142,38 @@ class Downloader: "SCRAPER_SLOT_MAX_ACTIVE_SIZE" ) assert deprecated_setting_priority is not None - if deprecated_setting_priority > 0: - warn( - ( - "The SCRAPER_SLOT_MAX_ACTIVE_SIZE setting is deprecated, " - "use RESPONSE_MAX_ACTIVE_SIZE instead." - ), - ScrapyDeprecationWarning, - stacklevel=2, - ) setting_priority = self.settings.getpriority("RESPONSE_MAX_ACTIVE_SIZE") assert setting_priority is not None - if setting_priority >= deprecated_setting_priority: - self._response_max_active_size = self.settings.getint( - "RESPONSE_MAX_ACTIVE_SIZE" - ) + if deprecated_setting_priority > 0: + if setting_priority >= deprecated_setting_priority: + warn( + ( + "The SCRAPER_SLOT_MAX_ACTIVE_SIZE setting is deprecated " + "and is being ignored because RESPONSE_MAX_ACTIVE_SIZE is " + "set with an equal or higher priority. Remove " + "SCRAPER_SLOT_MAX_ACTIVE_SIZE from your settings." + ), + ScrapyDeprecationWarning, + stacklevel=2, + ) + self._response_max_active_size = self.settings.getint( + "RESPONSE_MAX_ACTIVE_SIZE" + ) + else: + warn( + ( + "The SCRAPER_SLOT_MAX_ACTIVE_SIZE setting is deprecated, " + "use RESPONSE_MAX_ACTIVE_SIZE instead." + ), + ScrapyDeprecationWarning, + stacklevel=2, + ) + self._response_max_active_size = self.settings.getint( + "SCRAPER_SLOT_MAX_ACTIVE_SIZE" + ) else: self._response_max_active_size = self.settings.getint( - "SCRAPER_SLOT_MAX_ACTIVE_SIZE" + "RESPONSE_MAX_ACTIVE_SIZE" ) self._response_max_active_size_warned = False diff --git a/tests/test_downloader.py b/tests/test_downloader.py index 2808fdc2e..095873715 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -26,23 +26,35 @@ class OfflineSpider(Spider): pass -def _assert_scraper_slot_deprecation(warning_messages): +def _assert_scraper_slot_deprecation(warning_messages, *, ignored=False): """Assert that a crawl emitted exactly one Scrapy deprecation warning, the one about SCRAPER_SLOT_MAX_ACTIVE_SIZE. Only Scrapy deprecation warnings are counted: a crawl may emit unrelated warnings (e.g. a ResourceWarning for a socket garbage-collected while the - recorder is active), and those must not make the assertion flaky.""" + recorder is active), and those must not make the assertion flaky. + + Pass ``ignored=True`` when RESPONSE_MAX_ACTIVE_SIZE is set with an equal or + higher priority and therefore SCRAPER_SLOT_MAX_ACTIVE_SIZE is being + ignored.""" deprecations = [ message for message in warning_messages if issubclass(message.category, ScrapyDeprecationWarning) ] assert len(deprecations) == 1 - assert str(deprecations[0].message) == ( - "The SCRAPER_SLOT_MAX_ACTIVE_SIZE setting is deprecated, use " - "RESPONSE_MAX_ACTIVE_SIZE instead." - ) + if ignored: + assert str(deprecations[0].message) == ( + "The SCRAPER_SLOT_MAX_ACTIVE_SIZE setting is deprecated and is " + "being ignored because RESPONSE_MAX_ACTIVE_SIZE is set with an " + "equal or higher priority. Remove SCRAPER_SLOT_MAX_ACTIVE_SIZE " + "from your settings." + ) + else: + assert str(deprecations[0].message) == ( + "The SCRAPER_SLOT_MAX_ACTIVE_SIZE setting is deprecated, use " + "RESPONSE_MAX_ACTIVE_SIZE instead." + ) class gt: @@ -110,8 +122,9 @@ class TestResponseMaxActiveSize: async def test_both(self): """Setting RESPONSE_MAX_ACTIVE_SIZE and SCRAPER_SLOT_MAX_ACTIVE_SIZE to different values with the same setting priority triggers a deprecation - warning about SCRAPER_SLOT_MAX_ACTIVE_SIZE and makes the value of - RESPONSE_MAX_ACTIVE_SIZE the effective response max active size.""" + warning about SCRAPER_SLOT_MAX_ACTIVE_SIZE being ignored, and makes + the value of RESPONSE_MAX_ACTIVE_SIZE the effective response max active + size.""" crawler = get_crawler( OfflineSpider, settings_dict={ @@ -122,7 +135,7 @@ class TestResponseMaxActiveSize: with pytest.warns(ScrapyDeprecationWarning) as warning_messages: await maybe_deferred_to_future(crawler.crawl()) assert crawler.engine.downloader._response_max_active_size == 1 - _assert_scraper_slot_deprecation(warning_messages) + _assert_scraper_slot_deprecation(warning_messages, ignored=True) @coroutine_test async def test_both_deprecated_priority(self):