RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improve deprecation messaging around SCRAPER_SLOT_MAX_ACTIVE_SIZE

This commit is contained in:
Adrian Chaves 2026-06-19 17:46:22 +02:00
parent e746df0297
commit 0292d23d7a
2 changed files with 50 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
scrapy/core/downloader/__init__.py
View File

@ -142,24 +142,38 @@ class Downloader:
"SCRAPER_SLOT_MAX_ACTIVE_SIZE"
)
assert deprecated_setting_priority is not None
if deprecated_setting_priority > 0:
warn(
(
"The SCRAPER_SLOT_MAX_ACTIVE_SIZE setting is deprecated, "
"use RESPONSE_MAX_ACTIVE_SIZE instead."
),
ScrapyDeprecationWarning,
stacklevel=2,
)
setting_priority = self.settings.getpriority("RESPONSE_MAX_ACTIVE_SIZE")
assert setting_priority is not None
if setting_priority >= deprecated_setting_priority:
self._response_max_active_size = self.settings.getint(
"RESPONSE_MAX_ACTIVE_SIZE"
)
if deprecated_setting_priority > 0:
if setting_priority >= deprecated_setting_priority:
warn(
(
"The SCRAPER_SLOT_MAX_ACTIVE_SIZE setting is deprecated "
"and is being ignored because RESPONSE_MAX_ACTIVE_SIZE is "
"set with an equal or higher priority. Remove "
"SCRAPER_SLOT_MAX_ACTIVE_SIZE from your settings."
),
ScrapyDeprecationWarning,
stacklevel=2,
)
self._response_max_active_size = self.settings.getint(
"RESPONSE_MAX_ACTIVE_SIZE"
)
else:
warn(
(
"The SCRAPER_SLOT_MAX_ACTIVE_SIZE setting is deprecated, "
"use RESPONSE_MAX_ACTIVE_SIZE instead."
),
ScrapyDeprecationWarning,
stacklevel=2,
)
self._response_max_active_size = self.settings.getint(
"SCRAPER_SLOT_MAX_ACTIVE_SIZE"
)
else:
self._response_max_active_size = self.settings.getint(
"SCRAPER_SLOT_MAX_ACTIVE_SIZE"
"RESPONSE_MAX_ACTIVE_SIZE"
)
self._response_max_active_size_warned = False

31
tests/test_downloader.py
View File

@ -26,23 +26,35 @@ class OfflineSpider(Spider):
pass
def _assert_scraper_slot_deprecation(warning_messages):
def _assert_scraper_slot_deprecation(warning_messages, *, ignored=False):
"""Assert that a crawl emitted exactly one Scrapy deprecation warning, the
one about SCRAPER_SLOT_MAX_ACTIVE_SIZE.
Only Scrapy deprecation warnings are counted: a crawl may emit unrelated
warnings (e.g. a ResourceWarning for a socket garbage-collected while the
recorder is active), and those must not make the assertion flaky."""
recorder is active), and those must not make the assertion flaky.
Pass ``ignored=True`` when RESPONSE_MAX_ACTIVE_SIZE is set with an equal or
higher priority and therefore SCRAPER_SLOT_MAX_ACTIVE_SIZE is being
ignored."""
deprecations = [
message
for message in warning_messages
if issubclass(message.category, ScrapyDeprecationWarning)
]
assert len(deprecations) == 1
assert str(deprecations[0].message) == (
"The SCRAPER_SLOT_MAX_ACTIVE_SIZE setting is deprecated, use "
"RESPONSE_MAX_ACTIVE_SIZE instead."
)
if ignored:
assert str(deprecations[0].message) == (
"The SCRAPER_SLOT_MAX_ACTIVE_SIZE setting is deprecated and is "
"being ignored because RESPONSE_MAX_ACTIVE_SIZE is set with an "
"equal or higher priority. Remove SCRAPER_SLOT_MAX_ACTIVE_SIZE "
"from your settings."
)
else:
assert str(deprecations[0].message) == (
"The SCRAPER_SLOT_MAX_ACTIVE_SIZE setting is deprecated, use "
"RESPONSE_MAX_ACTIVE_SIZE instead."
)
class gt:
@ -110,8 +122,9 @@ class TestResponseMaxActiveSize:
async def test_both(self):
"""Setting RESPONSE_MAX_ACTIVE_SIZE and SCRAPER_SLOT_MAX_ACTIVE_SIZE to
different values with the same setting priority triggers a deprecation
warning about SCRAPER_SLOT_MAX_ACTIVE_SIZE and makes the value of
RESPONSE_MAX_ACTIVE_SIZE the effective response max active size."""
warning about SCRAPER_SLOT_MAX_ACTIVE_SIZE being ignored, and makes
the value of RESPONSE_MAX_ACTIVE_SIZE the effective response max active
size."""
crawler = get_crawler(
OfflineSpider,
settings_dict={
@ -122,7 +135,7 @@ class TestResponseMaxActiveSize:
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
await maybe_deferred_to_future(crawler.crawl())
assert crawler.engine.downloader._response_max_active_size == 1
_assert_scraper_slot_deprecation(warning_messages)
_assert_scraper_slot_deprecation(warning_messages, ignored=True)
@coroutine_test
async def test_both_deprecated_priority(self):