From 30c98924e35f3ccac4b433c123ed8cc6e77bac8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 8 Jul 2024 12:06:31 +0200 Subject: [PATCH] Test RESPONSE_MAX_ACTIVE_SIZE and SCRAPER_SLOT_MAX_ACTIVE_SIZE setting behavior --- scrapy/core/downloader/__init__.py | 20 +++-- scrapy/settings/default_settings.py | 3 +- tests/test_core_downloader.py | 12 --- tests/test_downloader.py | 116 ++++++++++++++++++++++++++++ 4 files changed, 130 insertions(+), 21 deletions(-) delete mode 100644 tests/test_core_downloader.py create mode 100644 tests/test_downloader.py diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 6fda7b502..dc76a6971 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -134,11 +134,10 @@ class Downloader: ) self._stats = crawler.stats - default_response_max_active_size = 5_000_000 - scraper_max_active_size = self.settings.getint( - "SCRAPER_SLOT_MAX_ACTIVE_SIZE", default_response_max_active_size + deprecated_setting_priority = self.settings.getpriority( + "SCRAPER_SLOT_MAX_ACTIVE_SIZE" ) - if scraper_max_active_size != default_response_max_active_size: + if deprecated_setting_priority > 0: warn( ( "The SCRAPER_SLOT_MAX_ACTIVE_SIZE setting is deprecated, " @@ -146,10 +145,15 @@ class Downloader: ), ScrapyDeprecationWarning, ) - default_response_max_active_size = scraper_max_active_size - self._response_max_active_size = self.settings.getint( - "RESPONSE_MAX_ACTIVE_SIZE", default_response_max_active_size - ) + setting_priority = self.settings.getpriority("RESPONSE_MAX_ACTIVE_SIZE") + if setting_priority >= deprecated_setting_priority: + self._response_max_active_size = self.settings.getint( + "RESPONSE_MAX_ACTIVE_SIZE" + ) + else: + self._response_max_active_size = self.settings.getint( + "SCRAPER_SLOT_MAX_ACTIVE_SIZE" + ) self._response_max_active_size_warned = False def fetch( diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 932475fb5..ed1ddf16c 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -292,7 +292,8 @@ SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleLifoDiskQueue" SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.LifoMemoryQueue" SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.ScrapyPriorityQueue" -SCRAPER_SLOT_MAX_ACTIVE_SIZE = 5000000 +SCRAPER_SLOT_MAX_ACTIVE_SIZE = 5_000_000 +RESPONSE_MAX_ACTIVE_SIZE = 5_000_000 SPIDER_LOADER_CLASS = "scrapy.spiderloader.SpiderLoader" SPIDER_LOADER_WARN_ONLY = False diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py deleted file mode 100644 index 81cff4947..000000000 --- a/tests/test_core_downloader.py +++ /dev/null @@ -1,12 +0,0 @@ -from twisted.trial import unittest - -from scrapy.core.downloader import Slot - - -class SlotTest(unittest.TestCase): - def test_repr(self): - slot = Slot(concurrency=8, delay=0.1, randomize_delay=True) - self.assertEqual( - repr(slot), - "Slot(concurrency=8, delay=0.10, randomize_delay=True, throttle=None)", - ) diff --git a/tests/test_downloader.py b/tests/test_downloader.py new file mode 100644 index 000000000..e12c6f29a --- /dev/null +++ b/tests/test_downloader.py @@ -0,0 +1,116 @@ +import warnings + +import pytest +from twisted.trial import unittest + +from scrapy import Spider +from scrapy.core.downloader import Slot +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.test import get_crawler + + +class SlotTest(unittest.TestCase): + def test_repr(self): + slot = Slot(concurrency=8, delay=0.1, randomize_delay=True) + self.assertEqual( + repr(slot), + "Slot(concurrency=8, delay=0.10, randomize_delay=True, throttle=None)", + ) + + +class OfflineSpider(Spider): + name = "offline" + start_urls = ["data:,"] + + def parse(self, response): + pass + + +class ResponseMaxActiveSizeTest(unittest.TestCase): + + @deferred_f_from_coro_f + async def test_default(self): + """A crawl without custom settings has its effective response max + active size set to 5 000 000, and triggers no deprecation warning.""" + crawler = get_crawler(OfflineSpider) + with warnings.catch_warnings(): + warnings.simplefilter("error") + await crawler.crawl() + self.assertEqual(crawler.engine.downloader._response_max_active_size, 5_000_000) + + @deferred_f_from_coro_f + async def test_custom(self): + """Setting RESPONSE_MAX_ACTIVE_SIZE to a custom value changes the + effective response max active size.""" + crawler = get_crawler( + OfflineSpider, settings_dict={"RESPONSE_MAX_ACTIVE_SIZE": 0} + ) + with warnings.catch_warnings(): + warnings.simplefilter("error") + await crawler.crawl() + self.assertEqual(crawler.engine.downloader._response_max_active_size, 0) + + @deferred_f_from_coro_f + async def test_deprecated_default(self): + """Setting SCRAPER_SLOT_MAX_ACTIVE_SIZE triggers a deprecation warning, + even if it is the default value.""" + crawler = get_crawler( + OfflineSpider, settings_dict={"SCRAPER_SLOT_MAX_ACTIVE_SIZE": 5_000_000} + ) + with pytest.warns(ScrapyDeprecationWarning) as warning_messages: + await crawler.crawl() + self.assertEqual(crawler.engine.downloader._response_max_active_size, 5_000_000) + self.assertEqual(len(warning_messages), 1) + self.assertEqual( + str(warning_messages[0].message), + ( + "The SCRAPER_SLOT_MAX_ACTIVE_SIZE setting is deprecated, use " + "RESPONSE_MAX_ACTIVE_SIZE instead." + ), + ) + + @deferred_f_from_coro_f + async def test_deprecated_custom(self): + """Setting SCRAPER_SLOT_MAX_ACTIVE_SIZE to a custom value triggers a + deprecation warning, and changes the effective response max active + size.""" + crawler = get_crawler( + OfflineSpider, settings_dict={"SCRAPER_SLOT_MAX_ACTIVE_SIZE": 0} + ) + with pytest.warns(ScrapyDeprecationWarning) as warning_messages: + await crawler.crawl() + self.assertEqual(crawler.engine.downloader._response_max_active_size, 0) + self.assertEqual(len(warning_messages), 1) + self.assertEqual( + str(warning_messages[0].message), + ( + "The SCRAPER_SLOT_MAX_ACTIVE_SIZE setting is deprecated, use " + "RESPONSE_MAX_ACTIVE_SIZE instead." + ), + ) + + @deferred_f_from_coro_f + async def test_both(self): + """Setting RESPONSE_MAX_ACTIVE_SIZE and SCRAPER_SLOT_MAX_ACTIVE_SIZE to + different values with the same setting priority triggers a deprecation + warning about SCRAPER_SLOT_MAX_ACTIVE_SIZE and makes the value of + RESPONSE_MAX_ACTIVE_SIZE the effective response max active size.""" + crawler = get_crawler( + OfflineSpider, + settings_dict={ + "RESPONSE_MAX_ACTIVE_SIZE": 1, + "SCRAPER_SLOT_MAX_ACTIVE_SIZE": 2, + }, + ) + with pytest.warns(ScrapyDeprecationWarning) as warning_messages: + await crawler.crawl() + self.assertEqual(crawler.engine.downloader._response_max_active_size, 1) + self.assertEqual(len(warning_messages), 1) + self.assertEqual( + str(warning_messages[0].message), + ( + "The SCRAPER_SLOT_MAX_ACTIVE_SIZE setting is deprecated, use " + "RESPONSE_MAX_ACTIVE_SIZE instead." + ), + )