RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Test RESPONSE_MAX_ACTIVE_SIZE and SCRAPER_SLOT_MAX_ACTIVE_SIZE setting behavior

This commit is contained in:
Adrián Chaves 2024-07-08 12:06:31 +02:00
parent fdc79f9536
commit 30c98924e3
4 changed files with 130 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
scrapy/core/downloader/__init__.py
View File

@ -134,11 +134,10 @@ class Downloader:
)
self._stats = crawler.stats
default_response_max_active_size = 5_000_000
scraper_max_active_size = self.settings.getint(
"SCRAPER_SLOT_MAX_ACTIVE_SIZE", default_response_max_active_size
deprecated_setting_priority = self.settings.getpriority(
"SCRAPER_SLOT_MAX_ACTIVE_SIZE"
)
if scraper_max_active_size != default_response_max_active_size:
if deprecated_setting_priority > 0:
warn(
(
"The SCRAPER_SLOT_MAX_ACTIVE_SIZE setting is deprecated, "
@ -146,10 +145,15 @@ class Downloader:
),
ScrapyDeprecationWarning,
)
default_response_max_active_size = scraper_max_active_size
self._response_max_active_size = self.settings.getint(
"RESPONSE_MAX_ACTIVE_SIZE", default_response_max_active_size
)
setting_priority = self.settings.getpriority("RESPONSE_MAX_ACTIVE_SIZE")
if setting_priority >= deprecated_setting_priority:
self._response_max_active_size = self.settings.getint(
"RESPONSE_MAX_ACTIVE_SIZE"
)
else:
self._response_max_active_size = self.settings.getint(
"SCRAPER_SLOT_MAX_ACTIVE_SIZE"
)
self._response_max_active_size_warned = False
def fetch(

3
scrapy/settings/default_settings.py
View File

@ -292,7 +292,8 @@ SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleLifoDiskQueue"
SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.LifoMemoryQueue"
SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.ScrapyPriorityQueue"
SCRAPER_SLOT_MAX_ACTIVE_SIZE = 5000000
SCRAPER_SLOT_MAX_ACTIVE_SIZE = 5_000_000
RESPONSE_MAX_ACTIVE_SIZE = 5_000_000
SPIDER_LOADER_CLASS = "scrapy.spiderloader.SpiderLoader"
SPIDER_LOADER_WARN_ONLY = False

12
tests/test_core_downloader.py
View File

@ -1,12 +0,0 @@
from twisted.trial import unittest
from scrapy.core.downloader import Slot
class SlotTest(unittest.TestCase):
def test_repr(self):
slot = Slot(concurrency=8, delay=0.1, randomize_delay=True)
self.assertEqual(
repr(slot),
"Slot(concurrency=8, delay=0.10, randomize_delay=True, throttle=None)",
)

116
tests/test_downloader.py Normal file
View File

@ -0,0 +1,116 @@
import warnings
import pytest
from twisted.trial import unittest
from scrapy import Spider
from scrapy.core.downloader import Slot
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.defer import deferred_f_from_coro_f
from scrapy.utils.test import get_crawler
class SlotTest(unittest.TestCase):
def test_repr(self):
slot = Slot(concurrency=8, delay=0.1, randomize_delay=True)
self.assertEqual(
repr(slot),
"Slot(concurrency=8, delay=0.10, randomize_delay=True, throttle=None)",
)
class OfflineSpider(Spider):
name = "offline"
start_urls = ["data:,"]
def parse(self, response):
pass
class ResponseMaxActiveSizeTest(unittest.TestCase):
@deferred_f_from_coro_f
async def test_default(self):
"""A crawl without custom settings has its effective response max
active size set to 5 000 000, and triggers no deprecation warning."""
crawler = get_crawler(OfflineSpider)
with warnings.catch_warnings():
warnings.simplefilter("error")
await crawler.crawl()
self.assertEqual(crawler.engine.downloader._response_max_active_size, 5_000_000)
@deferred_f_from_coro_f
async def test_custom(self):
"""Setting RESPONSE_MAX_ACTIVE_SIZE to a custom value changes the
effective response max active size."""
crawler = get_crawler(
OfflineSpider, settings_dict={"RESPONSE_MAX_ACTIVE_SIZE": 0}
)
with warnings.catch_warnings():
warnings.simplefilter("error")
await crawler.crawl()
self.assertEqual(crawler.engine.downloader._response_max_active_size, 0)
@deferred_f_from_coro_f
async def test_deprecated_default(self):
"""Setting SCRAPER_SLOT_MAX_ACTIVE_SIZE triggers a deprecation warning,
even if it is the default value."""
crawler = get_crawler(
OfflineSpider, settings_dict={"SCRAPER_SLOT_MAX_ACTIVE_SIZE": 5_000_000}
)
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
await crawler.crawl()
self.assertEqual(crawler.engine.downloader._response_max_active_size, 5_000_000)
self.assertEqual(len(warning_messages), 1)
self.assertEqual(
str(warning_messages[0].message),
(
"The SCRAPER_SLOT_MAX_ACTIVE_SIZE setting is deprecated, use "
"RESPONSE_MAX_ACTIVE_SIZE instead."
),
)
@deferred_f_from_coro_f
async def test_deprecated_custom(self):
"""Setting SCRAPER_SLOT_MAX_ACTIVE_SIZE to a custom value triggers a
deprecation warning, and changes the effective response max active
size."""
crawler = get_crawler(
OfflineSpider, settings_dict={"SCRAPER_SLOT_MAX_ACTIVE_SIZE": 0}
)
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
await crawler.crawl()
self.assertEqual(crawler.engine.downloader._response_max_active_size, 0)
self.assertEqual(len(warning_messages), 1)
self.assertEqual(
str(warning_messages[0].message),
(
"The SCRAPER_SLOT_MAX_ACTIVE_SIZE setting is deprecated, use "
"RESPONSE_MAX_ACTIVE_SIZE instead."
),
)
@deferred_f_from_coro_f
async def test_both(self):
"""Setting RESPONSE_MAX_ACTIVE_SIZE and SCRAPER_SLOT_MAX_ACTIVE_SIZE to
different values with the same setting priority triggers a deprecation
warning about SCRAPER_SLOT_MAX_ACTIVE_SIZE and makes the value of
RESPONSE_MAX_ACTIVE_SIZE the effective response max active size."""
crawler = get_crawler(
OfflineSpider,
settings_dict={
"RESPONSE_MAX_ACTIVE_SIZE": 1,
"SCRAPER_SLOT_MAX_ACTIVE_SIZE": 2,
},
)
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
await crawler.crawl()
self.assertEqual(crawler.engine.downloader._response_max_active_size, 1)
self.assertEqual(len(warning_messages), 1)
self.assertEqual(
str(warning_messages[0].message),
(
"The SCRAPER_SLOT_MAX_ACTIVE_SIZE setting is deprecated, use "
"RESPONSE_MAX_ACTIVE_SIZE instead."
),
)