import warnings import pytest from scrapy import Spider from scrapy.core.scraper import Slot from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.test import get_crawler from tests.utils.decorators import coroutine_test class TestScraper: @coroutine_test async def test_crawl(self): """A crawl should not trigger any deprecation warning.""" outcome = {} class TestSpider(Spider): name = "test" start_urls = ["data:,"] def parse(self, response): assert self.crawler.engine slot = self.crawler.engine.scraper.slot assert slot with pytest.warns(ScrapyDeprecationWarning): outcome["active_size"] = slot.active_size crawler = get_crawler( TestSpider, settings_dict={"TELNETCONSOLE_ENABLED": False} ) with warnings.catch_warnings(): warnings.simplefilter("error", ScrapyDeprecationWarning) await maybe_deferred_to_future(crawler.crawl()) assert crawler.engine slot = crawler.engine.scraper.slot assert slot with pytest.warns(ScrapyDeprecationWarning): expected = slot.MIN_RESPONSE_SIZE assert outcome["active_size"] == expected class TestSlot: def setup_method(self): self._saved_min_response_size = Slot._MIN_RESPONSE_SIZE def teardown_method(self): Slot._MIN_RESPONSE_SIZE = self._saved_min_response_size def test_min_response_time_read(self): slot = Slot() with pytest.warns(ScrapyDeprecationWarning) as warning_messages: actual = slot.MIN_RESPONSE_SIZE assert actual == 1024 assert len(warning_messages) == 1 assert ( str(warning_messages[0].message) == "scrapy.core.scraper.Slot.MIN_RESPONSE_SIZE is deprecated." ) def test_min_response_time_write(self): slot = Slot() with pytest.warns(ScrapyDeprecationWarning) as warning_messages: slot.MIN_RESPONSE_SIZE = 0 with pytest.warns(ScrapyDeprecationWarning): assert slot.MIN_RESPONSE_SIZE == 0 assert len(warning_messages) == 1 assert ( str(warning_messages[0].message) == "scrapy.core.scraper.Slot.MIN_RESPONSE_SIZE is deprecated." ) def test_min_response_size_class_read(self): with pytest.warns(ScrapyDeprecationWarning) as warning_messages: actual = Slot.MIN_RESPONSE_SIZE assert actual == 1024 assert len(warning_messages) == 1 assert ( str(warning_messages[0].message) == "scrapy.core.scraper.Slot.MIN_RESPONSE_SIZE is deprecated." ) def test_min_response_size_class_write(self): with pytest.warns(ScrapyDeprecationWarning) as warning_messages: Slot.MIN_RESPONSE_SIZE = 0 with pytest.warns(ScrapyDeprecationWarning): assert Slot.MIN_RESPONSE_SIZE == 0 assert len(warning_messages) == 1 assert ( str(warning_messages[0].message) == "scrapy.core.scraper.Slot.MIN_RESPONSE_SIZE is deprecated." ) def test_slot_init_max_active_size_default(self): with pytest.warns(ScrapyDeprecationWarning) as warning_messages: slot = Slot(max_active_size=5_000_000) with pytest.warns(ScrapyDeprecationWarning): assert slot.max_active_size == 5_000_000 assert len(warning_messages) == 1 assert str(warning_messages[0].message) == ( "The max_active_size parameter of scrapy.core.scraper.Slot is " "deprecated. Use the RESPONSE_MAX_ACTIVE_SIZE setting instead." ) def test_slot_init_max_active_size_custom(self): with pytest.warns(ScrapyDeprecationWarning) as warning_messages: slot = Slot(max_active_size=0) with pytest.warns(ScrapyDeprecationWarning): assert slot.max_active_size == 0 assert len(warning_messages) == 1 assert str(warning_messages[0].message) == ( "The max_active_size parameter of scrapy.core.scraper.Slot is " "deprecated. Use the RESPONSE_MAX_ACTIVE_SIZE setting instead." ) def test_max_active_size_read(self): slot = Slot() with pytest.warns(ScrapyDeprecationWarning) as warning_messages: actual = slot.max_active_size assert actual == 5_000_000 assert len(warning_messages) == 1 assert str(warning_messages[0].message) == ( "scrapy.core.scraper.Slot.max_active_size is deprecated. Read " "the RESPONSE_MAX_ACTIVE_SIZE setting instead." ) def test_max_active_size_write(self): slot = Slot() with pytest.warns(ScrapyDeprecationWarning) as warning_messages: slot.max_active_size = 0 with pytest.warns(ScrapyDeprecationWarning): assert slot.max_active_size == 0 assert len(warning_messages) == 1 assert str(warning_messages[0].message) == ( "scrapy.core.scraper.Slot.max_active_size is deprecated. Set " "the RESPONSE_MAX_ACTIVE_SIZE setting instead." ) def test_active_size_read(self): slot = Slot() with pytest.warns(ScrapyDeprecationWarning) as warning_messages: actual = slot.active_size assert actual == 0 assert len(warning_messages) == 1 assert str(warning_messages[0].message) == ( "scrapy.core.scraper.Slot.active_size is deprecated. The size of " "responses in memory is now tracked by the downloader, and no " "longer has a public API. If you have a use case for one, please " "open a GitHub issue." ) def test_active_size_write(self): slot = Slot() with pytest.warns(ScrapyDeprecationWarning) as warning_messages: slot.active_size = 1 with pytest.warns(ScrapyDeprecationWarning): assert slot.active_size == 1 assert len(warning_messages) == 1 assert str(warning_messages[0].message) == ( "scrapy.core.scraper.Slot.active_size is deprecated, and setting " "it no longer has any effect on request processing." ) def test_needs_backout_false(self): slot = Slot() with pytest.warns(ScrapyDeprecationWarning): slot.active_size = 5_000_000 with pytest.warns(ScrapyDeprecationWarning) as warning_messages: actual = slot.needs_backout() assert actual is False assert len(warning_messages) == 1 assert ( str(warning_messages[0].message) == "scrapy.core.scraper.Slot.needs_backout is deprecated." ) def test_needs_backout_true(self): slot = Slot() with pytest.warns(ScrapyDeprecationWarning): slot.active_size = 5_000_001 with pytest.warns(ScrapyDeprecationWarning) as warning_messages: actual = slot.needs_backout() assert actual is True assert len(warning_messages) == 1 assert ( str(warning_messages[0].message) == "scrapy.core.scraper.Slot.needs_backout is deprecated." )