mirror of https://github.com/scrapy/scrapy.git
191 lines
7.2 KiB
Python
191 lines
7.2 KiB
Python
import warnings
|
|
|
|
import pytest
|
|
|
|
from scrapy import Spider
|
|
from scrapy.core.scraper import Slot
|
|
from scrapy.exceptions import ScrapyDeprecationWarning
|
|
from scrapy.utils.defer import maybe_deferred_to_future
|
|
from scrapy.utils.test import get_crawler
|
|
from tests.utils.decorators import coroutine_test
|
|
|
|
|
|
class TestScraper:
|
|
@coroutine_test
|
|
async def test_crawl(self):
|
|
"""A crawl should not trigger any deprecation warning."""
|
|
outcome = {}
|
|
|
|
class TestSpider(Spider):
|
|
name = "test"
|
|
start_urls = ["data:,"]
|
|
|
|
def parse(self, response):
|
|
assert self.crawler.engine
|
|
slot = self.crawler.engine.scraper.slot
|
|
assert slot
|
|
with pytest.warns(ScrapyDeprecationWarning):
|
|
outcome["active_size"] = slot.active_size
|
|
|
|
crawler = get_crawler(
|
|
TestSpider, settings_dict={"TELNETCONSOLE_ENABLED": False}
|
|
)
|
|
with warnings.catch_warnings():
|
|
warnings.simplefilter("error", ScrapyDeprecationWarning)
|
|
await maybe_deferred_to_future(crawler.crawl())
|
|
|
|
assert crawler.engine
|
|
slot = crawler.engine.scraper.slot
|
|
assert slot
|
|
with pytest.warns(ScrapyDeprecationWarning):
|
|
expected = slot.MIN_RESPONSE_SIZE
|
|
assert outcome["active_size"] == expected
|
|
|
|
|
|
class TestSlot:
|
|
def setup_method(self):
|
|
self._saved_min_response_size = Slot._MIN_RESPONSE_SIZE
|
|
|
|
def teardown_method(self):
|
|
Slot._MIN_RESPONSE_SIZE = self._saved_min_response_size
|
|
|
|
def test_min_response_time_read(self):
|
|
slot = Slot()
|
|
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
|
|
actual = slot.MIN_RESPONSE_SIZE
|
|
assert actual == 1024
|
|
assert len(warning_messages) == 1
|
|
assert (
|
|
str(warning_messages[0].message)
|
|
== "scrapy.core.scraper.Slot.MIN_RESPONSE_SIZE is deprecated."
|
|
)
|
|
|
|
def test_min_response_time_write(self):
|
|
slot = Slot()
|
|
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
|
|
slot.MIN_RESPONSE_SIZE = 0
|
|
with pytest.warns(ScrapyDeprecationWarning):
|
|
assert slot.MIN_RESPONSE_SIZE == 0
|
|
assert len(warning_messages) == 1
|
|
assert (
|
|
str(warning_messages[0].message)
|
|
== "scrapy.core.scraper.Slot.MIN_RESPONSE_SIZE is deprecated."
|
|
)
|
|
|
|
def test_min_response_size_class_read(self):
|
|
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
|
|
actual = Slot.MIN_RESPONSE_SIZE
|
|
assert actual == 1024
|
|
assert len(warning_messages) == 1
|
|
assert (
|
|
str(warning_messages[0].message)
|
|
== "scrapy.core.scraper.Slot.MIN_RESPONSE_SIZE is deprecated."
|
|
)
|
|
|
|
def test_min_response_size_class_write(self):
|
|
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
|
|
Slot.MIN_RESPONSE_SIZE = 0
|
|
with pytest.warns(ScrapyDeprecationWarning):
|
|
assert Slot.MIN_RESPONSE_SIZE == 0
|
|
assert len(warning_messages) == 1
|
|
assert (
|
|
str(warning_messages[0].message)
|
|
== "scrapy.core.scraper.Slot.MIN_RESPONSE_SIZE is deprecated."
|
|
)
|
|
|
|
def test_slot_init_max_active_size_default(self):
|
|
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
|
|
slot = Slot(max_active_size=5_000_000)
|
|
with pytest.warns(ScrapyDeprecationWarning):
|
|
assert slot.max_active_size == 5_000_000
|
|
assert len(warning_messages) == 1
|
|
assert str(warning_messages[0].message) == (
|
|
"The max_active_size parameter of scrapy.core.scraper.Slot is "
|
|
"deprecated. Use the RESPONSE_MAX_ACTIVE_SIZE setting instead."
|
|
)
|
|
|
|
def test_slot_init_max_active_size_custom(self):
|
|
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
|
|
slot = Slot(max_active_size=0)
|
|
with pytest.warns(ScrapyDeprecationWarning):
|
|
assert slot.max_active_size == 0
|
|
assert len(warning_messages) == 1
|
|
assert str(warning_messages[0].message) == (
|
|
"The max_active_size parameter of scrapy.core.scraper.Slot is "
|
|
"deprecated. Use the RESPONSE_MAX_ACTIVE_SIZE setting instead."
|
|
)
|
|
|
|
def test_max_active_size_read(self):
|
|
slot = Slot()
|
|
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
|
|
actual = slot.max_active_size
|
|
assert actual == 5_000_000
|
|
assert len(warning_messages) == 1
|
|
assert str(warning_messages[0].message) == (
|
|
"scrapy.core.scraper.Slot.max_active_size is deprecated. Read "
|
|
"the RESPONSE_MAX_ACTIVE_SIZE setting instead."
|
|
)
|
|
|
|
def test_max_active_size_write(self):
|
|
slot = Slot()
|
|
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
|
|
slot.max_active_size = 0
|
|
with pytest.warns(ScrapyDeprecationWarning):
|
|
assert slot.max_active_size == 0
|
|
assert len(warning_messages) == 1
|
|
assert str(warning_messages[0].message) == (
|
|
"scrapy.core.scraper.Slot.max_active_size is deprecated. Set "
|
|
"the RESPONSE_MAX_ACTIVE_SIZE setting instead."
|
|
)
|
|
|
|
def test_active_size_read(self):
|
|
slot = Slot()
|
|
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
|
|
actual = slot.active_size
|
|
assert actual == 0
|
|
assert len(warning_messages) == 1
|
|
assert str(warning_messages[0].message) == (
|
|
"scrapy.core.scraper.Slot.active_size is deprecated. The size of "
|
|
"responses in memory is now tracked by the downloader, and no "
|
|
"longer has a public API. If you have a use case for one, please "
|
|
"open a GitHub issue."
|
|
)
|
|
|
|
def test_active_size_write(self):
|
|
slot = Slot()
|
|
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
|
|
slot.active_size = 1
|
|
with pytest.warns(ScrapyDeprecationWarning):
|
|
assert slot.active_size == 1
|
|
assert len(warning_messages) == 1
|
|
assert str(warning_messages[0].message) == (
|
|
"scrapy.core.scraper.Slot.active_size is deprecated, and setting "
|
|
"it no longer has any effect on request processing."
|
|
)
|
|
|
|
def test_needs_backout_false(self):
|
|
slot = Slot()
|
|
with pytest.warns(ScrapyDeprecationWarning):
|
|
slot.active_size = 5_000_000
|
|
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
|
|
actual = slot.needs_backout()
|
|
assert actual is False
|
|
assert len(warning_messages) == 1
|
|
assert (
|
|
str(warning_messages[0].message)
|
|
== "scrapy.core.scraper.Slot.needs_backout is deprecated."
|
|
)
|
|
|
|
def test_needs_backout_true(self):
|
|
slot = Slot()
|
|
with pytest.warns(ScrapyDeprecationWarning):
|
|
slot.active_size = 5_000_001
|
|
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
|
|
actual = slot.needs_backout()
|
|
assert actual is True
|
|
assert len(warning_messages) == 1
|
|
assert (
|
|
str(warning_messages[0].message)
|
|
== "scrapy.core.scraper.Slot.needs_backout is deprecated."
|
|
)
|