mirror of https://github.com/scrapy/scrapy.git
192 lines
7.4 KiB
Python
192 lines
7.4 KiB
Python
import warnings
|
|
|
|
import pytest
|
|
|
|
from scrapy import Spider
|
|
from scrapy.core.scraper import Slot
|
|
from scrapy.exceptions import ScrapyDeprecationWarning
|
|
from scrapy.utils.defer import maybe_deferred_to_future
|
|
from scrapy.utils.test import get_crawler
|
|
from tests.utils.decorators import coroutine_test
|
|
|
|
|
|
class TestScraper:
|
|
@coroutine_test
|
|
async def test_crawl(self):
|
|
"""A crawl should not trigger any deprecation warning."""
|
|
outcome = {}
|
|
|
|
class TestSpider(Spider):
|
|
name = "test"
|
|
start_urls = ["data:,"]
|
|
|
|
def parse(self, response):
|
|
with pytest.warns(ScrapyDeprecationWarning):
|
|
outcome["active_size"] = (
|
|
self.crawler.engine.scraper.slot.active_size
|
|
)
|
|
|
|
crawler = get_crawler(
|
|
TestSpider, settings_dict={"TELNETCONSOLE_ENABLED": False}
|
|
)
|
|
with warnings.catch_warnings():
|
|
warnings.simplefilter("error", ScrapyDeprecationWarning)
|
|
await maybe_deferred_to_future(crawler.crawl())
|
|
|
|
with pytest.warns(ScrapyDeprecationWarning):
|
|
expected = crawler.engine.scraper.slot.MIN_RESPONSE_SIZE
|
|
assert outcome["active_size"] == expected
|
|
|
|
|
|
class TestSlot:
|
|
def setup_method(self):
|
|
self._saved_min_response_size = Slot._MIN_RESPONSE_SIZE
|
|
|
|
def teardown_method(self):
|
|
Slot._MIN_RESPONSE_SIZE = self._saved_min_response_size
|
|
|
|
def test_min_response_time_read(self):
|
|
slot = Slot()
|
|
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
|
|
actual = slot.MIN_RESPONSE_SIZE
|
|
assert actual == 1024
|
|
assert len(warning_messages) == 1
|
|
assert (
|
|
str(warning_messages[0].message)
|
|
== "scrapy.core.scraper.Slot.MIN_RESPONSE_SIZE is deprecated."
|
|
)
|
|
|
|
def test_min_response_time_write(self):
|
|
slot = Slot()
|
|
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
|
|
slot.MIN_RESPONSE_SIZE = 0
|
|
with pytest.warns(ScrapyDeprecationWarning):
|
|
assert slot.MIN_RESPONSE_SIZE == 0
|
|
assert len(warning_messages) == 1
|
|
assert (
|
|
str(warning_messages[0].message)
|
|
== "scrapy.core.scraper.Slot.MIN_RESPONSE_SIZE is deprecated."
|
|
)
|
|
|
|
def test_min_response_size_class_read(self):
|
|
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
|
|
actual = Slot.MIN_RESPONSE_SIZE
|
|
assert actual == 1024
|
|
assert len(warning_messages) == 1
|
|
assert (
|
|
str(warning_messages[0].message)
|
|
== "scrapy.core.scraper.Slot.MIN_RESPONSE_SIZE is deprecated."
|
|
)
|
|
|
|
def test_min_response_size_class_write(self):
|
|
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
|
|
Slot.MIN_RESPONSE_SIZE = 0
|
|
with pytest.warns(ScrapyDeprecationWarning):
|
|
assert Slot.MIN_RESPONSE_SIZE == 0
|
|
assert len(warning_messages) == 1
|
|
assert (
|
|
str(warning_messages[0].message)
|
|
== "scrapy.core.scraper.Slot.MIN_RESPONSE_SIZE is deprecated."
|
|
)
|
|
|
|
def test_slot_init_max_active_size_default(self):
|
|
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
|
|
slot = Slot(max_active_size=5_000_000)
|
|
with pytest.warns(ScrapyDeprecationWarning):
|
|
assert slot.max_active_size == 5_000_000
|
|
assert len(warning_messages) == 1
|
|
assert str(warning_messages[0].message) == (
|
|
"The max_active_size parameter of scrapy.core.scraper.Slot is "
|
|
"deprecated. Use the RESPONSE_MAX_ACTIVE_SIZE setting instead."
|
|
)
|
|
|
|
def test_slot_init_max_active_size_custom(self):
|
|
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
|
|
slot = Slot(max_active_size=0)
|
|
with pytest.warns(ScrapyDeprecationWarning):
|
|
assert slot.max_active_size == 0
|
|
assert len(warning_messages) == 1
|
|
assert str(warning_messages[0].message) == (
|
|
"The max_active_size parameter of scrapy.core.scraper.Slot is "
|
|
"deprecated. Use the RESPONSE_MAX_ACTIVE_SIZE setting instead."
|
|
)
|
|
|
|
def test_max_active_size_read(self):
|
|
slot = Slot()
|
|
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
|
|
actual = slot.max_active_size
|
|
assert actual == 5_000_000
|
|
assert len(warning_messages) == 1
|
|
assert str(warning_messages[0].message) == (
|
|
"scrapy.core.scraper.Slot.max_active_size is deprecated. Read "
|
|
"the RESPONSE_MAX_ACTIVE_SIZE setting instead."
|
|
)
|
|
|
|
def test_max_active_size_write(self):
|
|
slot = Slot()
|
|
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
|
|
slot.max_active_size = 0
|
|
with pytest.warns(ScrapyDeprecationWarning):
|
|
assert slot.max_active_size == 0
|
|
assert len(warning_messages) == 1
|
|
assert str(warning_messages[0].message) == (
|
|
"scrapy.core.scraper.Slot.max_active_size is deprecated. Set "
|
|
"the RESPONSE_MAX_ACTIVE_SIZE setting instead."
|
|
)
|
|
|
|
def test_active_size_read(self):
|
|
slot = Slot()
|
|
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
|
|
actual = slot.active_size
|
|
assert actual == 0
|
|
assert len(warning_messages) == 1
|
|
assert str(warning_messages[0].message) == (
|
|
"scrapy.core.scraper.Slot.active_size is deprecated. Read "
|
|
"scrapy.core.downloader.DownloaderMiddlewareManager.response_active_size "
|
|
"instead."
|
|
)
|
|
|
|
def test_active_size_write(self):
|
|
slot = Slot()
|
|
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
|
|
slot.active_size = 1
|
|
with pytest.warns(ScrapyDeprecationWarning):
|
|
assert slot.active_size == 1
|
|
assert len(warning_messages) == 1
|
|
assert str(warning_messages[0].message) == (
|
|
"scrapy.core.scraper.Slot.active_size is deprecated. "
|
|
"scrapy.core.downloader.DownloaderMiddlewareManager.response_active_size "
|
|
"might work as a replacement, but modifying that attribute "
|
|
"might not be a good idea. If you have a use case for it, you "
|
|
"might want to bring it up in a GitHub issue, to discuss with "
|
|
"Scrapy developers if there is a better approach, or some "
|
|
"change we could implement in Scrapy to improve support for "
|
|
"your use case."
|
|
)
|
|
|
|
def test_needs_backout_false(self):
|
|
slot = Slot()
|
|
with pytest.warns(ScrapyDeprecationWarning):
|
|
slot.active_size = 5_000_000
|
|
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
|
|
actual = slot.needs_backout()
|
|
assert actual is False
|
|
assert len(warning_messages) == 1
|
|
assert (
|
|
str(warning_messages[0].message)
|
|
== "scrapy.core.scraper.Slot.needs_backout is deprecated."
|
|
)
|
|
|
|
def test_needs_backout_true(self):
|
|
slot = Slot()
|
|
with pytest.warns(ScrapyDeprecationWarning):
|
|
slot.active_size = 5_000_001
|
|
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
|
|
actual = slot.needs_backout()
|
|
assert actual is True
|
|
assert len(warning_messages) == 1
|
|
assert (
|
|
str(warning_messages[0].message)
|
|
== "scrapy.core.scraper.Slot.needs_backout is deprecated."
|
|
)
|