RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactoring of test_utils_*. (#6905)

This commit is contained in:
Andrey Rakhmatullin 2025-06-23 23:39:24 +05:00 committed by GitHub
parent 0d75355b41
commit d70f8a3f14
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
24 changed files with 1251 additions and 1318 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tests/test_command_startproject.py
View File

@ -106,8 +106,6 @@ def get_permissions_dict(
class TestStartprojectTemplates(TestProjectBase):
maxDiff = None
def setup_method(self):
super().setup_method()
self.tmpl = str(Path(self.temp_path, "templates"))

2
tests/test_request_cb_kwargs.py
View File

@ -150,8 +150,6 @@ class KeywordArgumentsSpider(MockServerSpider):
class TestCallbackKeywordArguments(TestCase):
maxDiff = None
@classmethod
def setUpClass(cls):
cls.mockserver = MockServer()

18
tests/test_toplevel.py
View File

@ -1,30 +1,34 @@
import scrapy
class TestToplevel:
def test_version(self):
def test_version():
assert isinstance(scrapy.__version__, str)
def test_version_info(self):
def test_version_info():
assert isinstance(scrapy.version_info, tuple)
def test_request_shortcut(self):
def test_request_shortcut():
from scrapy.http import FormRequest, Request
assert scrapy.Request is Request
assert scrapy.FormRequest is FormRequest
def test_spider_shortcut(self):
def test_spider_shortcut():
from scrapy.spiders import Spider
assert scrapy.Spider is Spider
def test_selector_shortcut(self):
def test_selector_shortcut():
from scrapy.selector import Selector
assert scrapy.Selector is Selector
def test_item_shortcut(self):
def test_item_shortcut():
from scrapy.item import Field, Item
assert scrapy.Item is Item

10
tests/test_urlparse_monkeypatches.py
View File

@ -1,10 +0,0 @@
from urllib.parse import urlparse
class TestUrlparse:
def test_s3_url(self):
p = urlparse("s3://bucket/key/name?param=value")
assert p.scheme == "s3"
assert p.hostname == "bucket"
assert p.path == "/key/name"
assert p.query == "param=value"

3
tests/test_utils_conf.py
View File

@ -47,8 +47,7 @@ class TestBuildComponentList:
assert build_component_list(d, convert=lambda x: x) == ["b", "c", "a"]
class TestUtilsConf:
def test_arglist_to_dict(self):
def test_arglist_to_dict():
assert arglist_to_dict(["arg1=val1", "arg2=val2"]) == {
"arg1": "val1",
"arg2": "val2",

13
tests/test_utils_console.py
View File

@ -18,8 +18,7 @@ except ImportError:
ipy = False
class TestUtilsConsole:
def test_get_shell_embed_func(self):
def test_get_shell_embed_func():
shell = get_shell_embed_func(["invalid"])
assert shell is None
@ -27,14 +26,16 @@ class TestUtilsConsole:
assert callable(shell)
assert shell.__name__ == "_embed_standard_shell"
@pytest.mark.skipif(not bpy, reason="bpython not available in testenv")
def test_get_shell_embed_func2(self):
@pytest.mark.skipif(not bpy, reason="bpython not available in testenv")
def test_get_shell_embed_func_bpython():
shell = get_shell_embed_func(["bpython"])
assert callable(shell)
assert shell.__name__ == "_embed_bpython_shell"
@pytest.mark.skipif(not ipy, reason="IPython not available in testenv")
def test_get_shell_embed_func3(self):
@pytest.mark.skipif(not ipy, reason="IPython not available in testenv")
def test_get_shell_embed_func_ipython():
# default shell should be 'ipython'
shell = get_shell_embed_func()
assert shell.__name__ == "_embed_ipython_shell"

6
tests/test_utils_curl.py
View File

@ -1,4 +1,5 @@
import warnings
from typing import Any
import pytest
from w3lib.http import basic_auth_header
@ -8,9 +9,8 @@ from scrapy.utils.curl import curl_to_request_kwargs
class TestCurlToRequestKwargs:
maxDiff = 5000
def _test_command(self, curl_command, expected_result):
@staticmethod
def _test_command(curl_command: str, expected_result: dict[str, Any]) -> None:
result = curl_to_request_kwargs(curl_command)
assert result == expected_result
try:

12
tests/test_utils_datatypes.py
View File

@ -1,5 +1,6 @@
import copy
import warnings
from abc import ABC, abstractmethod
from collections.abc import Iterator, Mapping, MutableMapping
import pytest
@ -16,7 +17,12 @@ from scrapy.utils.datatypes import (
from scrapy.utils.python import garbage_collect
class CaseInsensitiveDictBase:
class TestCaseInsensitiveDictBase(ABC):
@property
@abstractmethod
def dict_class(self) -> type[MutableMapping]:
raise NotImplementedError
def test_init_dict(self):
seq = {"red": 1, "black": 3}
d = self.dict_class(seq)
@ -199,7 +205,7 @@ class CaseInsensitiveDictBase:
assert h1.get("header1") == h3.get("HEADER1")
class TestCaseInsensitiveDict(CaseInsensitiveDictBase):
class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):
dict_class = CaseInsensitiveDict
def test_repr(self):
@ -216,7 +222,7 @@ class TestCaseInsensitiveDict(CaseInsensitiveDictBase):
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
class TestCaselessDict(CaseInsensitiveDictBase):
class TestCaselessDict(TestCaseInsensitiveDictBase):
dict_class = CaselessDict
def test_deprecation_message(self):

5
tests/test_utils_deprecate.py
View File

@ -1,6 +1,7 @@
import inspect
import warnings
from unittest import mock
from warnings import WarningMessage
import pytest
@ -21,7 +22,9 @@ class NewName(SomeBaseClass):
class TestWarnWhenSubclassed:
def _mywarnings(self, w, category=MyWarning):
def _mywarnings(
self, w: list[WarningMessage], category: type[Warning] = MyWarning
) -> list[WarningMessage]:
return [x for x in w if x.category is MyWarning]
def test_no_warning_on_definition(self):

84
tests/test_utils_display.py
View File

@ -3,10 +3,8 @@ from unittest import mock
from scrapy.utils.display import pformat, pprint
class TestDisplay:
object = {"a": 1}
colorized_strings = {
value = {"a": 1}
colorized_strings = {
(
(
"{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'"
@ -19,56 +17,61 @@ class TestDisplay:
"\n", # pygments ≤ 2.13
"\x1b[37m\x1b[39;49;00m\n", # pygments ≥ 2.14
)
}
plain_string = "{'a': 1}"
}
plain_string = "{'a': 1}"
@mock.patch("sys.platform", "linux")
@mock.patch("sys.stdout.isatty")
def test_pformat(self, isatty):
@mock.patch("sys.platform", "linux")
@mock.patch("sys.stdout.isatty")
def test_pformat(isatty):
isatty.return_value = True
assert pformat(self.object) in self.colorized_strings
assert pformat(value) in colorized_strings
@mock.patch("sys.stdout.isatty")
def test_pformat_dont_colorize(self, isatty):
@mock.patch("sys.stdout.isatty")
def test_pformat_dont_colorize(isatty):
isatty.return_value = True
assert pformat(self.object, colorize=False) == self.plain_string
assert pformat(value, colorize=False) == plain_string
def test_pformat_not_tty(self):
assert pformat(self.object) == self.plain_string
@mock.patch("sys.platform", "win32")
@mock.patch("platform.version")
@mock.patch("sys.stdout.isatty")
def test_pformat_old_windows(self, isatty, version):
def test_pformat_not_tty():
assert pformat(value) == plain_string
@mock.patch("sys.platform", "win32")
@mock.patch("platform.version")
@mock.patch("sys.stdout.isatty")
def test_pformat_old_windows(isatty, version):
isatty.return_value = True
version.return_value = "10.0.14392"
assert pformat(self.object) in self.colorized_strings
assert pformat(value) in colorized_strings
@mock.patch("sys.platform", "win32")
@mock.patch("scrapy.utils.display._enable_windows_terminal_processing")
@mock.patch("platform.version")
@mock.patch("sys.stdout.isatty")
def test_pformat_windows_no_terminal_processing(
self, isatty, version, terminal_processing
):
@mock.patch("sys.platform", "win32")
@mock.patch("scrapy.utils.display._enable_windows_terminal_processing")
@mock.patch("platform.version")
@mock.patch("sys.stdout.isatty")
def test_pformat_windows_no_terminal_processing(isatty, version, terminal_processing):
isatty.return_value = True
version.return_value = "10.0.14393"
terminal_processing.return_value = False
assert pformat(self.object) == self.plain_string
assert pformat(value) == plain_string
@mock.patch("sys.platform", "win32")
@mock.patch("scrapy.utils.display._enable_windows_terminal_processing")
@mock.patch("platform.version")
@mock.patch("sys.stdout.isatty")
def test_pformat_windows(self, isatty, version, terminal_processing):
@mock.patch("sys.platform", "win32")
@mock.patch("scrapy.utils.display._enable_windows_terminal_processing")
@mock.patch("platform.version")
@mock.patch("sys.stdout.isatty")
def test_pformat_windows(isatty, version, terminal_processing):
isatty.return_value = True
version.return_value = "10.0.14393"
terminal_processing.return_value = True
assert pformat(self.object) in self.colorized_strings
assert pformat(value) in colorized_strings
@mock.patch("sys.platform", "linux")
@mock.patch("sys.stdout.isatty")
def test_pformat_no_pygments(self, isatty):
@mock.patch("sys.platform", "linux")
@mock.patch("sys.stdout.isatty")
def test_pformat_no_pygments(isatty):
isatty.return_value = True
import builtins
@ -81,10 +84,11 @@ class TestDisplay:
return real_import(name, globals, locals, fromlist, level)
builtins.__import__ = mock_import
assert pformat(self.object) == self.plain_string
assert pformat(value) == plain_string
builtins.__import__ = real_import
def test_pprint(self):
def test_pprint():
with mock.patch("sys.stdout", new=StringIO()) as mock_out:
pprint(self.object)
pprint(value)
assert mock_out.getvalue() == "{'a': 1}\n"

18
tests/test_utils_gz.py
View File

@ -11,8 +11,7 @@ from tests import tests_datadir
SAMPLEDIR = Path(tests_datadir, "compressed")
class TestGunzip:
def test_gunzip_basic(self):
def test_gunzip_basic():
r1 = Response(
"http://www.example.com",
body=(SAMPLEDIR / "feed-sample1.xml.gz").read_bytes(),
@ -23,15 +22,18 @@ class TestGunzip:
assert not gzip_magic_number(r2)
assert len(r2.body) == 9950
def test_gunzip_truncated(self):
def test_gunzip_truncated():
text = gunzip((SAMPLEDIR / "truncated-crc-error.gz").read_bytes())
assert text.endswith(b"</html")
def test_gunzip_no_gzip_file_raises(self):
def test_gunzip_no_gzip_file_raises():
with pytest.raises(BadGzipFile):
gunzip((SAMPLEDIR / "feed-sample1.xml").read_bytes())
def test_gunzip_truncated_short(self):
def test_gunzip_truncated_short():
r1 = Response(
"http://www.example.com",
body=(SAMPLEDIR / "truncated-crc-error-short.gz").read_bytes(),
@ -42,11 +44,13 @@ class TestGunzip:
assert r2.body.endswith(b"</html>")
assert not gzip_magic_number(r2)
def test_is_gzipped_empty(self):
def test_is_gzipped_empty():
r1 = Response("http://www.example.com")
assert not gzip_magic_number(r1)
def test_gunzip_illegal_eof(self):
def test_gunzip_illegal_eof():
text = html_to_unicode(
"charset=cp1252", gunzip((SAMPLEDIR / "unexpected-eof.gz").read_bytes())
)[1]

3
tests/test_utils_httpobj.py
View File

@ -4,8 +4,7 @@ from scrapy.http import Request
from scrapy.utils.httpobj import urlparse_cached
class TestHttpobjUtils:
def test_urlparse_cached(self):
def test_urlparse_cached():
url = "http://www.example.com/index.html"
request1 = Request(url)
request2 = Request(url)

64
tests/test_utils_iterators.py
View File

@ -1,3 +1,8 @@
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Any
import pytest
from scrapy.exceptions import ScrapyDeprecationWarning
@ -5,9 +10,19 @@ from scrapy.http import Response, TextResponse, XmlResponse
from scrapy.utils.iterators import _body_or_str, csviter, xmliter, xmliter_lxml
from tests import get_testdata
if TYPE_CHECKING:
from collections.abc import Iterator
from scrapy import Selector
class TestXmliterBase(ABC):
@abstractmethod
def xmliter(
self, obj: Response | str | bytes, nodename: str, *args: Any
) -> Iterator[Selector]:
raise NotImplementedError
class XmliterBase:
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter(self):
body = b"""
<?xml version="1.0" encoding="UTF-8"?>
@ -39,7 +54,6 @@ class XmliterBase:
("002", ["Name 2"], ["Type 2"]),
]
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter_unusual_node(self):
body = b"""<?xml version="1.0" encoding="UTF-8"?>
<root>
@ -53,7 +67,6 @@ class XmliterBase:
]
assert nodenames == [["matchme..."]]
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter_unicode(self):
# example taken from https://github.com/scrapy/scrapy/issues/1665
body = """<?xml version="1.0" encoding="UTF-8"?>
@ -113,7 +126,6 @@ class XmliterBase:
("27", ["A"], ["27"]),
]
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter_text(self):
body = (
'<?xml version="1.0" encoding="UTF-8"?>'
@ -125,7 +137,6 @@ class XmliterBase:
["two"],
]
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter_namespaces(self):
body = b"""
<?xml version="1.0" encoding="UTF-8"?>
@ -163,7 +174,6 @@ class XmliterBase:
assert node.xpath("id/text()").getall() == []
assert node.xpath("price/text()").getall() == []
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter_namespaced_nodename(self):
body = b"""
<?xml version="1.0" encoding="UTF-8"?>
@ -191,7 +201,6 @@ class XmliterBase:
"http://www.mydummycompany.com/images/item1.jpg"
]
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter_namespaced_nodename_missing(self):
body = b"""
<?xml version="1.0" encoding="UTF-8"?>
@ -216,7 +225,6 @@ class XmliterBase:
with pytest.raises(StopIteration):
next(my_iter)
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter_exception(self):
body = (
'<?xml version="1.0" encoding="UTF-8"?>'
@ -229,13 +237,11 @@ class XmliterBase:
with pytest.raises(StopIteration):
next(iter)
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter_objtype_exception(self):
i = self.xmliter(42, "product")
with pytest.raises(TypeError):
next(i)
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter_encoding(self):
body = (
b'<?xml version="1.0" encoding="ISO-8859-9"?>\n'
@ -250,8 +256,12 @@ class XmliterBase:
)
class TestXmliter(XmliterBase):
xmliter = staticmethod(xmliter)
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
class TestXmliter(TestXmliterBase):
def xmliter(
self, obj: Response | str | bytes, nodename: str, *args: Any
) -> Iterator[Selector]:
return xmliter(obj, nodename)
def test_deprecation(self):
body = b"""
@ -267,8 +277,11 @@ class TestXmliter(XmliterBase):
next(self.xmliter(body, "product"))
class TestLxmlXmliter(XmliterBase):
xmliter = staticmethod(xmliter_lxml)
class TestLxmlXmliter(TestXmliterBase):
def xmliter(
self, obj: Response | str | bytes, nodename: str, *args: Any
) -> Iterator[Selector]:
return xmliter_lxml(obj, nodename, *args)
def test_xmliter_iterate_namespace(self):
body = b"""
@ -493,14 +506,20 @@ class TestUtilsCsv:
]
class TestHelper:
class TestBodyOrStr:
bbody = b"utf8-body"
ubody = bbody.decode("utf8")
txtresponse = TextResponse(url="http://example.org/", body=bbody, encoding="utf-8")
response = Response(url="http://example.org/", body=bbody)
def test_body_or_str(self):
for obj in (self.bbody, self.ubody, self.txtresponse, self.response):
@pytest.mark.parametrize(
"obj",
[
bbody,
ubody,
TextResponse(url="http://example.org/", body=bbody, encoding="utf-8"),
Response(url="http://example.org/", body=bbody),
],
)
def test_body_or_str(self, obj: Response | str | bytes) -> None:
r1 = _body_or_str(obj)
self._assert_type_and_value(r1, self.ubody, obj)
r2 = _body_or_str(obj, unicode=True)
@ -510,6 +529,9 @@ class TestHelper:
assert type(r1) is type(r2)
assert type(r1) is not type(r3)
def _assert_type_and_value(self, a, b, obj):
@staticmethod
def _assert_type_and_value(
a: str | bytes, b: str | bytes, obj: Response | str | bytes
) -> None:
assert type(a) is type(b), f"Got {type(a)}, expected {type(b)} for {obj!r}"
assert a == b

185
tests/test_utils_log.py
View File

@ -22,7 +22,9 @@ from scrapy.utils.test import get_crawler
from tests.spiders import LogSpider
if TYPE_CHECKING:
from collections.abc import Mapping, MutableMapping
from collections.abc import Generator, Mapping, MutableMapping
from scrapy.crawler import Crawler
class TestFailureToExcInfo:
@ -70,33 +72,41 @@ class TestTopLevelFormatter:
class TestLogCounterHandler:
def setup_method(self):
@pytest.fixture
def crawler(self) -> Crawler:
settings = {"LOG_LEVEL": "WARNING"}
self.logger = logging.getLogger("test")
self.logger.setLevel(logging.NOTSET)
self.logger.propagate = False
self.crawler = get_crawler(settings_dict=settings)
self.handler = LogCounterHandler(self.crawler)
self.logger.addHandler(self.handler)
return get_crawler(settings_dict=settings)
def teardown_method(self):
self.logger.propagate = True
self.logger.removeHandler(self.handler)
@pytest.fixture
def logger(self, crawler: Crawler) -> Generator[logging.Logger]:
logger = logging.getLogger("test")
logger.setLevel(logging.NOTSET)
logger.propagate = False
handler = LogCounterHandler(crawler)
logger.addHandler(handler)
def test_init(self):
assert self.crawler.stats.get_value("log_count/DEBUG") is None
assert self.crawler.stats.get_value("log_count/INFO") is None
assert self.crawler.stats.get_value("log_count/WARNING") is None
assert self.crawler.stats.get_value("log_count/ERROR") is None
assert self.crawler.stats.get_value("log_count/CRITICAL") is None
yield logger
def test_accepted_level(self):
self.logger.error("test log msg")
assert self.crawler.stats.get_value("log_count/ERROR") == 1
logger.propagate = True
logger.removeHandler(handler)
def test_filtered_out_level(self):
self.logger.debug("test log msg")
assert self.crawler.stats.get_value("log_count/INFO") is None
def test_init(self, crawler: Crawler, logger: logging.Logger) -> None:
assert crawler.stats
assert crawler.stats.get_value("log_count/DEBUG") is None
assert crawler.stats.get_value("log_count/INFO") is None
assert crawler.stats.get_value("log_count/WARNING") is None
assert crawler.stats.get_value("log_count/ERROR") is None
assert crawler.stats.get_value("log_count/CRITICAL") is None
def test_accepted_level(self, crawler: Crawler, logger: logging.Logger) -> None:
logger.error("test log msg")
assert crawler.stats
assert crawler.stats.get_value("log_count/ERROR") == 1
def test_filtered_out_level(self, crawler: Crawler, logger: logging.Logger) -> None:
logger.debug("test log msg")
assert crawler.stats
assert crawler.stats.get_value("log_count/INFO") is None
class TestStreamLogger:
@ -135,7 +145,7 @@ class TestStreamLogger:
)
def test_spider_logger_adapter_process(
base_extra: Mapping[str, Any], log_extra: MutableMapping, expected_extra: dict
):
) -> None:
logger = logging.getLogger("test")
spider_logger_adapter = SpiderLoggerAdapter(logger, base_extra)
@ -149,59 +159,75 @@ def test_spider_logger_adapter_process(
class TestLogging:
def setup_method(self):
self.log_stream = StringIO()
handler = logging.StreamHandler(self.log_stream)
@pytest.fixture
def log_stream(self) -> StringIO:
return StringIO()
@pytest.fixture
def spider(self) -> LogSpider:
return LogSpider()
@pytest.fixture(autouse=True)
def logger(self, log_stream: StringIO) -> Generator[logging.Logger]:
handler = logging.StreamHandler(log_stream)
logger = logging.getLogger("log_spider")
logger.addHandler(handler)
logger.setLevel(logging.DEBUG)
self.handler = handler
self.logger = logger
self.spider = LogSpider()
def teardown_method(self):
self.logger.removeHandler(self.handler)
yield logger
def test_debug_logging(self):
logger.removeHandler(handler)
def test_debug_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
log_message = "Foo message"
self.spider.log_debug(log_message)
log_contents = self.log_stream.getvalue()
spider.log_debug(log_message)
log_contents = log_stream.getvalue()
assert log_contents == f"{log_message}\n"
def test_info_logging(self):
def test_info_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
log_message = "Bar message"
self.spider.log_info(log_message)
log_contents = self.log_stream.getvalue()
spider.log_info(log_message)
log_contents = log_stream.getvalue()
assert log_contents == f"{log_message}\n"
def test_warning_logging(self):
def test_warning_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
log_message = "Baz message"
self.spider.log_warning(log_message)
log_contents = self.log_stream.getvalue()
spider.log_warning(log_message)
log_contents = log_stream.getvalue()
assert log_contents == f"{log_message}\n"
def test_error_logging(self):
def test_error_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
log_message = "Foo bar message"
self.spider.log_error(log_message)
log_contents = self.log_stream.getvalue()
spider.log_error(log_message)
log_contents = log_stream.getvalue()
assert log_contents == f"{log_message}\n"
def test_critical_logging(self):
def test_critical_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
log_message = "Foo bar baz message"
self.spider.log_critical(log_message)
log_contents = self.log_stream.getvalue()
spider.log_critical(log_message)
log_contents = log_stream.getvalue()
assert log_contents == f"{log_message}\n"
class TestLoggingWithExtra:
def setup_method(self):
self.log_stream = StringIO()
handler = logging.StreamHandler(self.log_stream)
regex_pattern = re.compile(r"^<LogSpider\s'log_spider'\sat\s[^>]+>$")
@pytest.fixture
def log_stream(self) -> StringIO:
return StringIO()
@pytest.fixture
def spider(self) -> LogSpider:
return LogSpider()
@pytest.fixture(autouse=True)
def logger(self, log_stream: StringIO) -> Generator[logging.Logger]:
handler = logging.StreamHandler(log_stream)
formatter = logging.Formatter(
'{"levelname": "%(levelname)s", "message": "%(message)s", "spider": "%(spider)s", "important_info": "%(important_info)s"}'
)
@ -209,80 +235,79 @@ class TestLoggingWithExtra:
logger = logging.getLogger("log_spider")
logger.addHandler(handler)
logger.setLevel(logging.DEBUG)
self.handler = handler
self.logger = logger
self.spider = LogSpider()
self.regex_pattern = re.compile(r"^<LogSpider\s'log_spider'\sat\s[^>]+>$")
def teardown_method(self):
self.logger.removeHandler(self.handler)
yield logger
def test_debug_logging(self):
logger.removeHandler(handler)
def test_debug_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
log_message = "Foo message"
extra = {"important_info": "foo"}
self.spider.log_debug(log_message, extra)
log_contents = self.log_stream.getvalue()
log_contents = json.loads(log_contents)
spider.log_debug(log_message, extra)
log_contents_str = log_stream.getvalue()
log_contents = json.loads(log_contents_str)
assert log_contents["levelname"] == "DEBUG"
assert log_contents["message"] == log_message
assert self.regex_pattern.match(log_contents["spider"])
assert log_contents["important_info"] == extra["important_info"]
def test_info_logging(self):
def test_info_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
log_message = "Bar message"
extra = {"important_info": "bar"}
self.spider.log_info(log_message, extra)
log_contents = self.log_stream.getvalue()
log_contents = json.loads(log_contents)
spider.log_info(log_message, extra)
log_contents_str = log_stream.getvalue()
log_contents = json.loads(log_contents_str)
assert log_contents["levelname"] == "INFO"
assert log_contents["message"] == log_message
assert self.regex_pattern.match(log_contents["spider"])
assert log_contents["important_info"] == extra["important_info"]
def test_warning_logging(self):
def test_warning_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
log_message = "Baz message"
extra = {"important_info": "baz"}
self.spider.log_warning(log_message, extra)
log_contents = self.log_stream.getvalue()
log_contents = json.loads(log_contents)
spider.log_warning(log_message, extra)
log_contents_str = log_stream.getvalue()
log_contents = json.loads(log_contents_str)
assert log_contents["levelname"] == "WARNING"
assert log_contents["message"] == log_message
assert self.regex_pattern.match(log_contents["spider"])
assert log_contents["important_info"] == extra["important_info"]
def test_error_logging(self):
def test_error_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
log_message = "Foo bar message"
extra = {"important_info": "foo bar"}
self.spider.log_error(log_message, extra)
log_contents = self.log_stream.getvalue()
log_contents = json.loads(log_contents)
spider.log_error(log_message, extra)
log_contents_str = log_stream.getvalue()
log_contents = json.loads(log_contents_str)
assert log_contents["levelname"] == "ERROR"
assert log_contents["message"] == log_message
assert self.regex_pattern.match(log_contents["spider"])
assert log_contents["important_info"] == extra["important_info"]
def test_critical_logging(self):
def test_critical_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
log_message = "Foo bar baz message"
extra = {"important_info": "foo bar baz"}
self.spider.log_critical(log_message, extra)
log_contents = self.log_stream.getvalue()
log_contents = json.loads(log_contents)
spider.log_critical(log_message, extra)
log_contents_str = log_stream.getvalue()
log_contents = json.loads(log_contents_str)
assert log_contents["levelname"] == "CRITICAL"
assert log_contents["message"] == log_message
assert self.regex_pattern.match(log_contents["spider"])
assert log_contents["important_info"] == extra["important_info"]
def test_overwrite_spider_extra(self):
def test_overwrite_spider_extra(
self, log_stream: StringIO, spider: LogSpider
) -> None:
log_message = "Foo message"
extra = {"important_info": "foo", "spider": "shouldn't change"}
self.spider.log_error(log_message, extra)
log_contents = self.log_stream.getvalue()
log_contents = json.loads(log_contents)
spider.log_error(log_message, extra)
log_contents_str = log_stream.getvalue()
log_contents = json.loads(log_contents_str)
assert log_contents["levelname"] == "ERROR"
assert log_contents["message"] == log_message

21
tests/test_utils_project.py
View File

@ -1,18 +1,17 @@
import contextlib
import os
import shutil
import tempfile
import warnings
from pathlib import Path
import pytest
from scrapy.utils.misc import set_environ
from scrapy.utils.project import data_path, get_project_settings
@contextlib.contextmanager
def inside_a_project():
@pytest.fixture
def proj_path(tmp_path):
prev_dir = Path.cwd()
project_dir = tempfile.mkdtemp()
project_dir = tmp_path
try:
os.chdir(project_dir)
@ -21,18 +20,16 @@ def inside_a_project():
yield project_dir
finally:
os.chdir(prev_dir)
shutil.rmtree(project_dir)
class TestProjectUtils:
def test_data_path_outside_project(self):
def test_data_path_outside_project():
assert str(Path(".scrapy", "somepath")) == data_path("somepath")
abspath = str(Path(os.path.sep, "absolute", "path"))
assert abspath == data_path(abspath)
def test_data_path_inside_project(self):
with inside_a_project() as proj_path:
expected = Path(proj_path, ".scrapy", "somepath")
def test_data_path_inside_project(proj_path: Path) -> None:
expected = proj_path / ".scrapy" / "somepath"
assert expected.resolve() == Path(data_path("somepath")).resolve()
abspath = str(Path(os.path.sep, "absolute", "path").resolve())
assert abspath == data_path(abspath)

71
tests/test_utils_python.py
View File

@ -1,7 +1,10 @@
from __future__ import annotations
import functools
import operator
import platform
import sys
from typing import TYPE_CHECKING, TypeVar
import pytest
from twisted.trial import unittest
@ -20,9 +23,15 @@ from scrapy.utils.python import (
without_none_values,
)
if TYPE_CHECKING:
from collections.abc import Iterable, Mapping
class TestMutableChain:
def test_mutablechain(self):
_KT = TypeVar("_KT")
_VT = TypeVar("_VT")
def test_mutablechain():
m = MutableChain(range(2), [2, 3], (4, 5))
m.extend(range(6, 7))
m.extend([7, 8])
@ -112,8 +121,7 @@ class TestToBytes:
assert to_bytes("a\ufffdb", "latin-1", errors="replace") == b"a?b"
class TestMemoizedMethod:
def test_memoizemethod_noargs(self):
def test_memoizemethod_noargs():
class A:
@memoizemethod_noargs
def cached(self):
@ -130,23 +138,21 @@ class TestMemoizedMethod:
assert one is not three
class TestBinaryIsText:
def test_binaryistext(self):
assert binary_is_text(b"hello")
def test_utf_16_strings_contain_null_bytes(self):
assert binary_is_text("hello".encode("utf-16"))
def test_one_with_encoding(self):
assert binary_is_text(b"<div>Price \xa3</div>")
def test_real_binary_bytes(self):
assert not binary_is_text(b"\x02\xa3")
@pytest.mark.parametrize(
("value", "expected"),
[
(b"hello", True),
("hello".encode("utf-16"), True),
(b"<div>Price \xa3</div>", True),
(b"\x02\xa3", False),
],
)
def test_binaryistext(value: bytes, expected: bool) -> None:
assert binary_is_text(value) is expected
class TestUtilsPython:
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_equal_attributes(self):
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_equal_attributes():
class Obj:
pass
@ -194,7 +200,8 @@ class TestUtilsPython:
a.meta["z"] = 2
assert not equal_attributes(a, b, [compare_z, "x"])
def test_get_func_args(self):
def test_get_func_args():
def f1(a, b, c):
pass
@ -245,11 +252,19 @@ class TestUtilsPython:
["args", "kwargs"],
]
def test_without_none_values(self):
assert without_none_values([1, None, 3, 4]) == [1, 3, 4]
assert without_none_values((1, None, 3, 4)) == (1, 3, 4)
assert without_none_values({"one": 1, "none": None, "three": 3, "four": 4}) == {
"one": 1,
"three": 3,
"four": 4,
}
@pytest.mark.parametrize(
("value", "expected"),
[
([1, None, 3, 4], [1, 3, 4]),
((1, None, 3, 4), (1, 3, 4)),
(
{"one": 1, "none": None, "three": 3, "four": 4},
{"one": 1, "three": 3, "four": 4},
),
],
)
def test_without_none_values(
value: Mapping[_KT, _VT] | Iterable[_KT], expected: dict[_KT, _VT] | Iterable[_KT]
) -> None:
assert without_none_values(value) == expected

86
tests/test_utils_request.py
View File

@ -20,45 +20,52 @@ from scrapy.utils.request import (
from scrapy.utils.test import get_crawler
class TestUtilsRequest:
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_request_authenticate(self):
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_request_authenticate():
r = Request("http://www.example.com")
request_authenticate(r, "someuser", "somepass")
assert r.headers["Authorization"] == b"Basic c29tZXVzZXI6c29tZXBhc3M="
def test_request_httprepr(self):
r1 = Request("http://www.example.com")
assert (
request_httprepr(r1) == b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n"
)
r1 = Request("http://www.example.com/some/page.html?arg=1")
assert (
request_httprepr(r1)
== b"GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n"
)
r1 = Request(
@pytest.mark.parametrize(
("r", "expected"),
[
(
Request("http://www.example.com"),
b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n",
),
(
Request("http://www.example.com/some/page.html?arg=1"),
b"GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n",
),
(
Request(
"http://www.example.com",
method="POST",
headers={"Content-type": b"text/html"},
body=b"Some body",
)
assert (
request_httprepr(r1)
== b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body"
)
),
b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body",
),
],
)
def test_request_httprepr(r: Request, expected: bytes) -> None:
assert request_httprepr(r) == expected
def test_request_httprepr_for_non_http_request(self):
@pytest.mark.parametrize(
"r",
[
Request("file:///tmp/foo.txt"),
Request("ftp://localhost/tmp/foo.txt"),
],
)
def test_request_httprepr_for_non_http_request(r: Request) -> None:
# the representation is not important but it must not fail.
request_httprepr(Request("file:///tmp/foo.txt"))
request_httprepr(Request("ftp://localhost/tmp/foo.txt"))
request_httprepr(r)
class TestFingerprint:
maxDiff = None
function: staticmethod = staticmethod(fingerprint)
cache: (
WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes]]
@ -229,35 +236,6 @@ class TestFingerprint:
assert actual == expected
REQUEST_OBJECTS_TO_TEST = (
Request("http://www.example.com/"),
Request("http://www.example.com/query?id=111&cat=222"),
Request("http://www.example.com/query?cat=222&id=111"),
Request("http://www.example.com/hnnoticiaj1.aspx?78132,199"),
Request("http://www.example.com/hnnoticiaj1.aspx?78160,199"),
Request("http://www.example.com/members/offers.html"),
Request(
"http://www.example.com/members/offers.html",
headers={"SESSIONID": b"somehash"},
),
Request(
"http://www.example.com/",
headers={"Accept-Language": b"en"},
),
Request(
"http://www.example.com/",
headers={
"Accept-Language": b"en",
"SESSIONID": b"somehash",
},
),
Request("http://www.example.com/test.html"),
Request("http://www.example.com/test.html#fragment"),
Request("http://www.example.com", method="POST"),
Request("http://www.example.com", method="POST", body=b"request body"),
)
class TestRequestFingerprinter:
def test_default_implementation(self):
crawler = get_crawler()

88
tests/test_utils_response.py
View File

@ -4,7 +4,7 @@ from urllib.parse import urlparse
import pytest
from scrapy.http import HtmlResponse, Response, TextResponse
from scrapy.http import HtmlResponse, Response
from scrapy.utils.python import to_bytes
from scrapy.utils.response import (
_remove_html_comments,
@ -15,14 +15,13 @@ from scrapy.utils.response import (
)
class TestResponseUtils:
dummy_response = TextResponse(url="http://example.org/", body=b"dummy_response")
def test_open_in_browser(self):
def test_open_in_browser():
url = "http:///www.example.com/some/page.html"
body = b"<html> <head> <title>test page</title> </head> <body>test body</body> </html>"
body = (
b"<html> <head> <title>test page</title> </head> <body>test body</body> </html>"
)
def browser_open(burl):
def browser_open(burl: str) -> bool:
path = urlparse(burl).path
if not path or not Path(path).exists():
path = burl.replace("file://", "")
@ -37,7 +36,8 @@ class TestResponseUtils:
with pytest.raises(TypeError):
open_in_browser(resp, debug=True) # pylint: disable=unexpected-keyword-arg
def test_get_meta_refresh(self):
def test_get_meta_refresh():
r1 = HtmlResponse(
"http://www.example.com",
body=b"""
@ -59,19 +59,20 @@ class TestResponseUtils:
r3 = HtmlResponse(
"http://www.example.com",
body=b"""
<noscript><meta http-equiv="REFRESH" content="0;url=http://www.example.com/newpage</noscript>
<script type="text/javascript">
if(!checkCookies()){
<noscript><meta http-equiv="REFRESH" content="0;url=http://www.example.com/newpage</noscript>
<script type="text/javascript">
if(!checkCookies()){
document.write('<meta http-equiv="REFRESH" content="0;url=http://www.example.com/newpage">');
}
</script>
}
</script>
""",
)
assert get_meta_refresh(r1) == (5.0, "http://example.org/newpage")
assert get_meta_refresh(r2) == (None, None)
assert get_meta_refresh(r3) == (None, None)
def test_get_base_url(self):
def test_get_base_url():
resp = HtmlResponse(
"http://www.example.com",
body=b"""
@ -89,12 +90,14 @@ class TestResponseUtils:
)
assert get_base_url(resp2) == "http://www.example.com"
def test_response_status_message(self):
def test_response_status_message():
assert response_status_message(200) == "200 OK"
assert response_status_message(404) == "404 Not Found"
assert response_status_message(573) == "573 Unknown Status"
def test_inject_base_url(self):
def test_inject_base_url():
url = "http://www.example.com"
def check_base_url(burl):
@ -169,37 +172,31 @@ class TestResponseUtils:
"Inject unique base url with conditional comment"
)
def test_open_in_browser_redos_comment(self):
def test_open_in_browser_redos_comment():
MAX_CPU_TIME = 0.02
# Exploit input from
# https://makenowjust-labs.github.io/recheck/playground/
# for /<!--.*?-->/ (old pattern to remove comments).
body = b"-><!--\x00" * 25_000 + b"->\n<!---->"
response = HtmlResponse("https://example.com", body=body)
start_time = process_time()
open_in_browser(response, lambda url: True)
end_time = process_time()
assert end_time - start_time < MAX_CPU_TIME
def test_open_in_browser_redos_head(self):
def test_open_in_browser_redos_head():
MAX_CPU_TIME = 0.02
# Exploit input from
# https://makenowjust-labs.github.io/recheck/playground/
# for /(<head(?:>|\s.*?>))/ (old pattern to find the head element).
body = b"<head\t" * 8_000
response = HtmlResponse("https://example.com", body=body)
start_time = process_time()
open_in_browser(response, lambda url: True)
end_time = process_time()
assert end_time - start_time < MAX_CPU_TIME
@ -207,37 +204,14 @@ class TestResponseUtils:
@pytest.mark.parametrize(
("input_body", "output_body"),
[
(
b"a<!--",
b"a",
),
(
b"a<!---->b",
b"ab",
),
(
b"a<!--b-->c",
b"ac",
),
(
b"a<!--b-->c<!--",
b"ac",
),
(
b"a<!--b-->c<!--d",
b"ac",
),
(
b"a<!--b-->c<!---->d",
b"acd",
),
(
b"a<!--b--><!--c-->d",
b"ad",
),
(b"a<!--", b"a"),
(b"a<!---->b", b"ab"),
(b"a<!--b-->c", b"ac"),
(b"a<!--b-->c<!--", b"ac"),
(b"a<!--b-->c<!--d", b"ac"),
(b"a<!--b-->c<!---->d", b"acd"),
(b"a<!--b--><!--c-->d", b"ad"),
],
)
def test_remove_html_comments(input_body, output_body):
assert _remove_html_comments(input_body) == output_body, (
f"{_remove_html_comments(input_body)=} == {output_body=}"
)
assert _remove_html_comments(input_body) == output_body

30
tests/test_utils_serialize.py
View File

@ -4,6 +4,7 @@ import json
from decimal import Decimal
import attr
import pytest
from twisted.internet import defer
from scrapy.http import Request, Response
@ -11,10 +12,11 @@ from scrapy.utils.serialize import ScrapyJSONEncoder
class TestJsonEncoder:
def setup_method(self):
self.encoder = ScrapyJSONEncoder(sort_keys=True)
@pytest.fixture
def encoder(self) -> ScrapyJSONEncoder:
return ScrapyJSONEncoder(sort_keys=True)
def test_encode_decode(self):
def test_encode_decode(self, encoder: ScrapyJSONEncoder) -> None:
dt = datetime.datetime(2010, 1, 2, 10, 11, 12)
dts = "2010-01-02 10:11:12"
d = datetime.date(2010, 1, 2)
@ -38,24 +40,24 @@ class TestJsonEncoder:
(s, ss),
(dt_set, dt_sets),
]:
assert self.encoder.encode(input) == json.dumps(output, sort_keys=True)
assert encoder.encode(input) == json.dumps(output, sort_keys=True)
def test_encode_deferred(self):
assert "Deferred" in self.encoder.encode(defer.Deferred())
def test_encode_deferred(self, encoder: ScrapyJSONEncoder) -> None:
assert "Deferred" in encoder.encode(defer.Deferred())
def test_encode_request(self):
def test_encode_request(self, encoder: ScrapyJSONEncoder) -> None:
r = Request("http://www.example.com/lala")
rs = self.encoder.encode(r)
rs = encoder.encode(r)
assert r.method in rs
assert r.url in rs
def test_encode_response(self):
def test_encode_response(self, encoder: ScrapyJSONEncoder) -> None:
r = Response("http://www.example.com/lala")
rs = self.encoder.encode(r)
rs = encoder.encode(r)
assert r.url in rs
assert str(r.status) in rs
def test_encode_dataclass_item(self) -> None:
def test_encode_dataclass_item(self, encoder: ScrapyJSONEncoder) -> None:
@dataclasses.dataclass
class TestDataClass:
name: str
@ -63,10 +65,10 @@ class TestJsonEncoder:
price: int
item = TestDataClass(name="Product", url="http://product.org", price=1)
encoded = self.encoder.encode(item)
encoded = encoder.encode(item)
assert encoded == '{"name": "Product", "price": 1, "url": "http://product.org"}'
def test_encode_attrs_item(self):
def test_encode_attrs_item(self, encoder: ScrapyJSONEncoder) -> None:
@attr.s
class AttrsItem:
name = attr.ib(type=str)
@ -74,5 +76,5 @@ class TestJsonEncoder:
price = attr.ib(type=int)
item = AttrsItem(name="Product", url="http://product.org", price=1)
encoded = self.encoder.encode(item)
encoded = encoder.encode(item)
assert encoded == '{"name": "Product", "price": 1, "url": "http://product.org"}'

137
tests/test_utils_sitemap.py
View File

@ -1,23 +1,22 @@
from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
class TestSitemap:
def test_sitemap(self):
def test_sitemap():
s = Sitemap(
b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.google.com/schemas/sitemap/0.84">
<url>
<loc>http://www.example.com/</loc>
<lastmod>2009-08-16</lastmod>
<changefreq>daily</changefreq>
<priority>1</priority>
</url>
<url>
<loc>http://www.example.com/Special-Offers.html</loc>
<lastmod>2009-08-16</lastmod>
<changefreq>weekly</changefreq>
<priority>0.8</priority>
</url>
<url>
<loc>http://www.example.com/</loc>
<lastmod>2009-08-16</lastmod>
<changefreq>daily</changefreq>
<priority>1</priority>
</url>
<url>
<loc>http://www.example.com/Special-Offers.html</loc>
<lastmod>2009-08-16</lastmod>
<changefreq>weekly</changefreq>
<priority>0.8</priority>
</url>
</urlset>"""
)
assert s.type == "urlset"
@ -36,18 +35,19 @@ class TestSitemap:
},
]
def test_sitemap_index(self):
def test_sitemap_index():
s = Sitemap(
b"""<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<sitemap>
<sitemap>
<loc>http://www.example.com/sitemap1.xml.gz</loc>
<lastmod>2004-10-01T18:23:17+00:00</lastmod>
</sitemap>
<sitemap>
</sitemap>
<sitemap>
<loc>http://www.example.com/sitemap2.xml.gz</loc>
<lastmod>2005-01-01</lastmod>
</sitemap>
</sitemap>
</sitemapindex>"""
)
assert s.type == "sitemapindex"
@ -62,23 +62,24 @@ class TestSitemap:
},
]
def test_sitemap_strip(self):
def test_sitemap_strip():
"""Assert we can deal with trailing spaces inside <loc> tags - we've
seen those
"""
s = Sitemap(
b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.google.com/schemas/sitemap/0.84">
<url>
<loc> http://www.example.com/</loc>
<lastmod>2009-08-16</lastmod>
<changefreq>daily</changefreq>
<priority>1</priority>
</url>
<url>
<loc> http://www.example.com/2</loc>
<lastmod />
</url>
<url>
<loc> http://www.example.com/</loc>
<lastmod>2009-08-16</lastmod>
<changefreq>daily</changefreq>
<priority>1</priority>
</url>
<url>
<loc> http://www.example.com/2</loc>
<lastmod />
</url>
</urlset>
"""
)
@ -92,22 +93,23 @@ class TestSitemap:
{"loc": "http://www.example.com/2", "lastmod": ""},
]
def test_sitemap_wrong_ns(self):
def test_sitemap_wrong_ns():
"""We have seen sitemaps with wrongs ns. Presumably, Google still works
with these, though is not 100% confirmed"""
s = Sitemap(
b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.google.com/schemas/sitemap/0.84">
<url xmlns="">
<loc> http://www.example.com/</loc>
<lastmod>2009-08-16</lastmod>
<changefreq>daily</changefreq>
<priority>1</priority>
</url>
<url xmlns="">
<loc> http://www.example.com/2</loc>
<lastmod />
</url>
<url xmlns="">
<loc> http://www.example.com/</loc>
<lastmod>2009-08-16</lastmod>
<changefreq>daily</changefreq>
<priority>1</priority>
</url>
<url xmlns="">
<loc> http://www.example.com/2</loc>
<lastmod />
</url>
</urlset>
"""
)
@ -121,22 +123,23 @@ class TestSitemap:
{"loc": "http://www.example.com/2", "lastmod": ""},
]
def test_sitemap_wrong_ns2(self):
def test_sitemap_wrong_ns2():
"""We have seen sitemaps with wrongs ns. Presumably, Google still works
with these, though is not 100% confirmed"""
s = Sitemap(
b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset>
<url xmlns="">
<loc> http://www.example.com/</loc>
<lastmod>2009-08-16</lastmod>
<changefreq>daily</changefreq>
<priority>1</priority>
</url>
<url xmlns="">
<loc> http://www.example.com/2</loc>
<lastmod />
</url>
<url xmlns="">
<loc> http://www.example.com/</loc>
<lastmod>2009-08-16</lastmod>
<changefreq>daily</changefreq>
<priority>1</priority>
</url>
<url xmlns="">
<loc> http://www.example.com/2</loc>
<lastmod />
</url>
</urlset>
"""
)
@ -151,7 +154,8 @@ class TestSitemap:
{"loc": "http://www.example.com/2", "lastmod": ""},
]
def test_sitemap_urls_from_robots(self):
def test_sitemap_urls_from_robots():
robots = """User-agent: *
Disallow: /aff/
Disallow: /wl/
@ -170,16 +174,15 @@ Sitemap: /sitemap-relative-url.xml
Disallow: /forum/search/
Disallow: /forum/active/
"""
assert list(
sitemap_urls_from_robots(robots, base_url="http://example.com")
) == [
assert list(sitemap_urls_from_robots(robots, base_url="http://example.com")) == [
"http://example.com/sitemap.xml",
"http://example.com/sitemap-product-index.xml",
"http://example.com/sitemap-uppercase.xml",
"http://example.com/sitemap-relative-url.xml",
]
def test_sitemap_blanklines(self):
def test_sitemap_blanklines():
"""Assert we can deal with starting blank lines before <xml> tag"""
s = Sitemap(
b"""
@ -212,24 +215,25 @@ Disallow: /forum/active/
{"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap3.xml"},
]
def test_comment(self):
def test_comment():
s = Sitemap(
b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
xmlns:xhtml="http://www.w3.org/1999/xhtml">
<url>
<loc>http://www.example.com/</loc>
<!-- this is a comment on which the parser might raise an exception if implemented incorrectly -->
</url>
</urlset>"""
</urlset>"""
)
assert list(s) == [{"loc": "http://www.example.com/"}]
def test_alternate(self):
def test_alternate():
s = Sitemap(
b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
xmlns:xhtml="http://www.w3.org/1999/xhtml">
<url>
<loc>http://www.example.com/english/</loc>
@ -241,9 +245,8 @@ Disallow: /forum/active/
href="http://www.example.com/english/"/>
<xhtml:link rel="alternate" hreflang="en"/><!-- wrong tag without href -->
</url>
</urlset>"""
</urlset>"""
)
assert list(s) == [
{
"loc": "http://www.example.com/english/",
@ -255,7 +258,8 @@ Disallow: /forum/active/
}
]
def test_xml_entity_expansion(self):
def test_xml_entity_expansion():
s = Sitemap(
b"""<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE foo [
@ -269,5 +273,4 @@ Disallow: /forum/active/
</urlset>
"""
)
assert list(s) == [{"loc": "http://127.0.0.1:8000/"}]

6
tests/test_utils_spider.py
View File

@ -12,8 +12,7 @@ class MySpider2(Spider):
name = "myspider2"
class TestUtilsSpiders:
def test_iterate_spider_output(self):
def test_iterate_spider_output():
i = Item()
r = Request("http://scrapytest.org")
o = object()
@ -23,7 +22,8 @@ class TestUtilsSpiders:
assert list(iterate_spider_output(o)) == [o]
assert list(iterate_spider_output([r, i, o])) == [r, i, o]
def test_iter_spider_classes(self):
def test_iter_spider_classes():
import tests.test_utils_spider # noqa: PLW0406 # pylint: disable=import-self
it = iter_spider_classes(tests.test_utils_spider)

3
tests/test_utils_template.py
View File

@ -1,8 +1,7 @@
from scrapy.utils.template import render_templatefile
class TestUtilsRenderTemplateFile:
def test_simple_render(self, tmp_path):
def test_simple_render(tmp_path):
context = {"project_name": "proj", "name": "spi", "classname": "TheSpider"}
template = "from ${project_name}.spiders.${name} import ${classname}"
rendered = "from proj.spiders.spi import TheSpider"

23
tests/test_utils_trackref.py
View File

@ -15,11 +15,12 @@ class Bar(trackref.object_ref):
pass
class TestTrackref:
def setup_method(self):
@pytest.fixture(autouse=True)
def clear_refs() -> None:
trackref.live_refs.clear()
def test_format_live_refs(self):
def test_format_live_refs():
o1 = Foo() # noqa: F841
o2 = Bar() # noqa: F841
o3 = Foo() # noqa: F841
@ -42,13 +43,15 @@ Bar 1 oldest: 0s ago
"""
)
@mock.patch("sys.stdout", new_callable=StringIO)
def test_print_live_refs_empty(self, stdout):
@mock.patch("sys.stdout", new_callable=StringIO)
def test_print_live_refs_empty(stdout):
trackref.print_live_refs()
assert stdout.getvalue() == "Live References\n\n\n"
@mock.patch("sys.stdout", new_callable=StringIO)
def test_print_live_refs_with_objects(self, stdout):
@mock.patch("sys.stdout", new_callable=StringIO)
def test_print_live_refs_with_objects(stdout):
o1 = Foo() # noqa: F841
trackref.print_live_refs()
assert (
@ -59,7 +62,8 @@ Live References
Foo 1 oldest: 0s ago\n\n"""
)
def test_get_oldest(self):
def test_get_oldest():
o1 = Foo()
o1_time = time()
@ -78,7 +82,8 @@ Foo 1 oldest: 0s ago\n\n"""
assert trackref.get_oldest("Bar") is o2
assert trackref.get_oldest("XXX") is None
def test_iter_all(self):
def test_iter_all():
o1 = Foo()
o2 = Bar() # noqa: F841
o3 = Foo()

447
tests/test_utils_url.py
View File

@ -4,7 +4,6 @@ import pytest
from scrapy.linkextractors import IGNORED_EXTENSIONS
from scrapy.spiders import Spider
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.url import ( # type: ignore[attr-defined]
_is_filesystem_path,
_public_w3lib_objects,
@ -17,8 +16,7 @@ from scrapy.utils.url import ( # type: ignore[attr-defined]
)
class TestUrlUtils:
def test_url_is_from_any_domain(self):
def test_url_is_from_any_domain():
url = "http://www.wheele-bin-art.co.uk/get/product/123"
assert url_is_from_any_domain(url, ["wheele-bin-art.co.uk"])
assert not url_is_from_any_domain(url, ["art.co.uk"])
@ -42,14 +40,8 @@ class TestUrlUtils:
assert not url_is_from_any_domain(url, ["testdomain.com"])
assert not url_is_from_any_domain(url + ".testdomain.com", ["testdomain.com"])
def test_url_is_from_spider(self):
spider = Spider(name="example.com")
assert url_is_from_spider("http://www.example.com/some/page.html", spider)
assert url_is_from_spider("http://sub.example.com/some/page.html", spider)
assert not url_is_from_spider("http://www.example.org/some/page.html", spider)
assert not url_is_from_spider("http://www.example.net/some/page.html", spider)
def test_url_is_from_spider_class_attributes(self):
def test_url_is_from_spider():
class MySpider(Spider):
name = "example.com"
@ -58,31 +50,21 @@ class TestUrlUtils:
assert not url_is_from_spider("http://www.example.org/some/page.html", MySpider)
assert not url_is_from_spider("http://www.example.net/some/page.html", MySpider)
def test_url_is_from_spider_with_allowed_domains(self):
spider = Spider(
name="example.com", allowed_domains=["example.org", "example.net"]
)
assert url_is_from_spider("http://www.example.com/some/page.html", spider)
assert url_is_from_spider("http://sub.example.com/some/page.html", spider)
assert url_is_from_spider("http://example.com/some/page.html", spider)
assert url_is_from_spider("http://www.example.org/some/page.html", spider)
assert url_is_from_spider("http://www.example.net/some/page.html", spider)
assert not url_is_from_spider("http://www.example.us/some/page.html", spider)
spider = Spider(
name="example.com", allowed_domains={"example.com", "example.net"}
)
assert url_is_from_spider("http://www.example.com/some/page.html", spider)
spider = Spider(
name="example.com", allowed_domains=("example.com", "example.net")
)
assert url_is_from_spider("http://www.example.com/some/page.html", spider)
def test_url_is_from_spider_with_allowed_domains_class_attributes(self):
def test_url_is_from_spider_class_attributes():
class MySpider(Spider):
name = "example.com"
allowed_domains = ("example.org", "example.net")
assert url_is_from_spider("http://www.example.com/some/page.html", MySpider)
assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider)
assert not url_is_from_spider("http://www.example.org/some/page.html", MySpider)
assert not url_is_from_spider("http://www.example.net/some/page.html", MySpider)
def test_url_is_from_spider_with_allowed_domains():
class MySpider(Spider):
name = "example.com"
allowed_domains = ["example.org", "example.net"]
assert url_is_from_spider("http://www.example.com/some/page.html", MySpider)
assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider)
@ -91,187 +73,99 @@ class TestUrlUtils:
assert url_is_from_spider("http://www.example.net/some/page.html", MySpider)
assert not url_is_from_spider("http://www.example.us/some/page.html", MySpider)
def test_url_has_any_extension(self):
deny_extensions = {"." + e for e in arg_to_iter(IGNORED_EXTENSIONS)}
assert url_has_any_extension(
"http://www.example.com/archive.tar.gz", deny_extensions
)
assert url_has_any_extension("http://www.example.com/page.doc", deny_extensions)
assert url_has_any_extension("http://www.example.com/page.pdf", deny_extensions)
assert not url_has_any_extension(
"http://www.example.com/page.htm", deny_extensions
)
assert not url_has_any_extension("http://www.example.com/", deny_extensions)
assert not url_has_any_extension(
"http://www.example.com/page.doc.html", deny_extensions
)
class MySpider2(Spider):
name = "example.com"
allowed_domains = {"example.com", "example.net"}
assert url_is_from_spider("http://www.example.com/some/page.html", MySpider2)
class MySpider3(Spider):
name = "example.com"
allowed_domains = ("example.com", "example.net")
assert url_is_from_spider("http://www.example.com/some/page.html", MySpider3)
class TestAddHttpIfNoScheme:
def test_add_scheme(self):
assert add_http_if_no_scheme("www.example.com") == "http://www.example.com"
def test_without_subdomain(self):
assert add_http_if_no_scheme("example.com") == "http://example.com"
def test_path(self):
assert (
add_http_if_no_scheme("www.example.com/some/page.html")
== "http://www.example.com/some/page.html"
)
def test_port(self):
assert (
add_http_if_no_scheme("www.example.com:80") == "http://www.example.com:80"
)
def test_fragment(self):
assert (
add_http_if_no_scheme("www.example.com/some/page#frag")
== "http://www.example.com/some/page#frag"
)
def test_query(self):
assert (
add_http_if_no_scheme("www.example.com/do?a=1&b=2&c=3")
== "http://www.example.com/do?a=1&b=2&c=3"
)
def test_username_password(self):
assert (
add_http_if_no_scheme("username:password@www.example.com")
== "http://username:password@www.example.com"
)
def test_complete_url(self):
assert (
add_http_if_no_scheme(
"username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag"
)
== "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag"
)
def test_preserve_http(self):
assert (
add_http_if_no_scheme("http://www.example.com") == "http://www.example.com"
)
def test_preserve_http_without_subdomain(self):
assert add_http_if_no_scheme("http://example.com") == "http://example.com"
def test_preserve_http_path(self):
assert (
add_http_if_no_scheme("http://www.example.com/some/page.html")
== "http://www.example.com/some/page.html"
)
def test_preserve_http_port(self):
assert (
add_http_if_no_scheme("http://www.example.com:80")
== "http://www.example.com:80"
)
def test_preserve_http_fragment(self):
assert (
add_http_if_no_scheme("http://www.example.com/some/page#frag")
== "http://www.example.com/some/page#frag"
)
def test_preserve_http_query(self):
assert (
add_http_if_no_scheme("http://www.example.com/do?a=1&b=2&c=3")
== "http://www.example.com/do?a=1&b=2&c=3"
)
def test_preserve_http_username_password(self):
assert (
add_http_if_no_scheme("http://username:password@www.example.com")
== "http://username:password@www.example.com"
)
def test_preserve_http_complete_url(self):
assert (
add_http_if_no_scheme(
"http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag"
)
== "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag"
)
def test_protocol_relative(self):
assert add_http_if_no_scheme("//www.example.com") == "http://www.example.com"
def test_protocol_relative_without_subdomain(self):
assert add_http_if_no_scheme("//example.com") == "http://example.com"
def test_protocol_relative_path(self):
assert (
add_http_if_no_scheme("//www.example.com/some/page.html")
== "http://www.example.com/some/page.html"
)
def test_protocol_relative_port(self):
assert (
add_http_if_no_scheme("//www.example.com:80") == "http://www.example.com:80"
)
def test_protocol_relative_fragment(self):
assert (
add_http_if_no_scheme("//www.example.com/some/page#frag")
== "http://www.example.com/some/page#frag"
)
def test_protocol_relative_query(self):
assert (
add_http_if_no_scheme("//www.example.com/do?a=1&b=2&c=3")
== "http://www.example.com/do?a=1&b=2&c=3"
)
def test_protocol_relative_username_password(self):
assert (
add_http_if_no_scheme("//username:password@www.example.com")
== "http://username:password@www.example.com"
)
def test_protocol_relative_complete_url(self):
assert (
add_http_if_no_scheme(
"//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag"
)
== "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag"
)
def test_preserve_https(self):
assert (
add_http_if_no_scheme("https://www.example.com")
== "https://www.example.com"
)
def test_preserve_ftp(self):
assert add_http_if_no_scheme("ftp://www.example.com") == "ftp://www.example.com"
@pytest.mark.parametrize(
("url", "expected"),
[
("http://www.example.com/archive.tar.gz", True),
("http://www.example.com/page.doc", True),
("http://www.example.com/page.pdf", True),
("http://www.example.com/page.htm", False),
("http://www.example.com/", False),
("http://www.example.com/page.doc.html", False),
],
)
def test_url_has_any_extension(url: str, expected: bool) -> None:
deny_extensions = {"." + e for e in IGNORED_EXTENSIONS}
assert url_has_any_extension(url, deny_extensions) is expected
class TestGuessScheme:
pass
@pytest.mark.parametrize(
("url", "expected"),
[
("www.example.com", "http://www.example.com"),
("example.com", "http://example.com"),
("www.example.com/some/page.html", "http://www.example.com/some/page.html"),
("www.example.com:80", "http://www.example.com:80"),
("www.example.com/some/page#frag", "http://www.example.com/some/page#frag"),
("www.example.com/do?a=1&b=2&c=3", "http://www.example.com/do?a=1&b=2&c=3"),
(
"username:password@www.example.com",
"http://username:password@www.example.com",
),
(
"username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag",
"http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag",
),
("http://www.example.com", "http://www.example.com"),
("http://example.com", "http://example.com"),
(
"http://www.example.com/some/page.html",
"http://www.example.com/some/page.html",
),
("http://www.example.com:80", "http://www.example.com:80"),
(
"http://www.example.com/some/page#frag",
"http://www.example.com/some/page#frag",
),
(
"http://www.example.com/do?a=1&b=2&c=3",
"http://www.example.com/do?a=1&b=2&c=3",
),
(
"http://username:password@www.example.com",
"http://username:password@www.example.com",
),
(
"http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag",
"http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag",
),
("//www.example.com", "http://www.example.com"),
("//example.com", "http://example.com"),
("//www.example.com/some/page.html", "http://www.example.com/some/page.html"),
("//www.example.com:80", "http://www.example.com:80"),
("//www.example.com/some/page#frag", "http://www.example.com/some/page#frag"),
("//www.example.com/do?a=1&b=2&c=3", "http://www.example.com/do?a=1&b=2&c=3"),
(
"//username:password@www.example.com",
"http://username:password@www.example.com",
),
(
"//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag",
"http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag",
),
("https://www.example.com", "https://www.example.com"),
("ftp://www.example.com", "ftp://www.example.com"),
],
)
def test_add_http_if_no_scheme(url: str, expected: str) -> None:
assert add_http_if_no_scheme(url) == expected
def create_guess_scheme_t(args):
def do_expected(self):
url = guess_scheme(args[0])
assert url.startswith(args[1]), (
f"Wrong scheme guessed: for `{args[0]}` got `{url}`, expected `{args[1]}...`"
)
return do_expected
def create_skipped_scheme_t(args):
def do_expected(self):
pytest.skip(args[2])
return do_expected
for k, args in enumerate(
@pytest.mark.parametrize(
("url", "expected"),
[
("/index", "file://"),
("/index.html", "file://"),
@ -295,14 +189,13 @@ for k, args in enumerate(
("/", "http://"),
(".../test", "http://"),
],
start=1,
):
t_method = create_guess_scheme_t(args)
t_method.__name__ = f"test_uri_{k:03}"
setattr(TestGuessScheme, t_method.__name__, t_method)
)
def test_guess_scheme(url: str, expected: str):
assert guess_scheme(url).startswith(expected)
# TODO: the following tests do not pass with current implementation
for k, skip_args in enumerate(
@pytest.mark.parametrize(
("url", "expected", "reason"),
[
(
r"C:\absolute\path\to\a\file.html",
@ -310,25 +203,21 @@ for k, skip_args in enumerate(
"Windows filepath are not supported for scrapy shell",
),
],
start=1,
):
t_method = create_skipped_scheme_t(skip_args)
t_method.__name__ = f"test_uri_skipped_{k:03}"
setattr(TestGuessScheme, t_method.__name__, t_method)
)
def test_guess_scheme_skipped(url: str, expected: str, reason: str):
pytest.skip(reason)
class TestStripUrl:
def test_noop(self):
assert (
strip_url("http://www.example.com/index.html")
== "http://www.example.com/index.html"
)
def test_noop_query_string(self):
assert (
strip_url("http://www.example.com/index.html?somekey=somevalue")
== "http://www.example.com/index.html?somekey=somevalue"
@pytest.mark.parametrize(
"url",
[
"http://www.example.com/index.html",
"http://www.example.com/index.html?somekey=somevalue",
],
)
def test_noop(self, url: str) -> None:
assert strip_url(url) == url
def test_fragments(self):
assert (
@ -339,16 +228,20 @@ class TestStripUrl:
== "http://www.example.com/index.html?somekey=somevalue#section"
)
def test_path(self):
for input_url, origin, output_url in [
@pytest.mark.parametrize(
("url", "origin", "expected"),
[
("http://www.example.com/", False, "http://www.example.com/"),
("http://www.example.com", False, "http://www.example.com"),
("http://www.example.com", True, "http://www.example.com/"),
]:
assert strip_url(input_url, origin_only=origin) == output_url
],
)
def test_path(self, url: str, origin: bool, expected: str) -> None:
assert strip_url(url, origin_only=origin) == expected
def test_credentials(self):
for i, o in [
@pytest.mark.parametrize(
("url", "expected"),
[
(
"http://username@www.example.com/index.html?somekey=somevalue#section",
"http://www.example.com/index.html?somekey=somevalue",
@ -361,34 +254,29 @@ class TestStripUrl:
"ftp://username:password@www.example.com/index.html?somekey=somevalue#section",
"ftp://www.example.com/index.html?somekey=somevalue",
),
]:
assert strip_url(i, strip_credentials=True) == o
def test_credentials_encoded_delims(self):
for i, o in [
# user: "username@"
# password: none
# user: "username@", password: none
(
"http://username%40@www.example.com/index.html?somekey=somevalue#section",
"http://www.example.com/index.html?somekey=somevalue",
),
# user: "username:pass"
# password: ""
# user: "username:pass", password: ""
(
"https://username%3Apass:@www.example.com/index.html?somekey=somevalue#section",
"https://www.example.com/index.html?somekey=somevalue",
),
# user: "me"
# password: "user@domain.com"
# user: "me", password: "user@domain.com"
(
"ftp://me:user%40domain.com@www.example.com/index.html?somekey=somevalue#section",
"ftp://www.example.com/index.html?somekey=somevalue",
),
]:
assert strip_url(i, strip_credentials=True) == o
],
)
def test_credentials(self, url: str, expected: str) -> None:
assert strip_url(url, strip_credentials=True) == expected
def test_default_ports_creds_off(self):
for i, o in [
@pytest.mark.parametrize(
("url", "expected"),
[
(
"http://username:password@www.example.com:80/index.html?somekey=somevalue#section",
"http://www.example.com/index.html?somekey=somevalue",
@ -421,11 +309,14 @@ class TestStripUrl:
"ftp://username:password@www.example.com:221/file.txt",
"ftp://www.example.com:221/file.txt",
),
]:
assert strip_url(i) == o
],
)
def test_default_ports_creds_off(self, url: str, expected: str) -> None:
assert strip_url(url) == expected
def test_default_ports(self):
for i, o in [
@pytest.mark.parametrize(
("url", "expected"),
[
(
"http://username:password@www.example.com:80/index.html",
"http://username:password@www.example.com/index.html",
@ -458,11 +349,16 @@ class TestStripUrl:
"ftp://username:password@www.example.com:221/file.txt",
"ftp://username:password@www.example.com:221/file.txt",
),
]:
assert strip_url(i, strip_default_port=True, strip_credentials=False) == o
],
)
def test_default_ports(self, url: str, expected: str) -> None:
assert (
strip_url(url, strip_default_port=True, strip_credentials=False) == expected
)
def test_default_ports_keep(self):
for i, o in [
@pytest.mark.parametrize(
("url", "expected"),
[
(
"http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov#section",
"http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov",
@ -495,11 +391,17 @@ class TestStripUrl:
"ftp://username:password@www.example.com:221/file.txt",
"ftp://username:password@www.example.com:221/file.txt",
),
]:
assert strip_url(i, strip_default_port=False, strip_credentials=False) == o
],
)
def test_default_ports_keep(self, url: str, expected: str) -> None:
assert (
strip_url(url, strip_default_port=False, strip_credentials=False)
== expected
)
def test_origin_only(self):
for i, o in [
@pytest.mark.parametrize(
("url", "expected"),
[
(
"http://username:password@www.example.com/index.html",
"http://www.example.com/",
@ -516,13 +418,15 @@ class TestStripUrl:
"https://username:password@www.example.com:443/index.html",
"https://www.example.com/",
),
]:
assert strip_url(i, origin_only=True) == o
],
)
def test_origin_only(self, url: str, expected: str) -> None:
assert strip_url(url, origin_only=True) == expected
class TestIsPath:
def test_path(self):
for input_value, output_value in (
@pytest.mark.parametrize(
("path", "expected"),
[
# https://en.wikipedia.org/wiki/Path_(computing)#Representations_of_paths_by_operating_system_and_shell
# Unix-like OS, Microsoft Windows / cmd.exe
("/home/user/docs/Letter.txt", True),
@ -537,8 +441,10 @@ class TestIsPath:
(r"\\?\C:\user\docs\Letter.txt", True),
(r"C:\user\docs\somefile.ext:alternate_stream_name", True),
(r"https://example.com", False),
):
assert _is_filesystem_path(input_value) == output_value, input_value
],
)
def test__is_filesystem_path(path: str, expected: bool) -> None:
assert _is_filesystem_path(path) == expected
@pytest.mark.parametrize(
@ -550,7 +456,7 @@ class TestIsPath:
*_public_w3lib_objects,
],
)
def test_deprecated_imports_from_w3lib(obj_name):
def test_deprecated_imports_from_w3lib(obj_name: str) -> None:
with warnings.catch_warnings(record=True) as warns:
obj_type = "attribute" if obj_name == "_safe_chars" else "function"
message = f"The scrapy.utils.url.{obj_name} {obj_type} is deprecated, use w3lib.url.{obj_name} instead."
@ -559,4 +465,5 @@ def test_deprecated_imports_from_w3lib(obj_name):
getattr(import_module("scrapy.utils.url"), obj_name)
assert isinstance(warns[0].message, Warning)
assert message in warns[0].message.args