mirror of https://github.com/scrapy/scrapy.git
Refactoring of test_utils_*. (#6905)
This commit is contained in:
Andrey Rakhmatullin
GitHub
parent
0d75355b41
commit
d70f8a3f14
|
|
@ -106,8 +106,6 @@ def get_permissions_dict(
|
|||
|
||||
|
||||
class TestStartprojectTemplates(TestProjectBase):
|
||||
maxDiff = None
|
||||
|
||||
def setup_method(self):
|
||||
super().setup_method()
|
||||
self.tmpl = str(Path(self.temp_path, "templates"))
|
||||
|
|
|
|||
|
|
@ -150,8 +150,6 @@ class KeywordArgumentsSpider(MockServerSpider):
|
|||
|
||||
|
||||
class TestCallbackKeywordArguments(TestCase):
|
||||
maxDiff = None
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.mockserver = MockServer()
|
||||
|
|
|
|||
|
|
@ -1,30 +1,34 @@
|
|||
import scrapy
|
||||
|
||||
|
||||
class TestToplevel:
|
||||
def test_version(self):
|
||||
def test_version():
|
||||
assert isinstance(scrapy.__version__, str)
|
||||
|
||||
def test_version_info(self):
|
||||
|
||||
def test_version_info():
|
||||
assert isinstance(scrapy.version_info, tuple)
|
||||
|
||||
def test_request_shortcut(self):
|
||||
|
||||
def test_request_shortcut():
|
||||
from scrapy.http import FormRequest, Request
|
||||
|
||||
assert scrapy.Request is Request
|
||||
assert scrapy.FormRequest is FormRequest
|
||||
|
||||
def test_spider_shortcut(self):
|
||||
|
||||
def test_spider_shortcut():
|
||||
from scrapy.spiders import Spider
|
||||
|
||||
assert scrapy.Spider is Spider
|
||||
|
||||
def test_selector_shortcut(self):
|
||||
|
||||
def test_selector_shortcut():
|
||||
from scrapy.selector import Selector
|
||||
|
||||
assert scrapy.Selector is Selector
|
||||
|
||||
def test_item_shortcut(self):
|
||||
|
||||
def test_item_shortcut():
|
||||
from scrapy.item import Field, Item
|
||||
|
||||
assert scrapy.Item is Item
|
||||
|
|
|
|||
|
|
@ -1,10 +0,0 @@
|
|||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
class TestUrlparse:
|
||||
def test_s3_url(self):
|
||||
p = urlparse("s3://bucket/key/name?param=value")
|
||||
assert p.scheme == "s3"
|
||||
assert p.hostname == "bucket"
|
||||
assert p.path == "/key/name"
|
||||
assert p.query == "param=value"
|
||||
|
|
@ -47,8 +47,7 @@ class TestBuildComponentList:
|
|||
assert build_component_list(d, convert=lambda x: x) == ["b", "c", "a"]
|
||||
|
||||
|
||||
class TestUtilsConf:
|
||||
def test_arglist_to_dict(self):
|
||||
def test_arglist_to_dict():
|
||||
assert arglist_to_dict(["arg1=val1", "arg2=val2"]) == {
|
||||
"arg1": "val1",
|
||||
"arg2": "val2",
|
||||
|
|
|
|||
|
|
@ -18,8 +18,7 @@ except ImportError:
|
|||
ipy = False
|
||||
|
||||
|
||||
class TestUtilsConsole:
|
||||
def test_get_shell_embed_func(self):
|
||||
def test_get_shell_embed_func():
|
||||
shell = get_shell_embed_func(["invalid"])
|
||||
assert shell is None
|
||||
|
||||
|
|
@ -27,14 +26,16 @@ class TestUtilsConsole:
|
|||
assert callable(shell)
|
||||
assert shell.__name__ == "_embed_standard_shell"
|
||||
|
||||
|
||||
@pytest.mark.skipif(not bpy, reason="bpython not available in testenv")
|
||||
def test_get_shell_embed_func2(self):
|
||||
def test_get_shell_embed_func_bpython():
|
||||
shell = get_shell_embed_func(["bpython"])
|
||||
assert callable(shell)
|
||||
assert shell.__name__ == "_embed_bpython_shell"
|
||||
|
||||
|
||||
@pytest.mark.skipif(not ipy, reason="IPython not available in testenv")
|
||||
def test_get_shell_embed_func3(self):
|
||||
def test_get_shell_embed_func_ipython():
|
||||
# default shell should be 'ipython'
|
||||
shell = get_shell_embed_func()
|
||||
assert shell.__name__ == "_embed_ipython_shell"
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import warnings
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from w3lib.http import basic_auth_header
|
||||
|
|
@ -8,9 +9,8 @@ from scrapy.utils.curl import curl_to_request_kwargs
|
|||
|
||||
|
||||
class TestCurlToRequestKwargs:
|
||||
maxDiff = 5000
|
||||
|
||||
def _test_command(self, curl_command, expected_result):
|
||||
@staticmethod
|
||||
def _test_command(curl_command: str, expected_result: dict[str, Any]) -> None:
|
||||
result = curl_to_request_kwargs(curl_command)
|
||||
assert result == expected_result
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import copy
|
||||
import warnings
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Iterator, Mapping, MutableMapping
|
||||
|
||||
import pytest
|
||||
|
|
@ -16,7 +17,12 @@ from scrapy.utils.datatypes import (
|
|||
from scrapy.utils.python import garbage_collect
|
||||
|
||||
|
||||
class CaseInsensitiveDictBase:
|
||||
class TestCaseInsensitiveDictBase(ABC):
|
||||
@property
|
||||
@abstractmethod
|
||||
def dict_class(self) -> type[MutableMapping]:
|
||||
raise NotImplementedError
|
||||
|
||||
def test_init_dict(self):
|
||||
seq = {"red": 1, "black": 3}
|
||||
d = self.dict_class(seq)
|
||||
|
|
@ -199,7 +205,7 @@ class CaseInsensitiveDictBase:
|
|||
assert h1.get("header1") == h3.get("HEADER1")
|
||||
|
||||
|
||||
class TestCaseInsensitiveDict(CaseInsensitiveDictBase):
|
||||
class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):
|
||||
dict_class = CaseInsensitiveDict
|
||||
|
||||
def test_repr(self):
|
||||
|
|
@ -216,7 +222,7 @@ class TestCaseInsensitiveDict(CaseInsensitiveDictBase):
|
|||
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
class TestCaselessDict(CaseInsensitiveDictBase):
|
||||
class TestCaselessDict(TestCaseInsensitiveDictBase):
|
||||
dict_class = CaselessDict
|
||||
|
||||
def test_deprecation_message(self):
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import inspect
|
||||
import warnings
|
||||
from unittest import mock
|
||||
from warnings import WarningMessage
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -21,7 +22,9 @@ class NewName(SomeBaseClass):
|
|||
|
||||
|
||||
class TestWarnWhenSubclassed:
|
||||
def _mywarnings(self, w, category=MyWarning):
|
||||
def _mywarnings(
|
||||
self, w: list[WarningMessage], category: type[Warning] = MyWarning
|
||||
) -> list[WarningMessage]:
|
||||
return [x for x in w if x.category is MyWarning]
|
||||
|
||||
def test_no_warning_on_definition(self):
|
||||
|
|
|
|||
|
|
@ -3,9 +3,7 @@ from unittest import mock
|
|||
|
||||
from scrapy.utils.display import pformat, pprint
|
||||
|
||||
|
||||
class TestDisplay:
|
||||
object = {"a": 1}
|
||||
value = {"a": 1}
|
||||
colorized_strings = {
|
||||
(
|
||||
(
|
||||
|
|
@ -22,53 +20,58 @@ class TestDisplay:
|
|||
}
|
||||
plain_string = "{'a': 1}"
|
||||
|
||||
|
||||
@mock.patch("sys.platform", "linux")
|
||||
@mock.patch("sys.stdout.isatty")
|
||||
def test_pformat(self, isatty):
|
||||
def test_pformat(isatty):
|
||||
isatty.return_value = True
|
||||
assert pformat(self.object) in self.colorized_strings
|
||||
assert pformat(value) in colorized_strings
|
||||
|
||||
|
||||
@mock.patch("sys.stdout.isatty")
|
||||
def test_pformat_dont_colorize(self, isatty):
|
||||
def test_pformat_dont_colorize(isatty):
|
||||
isatty.return_value = True
|
||||
assert pformat(self.object, colorize=False) == self.plain_string
|
||||
assert pformat(value, colorize=False) == plain_string
|
||||
|
||||
|
||||
def test_pformat_not_tty():
|
||||
assert pformat(value) == plain_string
|
||||
|
||||
def test_pformat_not_tty(self):
|
||||
assert pformat(self.object) == self.plain_string
|
||||
|
||||
@mock.patch("sys.platform", "win32")
|
||||
@mock.patch("platform.version")
|
||||
@mock.patch("sys.stdout.isatty")
|
||||
def test_pformat_old_windows(self, isatty, version):
|
||||
def test_pformat_old_windows(isatty, version):
|
||||
isatty.return_value = True
|
||||
version.return_value = "10.0.14392"
|
||||
assert pformat(self.object) in self.colorized_strings
|
||||
assert pformat(value) in colorized_strings
|
||||
|
||||
|
||||
@mock.patch("sys.platform", "win32")
|
||||
@mock.patch("scrapy.utils.display._enable_windows_terminal_processing")
|
||||
@mock.patch("platform.version")
|
||||
@mock.patch("sys.stdout.isatty")
|
||||
def test_pformat_windows_no_terminal_processing(
|
||||
self, isatty, version, terminal_processing
|
||||
):
|
||||
def test_pformat_windows_no_terminal_processing(isatty, version, terminal_processing):
|
||||
isatty.return_value = True
|
||||
version.return_value = "10.0.14393"
|
||||
terminal_processing.return_value = False
|
||||
assert pformat(self.object) == self.plain_string
|
||||
assert pformat(value) == plain_string
|
||||
|
||||
|
||||
@mock.patch("sys.platform", "win32")
|
||||
@mock.patch("scrapy.utils.display._enable_windows_terminal_processing")
|
||||
@mock.patch("platform.version")
|
||||
@mock.patch("sys.stdout.isatty")
|
||||
def test_pformat_windows(self, isatty, version, terminal_processing):
|
||||
def test_pformat_windows(isatty, version, terminal_processing):
|
||||
isatty.return_value = True
|
||||
version.return_value = "10.0.14393"
|
||||
terminal_processing.return_value = True
|
||||
assert pformat(self.object) in self.colorized_strings
|
||||
assert pformat(value) in colorized_strings
|
||||
|
||||
|
||||
@mock.patch("sys.platform", "linux")
|
||||
@mock.patch("sys.stdout.isatty")
|
||||
def test_pformat_no_pygments(self, isatty):
|
||||
def test_pformat_no_pygments(isatty):
|
||||
isatty.return_value = True
|
||||
|
||||
import builtins
|
||||
|
|
@ -81,10 +84,11 @@ class TestDisplay:
|
|||
return real_import(name, globals, locals, fromlist, level)
|
||||
|
||||
builtins.__import__ = mock_import
|
||||
assert pformat(self.object) == self.plain_string
|
||||
assert pformat(value) == plain_string
|
||||
builtins.__import__ = real_import
|
||||
|
||||
def test_pprint(self):
|
||||
|
||||
def test_pprint():
|
||||
with mock.patch("sys.stdout", new=StringIO()) as mock_out:
|
||||
pprint(self.object)
|
||||
pprint(value)
|
||||
assert mock_out.getvalue() == "{'a': 1}\n"
|
||||
|
|
|
|||
|
|
@ -11,8 +11,7 @@ from tests import tests_datadir
|
|||
SAMPLEDIR = Path(tests_datadir, "compressed")
|
||||
|
||||
|
||||
class TestGunzip:
|
||||
def test_gunzip_basic(self):
|
||||
def test_gunzip_basic():
|
||||
r1 = Response(
|
||||
"http://www.example.com",
|
||||
body=(SAMPLEDIR / "feed-sample1.xml.gz").read_bytes(),
|
||||
|
|
@ -23,15 +22,18 @@ class TestGunzip:
|
|||
assert not gzip_magic_number(r2)
|
||||
assert len(r2.body) == 9950
|
||||
|
||||
def test_gunzip_truncated(self):
|
||||
|
||||
def test_gunzip_truncated():
|
||||
text = gunzip((SAMPLEDIR / "truncated-crc-error.gz").read_bytes())
|
||||
assert text.endswith(b"</html")
|
||||
|
||||
def test_gunzip_no_gzip_file_raises(self):
|
||||
|
||||
def test_gunzip_no_gzip_file_raises():
|
||||
with pytest.raises(BadGzipFile):
|
||||
gunzip((SAMPLEDIR / "feed-sample1.xml").read_bytes())
|
||||
|
||||
def test_gunzip_truncated_short(self):
|
||||
|
||||
def test_gunzip_truncated_short():
|
||||
r1 = Response(
|
||||
"http://www.example.com",
|
||||
body=(SAMPLEDIR / "truncated-crc-error-short.gz").read_bytes(),
|
||||
|
|
@ -42,11 +44,13 @@ class TestGunzip:
|
|||
assert r2.body.endswith(b"</html>")
|
||||
assert not gzip_magic_number(r2)
|
||||
|
||||
def test_is_gzipped_empty(self):
|
||||
|
||||
def test_is_gzipped_empty():
|
||||
r1 = Response("http://www.example.com")
|
||||
assert not gzip_magic_number(r1)
|
||||
|
||||
def test_gunzip_illegal_eof(self):
|
||||
|
||||
def test_gunzip_illegal_eof():
|
||||
text = html_to_unicode(
|
||||
"charset=cp1252", gunzip((SAMPLEDIR / "unexpected-eof.gz").read_bytes())
|
||||
)[1]
|
||||
|
|
|
|||
|
|
@ -4,8 +4,7 @@ from scrapy.http import Request
|
|||
from scrapy.utils.httpobj import urlparse_cached
|
||||
|
||||
|
||||
class TestHttpobjUtils:
|
||||
def test_urlparse_cached(self):
|
||||
def test_urlparse_cached():
|
||||
url = "http://www.example.com/index.html"
|
||||
request1 = Request(url)
|
||||
request2 = Request(url)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,8 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
|
|
@ -5,9 +10,19 @@ from scrapy.http import Response, TextResponse, XmlResponse
|
|||
from scrapy.utils.iterators import _body_or_str, csviter, xmliter, xmliter_lxml
|
||||
from tests import get_testdata
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
|
||||
from scrapy import Selector
|
||||
|
||||
|
||||
class TestXmliterBase(ABC):
|
||||
@abstractmethod
|
||||
def xmliter(
|
||||
self, obj: Response | str | bytes, nodename: str, *args: Any
|
||||
) -> Iterator[Selector]:
|
||||
raise NotImplementedError
|
||||
|
||||
class XmliterBase:
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
def test_xmliter(self):
|
||||
body = b"""
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
|
@ -39,7 +54,6 @@ class XmliterBase:
|
|||
("002", ["Name 2"], ["Type 2"]),
|
||||
]
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
def test_xmliter_unusual_node(self):
|
||||
body = b"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<root>
|
||||
|
|
@ -53,7 +67,6 @@ class XmliterBase:
|
|||
]
|
||||
assert nodenames == [["matchme..."]]
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
def test_xmliter_unicode(self):
|
||||
# example taken from https://github.com/scrapy/scrapy/issues/1665
|
||||
body = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
|
@ -113,7 +126,6 @@ class XmliterBase:
|
|||
("27", ["A"], ["27"]),
|
||||
]
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
def test_xmliter_text(self):
|
||||
body = (
|
||||
'<?xml version="1.0" encoding="UTF-8"?>'
|
||||
|
|
@ -125,7 +137,6 @@ class XmliterBase:
|
|||
["two"],
|
||||
]
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
def test_xmliter_namespaces(self):
|
||||
body = b"""
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
|
@ -163,7 +174,6 @@ class XmliterBase:
|
|||
assert node.xpath("id/text()").getall() == []
|
||||
assert node.xpath("price/text()").getall() == []
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
def test_xmliter_namespaced_nodename(self):
|
||||
body = b"""
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
|
@ -191,7 +201,6 @@ class XmliterBase:
|
|||
"http://www.mydummycompany.com/images/item1.jpg"
|
||||
]
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
def test_xmliter_namespaced_nodename_missing(self):
|
||||
body = b"""
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
|
@ -216,7 +225,6 @@ class XmliterBase:
|
|||
with pytest.raises(StopIteration):
|
||||
next(my_iter)
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
def test_xmliter_exception(self):
|
||||
body = (
|
||||
'<?xml version="1.0" encoding="UTF-8"?>'
|
||||
|
|
@ -229,13 +237,11 @@ class XmliterBase:
|
|||
with pytest.raises(StopIteration):
|
||||
next(iter)
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
def test_xmliter_objtype_exception(self):
|
||||
i = self.xmliter(42, "product")
|
||||
with pytest.raises(TypeError):
|
||||
next(i)
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
def test_xmliter_encoding(self):
|
||||
body = (
|
||||
b'<?xml version="1.0" encoding="ISO-8859-9"?>\n'
|
||||
|
|
@ -250,8 +256,12 @@ class XmliterBase:
|
|||
)
|
||||
|
||||
|
||||
class TestXmliter(XmliterBase):
|
||||
xmliter = staticmethod(xmliter)
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
class TestXmliter(TestXmliterBase):
|
||||
def xmliter(
|
||||
self, obj: Response | str | bytes, nodename: str, *args: Any
|
||||
) -> Iterator[Selector]:
|
||||
return xmliter(obj, nodename)
|
||||
|
||||
def test_deprecation(self):
|
||||
body = b"""
|
||||
|
|
@ -267,8 +277,11 @@ class TestXmliter(XmliterBase):
|
|||
next(self.xmliter(body, "product"))
|
||||
|
||||
|
||||
class TestLxmlXmliter(XmliterBase):
|
||||
xmliter = staticmethod(xmliter_lxml)
|
||||
class TestLxmlXmliter(TestXmliterBase):
|
||||
def xmliter(
|
||||
self, obj: Response | str | bytes, nodename: str, *args: Any
|
||||
) -> Iterator[Selector]:
|
||||
return xmliter_lxml(obj, nodename, *args)
|
||||
|
||||
def test_xmliter_iterate_namespace(self):
|
||||
body = b"""
|
||||
|
|
@ -493,14 +506,20 @@ class TestUtilsCsv:
|
|||
]
|
||||
|
||||
|
||||
class TestHelper:
|
||||
class TestBodyOrStr:
|
||||
bbody = b"utf8-body"
|
||||
ubody = bbody.decode("utf8")
|
||||
txtresponse = TextResponse(url="http://example.org/", body=bbody, encoding="utf-8")
|
||||
response = Response(url="http://example.org/", body=bbody)
|
||||
|
||||
def test_body_or_str(self):
|
||||
for obj in (self.bbody, self.ubody, self.txtresponse, self.response):
|
||||
@pytest.mark.parametrize(
|
||||
"obj",
|
||||
[
|
||||
bbody,
|
||||
ubody,
|
||||
TextResponse(url="http://example.org/", body=bbody, encoding="utf-8"),
|
||||
Response(url="http://example.org/", body=bbody),
|
||||
],
|
||||
)
|
||||
def test_body_or_str(self, obj: Response | str | bytes) -> None:
|
||||
r1 = _body_or_str(obj)
|
||||
self._assert_type_and_value(r1, self.ubody, obj)
|
||||
r2 = _body_or_str(obj, unicode=True)
|
||||
|
|
@ -510,6 +529,9 @@ class TestHelper:
|
|||
assert type(r1) is type(r2)
|
||||
assert type(r1) is not type(r3)
|
||||
|
||||
def _assert_type_and_value(self, a, b, obj):
|
||||
@staticmethod
|
||||
def _assert_type_and_value(
|
||||
a: str | bytes, b: str | bytes, obj: Response | str | bytes
|
||||
) -> None:
|
||||
assert type(a) is type(b), f"Got {type(a)}, expected {type(b)} for {obj!r}"
|
||||
assert a == b
|
||||
|
|
|
|||
|
|
@ -22,7 +22,9 @@ from scrapy.utils.test import get_crawler
|
|||
from tests.spiders import LogSpider
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Mapping, MutableMapping
|
||||
from collections.abc import Generator, Mapping, MutableMapping
|
||||
|
||||
from scrapy.crawler import Crawler
|
||||
|
||||
|
||||
class TestFailureToExcInfo:
|
||||
|
|
@ -70,33 +72,41 @@ class TestTopLevelFormatter:
|
|||
|
||||
|
||||
class TestLogCounterHandler:
|
||||
def setup_method(self):
|
||||
@pytest.fixture
|
||||
def crawler(self) -> Crawler:
|
||||
settings = {"LOG_LEVEL": "WARNING"}
|
||||
self.logger = logging.getLogger("test")
|
||||
self.logger.setLevel(logging.NOTSET)
|
||||
self.logger.propagate = False
|
||||
self.crawler = get_crawler(settings_dict=settings)
|
||||
self.handler = LogCounterHandler(self.crawler)
|
||||
self.logger.addHandler(self.handler)
|
||||
return get_crawler(settings_dict=settings)
|
||||
|
||||
def teardown_method(self):
|
||||
self.logger.propagate = True
|
||||
self.logger.removeHandler(self.handler)
|
||||
@pytest.fixture
|
||||
def logger(self, crawler: Crawler) -> Generator[logging.Logger]:
|
||||
logger = logging.getLogger("test")
|
||||
logger.setLevel(logging.NOTSET)
|
||||
logger.propagate = False
|
||||
handler = LogCounterHandler(crawler)
|
||||
logger.addHandler(handler)
|
||||
|
||||
def test_init(self):
|
||||
assert self.crawler.stats.get_value("log_count/DEBUG") is None
|
||||
assert self.crawler.stats.get_value("log_count/INFO") is None
|
||||
assert self.crawler.stats.get_value("log_count/WARNING") is None
|
||||
assert self.crawler.stats.get_value("log_count/ERROR") is None
|
||||
assert self.crawler.stats.get_value("log_count/CRITICAL") is None
|
||||
yield logger
|
||||
|
||||
def test_accepted_level(self):
|
||||
self.logger.error("test log msg")
|
||||
assert self.crawler.stats.get_value("log_count/ERROR") == 1
|
||||
logger.propagate = True
|
||||
logger.removeHandler(handler)
|
||||
|
||||
def test_filtered_out_level(self):
|
||||
self.logger.debug("test log msg")
|
||||
assert self.crawler.stats.get_value("log_count/INFO") is None
|
||||
def test_init(self, crawler: Crawler, logger: logging.Logger) -> None:
|
||||
assert crawler.stats
|
||||
assert crawler.stats.get_value("log_count/DEBUG") is None
|
||||
assert crawler.stats.get_value("log_count/INFO") is None
|
||||
assert crawler.stats.get_value("log_count/WARNING") is None
|
||||
assert crawler.stats.get_value("log_count/ERROR") is None
|
||||
assert crawler.stats.get_value("log_count/CRITICAL") is None
|
||||
|
||||
def test_accepted_level(self, crawler: Crawler, logger: logging.Logger) -> None:
|
||||
logger.error("test log msg")
|
||||
assert crawler.stats
|
||||
assert crawler.stats.get_value("log_count/ERROR") == 1
|
||||
|
||||
def test_filtered_out_level(self, crawler: Crawler, logger: logging.Logger) -> None:
|
||||
logger.debug("test log msg")
|
||||
assert crawler.stats
|
||||
assert crawler.stats.get_value("log_count/INFO") is None
|
||||
|
||||
|
||||
class TestStreamLogger:
|
||||
|
|
@ -135,7 +145,7 @@ class TestStreamLogger:
|
|||
)
|
||||
def test_spider_logger_adapter_process(
|
||||
base_extra: Mapping[str, Any], log_extra: MutableMapping, expected_extra: dict
|
||||
):
|
||||
) -> None:
|
||||
logger = logging.getLogger("test")
|
||||
spider_logger_adapter = SpiderLoggerAdapter(logger, base_extra)
|
||||
|
||||
|
|
@ -149,59 +159,75 @@ def test_spider_logger_adapter_process(
|
|||
|
||||
|
||||
class TestLogging:
|
||||
def setup_method(self):
|
||||
self.log_stream = StringIO()
|
||||
handler = logging.StreamHandler(self.log_stream)
|
||||
@pytest.fixture
|
||||
def log_stream(self) -> StringIO:
|
||||
return StringIO()
|
||||
|
||||
@pytest.fixture
|
||||
def spider(self) -> LogSpider:
|
||||
return LogSpider()
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def logger(self, log_stream: StringIO) -> Generator[logging.Logger]:
|
||||
handler = logging.StreamHandler(log_stream)
|
||||
logger = logging.getLogger("log_spider")
|
||||
logger.addHandler(handler)
|
||||
logger.setLevel(logging.DEBUG)
|
||||
self.handler = handler
|
||||
self.logger = logger
|
||||
self.spider = LogSpider()
|
||||
|
||||
def teardown_method(self):
|
||||
self.logger.removeHandler(self.handler)
|
||||
yield logger
|
||||
|
||||
def test_debug_logging(self):
|
||||
logger.removeHandler(handler)
|
||||
|
||||
def test_debug_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
|
||||
log_message = "Foo message"
|
||||
self.spider.log_debug(log_message)
|
||||
log_contents = self.log_stream.getvalue()
|
||||
spider.log_debug(log_message)
|
||||
log_contents = log_stream.getvalue()
|
||||
|
||||
assert log_contents == f"{log_message}\n"
|
||||
|
||||
def test_info_logging(self):
|
||||
def test_info_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
|
||||
log_message = "Bar message"
|
||||
self.spider.log_info(log_message)
|
||||
log_contents = self.log_stream.getvalue()
|
||||
spider.log_info(log_message)
|
||||
log_contents = log_stream.getvalue()
|
||||
|
||||
assert log_contents == f"{log_message}\n"
|
||||
|
||||
def test_warning_logging(self):
|
||||
def test_warning_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
|
||||
log_message = "Baz message"
|
||||
self.spider.log_warning(log_message)
|
||||
log_contents = self.log_stream.getvalue()
|
||||
spider.log_warning(log_message)
|
||||
log_contents = log_stream.getvalue()
|
||||
|
||||
assert log_contents == f"{log_message}\n"
|
||||
|
||||
def test_error_logging(self):
|
||||
def test_error_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
|
||||
log_message = "Foo bar message"
|
||||
self.spider.log_error(log_message)
|
||||
log_contents = self.log_stream.getvalue()
|
||||
spider.log_error(log_message)
|
||||
log_contents = log_stream.getvalue()
|
||||
|
||||
assert log_contents == f"{log_message}\n"
|
||||
|
||||
def test_critical_logging(self):
|
||||
def test_critical_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
|
||||
log_message = "Foo bar baz message"
|
||||
self.spider.log_critical(log_message)
|
||||
log_contents = self.log_stream.getvalue()
|
||||
spider.log_critical(log_message)
|
||||
log_contents = log_stream.getvalue()
|
||||
|
||||
assert log_contents == f"{log_message}\n"
|
||||
|
||||
|
||||
class TestLoggingWithExtra:
|
||||
def setup_method(self):
|
||||
self.log_stream = StringIO()
|
||||
handler = logging.StreamHandler(self.log_stream)
|
||||
regex_pattern = re.compile(r"^<LogSpider\s'log_spider'\sat\s[^>]+>$")
|
||||
|
||||
@pytest.fixture
|
||||
def log_stream(self) -> StringIO:
|
||||
return StringIO()
|
||||
|
||||
@pytest.fixture
|
||||
def spider(self) -> LogSpider:
|
||||
return LogSpider()
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def logger(self, log_stream: StringIO) -> Generator[logging.Logger]:
|
||||
handler = logging.StreamHandler(log_stream)
|
||||
formatter = logging.Formatter(
|
||||
'{"levelname": "%(levelname)s", "message": "%(message)s", "spider": "%(spider)s", "important_info": "%(important_info)s"}'
|
||||
)
|
||||
|
|
@ -209,80 +235,79 @@ class TestLoggingWithExtra:
|
|||
logger = logging.getLogger("log_spider")
|
||||
logger.addHandler(handler)
|
||||
logger.setLevel(logging.DEBUG)
|
||||
self.handler = handler
|
||||
self.logger = logger
|
||||
self.spider = LogSpider()
|
||||
self.regex_pattern = re.compile(r"^<LogSpider\s'log_spider'\sat\s[^>]+>$")
|
||||
|
||||
def teardown_method(self):
|
||||
self.logger.removeHandler(self.handler)
|
||||
yield logger
|
||||
|
||||
def test_debug_logging(self):
|
||||
logger.removeHandler(handler)
|
||||
|
||||
def test_debug_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
|
||||
log_message = "Foo message"
|
||||
extra = {"important_info": "foo"}
|
||||
self.spider.log_debug(log_message, extra)
|
||||
log_contents = self.log_stream.getvalue()
|
||||
log_contents = json.loads(log_contents)
|
||||
spider.log_debug(log_message, extra)
|
||||
log_contents_str = log_stream.getvalue()
|
||||
log_contents = json.loads(log_contents_str)
|
||||
|
||||
assert log_contents["levelname"] == "DEBUG"
|
||||
assert log_contents["message"] == log_message
|
||||
assert self.regex_pattern.match(log_contents["spider"])
|
||||
assert log_contents["important_info"] == extra["important_info"]
|
||||
|
||||
def test_info_logging(self):
|
||||
def test_info_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
|
||||
log_message = "Bar message"
|
||||
extra = {"important_info": "bar"}
|
||||
self.spider.log_info(log_message, extra)
|
||||
log_contents = self.log_stream.getvalue()
|
||||
log_contents = json.loads(log_contents)
|
||||
spider.log_info(log_message, extra)
|
||||
log_contents_str = log_stream.getvalue()
|
||||
log_contents = json.loads(log_contents_str)
|
||||
|
||||
assert log_contents["levelname"] == "INFO"
|
||||
assert log_contents["message"] == log_message
|
||||
assert self.regex_pattern.match(log_contents["spider"])
|
||||
assert log_contents["important_info"] == extra["important_info"]
|
||||
|
||||
def test_warning_logging(self):
|
||||
def test_warning_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
|
||||
log_message = "Baz message"
|
||||
extra = {"important_info": "baz"}
|
||||
self.spider.log_warning(log_message, extra)
|
||||
log_contents = self.log_stream.getvalue()
|
||||
log_contents = json.loads(log_contents)
|
||||
spider.log_warning(log_message, extra)
|
||||
log_contents_str = log_stream.getvalue()
|
||||
log_contents = json.loads(log_contents_str)
|
||||
|
||||
assert log_contents["levelname"] == "WARNING"
|
||||
assert log_contents["message"] == log_message
|
||||
assert self.regex_pattern.match(log_contents["spider"])
|
||||
assert log_contents["important_info"] == extra["important_info"]
|
||||
|
||||
def test_error_logging(self):
|
||||
def test_error_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
|
||||
log_message = "Foo bar message"
|
||||
extra = {"important_info": "foo bar"}
|
||||
self.spider.log_error(log_message, extra)
|
||||
log_contents = self.log_stream.getvalue()
|
||||
log_contents = json.loads(log_contents)
|
||||
spider.log_error(log_message, extra)
|
||||
log_contents_str = log_stream.getvalue()
|
||||
log_contents = json.loads(log_contents_str)
|
||||
|
||||
assert log_contents["levelname"] == "ERROR"
|
||||
assert log_contents["message"] == log_message
|
||||
assert self.regex_pattern.match(log_contents["spider"])
|
||||
assert log_contents["important_info"] == extra["important_info"]
|
||||
|
||||
def test_critical_logging(self):
|
||||
def test_critical_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
|
||||
log_message = "Foo bar baz message"
|
||||
extra = {"important_info": "foo bar baz"}
|
||||
self.spider.log_critical(log_message, extra)
|
||||
log_contents = self.log_stream.getvalue()
|
||||
log_contents = json.loads(log_contents)
|
||||
spider.log_critical(log_message, extra)
|
||||
log_contents_str = log_stream.getvalue()
|
||||
log_contents = json.loads(log_contents_str)
|
||||
|
||||
assert log_contents["levelname"] == "CRITICAL"
|
||||
assert log_contents["message"] == log_message
|
||||
assert self.regex_pattern.match(log_contents["spider"])
|
||||
assert log_contents["important_info"] == extra["important_info"]
|
||||
|
||||
def test_overwrite_spider_extra(self):
|
||||
def test_overwrite_spider_extra(
|
||||
self, log_stream: StringIO, spider: LogSpider
|
||||
) -> None:
|
||||
log_message = "Foo message"
|
||||
extra = {"important_info": "foo", "spider": "shouldn't change"}
|
||||
self.spider.log_error(log_message, extra)
|
||||
log_contents = self.log_stream.getvalue()
|
||||
log_contents = json.loads(log_contents)
|
||||
spider.log_error(log_message, extra)
|
||||
log_contents_str = log_stream.getvalue()
|
||||
log_contents = json.loads(log_contents_str)
|
||||
|
||||
assert log_contents["levelname"] == "ERROR"
|
||||
assert log_contents["message"] == log_message
|
||||
|
|
|
|||
|
|
@ -1,18 +1,17 @@
|
|||
import contextlib
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from scrapy.utils.misc import set_environ
|
||||
from scrapy.utils.project import data_path, get_project_settings
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def inside_a_project():
|
||||
@pytest.fixture
|
||||
def proj_path(tmp_path):
|
||||
prev_dir = Path.cwd()
|
||||
project_dir = tempfile.mkdtemp()
|
||||
project_dir = tmp_path
|
||||
|
||||
try:
|
||||
os.chdir(project_dir)
|
||||
|
|
@ -21,18 +20,16 @@ def inside_a_project():
|
|||
yield project_dir
|
||||
finally:
|
||||
os.chdir(prev_dir)
|
||||
shutil.rmtree(project_dir)
|
||||
|
||||
|
||||
class TestProjectUtils:
|
||||
def test_data_path_outside_project(self):
|
||||
def test_data_path_outside_project():
|
||||
assert str(Path(".scrapy", "somepath")) == data_path("somepath")
|
||||
abspath = str(Path(os.path.sep, "absolute", "path"))
|
||||
assert abspath == data_path(abspath)
|
||||
|
||||
def test_data_path_inside_project(self):
|
||||
with inside_a_project() as proj_path:
|
||||
expected = Path(proj_path, ".scrapy", "somepath")
|
||||
|
||||
def test_data_path_inside_project(proj_path: Path) -> None:
|
||||
expected = proj_path / ".scrapy" / "somepath"
|
||||
assert expected.resolve() == Path(data_path("somepath")).resolve()
|
||||
abspath = str(Path(os.path.sep, "absolute", "path").resolve())
|
||||
assert abspath == data_path(abspath)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,10 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import operator
|
||||
import platform
|
||||
import sys
|
||||
from typing import TYPE_CHECKING, TypeVar
|
||||
|
||||
import pytest
|
||||
from twisted.trial import unittest
|
||||
|
|
@ -20,9 +23,15 @@ from scrapy.utils.python import (
|
|||
without_none_values,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable, Mapping
|
||||
|
||||
class TestMutableChain:
|
||||
def test_mutablechain(self):
|
||||
|
||||
_KT = TypeVar("_KT")
|
||||
_VT = TypeVar("_VT")
|
||||
|
||||
|
||||
def test_mutablechain():
|
||||
m = MutableChain(range(2), [2, 3], (4, 5))
|
||||
m.extend(range(6, 7))
|
||||
m.extend([7, 8])
|
||||
|
|
@ -112,8 +121,7 @@ class TestToBytes:
|
|||
assert to_bytes("a\ufffdb", "latin-1", errors="replace") == b"a?b"
|
||||
|
||||
|
||||
class TestMemoizedMethod:
|
||||
def test_memoizemethod_noargs(self):
|
||||
def test_memoizemethod_noargs():
|
||||
class A:
|
||||
@memoizemethod_noargs
|
||||
def cached(self):
|
||||
|
|
@ -130,23 +138,21 @@ class TestMemoizedMethod:
|
|||
assert one is not three
|
||||
|
||||
|
||||
class TestBinaryIsText:
|
||||
def test_binaryistext(self):
|
||||
assert binary_is_text(b"hello")
|
||||
|
||||
def test_utf_16_strings_contain_null_bytes(self):
|
||||
assert binary_is_text("hello".encode("utf-16"))
|
||||
|
||||
def test_one_with_encoding(self):
|
||||
assert binary_is_text(b"<div>Price \xa3</div>")
|
||||
|
||||
def test_real_binary_bytes(self):
|
||||
assert not binary_is_text(b"\x02\xa3")
|
||||
@pytest.mark.parametrize(
|
||||
("value", "expected"),
|
||||
[
|
||||
(b"hello", True),
|
||||
("hello".encode("utf-16"), True),
|
||||
(b"<div>Price \xa3</div>", True),
|
||||
(b"\x02\xa3", False),
|
||||
],
|
||||
)
|
||||
def test_binaryistext(value: bytes, expected: bool) -> None:
|
||||
assert binary_is_text(value) is expected
|
||||
|
||||
|
||||
class TestUtilsPython:
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
def test_equal_attributes(self):
|
||||
def test_equal_attributes():
|
||||
class Obj:
|
||||
pass
|
||||
|
||||
|
|
@ -194,7 +200,8 @@ class TestUtilsPython:
|
|||
a.meta["z"] = 2
|
||||
assert not equal_attributes(a, b, [compare_z, "x"])
|
||||
|
||||
def test_get_func_args(self):
|
||||
|
||||
def test_get_func_args():
|
||||
def f1(a, b, c):
|
||||
pass
|
||||
|
||||
|
|
@ -245,11 +252,19 @@ class TestUtilsPython:
|
|||
["args", "kwargs"],
|
||||
]
|
||||
|
||||
def test_without_none_values(self):
|
||||
assert without_none_values([1, None, 3, 4]) == [1, 3, 4]
|
||||
assert without_none_values((1, None, 3, 4)) == (1, 3, 4)
|
||||
assert without_none_values({"one": 1, "none": None, "three": 3, "four": 4}) == {
|
||||
"one": 1,
|
||||
"three": 3,
|
||||
"four": 4,
|
||||
}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "expected"),
|
||||
[
|
||||
([1, None, 3, 4], [1, 3, 4]),
|
||||
((1, None, 3, 4), (1, 3, 4)),
|
||||
(
|
||||
{"one": 1, "none": None, "three": 3, "four": 4},
|
||||
{"one": 1, "three": 3, "four": 4},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_without_none_values(
|
||||
value: Mapping[_KT, _VT] | Iterable[_KT], expected: dict[_KT, _VT] | Iterable[_KT]
|
||||
) -> None:
|
||||
assert without_none_values(value) == expected
|
||||
|
|
|
|||
|
|
@ -20,45 +20,52 @@ from scrapy.utils.request import (
|
|||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
class TestUtilsRequest:
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
def test_request_authenticate(self):
|
||||
def test_request_authenticate():
|
||||
r = Request("http://www.example.com")
|
||||
request_authenticate(r, "someuser", "somepass")
|
||||
assert r.headers["Authorization"] == b"Basic c29tZXVzZXI6c29tZXBhc3M="
|
||||
|
||||
def test_request_httprepr(self):
|
||||
r1 = Request("http://www.example.com")
|
||||
assert (
|
||||
request_httprepr(r1) == b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n"
|
||||
)
|
||||
|
||||
r1 = Request("http://www.example.com/some/page.html?arg=1")
|
||||
assert (
|
||||
request_httprepr(r1)
|
||||
== b"GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n"
|
||||
)
|
||||
|
||||
r1 = Request(
|
||||
@pytest.mark.parametrize(
|
||||
("r", "expected"),
|
||||
[
|
||||
(
|
||||
Request("http://www.example.com"),
|
||||
b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n",
|
||||
),
|
||||
(
|
||||
Request("http://www.example.com/some/page.html?arg=1"),
|
||||
b"GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n",
|
||||
),
|
||||
(
|
||||
Request(
|
||||
"http://www.example.com",
|
||||
method="POST",
|
||||
headers={"Content-type": b"text/html"},
|
||||
body=b"Some body",
|
||||
),
|
||||
b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body",
|
||||
),
|
||||
],
|
||||
)
|
||||
assert (
|
||||
request_httprepr(r1)
|
||||
== b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body"
|
||||
)
|
||||
def test_request_httprepr(r: Request, expected: bytes) -> None:
|
||||
assert request_httprepr(r) == expected
|
||||
|
||||
def test_request_httprepr_for_non_http_request(self):
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"r",
|
||||
[
|
||||
Request("file:///tmp/foo.txt"),
|
||||
Request("ftp://localhost/tmp/foo.txt"),
|
||||
],
|
||||
)
|
||||
def test_request_httprepr_for_non_http_request(r: Request) -> None:
|
||||
# the representation is not important but it must not fail.
|
||||
request_httprepr(Request("file:///tmp/foo.txt"))
|
||||
request_httprepr(Request("ftp://localhost/tmp/foo.txt"))
|
||||
request_httprepr(r)
|
||||
|
||||
|
||||
class TestFingerprint:
|
||||
maxDiff = None
|
||||
|
||||
function: staticmethod = staticmethod(fingerprint)
|
||||
cache: (
|
||||
WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes]]
|
||||
|
|
@ -229,35 +236,6 @@ class TestFingerprint:
|
|||
assert actual == expected
|
||||
|
||||
|
||||
REQUEST_OBJECTS_TO_TEST = (
|
||||
Request("http://www.example.com/"),
|
||||
Request("http://www.example.com/query?id=111&cat=222"),
|
||||
Request("http://www.example.com/query?cat=222&id=111"),
|
||||
Request("http://www.example.com/hnnoticiaj1.aspx?78132,199"),
|
||||
Request("http://www.example.com/hnnoticiaj1.aspx?78160,199"),
|
||||
Request("http://www.example.com/members/offers.html"),
|
||||
Request(
|
||||
"http://www.example.com/members/offers.html",
|
||||
headers={"SESSIONID": b"somehash"},
|
||||
),
|
||||
Request(
|
||||
"http://www.example.com/",
|
||||
headers={"Accept-Language": b"en"},
|
||||
),
|
||||
Request(
|
||||
"http://www.example.com/",
|
||||
headers={
|
||||
"Accept-Language": b"en",
|
||||
"SESSIONID": b"somehash",
|
||||
},
|
||||
),
|
||||
Request("http://www.example.com/test.html"),
|
||||
Request("http://www.example.com/test.html#fragment"),
|
||||
Request("http://www.example.com", method="POST"),
|
||||
Request("http://www.example.com", method="POST", body=b"request body"),
|
||||
)
|
||||
|
||||
|
||||
class TestRequestFingerprinter:
|
||||
def test_default_implementation(self):
|
||||
crawler = get_crawler()
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ from urllib.parse import urlparse
|
|||
|
||||
import pytest
|
||||
|
||||
from scrapy.http import HtmlResponse, Response, TextResponse
|
||||
from scrapy.http import HtmlResponse, Response
|
||||
from scrapy.utils.python import to_bytes
|
||||
from scrapy.utils.response import (
|
||||
_remove_html_comments,
|
||||
|
|
@ -15,14 +15,13 @@ from scrapy.utils.response import (
|
|||
)
|
||||
|
||||
|
||||
class TestResponseUtils:
|
||||
dummy_response = TextResponse(url="http://example.org/", body=b"dummy_response")
|
||||
|
||||
def test_open_in_browser(self):
|
||||
def test_open_in_browser():
|
||||
url = "http:///www.example.com/some/page.html"
|
||||
body = b"<html> <head> <title>test page</title> </head> <body>test body</body> </html>"
|
||||
body = (
|
||||
b"<html> <head> <title>test page</title> </head> <body>test body</body> </html>"
|
||||
)
|
||||
|
||||
def browser_open(burl):
|
||||
def browser_open(burl: str) -> bool:
|
||||
path = urlparse(burl).path
|
||||
if not path or not Path(path).exists():
|
||||
path = burl.replace("file://", "")
|
||||
|
|
@ -37,7 +36,8 @@ class TestResponseUtils:
|
|||
with pytest.raises(TypeError):
|
||||
open_in_browser(resp, debug=True) # pylint: disable=unexpected-keyword-arg
|
||||
|
||||
def test_get_meta_refresh(self):
|
||||
|
||||
def test_get_meta_refresh():
|
||||
r1 = HtmlResponse(
|
||||
"http://www.example.com",
|
||||
body=b"""
|
||||
|
|
@ -71,7 +71,8 @@ class TestResponseUtils:
|
|||
assert get_meta_refresh(r2) == (None, None)
|
||||
assert get_meta_refresh(r3) == (None, None)
|
||||
|
||||
def test_get_base_url(self):
|
||||
|
||||
def test_get_base_url():
|
||||
resp = HtmlResponse(
|
||||
"http://www.example.com",
|
||||
body=b"""
|
||||
|
|
@ -89,12 +90,14 @@ class TestResponseUtils:
|
|||
)
|
||||
assert get_base_url(resp2) == "http://www.example.com"
|
||||
|
||||
def test_response_status_message(self):
|
||||
|
||||
def test_response_status_message():
|
||||
assert response_status_message(200) == "200 OK"
|
||||
assert response_status_message(404) == "404 Not Found"
|
||||
assert response_status_message(573) == "573 Unknown Status"
|
||||
|
||||
def test_inject_base_url(self):
|
||||
|
||||
def test_inject_base_url():
|
||||
url = "http://www.example.com"
|
||||
|
||||
def check_base_url(burl):
|
||||
|
|
@ -169,37 +172,31 @@ class TestResponseUtils:
|
|||
"Inject unique base url with conditional comment"
|
||||
)
|
||||
|
||||
def test_open_in_browser_redos_comment(self):
|
||||
|
||||
def test_open_in_browser_redos_comment():
|
||||
MAX_CPU_TIME = 0.02
|
||||
|
||||
# Exploit input from
|
||||
# https://makenowjust-labs.github.io/recheck/playground/
|
||||
# for /<!--.*?-->/ (old pattern to remove comments).
|
||||
body = b"-><!--\x00" * 25_000 + b"->\n<!---->"
|
||||
|
||||
response = HtmlResponse("https://example.com", body=body)
|
||||
|
||||
start_time = process_time()
|
||||
|
||||
open_in_browser(response, lambda url: True)
|
||||
|
||||
end_time = process_time()
|
||||
assert end_time - start_time < MAX_CPU_TIME
|
||||
|
||||
def test_open_in_browser_redos_head(self):
|
||||
|
||||
def test_open_in_browser_redos_head():
|
||||
MAX_CPU_TIME = 0.02
|
||||
|
||||
# Exploit input from
|
||||
# https://makenowjust-labs.github.io/recheck/playground/
|
||||
# for /(<head(?:>|\s.*?>))/ (old pattern to find the head element).
|
||||
body = b"<head\t" * 8_000
|
||||
|
||||
response = HtmlResponse("https://example.com", body=body)
|
||||
|
||||
start_time = process_time()
|
||||
|
||||
open_in_browser(response, lambda url: True)
|
||||
|
||||
end_time = process_time()
|
||||
assert end_time - start_time < MAX_CPU_TIME
|
||||
|
||||
|
|
@ -207,37 +204,14 @@ class TestResponseUtils:
|
|||
@pytest.mark.parametrize(
|
||||
("input_body", "output_body"),
|
||||
[
|
||||
(
|
||||
b"a<!--",
|
||||
b"a",
|
||||
),
|
||||
(
|
||||
b"a<!---->b",
|
||||
b"ab",
|
||||
),
|
||||
(
|
||||
b"a<!--b-->c",
|
||||
b"ac",
|
||||
),
|
||||
(
|
||||
b"a<!--b-->c<!--",
|
||||
b"ac",
|
||||
),
|
||||
(
|
||||
b"a<!--b-->c<!--d",
|
||||
b"ac",
|
||||
),
|
||||
(
|
||||
b"a<!--b-->c<!---->d",
|
||||
b"acd",
|
||||
),
|
||||
(
|
||||
b"a<!--b--><!--c-->d",
|
||||
b"ad",
|
||||
),
|
||||
(b"a<!--", b"a"),
|
||||
(b"a<!---->b", b"ab"),
|
||||
(b"a<!--b-->c", b"ac"),
|
||||
(b"a<!--b-->c<!--", b"ac"),
|
||||
(b"a<!--b-->c<!--d", b"ac"),
|
||||
(b"a<!--b-->c<!---->d", b"acd"),
|
||||
(b"a<!--b--><!--c-->d", b"ad"),
|
||||
],
|
||||
)
|
||||
def test_remove_html_comments(input_body, output_body):
|
||||
assert _remove_html_comments(input_body) == output_body, (
|
||||
f"{_remove_html_comments(input_body)=} == {output_body=}"
|
||||
)
|
||||
assert _remove_html_comments(input_body) == output_body
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import json
|
|||
from decimal import Decimal
|
||||
|
||||
import attr
|
||||
import pytest
|
||||
from twisted.internet import defer
|
||||
|
||||
from scrapy.http import Request, Response
|
||||
|
|
@ -11,10 +12,11 @@ from scrapy.utils.serialize import ScrapyJSONEncoder
|
|||
|
||||
|
||||
class TestJsonEncoder:
|
||||
def setup_method(self):
|
||||
self.encoder = ScrapyJSONEncoder(sort_keys=True)
|
||||
@pytest.fixture
|
||||
def encoder(self) -> ScrapyJSONEncoder:
|
||||
return ScrapyJSONEncoder(sort_keys=True)
|
||||
|
||||
def test_encode_decode(self):
|
||||
def test_encode_decode(self, encoder: ScrapyJSONEncoder) -> None:
|
||||
dt = datetime.datetime(2010, 1, 2, 10, 11, 12)
|
||||
dts = "2010-01-02 10:11:12"
|
||||
d = datetime.date(2010, 1, 2)
|
||||
|
|
@ -38,24 +40,24 @@ class TestJsonEncoder:
|
|||
(s, ss),
|
||||
(dt_set, dt_sets),
|
||||
]:
|
||||
assert self.encoder.encode(input) == json.dumps(output, sort_keys=True)
|
||||
assert encoder.encode(input) == json.dumps(output, sort_keys=True)
|
||||
|
||||
def test_encode_deferred(self):
|
||||
assert "Deferred" in self.encoder.encode(defer.Deferred())
|
||||
def test_encode_deferred(self, encoder: ScrapyJSONEncoder) -> None:
|
||||
assert "Deferred" in encoder.encode(defer.Deferred())
|
||||
|
||||
def test_encode_request(self):
|
||||
def test_encode_request(self, encoder: ScrapyJSONEncoder) -> None:
|
||||
r = Request("http://www.example.com/lala")
|
||||
rs = self.encoder.encode(r)
|
||||
rs = encoder.encode(r)
|
||||
assert r.method in rs
|
||||
assert r.url in rs
|
||||
|
||||
def test_encode_response(self):
|
||||
def test_encode_response(self, encoder: ScrapyJSONEncoder) -> None:
|
||||
r = Response("http://www.example.com/lala")
|
||||
rs = self.encoder.encode(r)
|
||||
rs = encoder.encode(r)
|
||||
assert r.url in rs
|
||||
assert str(r.status) in rs
|
||||
|
||||
def test_encode_dataclass_item(self) -> None:
|
||||
def test_encode_dataclass_item(self, encoder: ScrapyJSONEncoder) -> None:
|
||||
@dataclasses.dataclass
|
||||
class TestDataClass:
|
||||
name: str
|
||||
|
|
@ -63,10 +65,10 @@ class TestJsonEncoder:
|
|||
price: int
|
||||
|
||||
item = TestDataClass(name="Product", url="http://product.org", price=1)
|
||||
encoded = self.encoder.encode(item)
|
||||
encoded = encoder.encode(item)
|
||||
assert encoded == '{"name": "Product", "price": 1, "url": "http://product.org"}'
|
||||
|
||||
def test_encode_attrs_item(self):
|
||||
def test_encode_attrs_item(self, encoder: ScrapyJSONEncoder) -> None:
|
||||
@attr.s
|
||||
class AttrsItem:
|
||||
name = attr.ib(type=str)
|
||||
|
|
@ -74,5 +76,5 @@ class TestJsonEncoder:
|
|||
price = attr.ib(type=int)
|
||||
|
||||
item = AttrsItem(name="Product", url="http://product.org", price=1)
|
||||
encoded = self.encoder.encode(item)
|
||||
encoded = encoder.encode(item)
|
||||
assert encoded == '{"name": "Product", "price": 1, "url": "http://product.org"}'
|
||||
|
|
|
|||
|
|
@ -1,8 +1,7 @@
|
|||
from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
|
||||
|
||||
|
||||
class TestSitemap:
|
||||
def test_sitemap(self):
|
||||
def test_sitemap():
|
||||
s = Sitemap(
|
||||
b"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<urlset xmlns="http://www.google.com/schemas/sitemap/0.84">
|
||||
|
|
@ -36,7 +35,8 @@ class TestSitemap:
|
|||
},
|
||||
]
|
||||
|
||||
def test_sitemap_index(self):
|
||||
|
||||
def test_sitemap_index():
|
||||
s = Sitemap(
|
||||
b"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
||||
|
|
@ -62,7 +62,8 @@ class TestSitemap:
|
|||
},
|
||||
]
|
||||
|
||||
def test_sitemap_strip(self):
|
||||
|
||||
def test_sitemap_strip():
|
||||
"""Assert we can deal with trailing spaces inside <loc> tags - we've
|
||||
seen those
|
||||
"""
|
||||
|
|
@ -92,7 +93,8 @@ class TestSitemap:
|
|||
{"loc": "http://www.example.com/2", "lastmod": ""},
|
||||
]
|
||||
|
||||
def test_sitemap_wrong_ns(self):
|
||||
|
||||
def test_sitemap_wrong_ns():
|
||||
"""We have seen sitemaps with wrongs ns. Presumably, Google still works
|
||||
with these, though is not 100% confirmed"""
|
||||
s = Sitemap(
|
||||
|
|
@ -121,7 +123,8 @@ class TestSitemap:
|
|||
{"loc": "http://www.example.com/2", "lastmod": ""},
|
||||
]
|
||||
|
||||
def test_sitemap_wrong_ns2(self):
|
||||
|
||||
def test_sitemap_wrong_ns2():
|
||||
"""We have seen sitemaps with wrongs ns. Presumably, Google still works
|
||||
with these, though is not 100% confirmed"""
|
||||
s = Sitemap(
|
||||
|
|
@ -151,7 +154,8 @@ class TestSitemap:
|
|||
{"loc": "http://www.example.com/2", "lastmod": ""},
|
||||
]
|
||||
|
||||
def test_sitemap_urls_from_robots(self):
|
||||
|
||||
def test_sitemap_urls_from_robots():
|
||||
robots = """User-agent: *
|
||||
Disallow: /aff/
|
||||
Disallow: /wl/
|
||||
|
|
@ -170,16 +174,15 @@ Sitemap: /sitemap-relative-url.xml
|
|||
Disallow: /forum/search/
|
||||
Disallow: /forum/active/
|
||||
"""
|
||||
assert list(
|
||||
sitemap_urls_from_robots(robots, base_url="http://example.com")
|
||||
) == [
|
||||
assert list(sitemap_urls_from_robots(robots, base_url="http://example.com")) == [
|
||||
"http://example.com/sitemap.xml",
|
||||
"http://example.com/sitemap-product-index.xml",
|
||||
"http://example.com/sitemap-uppercase.xml",
|
||||
"http://example.com/sitemap-relative-url.xml",
|
||||
]
|
||||
|
||||
def test_sitemap_blanklines(self):
|
||||
|
||||
def test_sitemap_blanklines():
|
||||
"""Assert we can deal with starting blank lines before <xml> tag"""
|
||||
s = Sitemap(
|
||||
b"""
|
||||
|
|
@ -212,7 +215,8 @@ Disallow: /forum/active/
|
|||
{"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap3.xml"},
|
||||
]
|
||||
|
||||
def test_comment(self):
|
||||
|
||||
def test_comment():
|
||||
s = Sitemap(
|
||||
b"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
|
||||
|
|
@ -223,10 +227,10 @@ Disallow: /forum/active/
|
|||
</url>
|
||||
</urlset>"""
|
||||
)
|
||||
|
||||
assert list(s) == [{"loc": "http://www.example.com/"}]
|
||||
|
||||
def test_alternate(self):
|
||||
|
||||
def test_alternate():
|
||||
s = Sitemap(
|
||||
b"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
|
||||
|
|
@ -243,7 +247,6 @@ Disallow: /forum/active/
|
|||
</url>
|
||||
</urlset>"""
|
||||
)
|
||||
|
||||
assert list(s) == [
|
||||
{
|
||||
"loc": "http://www.example.com/english/",
|
||||
|
|
@ -255,7 +258,8 @@ Disallow: /forum/active/
|
|||
}
|
||||
]
|
||||
|
||||
def test_xml_entity_expansion(self):
|
||||
|
||||
def test_xml_entity_expansion():
|
||||
s = Sitemap(
|
||||
b"""<?xml version="1.0" encoding="utf-8"?>
|
||||
<!DOCTYPE foo [
|
||||
|
|
@ -269,5 +273,4 @@ Disallow: /forum/active/
|
|||
</urlset>
|
||||
"""
|
||||
)
|
||||
|
||||
assert list(s) == [{"loc": "http://127.0.0.1:8000/"}]
|
||||
|
|
|
|||
|
|
@ -12,8 +12,7 @@ class MySpider2(Spider):
|
|||
name = "myspider2"
|
||||
|
||||
|
||||
class TestUtilsSpiders:
|
||||
def test_iterate_spider_output(self):
|
||||
def test_iterate_spider_output():
|
||||
i = Item()
|
||||
r = Request("http://scrapytest.org")
|
||||
o = object()
|
||||
|
|
@ -23,7 +22,8 @@ class TestUtilsSpiders:
|
|||
assert list(iterate_spider_output(o)) == [o]
|
||||
assert list(iterate_spider_output([r, i, o])) == [r, i, o]
|
||||
|
||||
def test_iter_spider_classes(self):
|
||||
|
||||
def test_iter_spider_classes():
|
||||
import tests.test_utils_spider # noqa: PLW0406 # pylint: disable=import-self
|
||||
|
||||
it = iter_spider_classes(tests.test_utils_spider)
|
||||
|
|
|
|||
|
|
@ -1,8 +1,7 @@
|
|||
from scrapy.utils.template import render_templatefile
|
||||
|
||||
|
||||
class TestUtilsRenderTemplateFile:
|
||||
def test_simple_render(self, tmp_path):
|
||||
def test_simple_render(tmp_path):
|
||||
context = {"project_name": "proj", "name": "spi", "classname": "TheSpider"}
|
||||
template = "from ${project_name}.spiders.${name} import ${classname}"
|
||||
rendered = "from proj.spiders.spi import TheSpider"
|
||||
|
|
|
|||
|
|
@ -15,11 +15,12 @@ class Bar(trackref.object_ref):
|
|||
pass
|
||||
|
||||
|
||||
class TestTrackref:
|
||||
def setup_method(self):
|
||||
@pytest.fixture(autouse=True)
|
||||
def clear_refs() -> None:
|
||||
trackref.live_refs.clear()
|
||||
|
||||
def test_format_live_refs(self):
|
||||
|
||||
def test_format_live_refs():
|
||||
o1 = Foo() # noqa: F841
|
||||
o2 = Bar() # noqa: F841
|
||||
o3 = Foo() # noqa: F841
|
||||
|
|
@ -42,13 +43,15 @@ Bar 1 oldest: 0s ago
|
|||
"""
|
||||
)
|
||||
|
||||
|
||||
@mock.patch("sys.stdout", new_callable=StringIO)
|
||||
def test_print_live_refs_empty(self, stdout):
|
||||
def test_print_live_refs_empty(stdout):
|
||||
trackref.print_live_refs()
|
||||
assert stdout.getvalue() == "Live References\n\n\n"
|
||||
|
||||
|
||||
@mock.patch("sys.stdout", new_callable=StringIO)
|
||||
def test_print_live_refs_with_objects(self, stdout):
|
||||
def test_print_live_refs_with_objects(stdout):
|
||||
o1 = Foo() # noqa: F841
|
||||
trackref.print_live_refs()
|
||||
assert (
|
||||
|
|
@ -59,7 +62,8 @@ Live References
|
|||
Foo 1 oldest: 0s ago\n\n"""
|
||||
)
|
||||
|
||||
def test_get_oldest(self):
|
||||
|
||||
def test_get_oldest():
|
||||
o1 = Foo()
|
||||
|
||||
o1_time = time()
|
||||
|
|
@ -78,7 +82,8 @@ Foo 1 oldest: 0s ago\n\n"""
|
|||
assert trackref.get_oldest("Bar") is o2
|
||||
assert trackref.get_oldest("XXX") is None
|
||||
|
||||
def test_iter_all(self):
|
||||
|
||||
def test_iter_all():
|
||||
o1 = Foo()
|
||||
o2 = Bar() # noqa: F841
|
||||
o3 = Foo()
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ import pytest
|
|||
|
||||
from scrapy.linkextractors import IGNORED_EXTENSIONS
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.misc import arg_to_iter
|
||||
from scrapy.utils.url import ( # type: ignore[attr-defined]
|
||||
_is_filesystem_path,
|
||||
_public_w3lib_objects,
|
||||
|
|
@ -17,8 +16,7 @@ from scrapy.utils.url import ( # type: ignore[attr-defined]
|
|||
)
|
||||
|
||||
|
||||
class TestUrlUtils:
|
||||
def test_url_is_from_any_domain(self):
|
||||
def test_url_is_from_any_domain():
|
||||
url = "http://www.wheele-bin-art.co.uk/get/product/123"
|
||||
assert url_is_from_any_domain(url, ["wheele-bin-art.co.uk"])
|
||||
assert not url_is_from_any_domain(url, ["art.co.uk"])
|
||||
|
|
@ -42,14 +40,8 @@ class TestUrlUtils:
|
|||
assert not url_is_from_any_domain(url, ["testdomain.com"])
|
||||
assert not url_is_from_any_domain(url + ".testdomain.com", ["testdomain.com"])
|
||||
|
||||
def test_url_is_from_spider(self):
|
||||
spider = Spider(name="example.com")
|
||||
assert url_is_from_spider("http://www.example.com/some/page.html", spider)
|
||||
assert url_is_from_spider("http://sub.example.com/some/page.html", spider)
|
||||
assert not url_is_from_spider("http://www.example.org/some/page.html", spider)
|
||||
assert not url_is_from_spider("http://www.example.net/some/page.html", spider)
|
||||
|
||||
def test_url_is_from_spider_class_attributes(self):
|
||||
def test_url_is_from_spider():
|
||||
class MySpider(Spider):
|
||||
name = "example.com"
|
||||
|
||||
|
|
@ -58,31 +50,21 @@ class TestUrlUtils:
|
|||
assert not url_is_from_spider("http://www.example.org/some/page.html", MySpider)
|
||||
assert not url_is_from_spider("http://www.example.net/some/page.html", MySpider)
|
||||
|
||||
def test_url_is_from_spider_with_allowed_domains(self):
|
||||
spider = Spider(
|
||||
name="example.com", allowed_domains=["example.org", "example.net"]
|
||||
)
|
||||
assert url_is_from_spider("http://www.example.com/some/page.html", spider)
|
||||
assert url_is_from_spider("http://sub.example.com/some/page.html", spider)
|
||||
assert url_is_from_spider("http://example.com/some/page.html", spider)
|
||||
assert url_is_from_spider("http://www.example.org/some/page.html", spider)
|
||||
assert url_is_from_spider("http://www.example.net/some/page.html", spider)
|
||||
assert not url_is_from_spider("http://www.example.us/some/page.html", spider)
|
||||
|
||||
spider = Spider(
|
||||
name="example.com", allowed_domains={"example.com", "example.net"}
|
||||
)
|
||||
assert url_is_from_spider("http://www.example.com/some/page.html", spider)
|
||||
|
||||
spider = Spider(
|
||||
name="example.com", allowed_domains=("example.com", "example.net")
|
||||
)
|
||||
assert url_is_from_spider("http://www.example.com/some/page.html", spider)
|
||||
|
||||
def test_url_is_from_spider_with_allowed_domains_class_attributes(self):
|
||||
def test_url_is_from_spider_class_attributes():
|
||||
class MySpider(Spider):
|
||||
name = "example.com"
|
||||
allowed_domains = ("example.org", "example.net")
|
||||
|
||||
assert url_is_from_spider("http://www.example.com/some/page.html", MySpider)
|
||||
assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider)
|
||||
assert not url_is_from_spider("http://www.example.org/some/page.html", MySpider)
|
||||
assert not url_is_from_spider("http://www.example.net/some/page.html", MySpider)
|
||||
|
||||
|
||||
def test_url_is_from_spider_with_allowed_domains():
|
||||
class MySpider(Spider):
|
||||
name = "example.com"
|
||||
allowed_domains = ["example.org", "example.net"]
|
||||
|
||||
assert url_is_from_spider("http://www.example.com/some/page.html", MySpider)
|
||||
assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider)
|
||||
|
|
@ -91,187 +73,99 @@ class TestUrlUtils:
|
|||
assert url_is_from_spider("http://www.example.net/some/page.html", MySpider)
|
||||
assert not url_is_from_spider("http://www.example.us/some/page.html", MySpider)
|
||||
|
||||
def test_url_has_any_extension(self):
|
||||
deny_extensions = {"." + e for e in arg_to_iter(IGNORED_EXTENSIONS)}
|
||||
assert url_has_any_extension(
|
||||
"http://www.example.com/archive.tar.gz", deny_extensions
|
||||
class MySpider2(Spider):
|
||||
name = "example.com"
|
||||
allowed_domains = {"example.com", "example.net"}
|
||||
|
||||
assert url_is_from_spider("http://www.example.com/some/page.html", MySpider2)
|
||||
|
||||
class MySpider3(Spider):
|
||||
name = "example.com"
|
||||
allowed_domains = ("example.com", "example.net")
|
||||
|
||||
assert url_is_from_spider("http://www.example.com/some/page.html", MySpider3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("url", "expected"),
|
||||
[
|
||||
("http://www.example.com/archive.tar.gz", True),
|
||||
("http://www.example.com/page.doc", True),
|
||||
("http://www.example.com/page.pdf", True),
|
||||
("http://www.example.com/page.htm", False),
|
||||
("http://www.example.com/", False),
|
||||
("http://www.example.com/page.doc.html", False),
|
||||
],
|
||||
)
|
||||
assert url_has_any_extension("http://www.example.com/page.doc", deny_extensions)
|
||||
assert url_has_any_extension("http://www.example.com/page.pdf", deny_extensions)
|
||||
assert not url_has_any_extension(
|
||||
"http://www.example.com/page.htm", deny_extensions
|
||||
)
|
||||
assert not url_has_any_extension("http://www.example.com/", deny_extensions)
|
||||
assert not url_has_any_extension(
|
||||
"http://www.example.com/page.doc.html", deny_extensions
|
||||
def test_url_has_any_extension(url: str, expected: bool) -> None:
|
||||
deny_extensions = {"." + e for e in IGNORED_EXTENSIONS}
|
||||
assert url_has_any_extension(url, deny_extensions) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("url", "expected"),
|
||||
[
|
||||
("www.example.com", "http://www.example.com"),
|
||||
("example.com", "http://example.com"),
|
||||
("www.example.com/some/page.html", "http://www.example.com/some/page.html"),
|
||||
("www.example.com:80", "http://www.example.com:80"),
|
||||
("www.example.com/some/page#frag", "http://www.example.com/some/page#frag"),
|
||||
("www.example.com/do?a=1&b=2&c=3", "http://www.example.com/do?a=1&b=2&c=3"),
|
||||
(
|
||||
"username:password@www.example.com",
|
||||
"http://username:password@www.example.com",
|
||||
),
|
||||
(
|
||||
"username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag",
|
||||
"http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag",
|
||||
),
|
||||
("http://www.example.com", "http://www.example.com"),
|
||||
("http://example.com", "http://example.com"),
|
||||
(
|
||||
"http://www.example.com/some/page.html",
|
||||
"http://www.example.com/some/page.html",
|
||||
),
|
||||
("http://www.example.com:80", "http://www.example.com:80"),
|
||||
(
|
||||
"http://www.example.com/some/page#frag",
|
||||
"http://www.example.com/some/page#frag",
|
||||
),
|
||||
(
|
||||
"http://www.example.com/do?a=1&b=2&c=3",
|
||||
"http://www.example.com/do?a=1&b=2&c=3",
|
||||
),
|
||||
(
|
||||
"http://username:password@www.example.com",
|
||||
"http://username:password@www.example.com",
|
||||
),
|
||||
(
|
||||
"http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag",
|
||||
"http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag",
|
||||
),
|
||||
("//www.example.com", "http://www.example.com"),
|
||||
("//example.com", "http://example.com"),
|
||||
("//www.example.com/some/page.html", "http://www.example.com/some/page.html"),
|
||||
("//www.example.com:80", "http://www.example.com:80"),
|
||||
("//www.example.com/some/page#frag", "http://www.example.com/some/page#frag"),
|
||||
("//www.example.com/do?a=1&b=2&c=3", "http://www.example.com/do?a=1&b=2&c=3"),
|
||||
(
|
||||
"//username:password@www.example.com",
|
||||
"http://username:password@www.example.com",
|
||||
),
|
||||
(
|
||||
"//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag",
|
||||
"http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag",
|
||||
),
|
||||
("https://www.example.com", "https://www.example.com"),
|
||||
("ftp://www.example.com", "ftp://www.example.com"),
|
||||
],
|
||||
)
|
||||
def test_add_http_if_no_scheme(url: str, expected: str) -> None:
|
||||
assert add_http_if_no_scheme(url) == expected
|
||||
|
||||
|
||||
class TestAddHttpIfNoScheme:
|
||||
def test_add_scheme(self):
|
||||
assert add_http_if_no_scheme("www.example.com") == "http://www.example.com"
|
||||
|
||||
def test_without_subdomain(self):
|
||||
assert add_http_if_no_scheme("example.com") == "http://example.com"
|
||||
|
||||
def test_path(self):
|
||||
assert (
|
||||
add_http_if_no_scheme("www.example.com/some/page.html")
|
||||
== "http://www.example.com/some/page.html"
|
||||
)
|
||||
|
||||
def test_port(self):
|
||||
assert (
|
||||
add_http_if_no_scheme("www.example.com:80") == "http://www.example.com:80"
|
||||
)
|
||||
|
||||
def test_fragment(self):
|
||||
assert (
|
||||
add_http_if_no_scheme("www.example.com/some/page#frag")
|
||||
== "http://www.example.com/some/page#frag"
|
||||
)
|
||||
|
||||
def test_query(self):
|
||||
assert (
|
||||
add_http_if_no_scheme("www.example.com/do?a=1&b=2&c=3")
|
||||
== "http://www.example.com/do?a=1&b=2&c=3"
|
||||
)
|
||||
|
||||
def test_username_password(self):
|
||||
assert (
|
||||
add_http_if_no_scheme("username:password@www.example.com")
|
||||
== "http://username:password@www.example.com"
|
||||
)
|
||||
|
||||
def test_complete_url(self):
|
||||
assert (
|
||||
add_http_if_no_scheme(
|
||||
"username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag"
|
||||
)
|
||||
== "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag"
|
||||
)
|
||||
|
||||
def test_preserve_http(self):
|
||||
assert (
|
||||
add_http_if_no_scheme("http://www.example.com") == "http://www.example.com"
|
||||
)
|
||||
|
||||
def test_preserve_http_without_subdomain(self):
|
||||
assert add_http_if_no_scheme("http://example.com") == "http://example.com"
|
||||
|
||||
def test_preserve_http_path(self):
|
||||
assert (
|
||||
add_http_if_no_scheme("http://www.example.com/some/page.html")
|
||||
== "http://www.example.com/some/page.html"
|
||||
)
|
||||
|
||||
def test_preserve_http_port(self):
|
||||
assert (
|
||||
add_http_if_no_scheme("http://www.example.com:80")
|
||||
== "http://www.example.com:80"
|
||||
)
|
||||
|
||||
def test_preserve_http_fragment(self):
|
||||
assert (
|
||||
add_http_if_no_scheme("http://www.example.com/some/page#frag")
|
||||
== "http://www.example.com/some/page#frag"
|
||||
)
|
||||
|
||||
def test_preserve_http_query(self):
|
||||
assert (
|
||||
add_http_if_no_scheme("http://www.example.com/do?a=1&b=2&c=3")
|
||||
== "http://www.example.com/do?a=1&b=2&c=3"
|
||||
)
|
||||
|
||||
def test_preserve_http_username_password(self):
|
||||
assert (
|
||||
add_http_if_no_scheme("http://username:password@www.example.com")
|
||||
== "http://username:password@www.example.com"
|
||||
)
|
||||
|
||||
def test_preserve_http_complete_url(self):
|
||||
assert (
|
||||
add_http_if_no_scheme(
|
||||
"http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag"
|
||||
)
|
||||
== "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag"
|
||||
)
|
||||
|
||||
def test_protocol_relative(self):
|
||||
assert add_http_if_no_scheme("//www.example.com") == "http://www.example.com"
|
||||
|
||||
def test_protocol_relative_without_subdomain(self):
|
||||
assert add_http_if_no_scheme("//example.com") == "http://example.com"
|
||||
|
||||
def test_protocol_relative_path(self):
|
||||
assert (
|
||||
add_http_if_no_scheme("//www.example.com/some/page.html")
|
||||
== "http://www.example.com/some/page.html"
|
||||
)
|
||||
|
||||
def test_protocol_relative_port(self):
|
||||
assert (
|
||||
add_http_if_no_scheme("//www.example.com:80") == "http://www.example.com:80"
|
||||
)
|
||||
|
||||
def test_protocol_relative_fragment(self):
|
||||
assert (
|
||||
add_http_if_no_scheme("//www.example.com/some/page#frag")
|
||||
== "http://www.example.com/some/page#frag"
|
||||
)
|
||||
|
||||
def test_protocol_relative_query(self):
|
||||
assert (
|
||||
add_http_if_no_scheme("//www.example.com/do?a=1&b=2&c=3")
|
||||
== "http://www.example.com/do?a=1&b=2&c=3"
|
||||
)
|
||||
|
||||
def test_protocol_relative_username_password(self):
|
||||
assert (
|
||||
add_http_if_no_scheme("//username:password@www.example.com")
|
||||
== "http://username:password@www.example.com"
|
||||
)
|
||||
|
||||
def test_protocol_relative_complete_url(self):
|
||||
assert (
|
||||
add_http_if_no_scheme(
|
||||
"//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag"
|
||||
)
|
||||
== "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag"
|
||||
)
|
||||
|
||||
def test_preserve_https(self):
|
||||
assert (
|
||||
add_http_if_no_scheme("https://www.example.com")
|
||||
== "https://www.example.com"
|
||||
)
|
||||
|
||||
def test_preserve_ftp(self):
|
||||
assert add_http_if_no_scheme("ftp://www.example.com") == "ftp://www.example.com"
|
||||
|
||||
|
||||
class TestGuessScheme:
|
||||
pass
|
||||
|
||||
|
||||
def create_guess_scheme_t(args):
|
||||
def do_expected(self):
|
||||
url = guess_scheme(args[0])
|
||||
assert url.startswith(args[1]), (
|
||||
f"Wrong scheme guessed: for `{args[0]}` got `{url}`, expected `{args[1]}...`"
|
||||
)
|
||||
|
||||
return do_expected
|
||||
|
||||
|
||||
def create_skipped_scheme_t(args):
|
||||
def do_expected(self):
|
||||
pytest.skip(args[2])
|
||||
|
||||
return do_expected
|
||||
|
||||
|
||||
for k, args in enumerate(
|
||||
@pytest.mark.parametrize(
|
||||
("url", "expected"),
|
||||
[
|
||||
("/index", "file://"),
|
||||
("/index.html", "file://"),
|
||||
|
|
@ -295,14 +189,13 @@ for k, args in enumerate(
|
|||
("/", "http://"),
|
||||
(".../test", "http://"),
|
||||
],
|
||||
start=1,
|
||||
):
|
||||
t_method = create_guess_scheme_t(args)
|
||||
t_method.__name__ = f"test_uri_{k:03}"
|
||||
setattr(TestGuessScheme, t_method.__name__, t_method)
|
||||
)
|
||||
def test_guess_scheme(url: str, expected: str):
|
||||
assert guess_scheme(url).startswith(expected)
|
||||
|
||||
# TODO: the following tests do not pass with current implementation
|
||||
for k, skip_args in enumerate(
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("url", "expected", "reason"),
|
||||
[
|
||||
(
|
||||
r"C:\absolute\path\to\a\file.html",
|
||||
|
|
@ -310,25 +203,21 @@ for k, skip_args in enumerate(
|
|||
"Windows filepath are not supported for scrapy shell",
|
||||
),
|
||||
],
|
||||
start=1,
|
||||
):
|
||||
t_method = create_skipped_scheme_t(skip_args)
|
||||
t_method.__name__ = f"test_uri_skipped_{k:03}"
|
||||
setattr(TestGuessScheme, t_method.__name__, t_method)
|
||||
)
|
||||
def test_guess_scheme_skipped(url: str, expected: str, reason: str):
|
||||
pytest.skip(reason)
|
||||
|
||||
|
||||
class TestStripUrl:
|
||||
def test_noop(self):
|
||||
assert (
|
||||
strip_url("http://www.example.com/index.html")
|
||||
== "http://www.example.com/index.html"
|
||||
)
|
||||
|
||||
def test_noop_query_string(self):
|
||||
assert (
|
||||
strip_url("http://www.example.com/index.html?somekey=somevalue")
|
||||
== "http://www.example.com/index.html?somekey=somevalue"
|
||||
@pytest.mark.parametrize(
|
||||
"url",
|
||||
[
|
||||
"http://www.example.com/index.html",
|
||||
"http://www.example.com/index.html?somekey=somevalue",
|
||||
],
|
||||
)
|
||||
def test_noop(self, url: str) -> None:
|
||||
assert strip_url(url) == url
|
||||
|
||||
def test_fragments(self):
|
||||
assert (
|
||||
|
|
@ -339,16 +228,20 @@ class TestStripUrl:
|
|||
== "http://www.example.com/index.html?somekey=somevalue#section"
|
||||
)
|
||||
|
||||
def test_path(self):
|
||||
for input_url, origin, output_url in [
|
||||
@pytest.mark.parametrize(
|
||||
("url", "origin", "expected"),
|
||||
[
|
||||
("http://www.example.com/", False, "http://www.example.com/"),
|
||||
("http://www.example.com", False, "http://www.example.com"),
|
||||
("http://www.example.com", True, "http://www.example.com/"),
|
||||
]:
|
||||
assert strip_url(input_url, origin_only=origin) == output_url
|
||||
],
|
||||
)
|
||||
def test_path(self, url: str, origin: bool, expected: str) -> None:
|
||||
assert strip_url(url, origin_only=origin) == expected
|
||||
|
||||
def test_credentials(self):
|
||||
for i, o in [
|
||||
@pytest.mark.parametrize(
|
||||
("url", "expected"),
|
||||
[
|
||||
(
|
||||
"http://username@www.example.com/index.html?somekey=somevalue#section",
|
||||
"http://www.example.com/index.html?somekey=somevalue",
|
||||
|
|
@ -361,34 +254,29 @@ class TestStripUrl:
|
|||
"ftp://username:password@www.example.com/index.html?somekey=somevalue#section",
|
||||
"ftp://www.example.com/index.html?somekey=somevalue",
|
||||
),
|
||||
]:
|
||||
assert strip_url(i, strip_credentials=True) == o
|
||||
|
||||
def test_credentials_encoded_delims(self):
|
||||
for i, o in [
|
||||
# user: "username@"
|
||||
# password: none
|
||||
# user: "username@", password: none
|
||||
(
|
||||
"http://username%40@www.example.com/index.html?somekey=somevalue#section",
|
||||
"http://www.example.com/index.html?somekey=somevalue",
|
||||
),
|
||||
# user: "username:pass"
|
||||
# password: ""
|
||||
# user: "username:pass", password: ""
|
||||
(
|
||||
"https://username%3Apass:@www.example.com/index.html?somekey=somevalue#section",
|
||||
"https://www.example.com/index.html?somekey=somevalue",
|
||||
),
|
||||
# user: "me"
|
||||
# password: "user@domain.com"
|
||||
# user: "me", password: "user@domain.com"
|
||||
(
|
||||
"ftp://me:user%40domain.com@www.example.com/index.html?somekey=somevalue#section",
|
||||
"ftp://www.example.com/index.html?somekey=somevalue",
|
||||
),
|
||||
]:
|
||||
assert strip_url(i, strip_credentials=True) == o
|
||||
],
|
||||
)
|
||||
def test_credentials(self, url: str, expected: str) -> None:
|
||||
assert strip_url(url, strip_credentials=True) == expected
|
||||
|
||||
def test_default_ports_creds_off(self):
|
||||
for i, o in [
|
||||
@pytest.mark.parametrize(
|
||||
("url", "expected"),
|
||||
[
|
||||
(
|
||||
"http://username:password@www.example.com:80/index.html?somekey=somevalue#section",
|
||||
"http://www.example.com/index.html?somekey=somevalue",
|
||||
|
|
@ -421,11 +309,14 @@ class TestStripUrl:
|
|||
"ftp://username:password@www.example.com:221/file.txt",
|
||||
"ftp://www.example.com:221/file.txt",
|
||||
),
|
||||
]:
|
||||
assert strip_url(i) == o
|
||||
],
|
||||
)
|
||||
def test_default_ports_creds_off(self, url: str, expected: str) -> None:
|
||||
assert strip_url(url) == expected
|
||||
|
||||
def test_default_ports(self):
|
||||
for i, o in [
|
||||
@pytest.mark.parametrize(
|
||||
("url", "expected"),
|
||||
[
|
||||
(
|
||||
"http://username:password@www.example.com:80/index.html",
|
||||
"http://username:password@www.example.com/index.html",
|
||||
|
|
@ -458,11 +349,16 @@ class TestStripUrl:
|
|||
"ftp://username:password@www.example.com:221/file.txt",
|
||||
"ftp://username:password@www.example.com:221/file.txt",
|
||||
),
|
||||
]:
|
||||
assert strip_url(i, strip_default_port=True, strip_credentials=False) == o
|
||||
],
|
||||
)
|
||||
def test_default_ports(self, url: str, expected: str) -> None:
|
||||
assert (
|
||||
strip_url(url, strip_default_port=True, strip_credentials=False) == expected
|
||||
)
|
||||
|
||||
def test_default_ports_keep(self):
|
||||
for i, o in [
|
||||
@pytest.mark.parametrize(
|
||||
("url", "expected"),
|
||||
[
|
||||
(
|
||||
"http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov#section",
|
||||
"http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov",
|
||||
|
|
@ -495,11 +391,17 @@ class TestStripUrl:
|
|||
"ftp://username:password@www.example.com:221/file.txt",
|
||||
"ftp://username:password@www.example.com:221/file.txt",
|
||||
),
|
||||
]:
|
||||
assert strip_url(i, strip_default_port=False, strip_credentials=False) == o
|
||||
],
|
||||
)
|
||||
def test_default_ports_keep(self, url: str, expected: str) -> None:
|
||||
assert (
|
||||
strip_url(url, strip_default_port=False, strip_credentials=False)
|
||||
== expected
|
||||
)
|
||||
|
||||
def test_origin_only(self):
|
||||
for i, o in [
|
||||
@pytest.mark.parametrize(
|
||||
("url", "expected"),
|
||||
[
|
||||
(
|
||||
"http://username:password@www.example.com/index.html",
|
||||
"http://www.example.com/",
|
||||
|
|
@ -516,13 +418,15 @@ class TestStripUrl:
|
|||
"https://username:password@www.example.com:443/index.html",
|
||||
"https://www.example.com/",
|
||||
),
|
||||
]:
|
||||
assert strip_url(i, origin_only=True) == o
|
||||
],
|
||||
)
|
||||
def test_origin_only(self, url: str, expected: str) -> None:
|
||||
assert strip_url(url, origin_only=True) == expected
|
||||
|
||||
|
||||
class TestIsPath:
|
||||
def test_path(self):
|
||||
for input_value, output_value in (
|
||||
@pytest.mark.parametrize(
|
||||
("path", "expected"),
|
||||
[
|
||||
# https://en.wikipedia.org/wiki/Path_(computing)#Representations_of_paths_by_operating_system_and_shell
|
||||
# Unix-like OS, Microsoft Windows / cmd.exe
|
||||
("/home/user/docs/Letter.txt", True),
|
||||
|
|
@ -537,8 +441,10 @@ class TestIsPath:
|
|||
(r"\\?\C:\user\docs\Letter.txt", True),
|
||||
(r"C:\user\docs\somefile.ext:alternate_stream_name", True),
|
||||
(r"https://example.com", False),
|
||||
):
|
||||
assert _is_filesystem_path(input_value) == output_value, input_value
|
||||
],
|
||||
)
|
||||
def test__is_filesystem_path(path: str, expected: bool) -> None:
|
||||
assert _is_filesystem_path(path) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -550,7 +456,7 @@ class TestIsPath:
|
|||
*_public_w3lib_objects,
|
||||
],
|
||||
)
|
||||
def test_deprecated_imports_from_w3lib(obj_name):
|
||||
def test_deprecated_imports_from_w3lib(obj_name: str) -> None:
|
||||
with warnings.catch_warnings(record=True) as warns:
|
||||
obj_type = "attribute" if obj_name == "_safe_chars" else "function"
|
||||
message = f"The scrapy.utils.url.{obj_name} {obj_type} is deprecated, use w3lib.url.{obj_name} instead."
|
||||
|
|
@ -559,4 +465,5 @@ def test_deprecated_imports_from_w3lib(obj_name):
|
|||
|
||||
getattr(import_module("scrapy.utils.url"), obj_name)
|
||||
|
||||
assert isinstance(warns[0].message, Warning)
|
||||
assert message in warns[0].message.args
|
||||
|
|
|
|||
Loading…
Reference in New Issue