diff --git a/tests/test_command_startproject.py b/tests/test_command_startproject.py index 988ad50b9..1edef0b4a 100644 --- a/tests/test_command_startproject.py +++ b/tests/test_command_startproject.py @@ -106,8 +106,6 @@ def get_permissions_dict( class TestStartprojectTemplates(TestProjectBase): - maxDiff = None - def setup_method(self): super().setup_method() self.tmpl = str(Path(self.temp_path, "templates")) diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 9d2e5f997..1714bd4db 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -150,8 +150,6 @@ class KeywordArgumentsSpider(MockServerSpider): class TestCallbackKeywordArguments(TestCase): - maxDiff = None - @classmethod def setUpClass(cls): cls.mockserver = MockServer() diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py index a4f31096e..66a6f5318 100644 --- a/tests/test_toplevel.py +++ b/tests/test_toplevel.py @@ -1,31 +1,35 @@ import scrapy -class TestToplevel: - def test_version(self): - assert isinstance(scrapy.__version__, str) +def test_version(): + assert isinstance(scrapy.__version__, str) - def test_version_info(self): - assert isinstance(scrapy.version_info, tuple) - def test_request_shortcut(self): - from scrapy.http import FormRequest, Request +def test_version_info(): + assert isinstance(scrapy.version_info, tuple) - assert scrapy.Request is Request - assert scrapy.FormRequest is FormRequest - def test_spider_shortcut(self): - from scrapy.spiders import Spider +def test_request_shortcut(): + from scrapy.http import FormRequest, Request - assert scrapy.Spider is Spider + assert scrapy.Request is Request + assert scrapy.FormRequest is FormRequest - def test_selector_shortcut(self): - from scrapy.selector import Selector - assert scrapy.Selector is Selector +def test_spider_shortcut(): + from scrapy.spiders import Spider - def test_item_shortcut(self): - from scrapy.item import Field, Item + assert scrapy.Spider is Spider - assert scrapy.Item is Item - assert scrapy.Field is Field + +def test_selector_shortcut(): + from scrapy.selector import Selector + + assert scrapy.Selector is Selector + + +def test_item_shortcut(): + from scrapy.item import Field, Item + + assert scrapy.Item is Item + assert scrapy.Field is Field diff --git a/tests/test_urlparse_monkeypatches.py b/tests/test_urlparse_monkeypatches.py deleted file mode 100644 index 0e1e89e81..000000000 --- a/tests/test_urlparse_monkeypatches.py +++ /dev/null @@ -1,10 +0,0 @@ -from urllib.parse import urlparse - - -class TestUrlparse: - def test_s3_url(self): - p = urlparse("s3://bucket/key/name?param=value") - assert p.scheme == "s3" - assert p.hostname == "bucket" - assert p.path == "/key/name" - assert p.query == "param=value" diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index 26f158380..ed7dda18d 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -47,12 +47,11 @@ class TestBuildComponentList: assert build_component_list(d, convert=lambda x: x) == ["b", "c", "a"] -class TestUtilsConf: - def test_arglist_to_dict(self): - assert arglist_to_dict(["arg1=val1", "arg2=val2"]) == { - "arg1": "val1", - "arg2": "val2", - } +def test_arglist_to_dict(): + assert arglist_to_dict(["arg1=val1", "arg2=val2"]) == { + "arg1": "val1", + "arg2": "val2", + } class TestFeedExportConfig: diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py index 6598bdce7..dc1d96f66 100644 --- a/tests/test_utils_console.py +++ b/tests/test_utils_console.py @@ -18,23 +18,24 @@ except ImportError: ipy = False -class TestUtilsConsole: - def test_get_shell_embed_func(self): - shell = get_shell_embed_func(["invalid"]) - assert shell is None +def test_get_shell_embed_func(): + shell = get_shell_embed_func(["invalid"]) + assert shell is None - shell = get_shell_embed_func(["invalid", "python"]) - assert callable(shell) - assert shell.__name__ == "_embed_standard_shell" + shell = get_shell_embed_func(["invalid", "python"]) + assert callable(shell) + assert shell.__name__ == "_embed_standard_shell" - @pytest.mark.skipif(not bpy, reason="bpython not available in testenv") - def test_get_shell_embed_func2(self): - shell = get_shell_embed_func(["bpython"]) - assert callable(shell) - assert shell.__name__ == "_embed_bpython_shell" - @pytest.mark.skipif(not ipy, reason="IPython not available in testenv") - def test_get_shell_embed_func3(self): - # default shell should be 'ipython' - shell = get_shell_embed_func() - assert shell.__name__ == "_embed_ipython_shell" +@pytest.mark.skipif(not bpy, reason="bpython not available in testenv") +def test_get_shell_embed_func_bpython(): + shell = get_shell_embed_func(["bpython"]) + assert callable(shell) + assert shell.__name__ == "_embed_bpython_shell" + + +@pytest.mark.skipif(not ipy, reason="IPython not available in testenv") +def test_get_shell_embed_func_ipython(): + # default shell should be 'ipython' + shell = get_shell_embed_func() + assert shell.__name__ == "_embed_ipython_shell" diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index e8dd88049..02362693a 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -1,4 +1,5 @@ import warnings +from typing import Any import pytest from w3lib.http import basic_auth_header @@ -8,9 +9,8 @@ from scrapy.utils.curl import curl_to_request_kwargs class TestCurlToRequestKwargs: - maxDiff = 5000 - - def _test_command(self, curl_command, expected_result): + @staticmethod + def _test_command(curl_command: str, expected_result: dict[str, Any]) -> None: result = curl_to_request_kwargs(curl_command) assert result == expected_result try: diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 75b6b0e99..352e49165 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,5 +1,6 @@ import copy import warnings +from abc import ABC, abstractmethod from collections.abc import Iterator, Mapping, MutableMapping import pytest @@ -16,7 +17,12 @@ from scrapy.utils.datatypes import ( from scrapy.utils.python import garbage_collect -class CaseInsensitiveDictBase: +class TestCaseInsensitiveDictBase(ABC): + @property + @abstractmethod + def dict_class(self) -> type[MutableMapping]: + raise NotImplementedError + def test_init_dict(self): seq = {"red": 1, "black": 3} d = self.dict_class(seq) @@ -199,7 +205,7 @@ class CaseInsensitiveDictBase: assert h1.get("header1") == h3.get("HEADER1") -class TestCaseInsensitiveDict(CaseInsensitiveDictBase): +class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase): dict_class = CaseInsensitiveDict def test_repr(self): @@ -216,7 +222,7 @@ class TestCaseInsensitiveDict(CaseInsensitiveDictBase): @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestCaselessDict(CaseInsensitiveDictBase): +class TestCaselessDict(TestCaseInsensitiveDictBase): dict_class = CaselessDict def test_deprecation_message(self): diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 662de0dc3..a88b5e008 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -1,6 +1,7 @@ import inspect import warnings from unittest import mock +from warnings import WarningMessage import pytest @@ -21,7 +22,9 @@ class NewName(SomeBaseClass): class TestWarnWhenSubclassed: - def _mywarnings(self, w, category=MyWarning): + def _mywarnings( + self, w: list[WarningMessage], category: type[Warning] = MyWarning + ) -> list[WarningMessage]: return [x for x in w if x.category is MyWarning] def test_no_warning_on_definition(self): diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py index cea564653..20251ca59 100644 --- a/tests/test_utils_display.py +++ b/tests/test_utils_display.py @@ -3,88 +3,92 @@ from unittest import mock from scrapy.utils.display import pformat, pprint - -class TestDisplay: - object = {"a": 1} - colorized_strings = { +value = {"a": 1} +colorized_strings = { + ( ( - ( - "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'" - "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}" - ) - + suffix + "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'" + "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}" ) - for suffix in ( - # https://github.com/pygments/pygments/issues/2313 - "\n", # pygments ≤ 2.13 - "\x1b[37m\x1b[39;49;00m\n", # pygments ≥ 2.14 - ) - } - plain_string = "{'a': 1}" + + suffix + ) + for suffix in ( + # https://github.com/pygments/pygments/issues/2313 + "\n", # pygments ≤ 2.13 + "\x1b[37m\x1b[39;49;00m\n", # pygments ≥ 2.14 + ) +} +plain_string = "{'a': 1}" - @mock.patch("sys.platform", "linux") - @mock.patch("sys.stdout.isatty") - def test_pformat(self, isatty): - isatty.return_value = True - assert pformat(self.object) in self.colorized_strings - @mock.patch("sys.stdout.isatty") - def test_pformat_dont_colorize(self, isatty): - isatty.return_value = True - assert pformat(self.object, colorize=False) == self.plain_string +@mock.patch("sys.platform", "linux") +@mock.patch("sys.stdout.isatty") +def test_pformat(isatty): + isatty.return_value = True + assert pformat(value) in colorized_strings - def test_pformat_not_tty(self): - assert pformat(self.object) == self.plain_string - @mock.patch("sys.platform", "win32") - @mock.patch("platform.version") - @mock.patch("sys.stdout.isatty") - def test_pformat_old_windows(self, isatty, version): - isatty.return_value = True - version.return_value = "10.0.14392" - assert pformat(self.object) in self.colorized_strings +@mock.patch("sys.stdout.isatty") +def test_pformat_dont_colorize(isatty): + isatty.return_value = True + assert pformat(value, colorize=False) == plain_string - @mock.patch("sys.platform", "win32") - @mock.patch("scrapy.utils.display._enable_windows_terminal_processing") - @mock.patch("platform.version") - @mock.patch("sys.stdout.isatty") - def test_pformat_windows_no_terminal_processing( - self, isatty, version, terminal_processing - ): - isatty.return_value = True - version.return_value = "10.0.14393" - terminal_processing.return_value = False - assert pformat(self.object) == self.plain_string - @mock.patch("sys.platform", "win32") - @mock.patch("scrapy.utils.display._enable_windows_terminal_processing") - @mock.patch("platform.version") - @mock.patch("sys.stdout.isatty") - def test_pformat_windows(self, isatty, version, terminal_processing): - isatty.return_value = True - version.return_value = "10.0.14393" - terminal_processing.return_value = True - assert pformat(self.object) in self.colorized_strings +def test_pformat_not_tty(): + assert pformat(value) == plain_string - @mock.patch("sys.platform", "linux") - @mock.patch("sys.stdout.isatty") - def test_pformat_no_pygments(self, isatty): - isatty.return_value = True - import builtins +@mock.patch("sys.platform", "win32") +@mock.patch("platform.version") +@mock.patch("sys.stdout.isatty") +def test_pformat_old_windows(isatty, version): + isatty.return_value = True + version.return_value = "10.0.14392" + assert pformat(value) in colorized_strings - real_import = builtins.__import__ - def mock_import(name, globals, locals, fromlist, level): - if "pygments" in name: - raise ImportError - return real_import(name, globals, locals, fromlist, level) +@mock.patch("sys.platform", "win32") +@mock.patch("scrapy.utils.display._enable_windows_terminal_processing") +@mock.patch("platform.version") +@mock.patch("sys.stdout.isatty") +def test_pformat_windows_no_terminal_processing(isatty, version, terminal_processing): + isatty.return_value = True + version.return_value = "10.0.14393" + terminal_processing.return_value = False + assert pformat(value) == plain_string - builtins.__import__ = mock_import - assert pformat(self.object) == self.plain_string - builtins.__import__ = real_import - def test_pprint(self): - with mock.patch("sys.stdout", new=StringIO()) as mock_out: - pprint(self.object) - assert mock_out.getvalue() == "{'a': 1}\n" +@mock.patch("sys.platform", "win32") +@mock.patch("scrapy.utils.display._enable_windows_terminal_processing") +@mock.patch("platform.version") +@mock.patch("sys.stdout.isatty") +def test_pformat_windows(isatty, version, terminal_processing): + isatty.return_value = True + version.return_value = "10.0.14393" + terminal_processing.return_value = True + assert pformat(value) in colorized_strings + + +@mock.patch("sys.platform", "linux") +@mock.patch("sys.stdout.isatty") +def test_pformat_no_pygments(isatty): + isatty.return_value = True + + import builtins + + real_import = builtins.__import__ + + def mock_import(name, globals, locals, fromlist, level): + if "pygments" in name: + raise ImportError + return real_import(name, globals, locals, fromlist, level) + + builtins.__import__ = mock_import + assert pformat(value) == plain_string + builtins.__import__ = real_import + + +def test_pprint(): + with mock.patch("sys.stdout", new=StringIO()) as mock_out: + pprint(value) + assert mock_out.getvalue() == "{'a': 1}\n" diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index c43ed152b..06fdf9cba 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -11,47 +11,51 @@ from tests import tests_datadir SAMPLEDIR = Path(tests_datadir, "compressed") -class TestGunzip: - def test_gunzip_basic(self): - r1 = Response( - "http://www.example.com", - body=(SAMPLEDIR / "feed-sample1.xml.gz").read_bytes(), - ) - assert gzip_magic_number(r1) +def test_gunzip_basic(): + r1 = Response( + "http://www.example.com", + body=(SAMPLEDIR / "feed-sample1.xml.gz").read_bytes(), + ) + assert gzip_magic_number(r1) - r2 = Response("http://www.example.com", body=gunzip(r1.body)) - assert not gzip_magic_number(r2) - assert len(r2.body) == 9950 + r2 = Response("http://www.example.com", body=gunzip(r1.body)) + assert not gzip_magic_number(r2) + assert len(r2.body) == 9950 - def test_gunzip_truncated(self): - text = gunzip((SAMPLEDIR / "truncated-crc-error.gz").read_bytes()) - assert text.endswith(b"") - assert not gzip_magic_number(r2) +def test_gunzip_no_gzip_file_raises(): + with pytest.raises(BadGzipFile): + gunzip((SAMPLEDIR / "feed-sample1.xml").read_bytes()) - def test_is_gzipped_empty(self): - r1 = Response("http://www.example.com") - assert not gzip_magic_number(r1) - def test_gunzip_illegal_eof(self): - text = html_to_unicode( - "charset=cp1252", gunzip((SAMPLEDIR / "unexpected-eof.gz").read_bytes()) - )[1] - expected_text = (SAMPLEDIR / "unexpected-eof-output.txt").read_text( - encoding="utf-8" - ) - assert len(text) == len(expected_text) - assert text == expected_text +def test_gunzip_truncated_short(): + r1 = Response( + "http://www.example.com", + body=(SAMPLEDIR / "truncated-crc-error-short.gz").read_bytes(), + ) + assert gzip_magic_number(r1) + + r2 = Response("http://www.example.com", body=gunzip(r1.body)) + assert r2.body.endswith(b"") + assert not gzip_magic_number(r2) + + +def test_is_gzipped_empty(): + r1 = Response("http://www.example.com") + assert not gzip_magic_number(r1) + + +def test_gunzip_illegal_eof(): + text = html_to_unicode( + "charset=cp1252", gunzip((SAMPLEDIR / "unexpected-eof.gz").read_bytes()) + )[1] + expected_text = (SAMPLEDIR / "unexpected-eof-output.txt").read_text( + encoding="utf-8" + ) + assert len(text) == len(expected_text) + assert text == expected_text diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py index 0c05ef7d6..9bd86f7fb 100644 --- a/tests/test_utils_httpobj.py +++ b/tests/test_utils_httpobj.py @@ -4,18 +4,17 @@ from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached -class TestHttpobjUtils: - def test_urlparse_cached(self): - url = "http://www.example.com/index.html" - request1 = Request(url) - request2 = Request(url) - req1a = urlparse_cached(request1) - req1b = urlparse_cached(request1) - req2 = urlparse_cached(request2) - urlp = urlparse(url) +def test_urlparse_cached(): + url = "http://www.example.com/index.html" + request1 = Request(url) + request2 = Request(url) + req1a = urlparse_cached(request1) + req1b = urlparse_cached(request1) + req2 = urlparse_cached(request2) + urlp = urlparse(url) - assert req1a == req2 - assert req1a == urlp - assert req1a is req1b - assert req1a is not req2 - assert req1a is not req2 + assert req1a == req2 + assert req1a == urlp + assert req1a is req1b + assert req1a is not req2 + assert req1a is not req2 diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index fa0d37866..ac32fff2c 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,3 +1,8 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any + import pytest from scrapy.exceptions import ScrapyDeprecationWarning @@ -5,9 +10,19 @@ from scrapy.http import Response, TextResponse, XmlResponse from scrapy.utils.iterators import _body_or_str, csviter, xmliter, xmliter_lxml from tests import get_testdata +if TYPE_CHECKING: + from collections.abc import Iterator + + from scrapy import Selector + + +class TestXmliterBase(ABC): + @abstractmethod + def xmliter( + self, obj: Response | str | bytes, nodename: str, *args: Any + ) -> Iterator[Selector]: + raise NotImplementedError -class XmliterBase: - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter(self): body = b""" @@ -39,7 +54,6 @@ class XmliterBase: ("002", ["Name 2"], ["Type 2"]), ] - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_unusual_node(self): body = b""" @@ -53,7 +67,6 @@ class XmliterBase: ] assert nodenames == [["matchme..."]] - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_unicode(self): # example taken from https://github.com/scrapy/scrapy/issues/1665 body = """ @@ -113,7 +126,6 @@ class XmliterBase: ("27", ["A"], ["27"]), ] - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_text(self): body = ( '' @@ -125,7 +137,6 @@ class XmliterBase: ["two"], ] - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_namespaces(self): body = b""" @@ -163,7 +174,6 @@ class XmliterBase: assert node.xpath("id/text()").getall() == [] assert node.xpath("price/text()").getall() == [] - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_namespaced_nodename(self): body = b""" @@ -191,7 +201,6 @@ class XmliterBase: "http://www.mydummycompany.com/images/item1.jpg" ] - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_namespaced_nodename_missing(self): body = b""" @@ -216,7 +225,6 @@ class XmliterBase: with pytest.raises(StopIteration): next(my_iter) - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_exception(self): body = ( '' @@ -229,13 +237,11 @@ class XmliterBase: with pytest.raises(StopIteration): next(iter) - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_objtype_exception(self): i = self.xmliter(42, "product") with pytest.raises(TypeError): next(i) - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_encoding(self): body = ( b'\n' @@ -250,8 +256,12 @@ class XmliterBase: ) -class TestXmliter(XmliterBase): - xmliter = staticmethod(xmliter) +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") +class TestXmliter(TestXmliterBase): + def xmliter( + self, obj: Response | str | bytes, nodename: str, *args: Any + ) -> Iterator[Selector]: + return xmliter(obj, nodename) def test_deprecation(self): body = b""" @@ -267,8 +277,11 @@ class TestXmliter(XmliterBase): next(self.xmliter(body, "product")) -class TestLxmlXmliter(XmliterBase): - xmliter = staticmethod(xmliter_lxml) +class TestLxmlXmliter(TestXmliterBase): + def xmliter( + self, obj: Response | str | bytes, nodename: str, *args: Any + ) -> Iterator[Selector]: + return xmliter_lxml(obj, nodename, *args) def test_xmliter_iterate_namespace(self): body = b""" @@ -493,23 +506,32 @@ class TestUtilsCsv: ] -class TestHelper: +class TestBodyOrStr: bbody = b"utf8-body" ubody = bbody.decode("utf8") - txtresponse = TextResponse(url="http://example.org/", body=bbody, encoding="utf-8") - response = Response(url="http://example.org/", body=bbody) - def test_body_or_str(self): - for obj in (self.bbody, self.ubody, self.txtresponse, self.response): - r1 = _body_or_str(obj) - self._assert_type_and_value(r1, self.ubody, obj) - r2 = _body_or_str(obj, unicode=True) - self._assert_type_and_value(r2, self.ubody, obj) - r3 = _body_or_str(obj, unicode=False) - self._assert_type_and_value(r3, self.bbody, obj) - assert type(r1) is type(r2) - assert type(r1) is not type(r3) + @pytest.mark.parametrize( + "obj", + [ + bbody, + ubody, + TextResponse(url="http://example.org/", body=bbody, encoding="utf-8"), + Response(url="http://example.org/", body=bbody), + ], + ) + def test_body_or_str(self, obj: Response | str | bytes) -> None: + r1 = _body_or_str(obj) + self._assert_type_and_value(r1, self.ubody, obj) + r2 = _body_or_str(obj, unicode=True) + self._assert_type_and_value(r2, self.ubody, obj) + r3 = _body_or_str(obj, unicode=False) + self._assert_type_and_value(r3, self.bbody, obj) + assert type(r1) is type(r2) + assert type(r1) is not type(r3) - def _assert_type_and_value(self, a, b, obj): + @staticmethod + def _assert_type_and_value( + a: str | bytes, b: str | bytes, obj: Response | str | bytes + ) -> None: assert type(a) is type(b), f"Got {type(a)}, expected {type(b)} for {obj!r}" assert a == b diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 56375606c..f40e424ff 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -22,7 +22,9 @@ from scrapy.utils.test import get_crawler from tests.spiders import LogSpider if TYPE_CHECKING: - from collections.abc import Mapping, MutableMapping + from collections.abc import Generator, Mapping, MutableMapping + + from scrapy.crawler import Crawler class TestFailureToExcInfo: @@ -70,33 +72,41 @@ class TestTopLevelFormatter: class TestLogCounterHandler: - def setup_method(self): + @pytest.fixture + def crawler(self) -> Crawler: settings = {"LOG_LEVEL": "WARNING"} - self.logger = logging.getLogger("test") - self.logger.setLevel(logging.NOTSET) - self.logger.propagate = False - self.crawler = get_crawler(settings_dict=settings) - self.handler = LogCounterHandler(self.crawler) - self.logger.addHandler(self.handler) + return get_crawler(settings_dict=settings) - def teardown_method(self): - self.logger.propagate = True - self.logger.removeHandler(self.handler) + @pytest.fixture + def logger(self, crawler: Crawler) -> Generator[logging.Logger]: + logger = logging.getLogger("test") + logger.setLevel(logging.NOTSET) + logger.propagate = False + handler = LogCounterHandler(crawler) + logger.addHandler(handler) - def test_init(self): - assert self.crawler.stats.get_value("log_count/DEBUG") is None - assert self.crawler.stats.get_value("log_count/INFO") is None - assert self.crawler.stats.get_value("log_count/WARNING") is None - assert self.crawler.stats.get_value("log_count/ERROR") is None - assert self.crawler.stats.get_value("log_count/CRITICAL") is None + yield logger - def test_accepted_level(self): - self.logger.error("test log msg") - assert self.crawler.stats.get_value("log_count/ERROR") == 1 + logger.propagate = True + logger.removeHandler(handler) - def test_filtered_out_level(self): - self.logger.debug("test log msg") - assert self.crawler.stats.get_value("log_count/INFO") is None + def test_init(self, crawler: Crawler, logger: logging.Logger) -> None: + assert crawler.stats + assert crawler.stats.get_value("log_count/DEBUG") is None + assert crawler.stats.get_value("log_count/INFO") is None + assert crawler.stats.get_value("log_count/WARNING") is None + assert crawler.stats.get_value("log_count/ERROR") is None + assert crawler.stats.get_value("log_count/CRITICAL") is None + + def test_accepted_level(self, crawler: Crawler, logger: logging.Logger) -> None: + logger.error("test log msg") + assert crawler.stats + assert crawler.stats.get_value("log_count/ERROR") == 1 + + def test_filtered_out_level(self, crawler: Crawler, logger: logging.Logger) -> None: + logger.debug("test log msg") + assert crawler.stats + assert crawler.stats.get_value("log_count/INFO") is None class TestStreamLogger: @@ -135,7 +145,7 @@ class TestStreamLogger: ) def test_spider_logger_adapter_process( base_extra: Mapping[str, Any], log_extra: MutableMapping, expected_extra: dict -): +) -> None: logger = logging.getLogger("test") spider_logger_adapter = SpiderLoggerAdapter(logger, base_extra) @@ -149,59 +159,75 @@ def test_spider_logger_adapter_process( class TestLogging: - def setup_method(self): - self.log_stream = StringIO() - handler = logging.StreamHandler(self.log_stream) + @pytest.fixture + def log_stream(self) -> StringIO: + return StringIO() + + @pytest.fixture + def spider(self) -> LogSpider: + return LogSpider() + + @pytest.fixture(autouse=True) + def logger(self, log_stream: StringIO) -> Generator[logging.Logger]: + handler = logging.StreamHandler(log_stream) logger = logging.getLogger("log_spider") logger.addHandler(handler) logger.setLevel(logging.DEBUG) - self.handler = handler - self.logger = logger - self.spider = LogSpider() - def teardown_method(self): - self.logger.removeHandler(self.handler) + yield logger - def test_debug_logging(self): + logger.removeHandler(handler) + + def test_debug_logging(self, log_stream: StringIO, spider: LogSpider) -> None: log_message = "Foo message" - self.spider.log_debug(log_message) - log_contents = self.log_stream.getvalue() + spider.log_debug(log_message) + log_contents = log_stream.getvalue() assert log_contents == f"{log_message}\n" - def test_info_logging(self): + def test_info_logging(self, log_stream: StringIO, spider: LogSpider) -> None: log_message = "Bar message" - self.spider.log_info(log_message) - log_contents = self.log_stream.getvalue() + spider.log_info(log_message) + log_contents = log_stream.getvalue() assert log_contents == f"{log_message}\n" - def test_warning_logging(self): + def test_warning_logging(self, log_stream: StringIO, spider: LogSpider) -> None: log_message = "Baz message" - self.spider.log_warning(log_message) - log_contents = self.log_stream.getvalue() + spider.log_warning(log_message) + log_contents = log_stream.getvalue() assert log_contents == f"{log_message}\n" - def test_error_logging(self): + def test_error_logging(self, log_stream: StringIO, spider: LogSpider) -> None: log_message = "Foo bar message" - self.spider.log_error(log_message) - log_contents = self.log_stream.getvalue() + spider.log_error(log_message) + log_contents = log_stream.getvalue() assert log_contents == f"{log_message}\n" - def test_critical_logging(self): + def test_critical_logging(self, log_stream: StringIO, spider: LogSpider) -> None: log_message = "Foo bar baz message" - self.spider.log_critical(log_message) - log_contents = self.log_stream.getvalue() + spider.log_critical(log_message) + log_contents = log_stream.getvalue() assert log_contents == f"{log_message}\n" class TestLoggingWithExtra: - def setup_method(self): - self.log_stream = StringIO() - handler = logging.StreamHandler(self.log_stream) + regex_pattern = re.compile(r"^]+>$") + + @pytest.fixture + def log_stream(self) -> StringIO: + return StringIO() + + @pytest.fixture + def spider(self) -> LogSpider: + return LogSpider() + + @pytest.fixture(autouse=True) + def logger(self, log_stream: StringIO) -> Generator[logging.Logger]: + handler = logging.StreamHandler(log_stream) formatter = logging.Formatter( '{"levelname": "%(levelname)s", "message": "%(message)s", "spider": "%(spider)s", "important_info": "%(important_info)s"}' ) @@ -209,80 +235,79 @@ class TestLoggingWithExtra: logger = logging.getLogger("log_spider") logger.addHandler(handler) logger.setLevel(logging.DEBUG) - self.handler = handler - self.logger = logger - self.spider = LogSpider() - self.regex_pattern = re.compile(r"^]+>$") - def teardown_method(self): - self.logger.removeHandler(self.handler) + yield logger - def test_debug_logging(self): + logger.removeHandler(handler) + + def test_debug_logging(self, log_stream: StringIO, spider: LogSpider) -> None: log_message = "Foo message" extra = {"important_info": "foo"} - self.spider.log_debug(log_message, extra) - log_contents = self.log_stream.getvalue() - log_contents = json.loads(log_contents) + spider.log_debug(log_message, extra) + log_contents_str = log_stream.getvalue() + log_contents = json.loads(log_contents_str) assert log_contents["levelname"] == "DEBUG" assert log_contents["message"] == log_message assert self.regex_pattern.match(log_contents["spider"]) assert log_contents["important_info"] == extra["important_info"] - def test_info_logging(self): + def test_info_logging(self, log_stream: StringIO, spider: LogSpider) -> None: log_message = "Bar message" extra = {"important_info": "bar"} - self.spider.log_info(log_message, extra) - log_contents = self.log_stream.getvalue() - log_contents = json.loads(log_contents) + spider.log_info(log_message, extra) + log_contents_str = log_stream.getvalue() + log_contents = json.loads(log_contents_str) assert log_contents["levelname"] == "INFO" assert log_contents["message"] == log_message assert self.regex_pattern.match(log_contents["spider"]) assert log_contents["important_info"] == extra["important_info"] - def test_warning_logging(self): + def test_warning_logging(self, log_stream: StringIO, spider: LogSpider) -> None: log_message = "Baz message" extra = {"important_info": "baz"} - self.spider.log_warning(log_message, extra) - log_contents = self.log_stream.getvalue() - log_contents = json.loads(log_contents) + spider.log_warning(log_message, extra) + log_contents_str = log_stream.getvalue() + log_contents = json.loads(log_contents_str) assert log_contents["levelname"] == "WARNING" assert log_contents["message"] == log_message assert self.regex_pattern.match(log_contents["spider"]) assert log_contents["important_info"] == extra["important_info"] - def test_error_logging(self): + def test_error_logging(self, log_stream: StringIO, spider: LogSpider) -> None: log_message = "Foo bar message" extra = {"important_info": "foo bar"} - self.spider.log_error(log_message, extra) - log_contents = self.log_stream.getvalue() - log_contents = json.loads(log_contents) + spider.log_error(log_message, extra) + log_contents_str = log_stream.getvalue() + log_contents = json.loads(log_contents_str) assert log_contents["levelname"] == "ERROR" assert log_contents["message"] == log_message assert self.regex_pattern.match(log_contents["spider"]) assert log_contents["important_info"] == extra["important_info"] - def test_critical_logging(self): + def test_critical_logging(self, log_stream: StringIO, spider: LogSpider) -> None: log_message = "Foo bar baz message" extra = {"important_info": "foo bar baz"} - self.spider.log_critical(log_message, extra) - log_contents = self.log_stream.getvalue() - log_contents = json.loads(log_contents) + spider.log_critical(log_message, extra) + log_contents_str = log_stream.getvalue() + log_contents = json.loads(log_contents_str) assert log_contents["levelname"] == "CRITICAL" assert log_contents["message"] == log_message assert self.regex_pattern.match(log_contents["spider"]) assert log_contents["important_info"] == extra["important_info"] - def test_overwrite_spider_extra(self): + def test_overwrite_spider_extra( + self, log_stream: StringIO, spider: LogSpider + ) -> None: log_message = "Foo message" extra = {"important_info": "foo", "spider": "shouldn't change"} - self.spider.log_error(log_message, extra) - log_contents = self.log_stream.getvalue() - log_contents = json.loads(log_contents) + spider.log_error(log_message, extra) + log_contents_str = log_stream.getvalue() + log_contents = json.loads(log_contents_str) assert log_contents["levelname"] == "ERROR" assert log_contents["message"] == log_message diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index aa250be69..20a3d940c 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -1,18 +1,17 @@ -import contextlib import os -import shutil -import tempfile import warnings from pathlib import Path +import pytest + from scrapy.utils.misc import set_environ from scrapy.utils.project import data_path, get_project_settings -@contextlib.contextmanager -def inside_a_project(): +@pytest.fixture +def proj_path(tmp_path): prev_dir = Path.cwd() - project_dir = tempfile.mkdtemp() + project_dir = tmp_path try: os.chdir(project_dir) @@ -21,21 +20,19 @@ def inside_a_project(): yield project_dir finally: os.chdir(prev_dir) - shutil.rmtree(project_dir) -class TestProjectUtils: - def test_data_path_outside_project(self): - assert str(Path(".scrapy", "somepath")) == data_path("somepath") - abspath = str(Path(os.path.sep, "absolute", "path")) - assert abspath == data_path(abspath) +def test_data_path_outside_project(): + assert str(Path(".scrapy", "somepath")) == data_path("somepath") + abspath = str(Path(os.path.sep, "absolute", "path")) + assert abspath == data_path(abspath) - def test_data_path_inside_project(self): - with inside_a_project() as proj_path: - expected = Path(proj_path, ".scrapy", "somepath") - assert expected.resolve() == Path(data_path("somepath")).resolve() - abspath = str(Path(os.path.sep, "absolute", "path").resolve()) - assert abspath == data_path(abspath) + +def test_data_path_inside_project(proj_path: Path) -> None: + expected = proj_path / ".scrapy" / "somepath" + assert expected.resolve() == Path(data_path("somepath")).resolve() + abspath = str(Path(os.path.sep, "absolute", "path").resolve()) + assert abspath == data_path(abspath) class TestGetProjectSettings: diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 291646ad7..c933e0ac9 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -1,7 +1,10 @@ +from __future__ import annotations + import functools import operator import platform import sys +from typing import TYPE_CHECKING, TypeVar import pytest from twisted.trial import unittest @@ -20,16 +23,22 @@ from scrapy.utils.python import ( without_none_values, ) +if TYPE_CHECKING: + from collections.abc import Iterable, Mapping -class TestMutableChain: - def test_mutablechain(self): - m = MutableChain(range(2), [2, 3], (4, 5)) - m.extend(range(6, 7)) - m.extend([7, 8]) - m.extend([9, 10], (11, 12)) - assert next(m) == 0 - assert m.__next__() == 1 - assert list(m) == list(range(2, 13)) + +_KT = TypeVar("_KT") +_VT = TypeVar("_VT") + + +def test_mutablechain(): + m = MutableChain(range(2), [2, 3], (4, 5)) + m.extend(range(6, 7)) + m.extend([7, 8]) + m.extend([9, 10], (11, 12)) + assert next(m) == 0 + assert m.__next__() == 1 + assert list(m) == list(range(2, 13)) class TestMutableAsyncChain(unittest.TestCase): @@ -112,144 +121,150 @@ class TestToBytes: assert to_bytes("a\ufffdb", "latin-1", errors="replace") == b"a?b" -class TestMemoizedMethod: - def test_memoizemethod_noargs(self): - class A: - @memoizemethod_noargs - def cached(self): - return object() +def test_memoizemethod_noargs(): + class A: + @memoizemethod_noargs + def cached(self): + return object() - def noncached(self): - return object() + def noncached(self): + return object() - a = A() - one = a.cached() - two = a.cached() - three = a.noncached() - assert one is two - assert one is not three + a = A() + one = a.cached() + two = a.cached() + three = a.noncached() + assert one is two + assert one is not three -class TestBinaryIsText: - def test_binaryistext(self): - assert binary_is_text(b"hello") - - def test_utf_16_strings_contain_null_bytes(self): - assert binary_is_text("hello".encode("utf-16")) - - def test_one_with_encoding(self): - assert binary_is_text(b"
Price \xa3
") - - def test_real_binary_bytes(self): - assert not binary_is_text(b"\x02\xa3") +@pytest.mark.parametrize( + ("value", "expected"), + [ + (b"hello", True), + ("hello".encode("utf-16"), True), + (b"
Price \xa3
", True), + (b"\x02\xa3", False), + ], +) +def test_binaryistext(value: bytes, expected: bool) -> None: + assert binary_is_text(value) is expected -class TestUtilsPython: - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") - def test_equal_attributes(self): - class Obj: +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") +def test_equal_attributes(): + class Obj: + pass + + a = Obj() + b = Obj() + # no attributes given return False + assert not equal_attributes(a, b, []) + # nonexistent attributes + assert not equal_attributes(a, b, ["x", "y"]) + + a.x = 1 + b.x = 1 + # equal attribute + assert equal_attributes(a, b, ["x"]) + + b.y = 2 + # obj1 has no attribute y + assert not equal_attributes(a, b, ["x", "y"]) + + a.y = 2 + # equal attributes + assert equal_attributes(a, b, ["x", "y"]) + + a.y = 1 + # different attributes + assert not equal_attributes(a, b, ["x", "y"]) + + # test callable + a.meta = {} + b.meta = {} + assert equal_attributes(a, b, ["meta"]) + + # compare ['meta']['a'] + a.meta["z"] = 1 + b.meta["z"] = 1 + + get_z = operator.itemgetter("z") + get_meta = operator.attrgetter("meta") + + def compare_z(obj): + return get_z(get_meta(obj)) + + assert equal_attributes(a, b, [compare_z, "x"]) + # fail z equality + a.meta["z"] = 2 + assert not equal_attributes(a, b, [compare_z, "x"]) + + +def test_get_func_args(): + def f1(a, b, c): + pass + + def f2(a, b=None, c=None): + pass + + def f3(a, b=None, *, c=None): + pass + + class A: + def __init__(self, a, b, c): pass - a = Obj() - b = Obj() - # no attributes given return False - assert not equal_attributes(a, b, []) - # nonexistent attributes - assert not equal_attributes(a, b, ["x", "y"]) - - a.x = 1 - b.x = 1 - # equal attribute - assert equal_attributes(a, b, ["x"]) - - b.y = 2 - # obj1 has no attribute y - assert not equal_attributes(a, b, ["x", "y"]) - - a.y = 2 - # equal attributes - assert equal_attributes(a, b, ["x", "y"]) - - a.y = 1 - # different attributes - assert not equal_attributes(a, b, ["x", "y"]) - - # test callable - a.meta = {} - b.meta = {} - assert equal_attributes(a, b, ["meta"]) - - # compare ['meta']['a'] - a.meta["z"] = 1 - b.meta["z"] = 1 - - get_z = operator.itemgetter("z") - get_meta = operator.attrgetter("meta") - - def compare_z(obj): - return get_z(get_meta(obj)) - - assert equal_attributes(a, b, [compare_z, "x"]) - # fail z equality - a.meta["z"] = 2 - assert not equal_attributes(a, b, [compare_z, "x"]) - - def test_get_func_args(self): - def f1(a, b, c): + def method(self, a, b, c): pass - def f2(a, b=None, c=None): + class Callable: + def __call__(self, a, b, c): pass - def f3(a, b=None, *, c=None): - pass + a = A(1, 2, 3) + cal = Callable() + partial_f1 = functools.partial(f1, None) + partial_f2 = functools.partial(f1, b=None) + partial_f3 = functools.partial(partial_f2, None) - class A: - def __init__(self, a, b, c): - pass + assert get_func_args(f1) == ["a", "b", "c"] + assert get_func_args(f2) == ["a", "b", "c"] + assert get_func_args(f3) == ["a", "b", "c"] + assert get_func_args(A) == ["a", "b", "c"] + assert get_func_args(a.method) == ["a", "b", "c"] + assert get_func_args(partial_f1) == ["b", "c"] + assert get_func_args(partial_f2) == ["a", "c"] + assert get_func_args(partial_f3) == ["c"] + assert get_func_args(cal) == ["a", "b", "c"] + assert get_func_args(object) == [] + assert get_func_args(str.split, stripself=True) == ["sep", "maxsplit"] + assert get_func_args(" ".join, stripself=True) == ["iterable"] - def method(self, a, b, c): - pass + if sys.version_info >= (3, 13) or platform.python_implementation() == "PyPy": + # the correct and correctly extracted signature + assert get_func_args(operator.itemgetter(2), stripself=True) == ["obj"] + elif platform.python_implementation() == "CPython": + # ["args", "kwargs"] is a correct result for the pre-3.13 incorrect function signature + # [] is an incorrect result on even older CPython (https://github.com/python/cpython/issues/86951) + assert get_func_args(operator.itemgetter(2), stripself=True) in [ + [], + ["args", "kwargs"], + ] - class Callable: - def __call__(self, a, b, c): - pass - a = A(1, 2, 3) - cal = Callable() - partial_f1 = functools.partial(f1, None) - partial_f2 = functools.partial(f1, b=None) - partial_f3 = functools.partial(partial_f2, None) - - assert get_func_args(f1) == ["a", "b", "c"] - assert get_func_args(f2) == ["a", "b", "c"] - assert get_func_args(f3) == ["a", "b", "c"] - assert get_func_args(A) == ["a", "b", "c"] - assert get_func_args(a.method) == ["a", "b", "c"] - assert get_func_args(partial_f1) == ["b", "c"] - assert get_func_args(partial_f2) == ["a", "c"] - assert get_func_args(partial_f3) == ["c"] - assert get_func_args(cal) == ["a", "b", "c"] - assert get_func_args(object) == [] - assert get_func_args(str.split, stripself=True) == ["sep", "maxsplit"] - assert get_func_args(" ".join, stripself=True) == ["iterable"] - - if sys.version_info >= (3, 13) or platform.python_implementation() == "PyPy": - # the correct and correctly extracted signature - assert get_func_args(operator.itemgetter(2), stripself=True) == ["obj"] - elif platform.python_implementation() == "CPython": - # ["args", "kwargs"] is a correct result for the pre-3.13 incorrect function signature - # [] is an incorrect result on even older CPython (https://github.com/python/cpython/issues/86951) - assert get_func_args(operator.itemgetter(2), stripself=True) in [ - [], - ["args", "kwargs"], - ] - - def test_without_none_values(self): - assert without_none_values([1, None, 3, 4]) == [1, 3, 4] - assert without_none_values((1, None, 3, 4)) == (1, 3, 4) - assert without_none_values({"one": 1, "none": None, "three": 3, "four": 4}) == { - "one": 1, - "three": 3, - "four": 4, - } +@pytest.mark.parametrize( + ("value", "expected"), + [ + ([1, None, 3, 4], [1, 3, 4]), + ((1, None, 3, 4), (1, 3, 4)), + ( + {"one": 1, "none": None, "three": 3, "four": 4}, + {"one": 1, "three": 3, "four": 4}, + ), + ], +) +def test_without_none_values( + value: Mapping[_KT, _VT] | Iterable[_KT], expected: dict[_KT, _VT] | Iterable[_KT] +) -> None: + assert without_none_values(value) == expected diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 5b8509753..9c4cb7159 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -20,45 +20,52 @@ from scrapy.utils.request import ( from scrapy.utils.test import get_crawler -class TestUtilsRequest: - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") - def test_request_authenticate(self): - r = Request("http://www.example.com") - request_authenticate(r, "someuser", "somepass") - assert r.headers["Authorization"] == b"Basic c29tZXVzZXI6c29tZXBhc3M=" +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") +def test_request_authenticate(): + r = Request("http://www.example.com") + request_authenticate(r, "someuser", "somepass") + assert r.headers["Authorization"] == b"Basic c29tZXVzZXI6c29tZXBhc3M=" - def test_request_httprepr(self): - r1 = Request("http://www.example.com") - assert ( - request_httprepr(r1) == b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n" - ) - r1 = Request("http://www.example.com/some/page.html?arg=1") - assert ( - request_httprepr(r1) - == b"GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n" - ) +@pytest.mark.parametrize( + ("r", "expected"), + [ + ( + Request("http://www.example.com"), + b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + ), + ( + Request("http://www.example.com/some/page.html?arg=1"), + b"GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + ), + ( + Request( + "http://www.example.com", + method="POST", + headers={"Content-type": b"text/html"}, + body=b"Some body", + ), + b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body", + ), + ], +) +def test_request_httprepr(r: Request, expected: bytes) -> None: + assert request_httprepr(r) == expected - r1 = Request( - "http://www.example.com", - method="POST", - headers={"Content-type": b"text/html"}, - body=b"Some body", - ) - assert ( - request_httprepr(r1) - == b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body" - ) - def test_request_httprepr_for_non_http_request(self): - # the representation is not important but it must not fail. - request_httprepr(Request("file:///tmp/foo.txt")) - request_httprepr(Request("ftp://localhost/tmp/foo.txt")) +@pytest.mark.parametrize( + "r", + [ + Request("file:///tmp/foo.txt"), + Request("ftp://localhost/tmp/foo.txt"), + ], +) +def test_request_httprepr_for_non_http_request(r: Request) -> None: + # the representation is not important but it must not fail. + request_httprepr(r) class TestFingerprint: - maxDiff = None - function: staticmethod = staticmethod(fingerprint) cache: ( WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes]] @@ -229,35 +236,6 @@ class TestFingerprint: assert actual == expected -REQUEST_OBJECTS_TO_TEST = ( - Request("http://www.example.com/"), - Request("http://www.example.com/query?id=111&cat=222"), - Request("http://www.example.com/query?cat=222&id=111"), - Request("http://www.example.com/hnnoticiaj1.aspx?78132,199"), - Request("http://www.example.com/hnnoticiaj1.aspx?78160,199"), - Request("http://www.example.com/members/offers.html"), - Request( - "http://www.example.com/members/offers.html", - headers={"SESSIONID": b"somehash"}, - ), - Request( - "http://www.example.com/", - headers={"Accept-Language": b"en"}, - ), - Request( - "http://www.example.com/", - headers={ - "Accept-Language": b"en", - "SESSIONID": b"somehash", - }, - ), - Request("http://www.example.com/test.html"), - Request("http://www.example.com/test.html#fragment"), - Request("http://www.example.com", method="POST"), - Request("http://www.example.com", method="POST", body=b"request body"), -) - - class TestRequestFingerprinter: def test_default_implementation(self): crawler = get_crawler() diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 80f2f25d5..179ca49e4 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -4,7 +4,7 @@ from urllib.parse import urlparse import pytest -from scrapy.http import HtmlResponse, Response, TextResponse +from scrapy.http import HtmlResponse, Response from scrapy.utils.python import to_bytes from scrapy.utils.response import ( _remove_html_comments, @@ -15,229 +15,203 @@ from scrapy.utils.response import ( ) -class TestResponseUtils: - dummy_response = TextResponse(url="http://example.org/", body=b"dummy_response") +def test_open_in_browser(): + url = "http:///www.example.com/some/page.html" + body = ( + b" test page test body " + ) - def test_open_in_browser(self): - url = "http:///www.example.com/some/page.html" - body = b" test page test body " + def browser_open(burl: str) -> bool: + path = urlparse(burl).path + if not path or not Path(path).exists(): + path = burl.replace("file://", "") + bbody = Path(path).read_bytes() + assert b'' in bbody + return True - def browser_open(burl): - path = urlparse(burl).path - if not path or not Path(path).exists(): - path = burl.replace("file://", "") - bbody = Path(path).read_bytes() - assert b'' in bbody - return True + response = HtmlResponse(url, body=body) + assert open_in_browser(response, _openfunc=browser_open), "Browser not called" - response = HtmlResponse(url, body=body) - assert open_in_browser(response, _openfunc=browser_open), "Browser not called" + resp = Response(url, body=body) + with pytest.raises(TypeError): + open_in_browser(resp, debug=True) # pylint: disable=unexpected-keyword-arg - resp = Response(url, body=body) - with pytest.raises(TypeError): - open_in_browser(resp, debug=True) # pylint: disable=unexpected-keyword-arg - def test_get_meta_refresh(self): - r1 = HtmlResponse( - "http://www.example.com", - body=b""" - - Dummy - blahablsdfsal& - """, - ) - r2 = HtmlResponse( - "http://www.example.com", - body=b""" - - Dummy - blahablsdfsal& - """, - ) - r3 = HtmlResponse( - "http://www.example.com", - body=b""" -