diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index abb5f6a70..c068d9b1e 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -97,8 +97,7 @@ def open_in_browser( # XXX: this implementation is a bit dirty and could be improved body = response.body if isinstance(response, HtmlResponse): - if b"' body = re.sub(rb"]*?>)", to_bytes(repl), body, count=1) ext = ".html" diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 381a9c3ff..e02bdfb69 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -4,7 +4,7 @@ from urllib.parse import urlparse import pytest -from scrapy.http import HtmlResponse, Response +from scrapy.http import HtmlResponse, Response, TextResponse from scrapy.utils.python import to_bytes from scrapy.utils.response import ( _remove_html_comments, @@ -15,6 +15,13 @@ from scrapy.utils.response import ( ) +def _read_browser_output(burl: str): + path = urlparse(burl).path + if not path or not Path(path).exists(): + path = burl.replace("file://", "") + return Path(path).read_bytes() + + def test_open_in_browser(): url = "http:///www.example.com/some/page.html" body = ( @@ -22,10 +29,7 @@ def test_open_in_browser(): ) def browser_open(burl: str) -> bool: - path = urlparse(burl).path - if not path or not Path(path).exists(): - path = burl.replace("file://", "") - bbody = Path(path).read_bytes() + bbody = _read_browser_output(burl) assert b'' in bbody return True @@ -169,10 +173,7 @@ def test_inject_base_url(body: bytes) -> None: url = "http://www.example.com" def check_base_url(burl): - path = urlparse(burl).path - if not path or not Path(path).exists(): - path = burl.replace("file://", "") - bbody = Path(path).read_bytes() + bbody = _read_browser_output(burl) assert bbody.count(b'>') == 1 assert b"ccd", b"acd"), (b"ad", b"ad"), + (b"a -->b", b"a -->b"), + (b"real", b"real"), ], ) def test_remove_html_comments(input_body, output_body): assert _remove_html_comments(input_body) == output_body + + +def test_open_in_browser_preserves_html_comments(): + url = "http://www.example.com" + body = ( + b"" + b"" + b"Real" + b"content" + b"" + ) + + def check(burl): + bbody = _read_browser_output(burl) + assert b"" in bbody + return True + + response = HtmlResponse(url, body=body) + assert open_in_browser(response, _openfunc=check) + + +def test_open_in_browser_does_not_inject_base_when_present(): + url = "http://www.example.com" + body = ( + b"" + b'T' + b"hi" + b"" + ) + + def check(burl): + bbody = _read_browser_output(burl) + assert b'' not in bbody + assert b'' in bbody + return True + + response = HtmlResponse(url, body=body) + assert open_in_browser(response, _openfunc=check) + + +def test_open_in_browser_injects_base_when_only_in_comment(): + url = "http://www.example.com" + body = ( + b"" + b"" + b"Real" + b"content" + b"" + ) + + def check(burl): + bbody = _read_browser_output(burl) + assert b'' in bbody + return True + + response = HtmlResponse(url, body=body) + assert open_in_browser(response, _openfunc=check) + + +def test_open_in_browser_injects_base_at_real_head_not_commented_head(): + url = "http://www.example.com" + body = ( + b"" + b"" + b"Actual" + b"hello" + b"" + ) + + def check(burl): + bbody = _read_browser_output(burl) + assert bbody.count(b'') == 1 + base_pos = bbody.find(b'') + title_pos = bbody.find(b"Actual") + assert base_pos < title_pos + return True + + response = HtmlResponse(url, body=body) + assert open_in_browser(response, _openfunc=check) + + +def test_open_in_browser_text_response_uses_txt_extension(): + response = TextResponse("http://www.example.com", body=b"plain text content") + + def check(burl): + assert burl.endswith(".txt") + return True + + assert open_in_browser(response, _openfunc=check) + + +def test_open_in_browser_raises_for_unsupported_response_type(): + response = Response("http://www.example.com", body=b"binary") + with pytest.raises(TypeError): + open_in_browser(response, _openfunc=lambda _: True)