diff --git a/pyproject.toml b/pyproject.toml index 0bdcf6b51..96eeaa194 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ dependencies = [ "queuelib>=1.6.1", "service_identity>=23.1.0", "tldextract", - "w3lib>=1.17.0", + "w3lib>=2.1.0", "zope.interface>=5.1.0", # Platform-specific dependencies 'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"', diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index b3622c159..41bdac163 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -60,15 +60,7 @@ def response_status_message(status: bytes | float | str) -> str: return f"{status_int} {to_unicode(message)}" -def _remove_html_comments(body: bytes) -> bytes: - start = body.find(b"", start + 1) - if end == -1: - return body[:start] - body = body[:start] + body[end + 3 :] - start = body.find(b" +

Hello world.

""", + id="Only commented-out and ", + ), ], ) def test_inject_base_url(body: bytes) -> None: @@ -176,8 +200,13 @@ def test_inject_base_url(body: bytes) -> None: def check_base_url(burl): bbody = _read_browser_output(burl) - assert bbody.count(b'>') == 1 - assert b"' + assert bbody.count(base_tag) == 1 + index = bbody.index(base_tag) + # The base tag is not commented out. + assert bbody.rfind(b"", 0, index) + # The base tag comes after the doctype declaration, if any. + assert b"b", b"ab"), - (b"ac", b"ac"), - (b"acccd", b"acd"), - (b"ad", b"ad"), - (b"a -->b", b"a -->b"), - (b"real", b"real"), - ], -) -def test_remove_html_comments(input_body: bytes, output_body: bytes) -> None: - assert _remove_html_comments(input_body) == output_body - - def test_open_in_browser_preserves_html_comments(): url = "http://www.example.com" body = ( @@ -248,19 +259,24 @@ def test_open_in_browser_preserves_html_comments(): assert open_in_browser(response, _openfunc=check) -def test_open_in_browser_does_not_inject_base_when_present(): - url = "http://www.example.com" - body = ( - b"" - b'T' - b"hi" - b"" - ) +@pytest.mark.parametrize( + ("base_tag", "expected_base_url"), + [ + (b'', b"http://real.com/"), + (b'', b"http://real.com/"), + (b'', b"http://www.example.com/img/"), + (b'', b"http://www.example.com/page.html"), + ], +) +def test_open_in_browser_keeps_base_url_of_response( + base_tag: bytes, expected_base_url: bytes +): + url = "http://www.example.com/page.html" + body = b"" + base_tag + b"Thi" def check(burl): bbody = _read_browser_output(burl) - assert b'' not in bbody - assert b'' in bbody + assert bbody.startswith(b'') return True response = HtmlResponse(url, body=body) @@ -286,7 +302,7 @@ def test_open_in_browser_injects_base_when_only_in_comment(): assert open_in_browser(response, _openfunc=check) -def test_open_in_browser_injects_base_at_real_head_not_commented_head(): +def test_open_in_browser_injects_base_before_head_contents(): url = "http://www.example.com" body = ( b"" diff --git a/tox.ini b/tox.ini index 7ea1c57d7..1c6a47e30 100644 --- a/tox.ini +++ b/tox.ini @@ -149,7 +149,7 @@ deps = pyOpenSSL==22.0.0 queuelib==1.6.1 service_identity==23.1.0 - w3lib==1.17.0 + w3lib==2.1.0 zope.interface==5.1.0 {[test-requirements]deps} setenv = @@ -308,11 +308,6 @@ deps = pyOpenSSL==24.3.0 queuelib==1.6.1 service_identity==23.1.0 - # w3lib 1.17 fails to import on PyPy 3.11 because its encoding regex uses - # an inline flag placement that Python 3.11 treats as an error: global - # flags not at the start of the expression. w3lib 1.18 stopped encoding [] - # in URLs until 2.1.0 brought that behavior back. Tests for verbatim_url - # rely on that encoding. w3lib==2.1.0 zope.interface==5.1.0 commands =