From fb98ffc310b257e922f2089953d8e54b02c71a64 Mon Sep 17 00:00:00 2001 From: JSap0914 Date: Sun, 28 Jun 2026 19:43:41 +0900 Subject: [PATCH] Fix get_base_url/get_meta_refresh ignoring base tag beyond 4096 chars The 4096-character truncation in scrapy.utils.response.get_base_url and get_meta_refresh caused the tag to be silently missed whenever it appeared after that offset (e.g. when a large inline script, comment block, or whitespace precedes the section). In those cases Scrapy fell back to the response URL, producing wrong absolute links and mis-resolved relative URLs. Remove the truncation so the full response text is searched, which is the correct behaviour. The results are still cached per-response via WeakKeyDictionary, so the performance impact is bounded to a single regex pass over the document text. Closes #3017 --- scrapy/utils/response.py | 4 ++-- tests/test_utils_response.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index c068d9b1e..f5632da75 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -28,7 +28,7 @@ _baseurl_cache: WeakKeyDictionary[Response, str] = WeakKeyDictionary() def get_base_url(response: TextResponse) -> str: """Return the base url of the given response, joined with the response url""" if response not in _baseurl_cache: - text = response.text[0:4096] + text = response.text _baseurl_cache[response] = html.get_base_url( text, response.url, response.encoding ) @@ -46,7 +46,7 @@ def get_meta_refresh( ) -> tuple[None, None] | tuple[float, str]: """Parse the http-equiv refresh parameter from the given response""" if response not in _metaref_cache: - text = response.text[0:4096] + text = response.text _metaref_cache[response] = html.get_meta_refresh( text, get_base_url(response), response.encoding, ignore_tags=ignore_tags ) diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 4544cd29e..f79d5ecd4 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -321,3 +321,21 @@ def test_open_in_browser_raises_for_unsupported_response_type(): response = Response("http://www.example.com", body=b"binary") with pytest.raises(TypeError): open_in_browser(response, _openfunc=lambda _: True) + + +def test_get_base_url_beyond_4096_chars(): + """get_base_url must find even when it appears after + the first 4096 characters of the response body. + + Regression test for https://github.com/scrapy/scrapy/issues/3017 + """ + # Build a body where the base tag is past the 4096-char mark by + # padding with an HTML comment before the . + padding = "" + body = ( + f"{padding}" + f'' + f"content" + ).encode() + resp = HtmlResponse("http://www.example.com", body=body) + assert get_base_url(resp) == "http://www.example.com/img/"