RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix get_base_url/get_meta_refresh ignoring base tag beyond 4096 chars 

The 4096-character truncation in scrapy.utils.response.get_base_url
and get_meta_refresh caused the <base href> tag to be silently missed
whenever it appeared after that offset (e.g. when a large inline
script, comment block, or whitespace precedes the <head> section).
In those cases Scrapy fell back to the response URL, producing wrong
absolute links and mis-resolved relative URLs.

Remove the truncation so the full response text is searched, which
is the correct behaviour. The results are still cached per-response
via WeakKeyDictionary, so the performance impact is bounded to a
single regex pass over the document text.

Closes #3017
This commit is contained in:
JSap0914 2026-06-28 19:43:41 +09:00
parent 185d6b9a20
commit fb98ffc310
2 changed files with 20 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scrapy/utils/response.py
View File

@ -28,7 +28,7 @@ _baseurl_cache: WeakKeyDictionary[Response, str] = WeakKeyDictionary()
def get_base_url(response: TextResponse) -> str:
"""Return the base url of the given response, joined with the response url"""
if response not in _baseurl_cache:
text = response.text[0:4096]
text = response.text
_baseurl_cache[response] = html.get_base_url(
text, response.url, response.encoding
)
@ -46,7 +46,7 @@ def get_meta_refresh(
) -> tuple[None, None] | tuple[float, str]:
"""Parse the http-equiv refresh parameter from the given response"""
if response not in _metaref_cache:
text = response.text[0:4096]
text = response.text
_metaref_cache[response] = html.get_meta_refresh(
text, get_base_url(response), response.encoding, ignore_tags=ignore_tags
)

18
tests/test_utils_response.py
View File

@ -321,3 +321,21 @@ def test_open_in_browser_raises_for_unsupported_response_type():
response = Response("http://www.example.com", body=b"binary")
with pytest.raises(TypeError):
open_in_browser(response, _openfunc=lambda _: True)
def test_get_base_url_beyond_4096_chars():
"""get_base_url must find <base href> even when it appears after
the first 4096 characters of the response body.
Regression test for https://github.com/scrapy/scrapy/issues/3017
"""
# Build a body where the base tag is past the 4096-char mark by
# padding with an HTML comment before the <head>.
padding = "<!-- " + "x" * 4100 + " -->"
body = (
f"<html>{padding}"
f'<head><base href="http://www.example.com/img/"></head>'
f"<body>content</body></html>"
).encode()
resp = HtmlResponse("http://www.example.com", body=body)
assert get_base_url(resp) == "http://www.example.com/img/"