RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix get_meta_refresh to correctly resolve relative URLs using <base> tag (#7047)

This commit is contained in:
Ashar Khan 2025-09-10 09:17:16 -04:00 committed by GitHub
parent 2ad81a0ef8
commit d239fcf936
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 15 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/utils/response.py
View File

@ -48,7 +48,7 @@ def get_meta_refresh(
if response not in _metaref_cache:
text = response.text[0:4096]
_metaref_cache[response] = html.get_meta_refresh(
text, response.url, response.encoding, ignore_tags=ignore_tags
text, get_base_url(response), response.encoding, ignore_tags=ignore_tags
)
return _metaref_cache[response]

14
tests/test_utils_response.py
View File

@ -67,9 +67,23 @@ if(!checkCookies()){
</script>
""",
)
r4 = HtmlResponse(
"http://www.example.com",
body=b"""
<html>
<head><title>Dummy</title>
<base href="http://www.another-domain.com/base/path/">
<meta http-equiv="refresh" content="5;url=target.html"</head>
<body>blahablsdfsal&amp;</body>
</html>""",
)
assert get_meta_refresh(r1) == (5.0, "http://example.org/newpage")
assert get_meta_refresh(r2) == (None, None)
assert get_meta_refresh(r3) == (None, None)
assert get_meta_refresh(r4) == (
5.0,
"http://www.another-domain.com/base/path/target.html",
)
def test_get_base_url():