RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #5320 from zessx/5319-oib-base-tag

This commit is contained in:
Andrey Rahmatullin 2021-12-21 17:49:31 +05:00 committed by GitHub
parent 0862b179e3 75ed765476
commit 9ec60e8e4a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 58 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scrapy/utils/response.py
View File

@ -3,8 +3,9 @@ This module provides some useful functions for working with
scrapy.http.Response objects
"""
import os
import webbrowser
import re
import tempfile
import webbrowser
from typing import Any, Callable, Iterable, Optional, Tuple, Union
from weakref import WeakKeyDictionary
@ -80,8 +81,9 @@ def open_in_browser(
body = response.body
if isinstance(response, HtmlResponse):
if b'<base' not in body:
repl = f'<head><base href="{response.url}">'
body = body.replace(b'<head>', to_bytes(repl))
repl = fr'\1<base href="{response.url}">'
body = re.sub(b"<!--.*?-->", b"", body, flags=re.DOTALL)
body = re.sub(rb"(<head(?:>|\s.*?>))", to_bytes(repl), body)
ext = '.html'
elif isinstance(response, TextResponse):
ext = '.txt'

53
tests/test_utils_response.py
View File

@ -83,3 +83,56 @@ class ResponseUtilsTest(unittest.TestCase):
self.assertEqual(response_status_message(200), '200 OK')
self.assertEqual(response_status_message(404), '404 Not Found')
self.assertEqual(response_status_message(573), "573 Unknown Status")
def test_inject_base_url(self):
url = "http://www.example.com"
def check_base_url(burl):
path = urlparse(burl).path
if not os.path.exists(path):
path = burl.replace('file://', '')
with open(path, "rb") as f:
bbody = f.read()
self.assertEqual(bbody.count(b'<base href="' + to_bytes(url) + b'">'), 1)
return True
r1 = HtmlResponse(url, body=b"""
<html>
<head><title>Dummy</title></head>
<body><p>Hello world.</p></body>
</html>""")
r2 = HtmlResponse(url, body=b"""
<html>
<head id="foo"><title>Dummy</title></head>
<body>Hello world.</body>
</html>""")
r3 = HtmlResponse(url, body=b"""
<html>
<head><title>Dummy</title></head>
<body>
<header>Hello header</header>
<p>Hello world.</p>
</body>
</html>""")
r4 = HtmlResponse(url, body=b"""
<html>
<!-- <head>Dummy comment</head> -->
<head><title>Dummy</title></head>
<body><p>Hello world.</p></body>
</html>""")
r5 = HtmlResponse(url, body=b"""
<html>
<!--[if IE]>
<head><title>IE head</title></head>
<![endif]-->
<!--[if !IE]>-->
<head><title>Standard head</title></head>
<!--<![endif]-->
<body><p>Hello world.</p></body>
</html>""")
assert open_in_browser(r1, _openfunc=check_base_url), "Inject base url"
assert open_in_browser(r2, _openfunc=check_base_url), "Inject base url with argumented head"
assert open_in_browser(r3, _openfunc=check_base_url), "Inject unique base url with misleading tag"
assert open_in_browser(r4, _openfunc=check_base_url), "Inject unique base url with misleading comment"
assert open_in_browser(r5, _openfunc=check_base_url), "Inject unique base url with conditional comment"