RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into crawler-process-reactor-later

This commit is contained in:
Andrey Rakhmatullin 2021-12-23 17:25:56 +05:00
parent 9c4bfb4836 d6a384b3cf
commit df04e934e6
5 changed files with 62 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
pylintrc
View File

@ -112,6 +112,7 @@ disable=abstract-method,
unused-private-member,
unused-variable,
unused-wildcard-import,
use-implicit-booleaness-not-comparison,
used-before-assignment,
useless-object-inheritance, # Required for Python 2 support
useless-return,

4
scrapy/commands/__init__.py
View File

@ -43,14 +43,14 @@ class ScrapyCommand:
def long_desc(self):
"""A long description of the command. Return short description when not
available. It cannot contain newlines, since contents will be formatted
available. It cannot contain newlines since contents will be formatted
by optparser which removes newlines and wraps text.
"""
return self.short_desc()
def help(self):
"""An extensive help for the command. It will be shown when using the
"help" command. It can contain newlines, since no post-formatting will
"help" command. It can contain newlines since no post-formatting will
be applied to its contents.
"""
return self.long_desc()

8
scrapy/utils/response.py
View File

@ -3,8 +3,9 @@ This module provides some useful functions for working with
scrapy.http.Response objects
"""
import os
import webbrowser
import re
import tempfile
import webbrowser
from typing import Any, Callable, Iterable, Optional, Tuple, Union
from weakref import WeakKeyDictionary
@ -80,8 +81,9 @@ def open_in_browser(
body = response.body
if isinstance(response, HtmlResponse):
if b'<base' not in body:
repl = f'<head><base href="{response.url}">'
body = body.replace(b'<head>', to_bytes(repl))
repl = fr'\1<base href="{response.url}">'
body = re.sub(b"<!--.*?-->", b"", body, flags=re.DOTALL)
body = re.sub(rb"(<head(?:>|\s.*?>))", to_bytes(repl), body)
ext = '.html'
elif isinstance(response, TextResponse):
ext = '.txt'

53
tests/test_utils_response.py
View File

@ -83,3 +83,56 @@ class ResponseUtilsTest(unittest.TestCase):
self.assertEqual(response_status_message(200), '200 OK')
self.assertEqual(response_status_message(404), '404 Not Found')
self.assertEqual(response_status_message(573), "573 Unknown Status")
def test_inject_base_url(self):
url = "http://www.example.com"
def check_base_url(burl):
path = urlparse(burl).path
if not os.path.exists(path):
path = burl.replace('file://', '')
with open(path, "rb") as f:
bbody = f.read()
self.assertEqual(bbody.count(b'<base href="' + to_bytes(url) + b'">'), 1)
return True
r1 = HtmlResponse(url, body=b"""
<html>
<head><title>Dummy</title></head>
<body><p>Hello world.</p></body>
</html>""")
r2 = HtmlResponse(url, body=b"""
<html>
<head id="foo"><title>Dummy</title></head>
<body>Hello world.</body>
</html>""")
r3 = HtmlResponse(url, body=b"""
<html>
<head><title>Dummy</title></head>
<body>
<header>Hello header</header>
<p>Hello world.</p>
</body>
</html>""")
r4 = HtmlResponse(url, body=b"""
<html>
<!-- <head>Dummy comment</head> -->
<head><title>Dummy</title></head>
<body><p>Hello world.</p></body>
</html>""")
r5 = HtmlResponse(url, body=b"""
<html>
<!--[if IE]>
<head><title>IE head</title></head>
<![endif]-->
<!--[if !IE]>-->
<head><title>Standard head</title></head>
<!--<![endif]-->
<body><p>Hello world.</p></body>
</html>""")
assert open_in_browser(r1, _openfunc=check_base_url), "Inject base url"
assert open_in_browser(r2, _openfunc=check_base_url), "Inject base url with argumented head"
assert open_in_browser(r3, _openfunc=check_base_url), "Inject unique base url with misleading tag"
assert open_in_browser(r4, _openfunc=check_base_url), "Inject unique base url with misleading comment"
assert open_in_browser(r5, _openfunc=check_base_url), "Inject unique base url with conditional comment"

2
tox.ini
View File

@ -66,7 +66,7 @@ commands =
basepython = python3
deps =
{[testenv:extra-deps]deps}
pylint
pylint==2.12.1
commands =
pylint conftest.py docs extras scrapy setup.py tests