RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into mw-manager-typing-fix

This commit is contained in:
Andrey Rakhmatullin 2021-12-31 18:34:29 +05:00
parent 4cc039628e 57dc58123b
commit a7846d2398
9 changed files with 66 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/topics/architecture.rst
View File

@ -67,7 +67,7 @@ this:
the :ref:`Scheduler <component-scheduler>` and asks for possible next Requests
to crawl.
9. The process repeats (from step 1) until there are no more requests from the
9. The process repeats (from step 3) until there are no more requests from the
:ref:`Scheduler <component-scheduler>`.
Components

5
docs/topics/asyncio.rst
View File

@ -10,11 +10,6 @@ Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the
asyncio reactor <install-asyncio>`, you may use :mod:`asyncio` and
:mod:`asyncio`-powered libraries in any :doc:`coroutine <coroutines>`.
.. warning:: :mod:`asyncio` support in Scrapy is experimental, and not yet
recommended for production environments. Future Scrapy versions
may introduce related changes without a deprecation period or
warning.
.. _install-asyncio:
Installing the asyncio reactor

1
pylintrc
View File

@ -112,6 +112,7 @@ disable=abstract-method,
unused-private-member,
unused-variable,
unused-wildcard-import,
use-implicit-booleaness-not-comparison,
used-before-assignment,
useless-object-inheritance, # Required for Python 2 support
useless-return,

4
scrapy/commands/__init__.py
View File

@ -43,14 +43,14 @@ class ScrapyCommand:
def long_desc(self):
"""A long description of the command. Return short description when not
available. It cannot contain newlines, since contents will be formatted
available. It cannot contain newlines since contents will be formatted
by optparser which removes newlines and wraps text.
"""
return self.short_desc()
def help(self):
"""An extensive help for the command. It will be shown when using the
"help" command. It can contain newlines, since no post-formatting will
"help" command. It can contain newlines since no post-formatting will
be applied to its contents.
"""
return self.long_desc()

3
scrapy/core/downloader/webclient.py
View File

@ -3,7 +3,7 @@ from time import time
from urllib.parse import urlparse, urlunparse, urldefrag
from twisted.web.http import HTTPClient
from twisted.internet import defer, reactor
from twisted.internet import defer
from twisted.internet.protocol import ClientFactory
from scrapy.http import Headers
@ -170,6 +170,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
p.followRedirect = self.followRedirect
p.afterFoundGet = self.afterFoundGet
if self.timeout:
from twisted.internet import reactor
timeoutCall = reactor.callLater(self.timeout, p.timeout)
self.deferred.addBoth(self._cancelTimeout, timeoutCall)
return p

2
scrapy/utils/conf.py
View File

@ -164,7 +164,7 @@ def feed_process_params_from_cli(settings, output, output_format=None,
message = (
'The -t command line option is deprecated in favor of '
'specifying the output format within the output URI. See the '
'documentation of the -o and -O options for more information.',
'documentation of the -o and -O options for more information.'
)
warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2)
return {output[0]: {'format': output_format}}

8
scrapy/utils/response.py
View File

@ -3,8 +3,9 @@ This module provides some useful functions for working with
scrapy.http.Response objects
"""
import os
import webbrowser
import re
import tempfile
import webbrowser
from typing import Any, Callable, Iterable, Optional, Tuple, Union
from weakref import WeakKeyDictionary
@ -80,8 +81,9 @@ def open_in_browser(
body = response.body
if isinstance(response, HtmlResponse):
if b'<base' not in body:
repl = f'<head><base href="{response.url}">'
body = body.replace(b'<head>', to_bytes(repl))
repl = fr'\1<base href="{response.url}">'
body = re.sub(b"<!--.*?-->", b"", body, flags=re.DOTALL)
body = re.sub(rb"(<head(?:>|\s.*?>))", to_bytes(repl), body)
ext = '.html'
elif isinstance(response, TextResponse):
ext = '.txt'

53
tests/test_utils_response.py
View File

@ -83,3 +83,56 @@ class ResponseUtilsTest(unittest.TestCase):
self.assertEqual(response_status_message(200), '200 OK')
self.assertEqual(response_status_message(404), '404 Not Found')
self.assertEqual(response_status_message(573), "573 Unknown Status")
def test_inject_base_url(self):
url = "http://www.example.com"
def check_base_url(burl):
path = urlparse(burl).path
if not os.path.exists(path):
path = burl.replace('file://', '')
with open(path, "rb") as f:
bbody = f.read()
self.assertEqual(bbody.count(b'<base href="' + to_bytes(url) + b'">'), 1)
return True
r1 = HtmlResponse(url, body=b"""
<html>
<head><title>Dummy</title></head>
<body><p>Hello world.</p></body>
</html>""")
r2 = HtmlResponse(url, body=b"""
<html>
<head id="foo"><title>Dummy</title></head>
<body>Hello world.</body>
</html>""")
r3 = HtmlResponse(url, body=b"""
<html>
<head><title>Dummy</title></head>
<body>
<header>Hello header</header>
<p>Hello world.</p>
</body>
</html>""")
r4 = HtmlResponse(url, body=b"""
<html>
<!-- <head>Dummy comment</head> -->
<head><title>Dummy</title></head>
<body><p>Hello world.</p></body>
</html>""")
r5 = HtmlResponse(url, body=b"""
<html>
<!--[if IE]>
<head><title>IE head</title></head>
<![endif]-->
<!--[if !IE]>-->
<head><title>Standard head</title></head>
<!--<![endif]-->
<body><p>Hello world.</p></body>
</html>""")
assert open_in_browser(r1, _openfunc=check_base_url), "Inject base url"
assert open_in_browser(r2, _openfunc=check_base_url), "Inject base url with argumented head"
assert open_in_browser(r3, _openfunc=check_base_url), "Inject unique base url with misleading tag"
assert open_in_browser(r4, _openfunc=check_base_url), "Inject unique base url with misleading comment"
assert open_in_browser(r5, _openfunc=check_base_url), "Inject unique base url with conditional comment"

2
tox.ini
View File

@ -66,7 +66,7 @@ commands =
basepython = python3
deps =
{[testenv:extra-deps]deps}
pylint
pylint==2.12.1
commands =
pylint conftest.py docs extras scrapy setup.py tests