From f2c800c5c9f88b4b583181e9cf49eb3cd8d538f0 Mon Sep 17 00:00:00 2001 From: Samuel Marchal Date: Mon, 15 Nov 2021 11:14:54 +0100 Subject: [PATCH 1/8] Improve open_in_browser base tag injection (#5319) --- scrapy/utils/response.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index b3ef7b463..8b109dced 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -3,8 +3,9 @@ This module provides some useful functions for working with scrapy.http.Response objects """ import os -import webbrowser +import re import tempfile +import webbrowser from typing import Any, Callable, Iterable, Optional, Tuple, Union from weakref import WeakKeyDictionary @@ -80,8 +81,9 @@ def open_in_browser( body = response.body if isinstance(response, HtmlResponse): if b'' - body = body.replace(b'', to_bytes(repl)) + repl = fr'\1' + body = re.sub(b"", b"", body, flags=re.DOTALL) + body = re.sub(rb"(|\s.*?>))", to_bytes(repl), body) ext = '.html' elif isinstance(response, TextResponse): ext = '.txt' From 75ed765476a2ac66ea8f52e7b29186864f65535c Mon Sep 17 00:00:00 2001 From: Samuel Marchal Date: Mon, 15 Nov 2021 14:31:24 +0100 Subject: [PATCH 2/8] Test coverage for open_in_browser base tag injection (#5319) --- tests/test_utils_response.py | 53 ++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index d6f4c0bb5..0a09f6109 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -83,3 +83,56 @@ class ResponseUtilsTest(unittest.TestCase): self.assertEqual(response_status_message(200), '200 OK') self.assertEqual(response_status_message(404), '404 Not Found') self.assertEqual(response_status_message(573), "573 Unknown Status") + + def test_inject_base_url(self): + url = "http://www.example.com" + + def check_base_url(burl): + path = urlparse(burl).path + if not os.path.exists(path): + path = burl.replace('file://', '') + with open(path, "rb") as f: + bbody = f.read() + self.assertEqual(bbody.count(b''), 1) + return True + + r1 = HtmlResponse(url, body=b""" + + Dummy +

Hello world.

+ """) + r2 = HtmlResponse(url, body=b""" + + Dummy + Hello world. + """) + r3 = HtmlResponse(url, body=b""" + + Dummy + +
Hello header
+

Hello world.

+ + """) + r4 = HtmlResponse(url, body=b""" + + + Dummy +

Hello world.

+ """) + r5 = HtmlResponse(url, body=b""" + + + + Standard head + +

Hello world.

+ """) + + assert open_in_browser(r1, _openfunc=check_base_url), "Inject base url" + assert open_in_browser(r2, _openfunc=check_base_url), "Inject base url with argumented head" + assert open_in_browser(r3, _openfunc=check_base_url), "Inject unique base url with misleading tag" + assert open_in_browser(r4, _openfunc=check_base_url), "Inject unique base url with misleading comment" + assert open_in_browser(r5, _openfunc=check_base_url), "Inject unique base url with conditional comment" From 6ec66c96fb962a276c2d285cd5ffa34d84925df1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 26 Nov 2021 12:25:45 +0500 Subject: [PATCH 3/8] Fix and pin pylint. --- pylintrc | 1 + tox.ini | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pylintrc b/pylintrc index 699686e16..2cdd6321e 100644 --- a/pylintrc +++ b/pylintrc @@ -112,6 +112,7 @@ disable=abstract-method, unused-private-member, unused-variable, unused-wildcard-import, + use-implicit-booleaness-not-comparison, used-before-assignment, useless-object-inheritance, # Required for Python 2 support useless-return, diff --git a/tox.ini b/tox.ini index e4514f512..2031a2d92 100644 --- a/tox.ini +++ b/tox.ini @@ -66,7 +66,7 @@ commands = basepython = python3 deps = {[testenv:extra-deps]deps} - pylint + pylint==2.12.1 commands = pylint conftest.py docs extras scrapy setup.py tests From d6a384b3cfdb36cd19b32942479487a9e47c244b Mon Sep 17 00:00:00 2001 From: yogender26 <95638485+yogender26@users.noreply.github.com> Date: Thu, 23 Dec 2021 04:09:05 +0530 Subject: [PATCH 4/8] corrrection of coma (#5347) --- scrapy/commands/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 6e77551c6..5f1dabd33 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -43,14 +43,14 @@ class ScrapyCommand: def long_desc(self): """A long description of the command. Return short description when not - available. It cannot contain newlines, since contents will be formatted + available. It cannot contain newlines since contents will be formatted by optparser which removes newlines and wraps text. """ return self.short_desc() def help(self): """An extensive help for the command. It will be shown when using the - "help" command. It can contain newlines, since no post-formatting will + "help" command. It can contain newlines since no post-formatting will be applied to its contents. """ return self.long_desc() From 7380888cad2c255e6f4a7efb6fe4cfd1240fb42c Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Thu, 30 Dec 2021 18:55:16 +0500 Subject: [PATCH 5/8] Fix a warning message. (#5359) --- scrapy/utils/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 24873f75d..24a6187b9 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -164,7 +164,7 @@ def feed_process_params_from_cli(settings, output, output_format=None, message = ( 'The -t command line option is deprecated in favor of ' 'specifying the output format within the output URI. See the ' - 'documentation of the -o and -O options for more information.', + 'documentation of the -o and -O options for more information.' ) warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2) return {output[0]: {'format': output_format}} From b81938684b55fca29e48faa766f2b6f6e3ab5d6a Mon Sep 17 00:00:00 2001 From: Andrey Oskin Date: Fri, 31 Dec 2021 21:49:18 +1100 Subject: [PATCH 6/8] Docs: correct process repetition start step (#5356) The process repeats from step 3, the scheduler feeds request to the engine. Steps 1 and 2 are not parts of the loop as their incarnations steps 7 and 8 are parts of the loop. --- docs/topics/architecture.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index 71d027c86..0c3a7ed88 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -67,7 +67,7 @@ this: the :ref:`Scheduler ` and asks for possible next Requests to crawl. -9. The process repeats (from step 1) until there are no more requests from the +9. The process repeats (from step 3) until there are no more requests from the :ref:`Scheduler `. Components From e4bdd1cb958b7d89b86ea66f0af1cec2d91a6d44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Miech?= Date: Fri, 31 Dec 2021 11:57:12 +0100 Subject: [PATCH 7/8] downloader.webclient: make reactor import local (#5357) --- scrapy/core/downloader/webclient.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 915cb5fe3..06cb96489 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -3,7 +3,7 @@ from time import time from urllib.parse import urlparse, urlunparse, urldefrag from twisted.web.http import HTTPClient -from twisted.internet import defer, reactor +from twisted.internet import defer from twisted.internet.protocol import ClientFactory from scrapy.http import Headers @@ -170,6 +170,7 @@ class ScrapyHTTPClientFactory(ClientFactory): p.followRedirect = self.followRedirect p.afterFoundGet = self.afterFoundGet if self.timeout: + from twisted.internet import reactor timeoutCall = reactor.callLater(self.timeout, p.timeout) self.deferred.addBoth(self._cancelTimeout, timeoutCall) return p From 57dc58123b98e2026025cc87bdee474bf0656dcb Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Fri, 31 Dec 2021 17:15:08 +0500 Subject: [PATCH 8/8] Remove the experimental note about asyncio (#5332) --- docs/topics/asyncio.rst | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 28241ae24..402352721 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -10,11 +10,6 @@ Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the asyncio reactor `, you may use :mod:`asyncio` and :mod:`asyncio`-powered libraries in any :doc:`coroutine `. -.. warning:: :mod:`asyncio` support in Scrapy is experimental, and not yet - recommended for production environments. Future Scrapy versions - may introduce related changes without a deprecation period or - warning. - .. _install-asyncio: Installing the asyncio reactor