From 6757973b61766cdb28225733f1c12702fe0886db Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Sun, 11 Dec 2022 09:11:55 -0300 Subject: [PATCH 1/7] fix test_follow_whitespace_* tests --- tests/test_http_response.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index f51f3d988..cb65a6a7e 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -2,14 +2,16 @@ import codecs import unittest from unittest import mock +from pkg_resources import parse_version +from w3lib import __version__ as w3lib_version from w3lib.encoding import resolve_encoding -from scrapy.http import (Request, Response, TextResponse, HtmlResponse, - XmlResponse, Headers) +from scrapy.exceptions import NotSupported +from scrapy.http import (Headers, HtmlResponse, Request, Response, + TextResponse, XmlResponse) +from scrapy.link import Link from scrapy.selector import Selector from scrapy.utils.python import to_unicode -from scrapy.exceptions import NotSupported -from scrapy.link import Link from tests import get_testdata @@ -180,12 +182,18 @@ class BaseResponseTest(unittest.TestCase): self.assertRaises(ValueError, r.follow, None) def test_follow_whitespace_url(self): + target_url = 'http://example.com/foo' + if parse_version(w3lib_version) < parse_version("2.1.1"): + target_url += '%20' self._assert_followed_url('foo ', - 'http://example.com/foo%20') + target_url) def test_follow_whitespace_link(self): + target_url = 'http://example.com/foo' + if parse_version(w3lib_version) < parse_version("2.1.1"): + target_url += '%20' self._assert_followed_url(Link('http://example.com/foo '), - 'http://example.com/foo%20') + target_url) def test_follow_flags(self): res = self.response_class('http://example.com/') From 0dbd1d9b81399b624d8f8081dbc4ba08e66993e7 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Mon, 12 Dec 2022 08:58:40 -0300 Subject: [PATCH 2/7] use xfail, switch to packaging to parse version --- tests/test_http_response.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index cb65a6a7e..891b38f6e 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -2,7 +2,8 @@ import codecs import unittest from unittest import mock -from pkg_resources import parse_version +from packaging.version import Version as parse_version +from pytest import mark from w3lib import __version__ as w3lib_version from w3lib.encoding import resolve_encoding @@ -181,19 +182,23 @@ class BaseResponseTest(unittest.TestCase): r = self.response_class("http://example.com") self.assertRaises(ValueError, r.follow, None) + @mark.xfail( + parse_version(w3lib_version) >= parse_version("2.1.1"), + reason="https://github.com/scrapy/w3lib/pull/207", + strict=True, + ) def test_follow_whitespace_url(self): - target_url = 'http://example.com/foo' - if parse_version(w3lib_version) < parse_version("2.1.1"): - target_url += '%20' self._assert_followed_url('foo ', - target_url) + 'http://example.com/foo%20') + @mark.xfail( + parse_version(w3lib_version) >= parse_version("2.1.1"), + reason="https://github.com/scrapy/w3lib/pull/207", + strict=True, + ) def test_follow_whitespace_link(self): - target_url = 'http://example.com/foo' - if parse_version(w3lib_version) < parse_version("2.1.1"): - target_url += '%20' self._assert_followed_url(Link('http://example.com/foo '), - target_url) + 'http://example.com/foo%20') def test_follow_flags(self): res = self.response_class('http://example.com/') From e099572cec6bb9aa7e164b3bff9fd8723286fbad Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Mon, 12 Dec 2022 09:56:50 -0300 Subject: [PATCH 3/7] change xfail condition --- tests/test_http_response.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 891b38f6e..74e170ec0 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -183,22 +183,22 @@ class BaseResponseTest(unittest.TestCase): self.assertRaises(ValueError, r.follow, None) @mark.xfail( - parse_version(w3lib_version) >= parse_version("2.1.1"), + parse_version(w3lib_version) < parse_version("2.1.1"), reason="https://github.com/scrapy/w3lib/pull/207", strict=True, ) def test_follow_whitespace_url(self): self._assert_followed_url('foo ', - 'http://example.com/foo%20') + 'http://example.com/foo') @mark.xfail( - parse_version(w3lib_version) >= parse_version("2.1.1"), + parse_version(w3lib_version) < parse_version("2.1.1"), reason="https://github.com/scrapy/w3lib/pull/207", strict=True, ) def test_follow_whitespace_link(self): self._assert_followed_url(Link('http://example.com/foo '), - 'http://example.com/foo%20') + 'http://example.com/foo') def test_follow_flags(self): res = self.response_class('http://example.com/') From 8b09b0e0d7eb133db648dfd02a8a531219db0a15 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 13 Dec 2022 20:41:48 -0800 Subject: [PATCH 4/7] set asyncio event loop at start of new thread --- scrapy/shell.py | 5 +++++ scrapy/utils/reactor.py | 27 +++++++++++++++++++-------- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/scrapy/shell.py b/scrapy/shell.py index f2dff2ae3..515b71bb6 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -21,6 +21,7 @@ from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.misc import load_object from scrapy.utils.response import open_in_browser +from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop class Shell: @@ -76,6 +77,10 @@ class Shell: banner=self.vars.pop('banner', '')) def _schedule(self, request, spider): + if is_asyncio_reactor_installed(): + # set the asyncio event loop for the current thread + event_loop_path = self.crawler.settings['ASYNCIO_EVENT_LOOP'] + set_asyncio_event_loop(event_loop_path) spider = self._open_spider(request, spider) d = _request_deferred(request) d.addCallback(lambda x: (x, spider)) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index ddf354d88..532d7b0d0 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -71,14 +71,7 @@ def install_reactor(reactor_path, event_loop_path=None): reactor_class = load_object(reactor_path) if reactor_class is asyncioreactor.AsyncioSelectorReactor: with suppress(error.ReactorAlreadyInstalledError): - policy = get_asyncio_event_loop_policy() - if event_loop_path is not None: - event_loop_class = load_object(event_loop_path) - event_loop = event_loop_class() - asyncio.set_event_loop(event_loop) - else: - event_loop = policy.get_event_loop() - + event_loop = set_asyncio_event_loop(event_loop_path) asyncioreactor.install(eventloop=event_loop) else: *module, _ = reactor_path.split(".") @@ -87,6 +80,24 @@ def install_reactor(reactor_path, event_loop_path=None): with suppress(error.ReactorAlreadyInstalledError): installer() +def set_asyncio_event_loop(event_loop_path): + """Sets and returns the event loop with specified import path.""" + policy = get_asyncio_event_loop_policy() + if event_loop_path is not None: + event_loop_class = load_object(event_loop_path) + event_loop = event_loop_class() + asyncio.set_event_loop(event_loop) + else: + try: + event_loop = policy.get_event_loop() + except RuntimeError: + # `get_event_loop` is expected to fail when called from a new thread + # with no asyncio event loop yet installed. Such is the case when + # called from `scrapy shell` + event_loop = policy.new_event_loop() + asyncio.set_event_loop(event_loop) + return event_loop + def verify_installed_reactor(reactor_path): """Raises :exc:`Exception` if the installed From c2de9372a27120abf81f2aaf6609e5acae82549e Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 13 Dec 2022 21:37:17 -0800 Subject: [PATCH 5/7] added unittest for scrapy shell command and fixed flake8 error --- scrapy/utils/reactor.py | 1 + tests/test_command_shell.py | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 532d7b0d0..e6b8de292 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -80,6 +80,7 @@ def install_reactor(reactor_path, event_loop_path=None): with suppress(error.ReactorAlreadyInstalledError): installer() + def set_asyncio_event_loop(event_loop_path): """Sets and returns the event loop with specified import path.""" policy = get_asyncio_event_loop_policy() diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 33c98ad69..f06a02f5f 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -115,3 +115,14 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): errcode, out, err = yield self.execute([url, '-c', 'item'], check_code=False) self.assertEqual(errcode, 1, out or err) self.assertIn(b'DNS lookup failed', err) + + @defer.inlineCallbacks + def test_shell_fetch_async(self): + reactor_path = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" + url = self.url('/html') + code = f"fetch('{url}')" + args = ["-c", code, "--set", f"TWISTED_REACTOR={reactor_path}"] + _, _, err = yield self.execute(args, check_code=True) + self.assertNotIn( + b"RuntimeError: There is no current event loop in thread", err + ) From 96e526aad8ac93966e383a054d151e6561270bc7 Mon Sep 17 00:00:00 2001 From: Yohanna Lisnichuk Date: Thu, 15 Dec 2022 10:01:55 -0300 Subject: [PATCH 6/7] docs: change DOWNLOAD_DELAY description format To be consistent with the other settings' descriptions --- docs/topics/settings.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 40bcda288..c935e74a7 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -652,8 +652,10 @@ per ip address instead of per domain. .. _spider-download_delay-attribute: -You can also change this setting per spider by setting ``download_delay`` -spider attribute. +Note + +This delay can be set per spider using :attr:`download_delay` spider attribute. + .. setting:: DOWNLOAD_HANDLERS From b63ca6f834d06ba9b63108079129705f6326354c Mon Sep 17 00:00:00 2001 From: Yohanna Lisnichuk Date: Thu, 15 Dec 2022 13:20:45 -0300 Subject: [PATCH 7/7] docs: use the right notes directive --- docs/topics/settings.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index c935e74a7..022265992 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -652,10 +652,9 @@ per ip address instead of per domain. .. _spider-download_delay-attribute: -Note - -This delay can be set per spider using :attr:`download_delay` spider attribute. +.. note:: + This delay can be set per spider using :attr:`download_delay` spider attribute. .. setting:: DOWNLOAD_HANDLERS