diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 40bcda288..022265992 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -652,8 +652,9 @@ per ip address instead of per domain. .. _spider-download_delay-attribute: -You can also change this setting per spider by setting ``download_delay`` -spider attribute. +.. note:: + + This delay can be set per spider using :attr:`download_delay` spider attribute. .. setting:: DOWNLOAD_HANDLERS diff --git a/scrapy/shell.py b/scrapy/shell.py index f2dff2ae3..515b71bb6 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -21,6 +21,7 @@ from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.misc import load_object from scrapy.utils.response import open_in_browser +from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop class Shell: @@ -76,6 +77,10 @@ class Shell: banner=self.vars.pop('banner', '')) def _schedule(self, request, spider): + if is_asyncio_reactor_installed(): + # set the asyncio event loop for the current thread + event_loop_path = self.crawler.settings['ASYNCIO_EVENT_LOOP'] + set_asyncio_event_loop(event_loop_path) spider = self._open_spider(request, spider) d = _request_deferred(request) d.addCallback(lambda x: (x, spider)) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index ddf354d88..e6b8de292 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -71,14 +71,7 @@ def install_reactor(reactor_path, event_loop_path=None): reactor_class = load_object(reactor_path) if reactor_class is asyncioreactor.AsyncioSelectorReactor: with suppress(error.ReactorAlreadyInstalledError): - policy = get_asyncio_event_loop_policy() - if event_loop_path is not None: - event_loop_class = load_object(event_loop_path) - event_loop = event_loop_class() - asyncio.set_event_loop(event_loop) - else: - event_loop = policy.get_event_loop() - + event_loop = set_asyncio_event_loop(event_loop_path) asyncioreactor.install(eventloop=event_loop) else: *module, _ = reactor_path.split(".") @@ -88,6 +81,25 @@ def install_reactor(reactor_path, event_loop_path=None): installer() +def set_asyncio_event_loop(event_loop_path): + """Sets and returns the event loop with specified import path.""" + policy = get_asyncio_event_loop_policy() + if event_loop_path is not None: + event_loop_class = load_object(event_loop_path) + event_loop = event_loop_class() + asyncio.set_event_loop(event_loop) + else: + try: + event_loop = policy.get_event_loop() + except RuntimeError: + # `get_event_loop` is expected to fail when called from a new thread + # with no asyncio event loop yet installed. Such is the case when + # called from `scrapy shell` + event_loop = policy.new_event_loop() + asyncio.set_event_loop(event_loop) + return event_loop + + def verify_installed_reactor(reactor_path): """Raises :exc:`Exception` if the installed :mod:`~twisted.internet.reactor` does not match the specified import diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 33c98ad69..f06a02f5f 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -115,3 +115,14 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): errcode, out, err = yield self.execute([url, '-c', 'item'], check_code=False) self.assertEqual(errcode, 1, out or err) self.assertIn(b'DNS lookup failed', err) + + @defer.inlineCallbacks + def test_shell_fetch_async(self): + reactor_path = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" + url = self.url('/html') + code = f"fetch('{url}')" + args = ["-c", code, "--set", f"TWISTED_REACTOR={reactor_path}"] + _, _, err = yield self.execute(args, check_code=True) + self.assertNotIn( + b"RuntimeError: There is no current event loop in thread", err + ) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index f51f3d988..74e170ec0 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -2,14 +2,17 @@ import codecs import unittest from unittest import mock +from packaging.version import Version as parse_version +from pytest import mark +from w3lib import __version__ as w3lib_version from w3lib.encoding import resolve_encoding -from scrapy.http import (Request, Response, TextResponse, HtmlResponse, - XmlResponse, Headers) +from scrapy.exceptions import NotSupported +from scrapy.http import (Headers, HtmlResponse, Request, Response, + TextResponse, XmlResponse) +from scrapy.link import Link from scrapy.selector import Selector from scrapy.utils.python import to_unicode -from scrapy.exceptions import NotSupported -from scrapy.link import Link from tests import get_testdata @@ -179,13 +182,23 @@ class BaseResponseTest(unittest.TestCase): r = self.response_class("http://example.com") self.assertRaises(ValueError, r.follow, None) + @mark.xfail( + parse_version(w3lib_version) < parse_version("2.1.1"), + reason="https://github.com/scrapy/w3lib/pull/207", + strict=True, + ) def test_follow_whitespace_url(self): self._assert_followed_url('foo ', - 'http://example.com/foo%20') + 'http://example.com/foo') + @mark.xfail( + parse_version(w3lib_version) < parse_version("2.1.1"), + reason="https://github.com/scrapy/w3lib/pull/207", + strict=True, + ) def test_follow_whitespace_link(self): self._assert_followed_url(Link('http://example.com/foo '), - 'http://example.com/foo%20') + 'http://example.com/foo') def test_follow_flags(self): res = self.response_class('http://example.com/')