Merge branch 'master' into finish_exporting_for_each_start_exporting

docs/topics/settings.rst

@@ -652,8 +652,9 @@ per ip address instead of per domain.
 
 .. _spider-download_delay-attribute:
 
-You can also change this setting per spider by setting ``download_delay``
-spider attribute.
+.. note::
+
+    This delay can be set per spider using :attr:`download_delay` spider attribute.
 
 .. setting:: DOWNLOAD_HANDLERS
 

scrapy/shell.py

@@ -21,6 +21,7 @@ from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console
 from scrapy.utils.datatypes import SequenceExclude
 from scrapy.utils.misc import load_object
 from scrapy.utils.response import open_in_browser
+from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop
 
 
 class Shell:
@@ -76,6 +77,10 @@ class Shell:
                                  banner=self.vars.pop('banner', ''))
 
     def _schedule(self, request, spider):
+        if is_asyncio_reactor_installed():
+            # set the asyncio event loop for the current thread
+            event_loop_path = self.crawler.settings['ASYNCIO_EVENT_LOOP']
+            set_asyncio_event_loop(event_loop_path)
         spider = self._open_spider(request, spider)
         d = _request_deferred(request)
         d.addCallback(lambda x: (x, spider))

scrapy/utils/reactor.py

@@ -71,14 +71,7 @@ def install_reactor(reactor_path, event_loop_path=None):
     reactor_class = load_object(reactor_path)
     if reactor_class is asyncioreactor.AsyncioSelectorReactor:
         with suppress(error.ReactorAlreadyInstalledError):
-            policy = get_asyncio_event_loop_policy()
-            if event_loop_path is not None:
-                event_loop_class = load_object(event_loop_path)
-                event_loop = event_loop_class()
-                asyncio.set_event_loop(event_loop)
-            else:
-                event_loop = policy.get_event_loop()
-
+            event_loop = set_asyncio_event_loop(event_loop_path)
             asyncioreactor.install(eventloop=event_loop)
     else:
         *module, _ = reactor_path.split(".")
@@ -88,6 +81,25 @@ def install_reactor(reactor_path, event_loop_path=None):
             installer()
 
 
+def set_asyncio_event_loop(event_loop_path):
+    """Sets and returns the event loop with specified import path."""
+    policy = get_asyncio_event_loop_policy()
+    if event_loop_path is not None:
+        event_loop_class = load_object(event_loop_path)
+        event_loop = event_loop_class()
+        asyncio.set_event_loop(event_loop)
+    else:
+        try:
+            event_loop = policy.get_event_loop()
+        except RuntimeError:
+            # `get_event_loop` is expected to fail when called from a new thread
+            # with no asyncio event loop yet installed. Such is the case when
+            # called from `scrapy shell`
+            event_loop = policy.new_event_loop()
+            asyncio.set_event_loop(event_loop)
+    return event_loop
+
+
 def verify_installed_reactor(reactor_path):
     """Raises :exc:`Exception` if the installed
     :mod:`~twisted.internet.reactor` does not match the specified import

tests/test_command_shell.py

@@ -115,3 +115,14 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
         errcode, out, err = yield self.execute([url, '-c', 'item'], check_code=False)
         self.assertEqual(errcode, 1, out or err)
         self.assertIn(b'DNS lookup failed', err)
+
+    @defer.inlineCallbacks
+    def test_shell_fetch_async(self):
+        reactor_path = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
+        url = self.url('/html')
+        code = f"fetch('{url}')"
+        args = ["-c", code, "--set", f"TWISTED_REACTOR={reactor_path}"]
+        _, _, err = yield self.execute(args, check_code=True)
+        self.assertNotIn(
+            b"RuntimeError: There is no current event loop in thread", err
+        )

tests/test_http_response.py

@@ -2,14 +2,17 @@ import codecs
 import unittest
 from unittest import mock
 
+from packaging.version import Version as parse_version
+from pytest import mark
+from w3lib import __version__ as w3lib_version
 from w3lib.encoding import resolve_encoding
 
-from scrapy.http import (Request, Response, TextResponse, HtmlResponse,
-                         XmlResponse, Headers)
+from scrapy.exceptions import NotSupported
+from scrapy.http import (Headers, HtmlResponse, Request, Response,
+                         TextResponse, XmlResponse)
+from scrapy.link import Link
 from scrapy.selector import Selector
 from scrapy.utils.python import to_unicode
-from scrapy.exceptions import NotSupported
-from scrapy.link import Link
 from tests import get_testdata
 
 
@@ -179,13 +182,23 @@ class BaseResponseTest(unittest.TestCase):
         r = self.response_class("http://example.com")
         self.assertRaises(ValueError, r.follow, None)
 
+    @mark.xfail(
+        parse_version(w3lib_version) < parse_version("2.1.1"),
+        reason="https://github.com/scrapy/w3lib/pull/207",
+        strict=True,
+    )
     def test_follow_whitespace_url(self):
         self._assert_followed_url('foo ',
-                                  'http://example.com/foo%20')
+                                  'http://example.com/foo')
 
+    @mark.xfail(
+        parse_version(w3lib_version) < parse_version("2.1.1"),
+        reason="https://github.com/scrapy/w3lib/pull/207",
+        strict=True,
+    )
     def test_follow_whitespace_link(self):
         self._assert_followed_url(Link('http://example.com/foo '),
-                                  'http://example.com/foo%20')
+                                  'http://example.com/foo')
 
     def test_follow_flags(self):
         res = self.response_class('http://example.com/')