set asyncio event loop at start of new thread

scrapy/shell.py

@@ -21,6 +21,7 @@ from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console
 from scrapy.utils.datatypes import SequenceExclude
 from scrapy.utils.misc import load_object
 from scrapy.utils.response import open_in_browser
+from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop
 
 
 class Shell:
@@ -76,6 +77,10 @@ class Shell:
                                  banner=self.vars.pop('banner', ''))
 
     def _schedule(self, request, spider):
+        if is_asyncio_reactor_installed():
+            # set the asyncio event loop for the current thread
+            event_loop_path = self.crawler.settings['ASYNCIO_EVENT_LOOP']
+            set_asyncio_event_loop(event_loop_path)
         spider = self._open_spider(request, spider)
         d = _request_deferred(request)
         d.addCallback(lambda x: (x, spider))

scrapy/utils/reactor.py

@@ -71,14 +71,7 @@ def install_reactor(reactor_path, event_loop_path=None):
     reactor_class = load_object(reactor_path)
     if reactor_class is asyncioreactor.AsyncioSelectorReactor:
         with suppress(error.ReactorAlreadyInstalledError):
-            policy = get_asyncio_event_loop_policy()
-            if event_loop_path is not None:
-                event_loop_class = load_object(event_loop_path)
-                event_loop = event_loop_class()
-                asyncio.set_event_loop(event_loop)
-            else:
-                event_loop = policy.get_event_loop()
-
+            event_loop = set_asyncio_event_loop(event_loop_path)
             asyncioreactor.install(eventloop=event_loop)
     else:
         *module, _ = reactor_path.split(".")
@@ -87,6 +80,24 @@ def install_reactor(reactor_path, event_loop_path=None):
         with suppress(error.ReactorAlreadyInstalledError):
             installer()
 
+def set_asyncio_event_loop(event_loop_path):
+    """Sets and returns the event loop with specified import path."""
+    policy = get_asyncio_event_loop_policy()
+    if event_loop_path is not None:
+        event_loop_class = load_object(event_loop_path)
+        event_loop = event_loop_class()
+        asyncio.set_event_loop(event_loop)
+    else:
+        try:
+            event_loop = policy.get_event_loop()
+        except RuntimeError:
+            # `get_event_loop` is expected to fail when called from a new thread
+            # with no asyncio event loop yet installed. Such is the case when
+            # called from `scrapy shell`
+            event_loop = policy.new_event_loop()
+            asyncio.set_event_loop(event_loop)
+    return event_loop
+
 
 def verify_installed_reactor(reactor_path):
     """Raises :exc:`Exception` if the installed