From b04b541372b219a02bb5fda2cc15cfd9fa1aac66 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 21 Aug 2019 17:14:46 +0500
Subject: [PATCH] Install the asyncio reactor only in scrapy.cmdline.

---
 scrapy/__init__.py      | 22 ----------------------
 scrapy/cmdline.py       |  8 +++++++-
 scrapy/utils/asyncio.py | 26 ++++++++++++++++++++++++++
 scrapy/utils/defer.py   |  5 +++--
 tests/mockserver.py     |  2 --
 5 files changed, 36 insertions(+), 27 deletions(-)
 create mode 100644 scrapy/utils/asyncio.py

diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index 41eaee959..230e5cee3 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -23,28 +23,6 @@ import warnings
 warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
 del warnings
 
-# Install twisted asyncio loop
-def _install_asyncio_reactor():
-    global asyncio_supported
-    try:
-        import asyncio
-        from twisted.internet import asyncioreactor
-    except ImportError:
-        pass
-    else:
-        from twisted.internet.error import ReactorAlreadyInstalledError
-        try:
-            asyncioreactor.install(asyncio.get_event_loop())
-            asyncio_supported = True
-        except ReactorAlreadyInstalledError:
-            import twisted.internet.reactor
-            if isinstance(twisted.internet.reactor,
-                              asyncioreactor.AsyncioSelectorReactor):
-                asyncio_supported = True
-asyncio_supported = False
-_install_asyncio_reactor()
-del _install_asyncio_reactor
-
 # Apply monkey patches to fix issues in external libraries
 from . import _monkeypatches
 del _monkeypatches
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 418dc1ac9..d66f0cc2d 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -7,9 +7,9 @@ import inspect
 import pkg_resources
 
 import scrapy
-from scrapy.crawler import CrawlerProcess
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
+from scrapy.utils.asyncio import install_asyncio_reactor, is_asyncio_supported
 from scrapy.utils.misc import walk_modules
 from scrapy.utils.project import inside_project, get_project_settings
 from scrapy.utils.python import garbage_collect
@@ -121,6 +121,10 @@ def execute(argv=None, settings=None):
             settings['EDITOR'] = editor
     check_deprecated_settings(settings)
 
+    # needs to be before _get_commands_dict() as that imports the command modules
+    # which may import twisted.internet.reactor
+    install_asyncio_reactor()
+
     inproject = inside_project()
     cmds = _get_commands_dict(settings, inproject)
     cmdname = _pop_command_name(argv)
@@ -142,6 +146,8 @@ def execute(argv=None, settings=None):
     opts, args = parser.parse_args(args=argv[1:])
     _run_print_help(parser, cmd.process_options, args, opts)
 
+    # needs to be after install_asyncio_reactor() as it imports twisted.internet.reactor
+    from scrapy.crawler import CrawlerProcess
     cmd.crawler_process = CrawlerProcess(settings)
     _run_print_help(parser, _run_command, cmd, args, opts)
     sys.exit(cmd.exitcode)
diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py
new file mode 100644
index 000000000..e9e3bdd88
--- /dev/null
+++ b/scrapy/utils/asyncio.py
@@ -0,0 +1,26 @@
+#coding: utf-8
+
+
+def install_asyncio_reactor():
+    """ Tries to install AsyncioSelectorReactor
+    """
+    try:
+        import asyncio
+        from twisted.internet import asyncioreactor
+    except ImportError:
+        pass
+    else:
+        from twisted.internet.error import ReactorAlreadyInstalledError
+        try:
+            asyncioreactor.install(asyncio.get_event_loop())
+        except ReactorAlreadyInstalledError:
+            pass
+
+
+def is_asyncio_supported():
+    try:
+        import twisted.internet.reactor
+        from twisted.internet import asyncioreactor
+        return isinstance(twisted.internet.reactor, asyncioreactor.AsyncioSelectorReactor)
+    except ImportError:
+        return False
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 1f6a2584c..955fc820a 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -8,8 +8,9 @@ import inspect
 from twisted.internet import defer, reactor, task
 from twisted.python import failure
 
-from scrapy import asyncio_supported
 from scrapy.exceptions import IgnoreRequest
+from scrapy.utils.asyncio import is_asyncio_supported
+
 
 
 def defer_fail(_failure):
@@ -131,7 +132,7 @@ def deferred_from_coro(o):
     if isinstance(o, defer.Deferred):
         return o
     if asyncio.iscoroutine(o) or isfuture(o) or inspect.isawaitable(o):
-        if not asyncio_supported:
+        if not is_asyncio_supported():
             raise TypeError('Using coroutines requires installing AsyncioSelectorReactor')
         return defer.Deferred.fromFuture(asyncio.ensure_future(o))
     return o
diff --git a/tests/mockserver.py b/tests/mockserver.py
index b6aee009a..d09fbc171 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -7,8 +7,6 @@ from subprocess import Popen, PIPE
 from OpenSSL import SSL
 from six.moves.urllib.parse import urlencode
 
-import scrapy  # needed before importing twisted.internet.reactor
-
 from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.resource import Resource
 from twisted.web.static import File