Install the asyncio reactor only in scrapy.cmdline.

scrapy/__init__.py

@@ -23,28 +23,6 @@ import warnings
 warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
 del warnings
 
-# Install twisted asyncio loop
-def _install_asyncio_reactor():
-    global asyncio_supported
-    try:
-        import asyncio
-        from twisted.internet import asyncioreactor
-    except ImportError:
-        pass
-    else:
-        from twisted.internet.error import ReactorAlreadyInstalledError
-        try:
-            asyncioreactor.install(asyncio.get_event_loop())
-            asyncio_supported = True
-        except ReactorAlreadyInstalledError:
-            import twisted.internet.reactor
-            if isinstance(twisted.internet.reactor,
-                              asyncioreactor.AsyncioSelectorReactor):
-                asyncio_supported = True
-asyncio_supported = False
-_install_asyncio_reactor()
-del _install_asyncio_reactor
-
 # Apply monkey patches to fix issues in external libraries
 from . import _monkeypatches
 del _monkeypatches

scrapy/cmdline.py

@@ -7,9 +7,9 @@ import inspect
 import pkg_resources
 
 import scrapy
-from scrapy.crawler import CrawlerProcess
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
+from scrapy.utils.asyncio import install_asyncio_reactor, is_asyncio_supported
 from scrapy.utils.misc import walk_modules
 from scrapy.utils.project import inside_project, get_project_settings
 from scrapy.utils.python import garbage_collect
@@ -121,6 +121,10 @@ def execute(argv=None, settings=None):
             settings['EDITOR'] = editor
     check_deprecated_settings(settings)
 
+    # needs to be before _get_commands_dict() as that imports the command modules
+    # which may import twisted.internet.reactor
+    install_asyncio_reactor()
+
     inproject = inside_project()
     cmds = _get_commands_dict(settings, inproject)
     cmdname = _pop_command_name(argv)
@@ -142,6 +146,8 @@ def execute(argv=None, settings=None):
     opts, args = parser.parse_args(args=argv[1:])
     _run_print_help(parser, cmd.process_options, args, opts)
 
+    # needs to be after install_asyncio_reactor() as it imports twisted.internet.reactor
+    from scrapy.crawler import CrawlerProcess
     cmd.crawler_process = CrawlerProcess(settings)
     _run_print_help(parser, _run_command, cmd, args, opts)
     sys.exit(cmd.exitcode)

scrapy/utils/asyncio.py

@@ -0,0 +1,26 @@
+#coding: utf-8
+
+
+def install_asyncio_reactor():
+    """ Tries to install AsyncioSelectorReactor
+    """
+    try:
+        import asyncio
+        from twisted.internet import asyncioreactor
+    except ImportError:
+        pass
+    else:
+        from twisted.internet.error import ReactorAlreadyInstalledError
+        try:
+            asyncioreactor.install(asyncio.get_event_loop())
+        except ReactorAlreadyInstalledError:
+            pass
+
+
+def is_asyncio_supported():
+    try:
+        import twisted.internet.reactor
+        from twisted.internet import asyncioreactor
+        return isinstance(twisted.internet.reactor, asyncioreactor.AsyncioSelectorReactor)
+    except ImportError:
+        return False

scrapy/utils/defer.py

@@ -8,8 +8,9 @@ import inspect
 from twisted.internet import defer, reactor, task
 from twisted.python import failure
 
-from scrapy import asyncio_supported
 from scrapy.exceptions import IgnoreRequest
+from scrapy.utils.asyncio import is_asyncio_supported
+
 
 
 def defer_fail(_failure):
@@ -131,7 +132,7 @@ def deferred_from_coro(o):
     if isinstance(o, defer.Deferred):
         return o
     if asyncio.iscoroutine(o) or isfuture(o) or inspect.isawaitable(o):
-        if not asyncio_supported:
+        if not is_asyncio_supported():
             raise TypeError('Using coroutines requires installing AsyncioSelectorReactor')
         return defer.Deferred.fromFuture(asyncio.ensure_future(o))
     return o

tests/mockserver.py

@@ -7,8 +7,6 @@ from subprocess import Popen, PIPE
 from OpenSSL import SSL
 from six.moves.urllib.parse import urlencode
 
-import scrapy  # needed before importing twisted.internet.reactor
-
 from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.resource import Resource
 from twisted.web.static import File