Move a bunch of "from twisted.internet import reactor" inside functions.

scrapy/cmdline.py

@@ -6,6 +6,7 @@ import inspect
 import pkg_resources
 
 import scrapy
+from scrapy.crawler import CrawlerProcess
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
 from scrapy.utils.misc import walk_modules
@@ -140,9 +141,6 @@ def execute(argv=None, settings=None):
     opts, args = parser.parse_args(args=argv[1:])
     _run_print_help(parser, cmd.process_options, args, opts)
 
-    # needs to be after cmd.process_options() as it imports twisted.internet.reactor
-    # while commands may want to install the asyncio reactor
-    from scrapy.crawler import CrawlerProcess
     cmd.crawler_process = CrawlerProcess(settings)
     _run_print_help(parser, _run_command, cmd, args, opts)
     sys.exit(cmd.exitcode)

scrapy/commands/shell.py

@@ -7,6 +7,7 @@ from threading import Thread
 
 from scrapy.commands import ScrapyCommand
 from scrapy.http import Request
+from scrapy.shell import Shell
 from scrapy.utils.spider import spidercls_for_request, DefaultSpider
 from scrapy.utils.url import guess_scheme
 
@@ -69,8 +70,6 @@ class Command(ScrapyCommand):
 
         self._start_crawler_thread()
 
-        # moved from the top-level because it imports twisted.internet.reactor
-        from scrapy.shell import Shell
         shell = Shell(crawler, update_vars=self.update_vars, code=opts.code)
         shell.start(url=url, redirect=not opts.no_redirect)
 

scrapy/crawler.py

@@ -3,7 +3,7 @@ import pprint
 import signal
 import warnings
 
-from twisted.internet import reactor, defer
+from twisted.internet import defer
 from zope.interface.verify import verifyClass, DoesNotImplement
 
 from scrapy import Spider
@@ -261,6 +261,7 @@ class CrawlerProcess(CrawlerRunner):
         log_scrapy_info(self.settings)
 
     def _signal_shutdown(self, signum, _):
+        from twisted.internet import reactor
         install_shutdown_handlers(self._signal_kill)
         signame = signal_names[signum]
         logger.info("Received %(signame)s, shutting down gracefully. Send again to force ",
@@ -268,6 +269,7 @@ class CrawlerProcess(CrawlerRunner):
         reactor.callFromThread(self._graceful_stop_reactor)
 
     def _signal_kill(self, signum, _):
+        from twisted.internet import reactor
         install_shutdown_handlers(signal.SIG_IGN)
         signame = signal_names[signum]
         logger.info('Received %(signame)s twice, forcing unclean shutdown',
@@ -286,6 +288,7 @@ class CrawlerProcess(CrawlerRunner):
         :param boolean stop_after_crawl: stop or not the reactor when all
             crawlers have finished
         """
+        from twisted.internet import reactor
         if stop_after_crawl:
             d = self.join()
             # Don't start the reactor if the deferreds are already fired
@@ -300,6 +303,7 @@ class CrawlerProcess(CrawlerRunner):
         reactor.run(installSignalHandlers=False)  # blocking call
 
     def _get_dns_resolver(self):
+        from twisted.internet import reactor
         if self.settings.getbool('DNSCACHE_ENABLED'):
             cache_size = self.settings.getint('DNSCACHE_SIZE')
         else:
@@ -316,6 +320,7 @@ class CrawlerProcess(CrawlerRunner):
         return d
 
     def _stop_reactor(self, _=None):
+        from twisted.internet import reactor
         try:
             reactor.stop()
         except RuntimeError:  # raised if already stopped or in shutdown stage

scrapy/shell.py

@@ -7,7 +7,7 @@ import os
 import signal
 import warnings
 
-from twisted.internet import reactor, threads, defer
+from twisted.internet import threads, defer
 from twisted.python import threadable
 from w3lib.url import any_to_uri
 
@@ -98,6 +98,7 @@ class Shell(object):
         return spider
 
     def fetch(self, request_or_url, spider=None, redirect=True, **kwargs):
+        from twisted.internet import reactor
         if isinstance(request_or_url, Request):
             request = request_or_url
         else:

scrapy/utils/defer.py

@@ -5,7 +5,7 @@ import asyncio
 import asyncio.futures
 import inspect
 
-from twisted.internet import defer, reactor, task
+from twisted.internet import defer, task
 from twisted.python import failure
 
 from scrapy.exceptions import IgnoreRequest
@@ -19,6 +19,7 @@ def defer_fail(_failure):
     It delays by 100ms so reactor has a chance to go through readers and writers
     before attending pending delayed calls, so do not set delay to zero.
     """
+    from twisted.internet import reactor
     d = defer.Deferred()
     reactor.callLater(0.1, d.errback, _failure)
     return d
@@ -31,6 +32,7 @@ def defer_succeed(result):
     It delays by 100ms so reactor has a chance to go trough readers and writers
     before attending pending delayed calls, so do not set delay to zero.
     """
+    from twisted.internet import reactor
     d = defer.Deferred()
     reactor.callLater(0.1, d.callback, result)
     return d

scrapy/utils/ossignal.py

@@ -1,7 +1,5 @@
 import signal
 
-from twisted.internet import reactor
-
 
 signal_names = {}
 for signame in dir(signal):
@@ -17,6 +15,7 @@ def install_shutdown_handlers(function, override_sigint=True):
     SIGINT handler won't be install if there is already a handler in place
     (e.g.  Pdb)
     """
+    from twisted.internet import reactor
     reactor._handleSignals()
     signal.signal(signal.SIGTERM, function)
     if signal.getsignal(signal.SIGINT) == signal.default_int_handler or \

scrapy/utils/reactor.py

@@ -1,8 +1,9 @@
-from twisted.internet import reactor, error
+from twisted.internet import error
 
 
 def listen_tcp(portrange, host, factory):
     """Like reactor.listenTCP but tries different ports in a range."""
+    from twisted.internet import reactor
     assert len(portrange) <= 2, "invalid portrange: %s" % portrange
     if not portrange:
         return reactor.listenTCP(0, factory, interface=host)
@@ -30,6 +31,7 @@ class CallLaterOnce(object):
         self._call = None
 
     def schedule(self, delay=0):
+        from twisted.internet import reactor
         if self._call is None:
             self._call = reactor.callLater(delay, self)