Remove all top-level imports for twisted.internet.reactor (#4406)

docs/topics/settings.rst

@@ -1473,7 +1473,66 @@ If a reactor is already installed,
 
 :meth:`CrawlerRunner.__init__ <scrapy.crawler.CrawlerRunner.__init__>` raises
 :exc:`Exception` if the installed reactor does not match the
-:setting:`TWISTED_REACTOR` setting.
+:setting:`TWISTED_REACTOR` setting; therfore, having top-level
+:mod:`~twisted.internet.reactor` imports in project files and imported
+third-party libraries will make Scrapy raise :exc:`Exception` when
+it checks which reactor is installed.
+
+In order to use the reactor installed by Scrapy::
+
+    import scrapy
+    from twisted.internet import reactor
+
+
+    class QuotesSpider(scrapy.Spider):
+        name = 'quotes'
+
+        def __init__(self, *args, **kwargs):
+            self.timeout = int(kwargs.pop('timeout', '60'))
+            super(QuotesSpider, self).__init__(*args, **kwargs)
+
+        def start_requests(self):
+            reactor.callLater(self.timeout, self.stop)
+
+            urls = ['http://quotes.toscrape.com/page/1']
+            for url in urls:
+                yield scrapy.Request(url=url, callback=self.parse)
+
+        def parse(self, response):
+            for quote in response.css('div.quote'):
+                yield {'text': quote.css('span.text::text').get()}
+
+        def stop(self):
+            self.crawler.engine.close_spider(self, 'timeout')
+
+
+which raises :exc:`Exception`, becomes::
+
+    import scrapy
+
+
+    class QuotesSpider(scrapy.Spider):
+        name = 'quotes'
+
+        def __init__(self, *args, **kwargs):
+            self.timeout = int(kwargs.pop('timeout', '60'))
+            super(QuotesSpider, self).__init__(*args, **kwargs)
+
+        def start_requests(self):
+            from twisted.internet import reactor
+            reactor.callLater(self.timeout, self.stop)
+
+            urls = ['http://quotes.toscrape.com/page/1']
+            for url in urls:
+                yield scrapy.Request(url=url, callback=self.parse)
+
+        def parse(self, response):
+            for quote in response.css('div.quote'):
+                yield {'text': quote.css('span.text::text').get()}
+
+        def stop(self):
+            self.crawler.engine.close_spider(self, 'timeout')
+
 
 The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which
 means that Scrapy will not attempt to install any specific reactor, and the

scrapy/core/downloader/__init__.py

@@ -3,7 +3,7 @@ from time import time
 from datetime import datetime
 from collections import deque
 
-from twisted.internet import reactor, defer, task
+from twisted.internet import defer, task
 
 from scrapy.utils.defer import mustbe_deferred
 from scrapy.utils.httpobj import urlparse_cached
@@ -133,6 +133,7 @@ class Downloader(object):
         return deferred
 
     def _process_queue(self, spider, slot):
+        from twisted.internet import reactor
         if slot.latercall and slot.latercall.active():
             return
 

scrapy/core/downloader/handlers/ftp.py

@@ -32,7 +32,6 @@ import re
 from io import BytesIO
 from urllib.parse import unquote
 
-from twisted.internet import reactor
 from twisted.internet.protocol import ClientCreator, Protocol
 from twisted.protocols.ftp import CommandFailed, FTPClient
 
@@ -81,6 +80,7 @@ class FTPDownloadHandler:
         return cls(crawler.settings)
 
     def download_request(self, request, spider):
+        from twisted.internet import reactor
         parsed_url = urlparse_cached(request)
         user = request.meta.get("ftp_user", self.default_user)
         password = request.meta.get("ftp_password", self.default_password)

scrapy/core/downloader/handlers/http10.py

@@ -1,7 +1,5 @@
 """Download handlers for http and https schemes
 """
-from twisted.internet import reactor
-
 from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.python import to_unicode
 
@@ -26,6 +24,7 @@ class HTTP10DownloadHandler:
         return factory.deferred
 
     def _connect(self, factory):
+        from twisted.internet import reactor
         host, port = to_unicode(factory.host), factory.port
         if factory.scheme == b'https':
             client_context_factory = create_instance(

scrapy/core/downloader/handlers/http11.py

@@ -8,7 +8,7 @@ from io import BytesIO
 from time import time
 from urllib.parse import urldefrag
 
-from twisted.internet import defer, protocol, reactor, ssl
+from twisted.internet import defer, protocol, ssl
 from twisted.internet.endpoints import TCP4ClientEndpoint
 from twisted.internet.error import TimeoutError
 from twisted.web.client import Agent, HTTPConnectionPool, ResponseDone, ResponseFailed, URI
@@ -33,6 +33,7 @@ class HTTP11DownloadHandler:
     lazy = False
 
     def __init__(self, settings, crawler=None):
+        from twisted.internet import reactor
         self._pool = HTTPConnectionPool(reactor, persistent=True)
         self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
         self._pool._factory.noisy = False
@@ -81,6 +82,7 @@ class HTTP11DownloadHandler:
         return agent.download_request(request)
 
     def close(self):
+        from twisted.internet import reactor
         d = self._pool.closeCachedConnections()
         # closeCachedConnections will hang on network or server issues, so
         # we'll manually timeout the deferred.
@@ -284,6 +286,7 @@ class ScrapyAgent(object):
         self._txresponse = None
 
     def _get_agent(self, request, timeout):
+        from twisted.internet import reactor
         bindaddress = request.meta.get('bindaddress') or self._bindAddress
         proxy = request.meta.get('proxy')
         if proxy:
@@ -326,6 +329,7 @@ class ScrapyAgent(object):
         )
 
     def download_request(self, request):
+        from twisted.internet import reactor
         timeout = request.meta.get('download_timeout') or self._connectTimeout
         agent = self._get_agent(request, timeout)
 

scrapy/extensions/closespider.py

@@ -6,8 +6,6 @@ See documentation in docs/topics/extensions.rst
 
 from collections import defaultdict
 
-from twisted.internet import reactor
-
 from scrapy import signals
 from scrapy.exceptions import NotConfigured
 
@@ -54,6 +52,7 @@ class CloseSpider(object):
             self.crawler.engine.close_spider(spider, 'closespider_pagecount')
 
     def spider_opened(self, spider):
+        from twisted.internet import reactor
         self.task = reactor.callLater(self.close_on['timeout'],
                                       self.crawler.engine.close_spider, spider,
                                       reason='closespider_timeout')

scrapy/mail.py

@@ -12,7 +12,7 @@ from email.mime.text import MIMEText
 from email.utils import COMMASPACE, formatdate
 from io import BytesIO
 
-from twisted.internet import defer, reactor, ssl
+from twisted.internet import defer, ssl
 
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.python import to_bytes
@@ -28,7 +28,6 @@ def _to_bytes_or_none(text):
 
 
 class MailSender(object):
-
     def __init__(self, smtphost='localhost', mailfrom='scrapy@localhost',
             smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False):
         self.smtphost = smtphost
@@ -47,6 +46,7 @@ class MailSender(object):
             settings.getbool('MAIL_TLS'), settings.getbool('MAIL_SSL'))
 
     def send(self, to, subject, body, cc=None, attachs=(), mimetype='text/plain', charset=None, _callback=None):
+        from twisted.internet import reactor
         if attachs:
             msg = MIMEMultipart()
         else:
@@ -111,6 +111,7 @@ class MailSender(object):
 
     def _sendmail(self, to_addrs, msg):
         # Import twisted.mail here because it is not available in python3
+        from twisted.internet import reactor
         from twisted.mail.smtp import ESMTPSenderFactory
         msg = BytesIO(msg)
         d = defer.Deferred()

scrapy/utils/benchserver.py

@@ -3,7 +3,6 @@ from urllib.parse import urlencode
 
 from twisted.web.server import Site
 from twisted.web.resource import Resource
-from twisted.internet import reactor
 
 
 class Root(Resource):
@@ -34,6 +33,7 @@ def _getarg(request, name, default=None, type=str):
 
 
 if __name__ == '__main__':
+    from twisted.internet import reactor
     root = Root()
     factory = Site(root)
     httpPort = reactor.listenTCP(8998, Site(root))

scrapy/utils/testproc.py

@@ -1,7 +1,7 @@
 import sys
 import os
 
-from twisted.internet import reactor, defer, protocol
+from twisted.internet import defer, protocol
 
 
 class ProcessTest(object):
@@ -11,6 +11,7 @@ class ProcessTest(object):
     cwd = os.getcwd()  # trial chdirs to temp dir
 
     def execute(self, args, check_code=True, settings=None):
+        from twisted.internet import reactor
         env = os.environ.copy()
         if settings is not None:
             env['SCRAPY_SETTINGS_MODULE'] = settings

scrapy/utils/testsite.py

@@ -1,12 +1,12 @@
 from urllib.parse import urljoin
 
-from twisted.internet import reactor
 from twisted.web import server, resource, static, util
 
 
 class SiteTest(object):
 
     def setUp(self):
+        from twisted.internet import reactor
         super(SiteTest, self).setUp()
         self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
         self.baseurl = "http://localhost:%d/" % self.site.getHost().port
@@ -38,6 +38,7 @@ def test_site():
 
 
 if __name__ == '__main__':
+    from twisted.internet import reactor
     port = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
     print("http://localhost:%d/" % port.getHost().port)
     reactor.run()