Specify Twisted reactor (TWISTED_REACTOR setting) (#4294)

* Add the ability to install a specific reactor

* Add docs for the TWISTED_REACTOR setting

* Add tests for the TWISTED_REACTOR setting

* Update asyncio reactor test

* Ignore W503 globally

W503 is not PEP8-compliant:
c59c4376ad

* Line length adjustment

* Adjust asyncio reactor tests

* Merge ASYNCIO_ENABLED and TWISTED_REACTOR settings

* More docs about TWISTED_REACTOR

* Fix asyncio reactor test

* Docs: fix title

* Reword docs

* Check the TWISTED_REACTOR setting outside of the installing function

* Remove unrelated change

* Update scrapy/utils/log.py

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* Update docs/topics/settings.rst

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* Update docs/topics/settings.rst

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

Co-authored-by: Adrián Chaves <adrian@chaves.io>

docs/faq.rst

@@ -361,6 +361,17 @@ Note that by doing so, you lose the ability to set a specific timeout for DNS re
 (the value of the :setting:`DNS_TIMEOUT` setting is ignored).
 
 
+.. _faq-specific-reactor:
+
+How to deal with ``<class 'ValueError'>: filedescriptor out of range in select()`` exceptions?
+----------------------------------------------------------------------------------------------
+
+This issue `has been reported`_ to appear when running broad crawls in macOS, where the default
+Twisted reactor is :class:`twisted.internet.selectreactor.SelectReactor`. Switching to a
+different reactor is possible by using the :setting:`TWISTED_REACTOR` setting.
+
+
+.. _has been reported: https://github.com/scrapy/scrapy/issues/2905
 .. _user agents: https://en.wikipedia.org/wiki/User_agent
 .. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type)
 .. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search

docs/topics/broad-crawls.rst

@@ -211,3 +211,10 @@ If your broad crawl shows a high memory usage, in addition to :ref:`crawling in
 BFO order <broad-crawls-bfo>` and :ref:`lowering concurrency
 <broad-crawls-concurrency>` you should :ref:`debug your memory leaks
 <topics-leaks>`.
+
+
+Install a specific Twisted reactor
+==================================
+
+If the crawl is exceeding the system's capabilities, you might want to try
+installing a specific Twisted reactor, via the :setting:`TWISTED_REACTOR` setting.

docs/topics/settings.rst

@@ -160,27 +160,6 @@ to any particular component. In that case the module of that component will be
 shown, typically an extension, middleware or pipeline. It also means that the
 component must be enabled in order for the setting to have any effect.
 
-.. setting:: ASYNCIO_REACTOR
-
-ASYNCIO_REACTOR
----------------
-
-Default: ``False``
-
-Whether to install and require the Twisted reactor that uses the asyncio loop.
-
-When this option is set to ``True``, Scrapy will require
-:class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`. It will
-install this reactor if no reactor is installed yet, such as when using the
-``scrapy`` script or :class:`~scrapy.crawler.CrawlerProcess`. If you are using
-:class:`~scrapy.crawler.CrawlerRunner`, you need to install the correct reactor
-manually. If a different reactor is installed outside Scrapy, it will raise an
-exception.
-
-The default value for this option is currently ``False`` to maintain backward
-compatibility and avoid possible problems caused by using a different Twisted
-reactor.
-
 .. setting:: AWS_ACCESS_KEY_ID
 
 AWS_ACCESS_KEY_ID
@@ -1463,6 +1442,30 @@ command.
 The project name must not conflict with the name of custom files or directories
 in the ``project`` subdirectory.
 
+.. setting:: TWISTED_REACTOR
+
+TWISTED_REACTOR
+---------------
+
+Default: ``None``
+
+Import path of a given Twisted reactor, for instance:
+:class:`twisted.internet.asyncioreactor.AsyncioSelectorReactor`.
+
+Scrapy will install this reactor if no other is installed yet, such as when
+the ``scrapy`` CLI program is invoked or when using the
+:class:`~scrapy.crawler.CrawlerProcess` class. If you are using the
+:class:`~scrapy.crawler.CrawlerRunner` class, you need to install the correct
+reactor manually. An exception will be raised if the installation fails.
+
+The default value for this option is currently ``None``, which means that Scrapy
+will not attempt to install any specific reactor, and the default one defined by
+Twisted for the current platform will be used. This is to maintain backward
+compatibility and avoid possible problems caused by using a non-default reactor.
+
+For additional information, please see
+:doc:`core/howto/choosing-reactor`.
+
 
 .. setting:: URLLENGTH_LIMIT
 

pytest.ini

@@ -21,6 +21,7 @@ twisted = 1
 markers =
     only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
 flake8-ignore =
+    W503
     # Files that are only meant to provide top-level imports are expected not
     # to use any of their imports:
     scrapy/core/downloader/handlers/http.py F401
@@ -109,7 +110,7 @@ flake8-ignore =
     # scrapy/spidermiddlewares
     scrapy/spidermiddlewares/httperror.py E501
     scrapy/spidermiddlewares/offsite.py E501
-    scrapy/spidermiddlewares/referer.py E501 E129 W503 W504
+    scrapy/spidermiddlewares/referer.py E501 E129 W504
     scrapy/spidermiddlewares/urllength.py E501
     # scrapy/spiders
     scrapy/spiders/__init__.py E501 E402
@@ -129,13 +130,13 @@ flake8-ignore =
     scrapy/utils/http.py F403 E226
     scrapy/utils/httpobj.py E501
     scrapy/utils/iterators.py E501 E701
-    scrapy/utils/log.py E128 W503
+    scrapy/utils/log.py E128 E501
     scrapy/utils/markup.py F403
     scrapy/utils/misc.py E501 E226
     scrapy/utils/multipart.py F403
     scrapy/utils/project.py E501
     scrapy/utils/python.py E501
-    scrapy/utils/reactor.py E226
+    scrapy/utils/reactor.py E226 E501
     scrapy/utils/reqser.py E501
     scrapy/utils/request.py E127 E501
     scrapy/utils/response.py E501 E128

scrapy/crawler.py

@@ -13,7 +13,6 @@ from scrapy.extension import ExtensionManager
 from scrapy.interfaces import ISpiderLoader
 from scrapy.settings import overridden_settings, Settings
 from scrapy.signalmanager import SignalManager
-from scrapy.utils.asyncio import install_asyncio_reactor, is_asyncio_reactor_installed
 from scrapy.utils.log import (
     configure_logging,
     get_scrapy_root_handler,
@@ -23,6 +22,7 @@ from scrapy.utils.log import (
 )
 from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
+from scrapy.utils.reactor import install_reactor, verify_installed_reactor
 
 
 logger = logging.getLogger(__name__)
@@ -138,7 +138,7 @@ class CrawlerRunner:
         self._crawlers = set()
         self._active = set()
         self.bootstrap_failed = False
-        self._handle_asyncio_reactor()
+        self._handle_twisted_reactor()
 
     @property
     def spiders(self):
@@ -232,10 +232,9 @@ class CrawlerRunner:
         while self._active:
             yield defer.DeferredList(self._active)
 
-    def _handle_asyncio_reactor(self):
-        if self.settings.getbool('ASYNCIO_REACTOR') and not is_asyncio_reactor_installed():
-            raise Exception("ASYNCIO_REACTOR is on but the Twisted asyncio "
-                            "reactor is not installed.")
+    def _handle_twisted_reactor(self):
+        if self.settings.get("TWISTED_REACTOR"):
+            verify_installed_reactor(self.settings["TWISTED_REACTOR"])
 
 
 class CrawlerProcess(CrawlerRunner):
@@ -324,10 +323,10 @@ class CrawlerProcess(CrawlerRunner):
         except RuntimeError:  # raised if already stopped or in shutdown stage
             pass
 
-    def _handle_asyncio_reactor(self):
-        if self.settings.getbool('ASYNCIO_REACTOR'):
-            install_asyncio_reactor()
-        super()._handle_asyncio_reactor()
+    def _handle_twisted_reactor(self):
+        if self.settings.get("TWISTED_REACTOR"):
+            install_reactor(self.settings["TWISTED_REACTOR"])
+        super()._handle_twisted_reactor()
 
 
 def _get_spider_loader(settings):

scrapy/settings/default_settings.py

@@ -19,8 +19,6 @@ from os.path import join, abspath, dirname
 
 AJAXCRAWL_ENABLED = False
 
-ASYNCIO_REACTOR = False
-
 AUTOTHROTTLE_ENABLED = False
 AUTOTHROTTLE_DEBUG = False
 AUTOTHROTTLE_MAX_DELAY = 60.0
@@ -291,6 +289,8 @@ TELNETCONSOLE_HOST = '127.0.0.1'
 TELNETCONSOLE_USERNAME = 'scrapy'
 TELNETCONSOLE_PASSWORD = None
 
+TWISTED_REACTOR = None
+
 SPIDER_CONTRACTS = {}
 SPIDER_CONTRACTS_BASE = {
     'scrapy.contracts.default.UrlContract': 1,

scrapy/utils/asyncio.py

@@ -1,17 +0,0 @@
-import asyncio
-from contextlib import suppress
-
-from twisted.internet import asyncioreactor
-from twisted.internet.error import ReactorAlreadyInstalledError
-
-
-def install_asyncio_reactor():
-    """ Tries to install AsyncioSelectorReactor
-    """
-    with suppress(ReactorAlreadyInstalledError):
-        asyncioreactor.install(asyncio.get_event_loop())
-
-
-def is_asyncio_reactor_installed():
-    from twisted.internet import reactor
-    return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor)

scrapy/utils/defer.py

@@ -2,14 +2,14 @@
 Helper functions for dealing with Twisted deferreds
 """
 import asyncio
-from functools import wraps
 import inspect
+from functools import wraps
 
 from twisted.internet import defer, task
 from twisted.python import failure
 
 from scrapy.exceptions import IgnoreRequest
-from scrapy.utils.asyncio import is_asyncio_reactor_installed
+from scrapy.utils.reactor import is_asyncio_reactor_installed
 
 
 def defer_fail(_failure):

scrapy/utils/log.py

@@ -1,17 +1,16 @@
 # -*- coding: utf-8 -*-
 
-import sys
 import logging
+import sys
 import warnings
 from logging.config import dictConfig
 
-from twisted.python.failure import Failure
 from twisted.python import log as twisted_log
+from twisted.python.failure import Failure
 
 import scrapy
-from scrapy.settings import Settings
 from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.asyncio import is_asyncio_reactor_installed
+from scrapy.settings import Settings
 from scrapy.utils.versions import scrapy_components_versions
 
 
@@ -149,8 +148,8 @@ def log_scrapy_info(settings):
                 {'versions': ", ".join("%s %s" % (name, version)
                     for name, version in scrapy_components_versions()
                     if name != "Scrapy")})
-    if is_asyncio_reactor_installed():
-        logger.debug("Asyncio reactor is installed")
+    from twisted.internet import reactor
+    logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__)
 
 
 class StreamLogger(object):

scrapy/utils/reactor.py

@@ -1,4 +1,9 @@
-from twisted.internet import error
+import asyncio
+from contextlib import suppress
+
+from twisted.internet import asyncioreactor, error
+
+from scrapy.utils.misc import load_object
 
 
 def listen_tcp(portrange, host, factory):
@@ -42,3 +47,31 @@ class CallLaterOnce(object):
     def __call__(self):
         self._call = None
         return self._func(*self._a, **self._kw)
+
+
+def install_reactor(reactor_path):
+    reactor_class = load_object(reactor_path)
+    if reactor_class is asyncioreactor.AsyncioSelectorReactor:
+        with suppress(error.ReactorAlreadyInstalledError):
+            asyncioreactor.install(asyncio.get_event_loop())
+    else:
+        *module, _ = reactor_path.split(".")
+        installer_path = module + ["install"]
+        installer = load_object(".".join(installer_path))
+        with suppress(error.ReactorAlreadyInstalledError):
+            installer()
+
+
+def verify_installed_reactor(reactor_path):
+    from twisted.internet import reactor
+    reactor_class = load_object(reactor_path)
+    if not isinstance(reactor, reactor_class):
+        msg = "The installed reactor ({}.{}) does not match the requested one ({})".format(
+            reactor.__module__, reactor.__class__.__name__, reactor_path
+        )
+        raise Exception(msg)
+
+
+def is_asyncio_reactor_installed():
+    from twisted.internet import reactor
+    return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor)

tests/CrawlerProcess/asyncio_enabled_no_reactor.py

@@ -10,8 +10,7 @@ class NoRequestsSpider(scrapy.Spider):
 
 
 process = CrawlerProcess(settings={
-    'ASYNCIO_REACTOR': True,
+    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
 })
-
 process.crawl(NoRequestsSpider)
 process.start()

tests/CrawlerProcess/asyncio_enabled_reactor.py

@@ -15,8 +15,7 @@ class NoRequestsSpider(scrapy.Spider):
 
 
 process = CrawlerProcess(settings={
-    'ASYNCIO_REACTOR': True,
+    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
 })
-
 process.crawl(NoRequestsSpider)
 process.start()

tests/CrawlerProcess/twisted_reactor_asyncio.py

@@ -0,0 +1,13 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class AsyncioReactorSpider(scrapy.Spider):
+    name = 'asyncio_reactor'
+
+
+process = CrawlerProcess(settings={
+    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+})
+process.crawl(AsyncioReactorSpider)
+process.start()

tests/CrawlerProcess/twisted_reactor_poll.py

@@ -0,0 +1,13 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class PollReactorSpider(scrapy.Spider):
+    name = 'poll_reactor'
+
+
+process = CrawlerProcess(settings={
+    "TWISTED_REACTOR": "twisted.internet.pollreactor.PollReactor",
+})
+process.crawl(PollReactorSpider)
+process.start()

tests/CrawlerProcess/twisted_reactor_select.py

@@ -0,0 +1,13 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class SelectReactorSpider(scrapy.Spider):
+    name = 'epoll_reactor'
+
+
+process = CrawlerProcess(settings={
+    "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor",
+})
+process.crawl(SelectReactorSpider)
+process.start()

tests/test_commands.py

@@ -296,12 +296,14 @@ class BadSpider(scrapy.Spider):
         self.assertIn("badspider.py", log)
 
     def test_asyncio_enabled_true(self):
-        log = self.get_log(self.debug_log_spider, args=['-s', 'ASYNCIO_REACTOR=True'])
-        self.assertIn("DEBUG: Asyncio reactor is installed", log)
+        log = self.get_log(self.debug_log_spider, args=[
+            '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor'
+        ])
+        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
     def test_asyncio_enabled_false(self):
-        log = self.get_log(self.debug_log_spider, args=['-s', 'ASYNCIO_REACTOR=False'])
-        self.assertNotIn("DEBUG: Asyncio reactor is installed", log)
+        log = self.get_log(self.debug_log_spider, args=[])
+        self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
 
 class BenchCommandTest(CommandTest):

tests/test_crawler.py

@@ -254,30 +254,38 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
 
     def test_crawler_runner_asyncio_enabled_true(self):
         if self.reactor_pytest == 'asyncio':
-            runner = CrawlerRunner(settings={'ASYNCIO_REACTOR': True})
+            runner = CrawlerRunner(settings={
+                "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+            })
         else:
-            msg = "ASYNCIO_REACTOR is on but the Twisted asyncio reactor is not installed"
+            msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"
             with self.assertRaisesRegex(Exception, msg):
-                runner = CrawlerRunner(settings={'ASYNCIO_REACTOR': True})
+                runner = CrawlerRunner(settings={
+                    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+                })
 
     @defer.inlineCallbacks
     def test_crawler_process_asyncio_enabled_true(self):
         with LogCapture(level=logging.DEBUG) as log:
             if self.reactor_pytest == 'asyncio':
-                runner = CrawlerProcess(settings={'ASYNCIO_REACTOR': True})
+                runner = CrawlerProcess(settings={
+                    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+                })
                 yield runner.crawl(NoRequestsSpider)
-                self.assertIn("Asyncio reactor is installed", str(log))
+                self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log))
             else:
-                msg = "ASYNCIO_REACTOR is on but the Twisted asyncio reactor is not installed"
+                msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"
                 with self.assertRaisesRegex(Exception, msg):
-                    runner = CrawlerProcess(settings={'ASYNCIO_REACTOR': True})
+                    runner = CrawlerProcess(settings={
+                        "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+                    })
 
     @defer.inlineCallbacks
     def test_crawler_process_asyncio_enabled_false(self):
-        runner = CrawlerProcess(settings={'ASYNCIO_REACTOR': False})
+        runner = CrawlerProcess(settings={"TWISTED_REACTOR": None})
         with LogCapture(level=logging.DEBUG) as log:
             yield runner.crawl(NoRequestsSpider)
-            self.assertNotIn("Asyncio reactor is installed", str(log))
+            self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log))
 
 
 class CrawlerProcessSubprocess(unittest.TestCase):
@@ -294,17 +302,17 @@ class CrawlerProcessSubprocess(unittest.TestCase):
     def test_simple(self):
         log = self.run_script('simple.py')
         self.assertIn('Spider closed (finished)', log)
-        self.assertNotIn("DEBUG: Asyncio reactor is installed", log)
+        self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
     def test_asyncio_enabled_no_reactor(self):
         log = self.run_script('asyncio_enabled_no_reactor.py')
         self.assertIn('Spider closed (finished)', log)
-        self.assertIn("DEBUG: Asyncio reactor is installed", log)
+        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
     def test_asyncio_enabled_reactor(self):
         log = self.run_script('asyncio_enabled_reactor.py')
         self.assertIn('Spider closed (finished)', log)
-        self.assertIn("DEBUG: Asyncio reactor is installed", log)
+        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
     def test_ipv6_default_name_resolver(self):
         log = self.run_script('default_name_resolver.py')
@@ -323,3 +331,18 @@ class CrawlerProcessSubprocess(unittest.TestCase):
             "'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log,
             "'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log,
         ]))
+
+    def test_reactor_select(self):
+        log = self.run_script("twisted_reactor_select.py")
+        self.assertIn("Spider closed (finished)", log)
+        self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log)
+
+    def test_reactor_poll(self):
+        log = self.run_script("twisted_reactor_poll.py")
+        self.assertIn("Spider closed (finished)", log)
+        self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log)
+
+    def test_reactor_asyncio(self):
+        log = self.run_script("twisted_reactor_asyncio.py")
+        self.assertIn("Spider closed (finished)", log)
+        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)

tests/test_utils_asyncio.py

@@ -2,7 +2,7 @@ from unittest import TestCase
 
 from pytest import mark
 
-from scrapy.utils.asyncio import is_asyncio_reactor_installed, install_asyncio_reactor
+from scrapy.utils.reactor import is_asyncio_reactor_installed, install_reactor
 
 
 @mark.usefixtures('reactor_pytest')
@@ -14,4 +14,4 @@ class AsyncioTest(TestCase):
 
     def test_install_asyncio_reactor(self):
         # this should do nothing
-        install_asyncio_reactor()
+        install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")