From 35dafef7f106dbf1c022d997b4e29e3eee84de7c Mon Sep 17 00:00:00 2001 From: elacuesta Date: Thu, 6 Feb 2020 14:42:34 -0300 Subject: [PATCH] Specify Twisted reactor (TWISTED_REACTOR setting) (#4294) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add the ability to install a specific reactor * Add docs for the TWISTED_REACTOR setting * Add tests for the TWISTED_REACTOR setting * Update asyncio reactor test * Ignore W503 globally W503 is not PEP8-compliant: https://github.com/python/peps/commit/c59c4376ad233a62ca4b3a6060c81368bd21e85b * Line length adjustment * Adjust asyncio reactor tests * Merge ASYNCIO_ENABLED and TWISTED_REACTOR settings * More docs about TWISTED_REACTOR * Fix asyncio reactor test * Docs: fix title * Reword docs * Check the TWISTED_REACTOR setting outside of the installing function * Remove unrelated change * Update scrapy/utils/log.py Co-Authored-By: Adrián Chaves * Update docs/topics/settings.rst Co-Authored-By: Adrián Chaves * Update docs/topics/settings.rst Co-Authored-By: Adrián Chaves Co-authored-by: Adrián Chaves --- docs/faq.rst | 11 +++++ docs/topics/broad-crawls.rst | 7 +++ docs/topics/settings.rst | 45 +++++++++--------- pytest.ini | 7 +-- scrapy/crawler.py | 19 ++++---- scrapy/settings/default_settings.py | 4 +- scrapy/utils/asyncio.py | 17 ------- scrapy/utils/defer.py | 4 +- scrapy/utils/log.py | 11 ++--- scrapy/utils/reactor.py | 35 +++++++++++++- .../asyncio_enabled_no_reactor.py | 3 +- .../CrawlerProcess/asyncio_enabled_reactor.py | 3 +- .../CrawlerProcess/twisted_reactor_asyncio.py | 13 +++++ tests/CrawlerProcess/twisted_reactor_poll.py | 13 +++++ .../CrawlerProcess/twisted_reactor_select.py | 13 +++++ tests/test_commands.py | 10 ++-- tests/test_crawler.py | 47 ++++++++++++++----- tests/test_utils_asyncio.py | 4 +- 18 files changed, 182 insertions(+), 84 deletions(-) delete mode 100644 scrapy/utils/asyncio.py create mode 100644 tests/CrawlerProcess/twisted_reactor_asyncio.py create mode 100644 tests/CrawlerProcess/twisted_reactor_poll.py create mode 100644 tests/CrawlerProcess/twisted_reactor_select.py diff --git a/docs/faq.rst b/docs/faq.rst index b65908012..f72e4cf01 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -361,6 +361,17 @@ Note that by doing so, you lose the ability to set a specific timeout for DNS re (the value of the :setting:`DNS_TIMEOUT` setting is ignored). +.. _faq-specific-reactor: + +How to deal with ``: filedescriptor out of range in select()`` exceptions? +---------------------------------------------------------------------------------------------- + +This issue `has been reported`_ to appear when running broad crawls in macOS, where the default +Twisted reactor is :class:`twisted.internet.selectreactor.SelectReactor`. Switching to a +different reactor is possible by using the :setting:`TWISTED_REACTOR` setting. + + +.. _has been reported: https://github.com/scrapy/scrapy/issues/2905 .. _user agents: https://en.wikipedia.org/wiki/User_agent .. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type) .. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 1ab08d949..4922694ee 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -211,3 +211,10 @@ If your broad crawl shows a high memory usage, in addition to :ref:`crawling in BFO order ` and :ref:`lowering concurrency ` you should :ref:`debug your memory leaks `. + + +Install a specific Twisted reactor +================================== + +If the crawl is exceeding the system's capabilities, you might want to try +installing a specific Twisted reactor, via the :setting:`TWISTED_REACTOR` setting. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 4b770d249..fa63a5807 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -160,27 +160,6 @@ to any particular component. In that case the module of that component will be shown, typically an extension, middleware or pipeline. It also means that the component must be enabled in order for the setting to have any effect. -.. setting:: ASYNCIO_REACTOR - -ASYNCIO_REACTOR ---------------- - -Default: ``False`` - -Whether to install and require the Twisted reactor that uses the asyncio loop. - -When this option is set to ``True``, Scrapy will require -:class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`. It will -install this reactor if no reactor is installed yet, such as when using the -``scrapy`` script or :class:`~scrapy.crawler.CrawlerProcess`. If you are using -:class:`~scrapy.crawler.CrawlerRunner`, you need to install the correct reactor -manually. If a different reactor is installed outside Scrapy, it will raise an -exception. - -The default value for this option is currently ``False`` to maintain backward -compatibility and avoid possible problems caused by using a different Twisted -reactor. - .. setting:: AWS_ACCESS_KEY_ID AWS_ACCESS_KEY_ID @@ -1463,6 +1442,30 @@ command. The project name must not conflict with the name of custom files or directories in the ``project`` subdirectory. +.. setting:: TWISTED_REACTOR + +TWISTED_REACTOR +--------------- + +Default: ``None`` + +Import path of a given Twisted reactor, for instance: +:class:`twisted.internet.asyncioreactor.AsyncioSelectorReactor`. + +Scrapy will install this reactor if no other is installed yet, such as when +the ``scrapy`` CLI program is invoked or when using the +:class:`~scrapy.crawler.CrawlerProcess` class. If you are using the +:class:`~scrapy.crawler.CrawlerRunner` class, you need to install the correct +reactor manually. An exception will be raised if the installation fails. + +The default value for this option is currently ``None``, which means that Scrapy +will not attempt to install any specific reactor, and the default one defined by +Twisted for the current platform will be used. This is to maintain backward +compatibility and avoid possible problems caused by using a non-default reactor. + +For additional information, please see +:doc:`core/howto/choosing-reactor`. + .. setting:: URLLENGTH_LIMIT diff --git a/pytest.ini b/pytest.ini index bae68cd3a..552829d4e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -21,6 +21,7 @@ twisted = 1 markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed flake8-ignore = + W503 # Files that are only meant to provide top-level imports are expected not # to use any of their imports: scrapy/core/downloader/handlers/http.py F401 @@ -109,7 +110,7 @@ flake8-ignore = # scrapy/spidermiddlewares scrapy/spidermiddlewares/httperror.py E501 scrapy/spidermiddlewares/offsite.py E501 - scrapy/spidermiddlewares/referer.py E501 E129 W503 W504 + scrapy/spidermiddlewares/referer.py E501 E129 W504 scrapy/spidermiddlewares/urllength.py E501 # scrapy/spiders scrapy/spiders/__init__.py E501 E402 @@ -129,13 +130,13 @@ flake8-ignore = scrapy/utils/http.py F403 E226 scrapy/utils/httpobj.py E501 scrapy/utils/iterators.py E501 E701 - scrapy/utils/log.py E128 W503 + scrapy/utils/log.py E128 E501 scrapy/utils/markup.py F403 scrapy/utils/misc.py E501 E226 scrapy/utils/multipart.py F403 scrapy/utils/project.py E501 scrapy/utils/python.py E501 - scrapy/utils/reactor.py E226 + scrapy/utils/reactor.py E226 E501 scrapy/utils/reqser.py E501 scrapy/utils/request.py E127 E501 scrapy/utils/response.py E501 E128 diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 35c6b7716..49b8e4511 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -13,7 +13,6 @@ from scrapy.extension import ExtensionManager from scrapy.interfaces import ISpiderLoader from scrapy.settings import overridden_settings, Settings from scrapy.signalmanager import SignalManager -from scrapy.utils.asyncio import install_asyncio_reactor, is_asyncio_reactor_installed from scrapy.utils.log import ( configure_logging, get_scrapy_root_handler, @@ -23,6 +22,7 @@ from scrapy.utils.log import ( ) from scrapy.utils.misc import create_instance, load_object from scrapy.utils.ossignal import install_shutdown_handlers, signal_names +from scrapy.utils.reactor import install_reactor, verify_installed_reactor logger = logging.getLogger(__name__) @@ -138,7 +138,7 @@ class CrawlerRunner: self._crawlers = set() self._active = set() self.bootstrap_failed = False - self._handle_asyncio_reactor() + self._handle_twisted_reactor() @property def spiders(self): @@ -232,10 +232,9 @@ class CrawlerRunner: while self._active: yield defer.DeferredList(self._active) - def _handle_asyncio_reactor(self): - if self.settings.getbool('ASYNCIO_REACTOR') and not is_asyncio_reactor_installed(): - raise Exception("ASYNCIO_REACTOR is on but the Twisted asyncio " - "reactor is not installed.") + def _handle_twisted_reactor(self): + if self.settings.get("TWISTED_REACTOR"): + verify_installed_reactor(self.settings["TWISTED_REACTOR"]) class CrawlerProcess(CrawlerRunner): @@ -324,10 +323,10 @@ class CrawlerProcess(CrawlerRunner): except RuntimeError: # raised if already stopped or in shutdown stage pass - def _handle_asyncio_reactor(self): - if self.settings.getbool('ASYNCIO_REACTOR'): - install_asyncio_reactor() - super()._handle_asyncio_reactor() + def _handle_twisted_reactor(self): + if self.settings.get("TWISTED_REACTOR"): + install_reactor(self.settings["TWISTED_REACTOR"]) + super()._handle_twisted_reactor() def _get_spider_loader(settings): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index c10dc1a1c..c80833710 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -19,8 +19,6 @@ from os.path import join, abspath, dirname AJAXCRAWL_ENABLED = False -ASYNCIO_REACTOR = False - AUTOTHROTTLE_ENABLED = False AUTOTHROTTLE_DEBUG = False AUTOTHROTTLE_MAX_DELAY = 60.0 @@ -291,6 +289,8 @@ TELNETCONSOLE_HOST = '127.0.0.1' TELNETCONSOLE_USERNAME = 'scrapy' TELNETCONSOLE_PASSWORD = None +TWISTED_REACTOR = None + SPIDER_CONTRACTS = {} SPIDER_CONTRACTS_BASE = { 'scrapy.contracts.default.UrlContract': 1, diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py deleted file mode 100644 index 917973de2..000000000 --- a/scrapy/utils/asyncio.py +++ /dev/null @@ -1,17 +0,0 @@ -import asyncio -from contextlib import suppress - -from twisted.internet import asyncioreactor -from twisted.internet.error import ReactorAlreadyInstalledError - - -def install_asyncio_reactor(): - """ Tries to install AsyncioSelectorReactor - """ - with suppress(ReactorAlreadyInstalledError): - asyncioreactor.install(asyncio.get_event_loop()) - - -def is_asyncio_reactor_installed(): - from twisted.internet import reactor - return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 62b43a96c..6a21a4903 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -2,14 +2,14 @@ Helper functions for dealing with Twisted deferreds """ import asyncio -from functools import wraps import inspect +from functools import wraps from twisted.internet import defer, task from twisted.python import failure from scrapy.exceptions import IgnoreRequest -from scrapy.utils.asyncio import is_asyncio_reactor_installed +from scrapy.utils.reactor import is_asyncio_reactor_installed def defer_fail(_failure): diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index e4cf0196b..afef2c93f 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -1,17 +1,16 @@ # -*- coding: utf-8 -*- -import sys import logging +import sys import warnings from logging.config import dictConfig -from twisted.python.failure import Failure from twisted.python import log as twisted_log +from twisted.python.failure import Failure import scrapy -from scrapy.settings import Settings from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.asyncio import is_asyncio_reactor_installed +from scrapy.settings import Settings from scrapy.utils.versions import scrapy_components_versions @@ -149,8 +148,8 @@ def log_scrapy_info(settings): {'versions': ", ".join("%s %s" % (name, version) for name, version in scrapy_components_versions() if name != "Scrapy")}) - if is_asyncio_reactor_installed(): - logger.debug("Asyncio reactor is installed") + from twisted.internet import reactor + logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__) class StreamLogger(object): diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index b98fff6ec..80f52a4ef 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -1,4 +1,9 @@ -from twisted.internet import error +import asyncio +from contextlib import suppress + +from twisted.internet import asyncioreactor, error + +from scrapy.utils.misc import load_object def listen_tcp(portrange, host, factory): @@ -42,3 +47,31 @@ class CallLaterOnce(object): def __call__(self): self._call = None return self._func(*self._a, **self._kw) + + +def install_reactor(reactor_path): + reactor_class = load_object(reactor_path) + if reactor_class is asyncioreactor.AsyncioSelectorReactor: + with suppress(error.ReactorAlreadyInstalledError): + asyncioreactor.install(asyncio.get_event_loop()) + else: + *module, _ = reactor_path.split(".") + installer_path = module + ["install"] + installer = load_object(".".join(installer_path)) + with suppress(error.ReactorAlreadyInstalledError): + installer() + + +def verify_installed_reactor(reactor_path): + from twisted.internet import reactor + reactor_class = load_object(reactor_path) + if not isinstance(reactor, reactor_class): + msg = "The installed reactor ({}.{}) does not match the requested one ({})".format( + reactor.__module__, reactor.__class__.__name__, reactor_path + ) + raise Exception(msg) + + +def is_asyncio_reactor_installed(): + from twisted.internet import reactor + return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) diff --git a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py index db1b75931..d1e4a7bb5 100644 --- a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py @@ -10,8 +10,7 @@ class NoRequestsSpider(scrapy.Spider): process = CrawlerProcess(settings={ - 'ASYNCIO_REACTOR': True, + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", }) - process.crawl(NoRequestsSpider) process.start() diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index cec3c9c25..8568bd8b8 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -15,8 +15,7 @@ class NoRequestsSpider(scrapy.Spider): process = CrawlerProcess(settings={ - 'ASYNCIO_REACTOR': True, + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", }) - process.crawl(NoRequestsSpider) process.start() diff --git a/tests/CrawlerProcess/twisted_reactor_asyncio.py b/tests/CrawlerProcess/twisted_reactor_asyncio.py new file mode 100644 index 000000000..c6cbf949b --- /dev/null +++ b/tests/CrawlerProcess/twisted_reactor_asyncio.py @@ -0,0 +1,13 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class AsyncioReactorSpider(scrapy.Spider): + name = 'asyncio_reactor' + + +process = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", +}) +process.crawl(AsyncioReactorSpider) +process.start() diff --git a/tests/CrawlerProcess/twisted_reactor_poll.py b/tests/CrawlerProcess/twisted_reactor_poll.py new file mode 100644 index 000000000..27063260b --- /dev/null +++ b/tests/CrawlerProcess/twisted_reactor_poll.py @@ -0,0 +1,13 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class PollReactorSpider(scrapy.Spider): + name = 'poll_reactor' + + +process = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.pollreactor.PollReactor", +}) +process.crawl(PollReactorSpider) +process.start() diff --git a/tests/CrawlerProcess/twisted_reactor_select.py b/tests/CrawlerProcess/twisted_reactor_select.py new file mode 100644 index 000000000..9af8ceb4d --- /dev/null +++ b/tests/CrawlerProcess/twisted_reactor_select.py @@ -0,0 +1,13 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class SelectReactorSpider(scrapy.Spider): + name = 'epoll_reactor' + + +process = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor", +}) +process.crawl(SelectReactorSpider) +process.start() diff --git a/tests/test_commands.py b/tests/test_commands.py index 6024af71c..3612b70c9 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -296,12 +296,14 @@ class BadSpider(scrapy.Spider): self.assertIn("badspider.py", log) def test_asyncio_enabled_true(self): - log = self.get_log(self.debug_log_spider, args=['-s', 'ASYNCIO_REACTOR=True']) - self.assertIn("DEBUG: Asyncio reactor is installed", log) + log = self.get_log(self.debug_log_spider, args=[ + '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor' + ]) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) def test_asyncio_enabled_false(self): - log = self.get_log(self.debug_log_spider, args=['-s', 'ASYNCIO_REACTOR=False']) - self.assertNotIn("DEBUG: Asyncio reactor is installed", log) + log = self.get_log(self.debug_log_spider, args=[]) + self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) class BenchCommandTest(CommandTest): diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 0ce0674de..f8fa26def 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -254,30 +254,38 @@ class CrawlerRunnerHasSpider(unittest.TestCase): def test_crawler_runner_asyncio_enabled_true(self): if self.reactor_pytest == 'asyncio': - runner = CrawlerRunner(settings={'ASYNCIO_REACTOR': True}) + runner = CrawlerRunner(settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + }) else: - msg = "ASYNCIO_REACTOR is on but the Twisted asyncio reactor is not installed" + msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" with self.assertRaisesRegex(Exception, msg): - runner = CrawlerRunner(settings={'ASYNCIO_REACTOR': True}) + runner = CrawlerRunner(settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + }) @defer.inlineCallbacks def test_crawler_process_asyncio_enabled_true(self): with LogCapture(level=logging.DEBUG) as log: if self.reactor_pytest == 'asyncio': - runner = CrawlerProcess(settings={'ASYNCIO_REACTOR': True}) + runner = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + }) yield runner.crawl(NoRequestsSpider) - self.assertIn("Asyncio reactor is installed", str(log)) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log)) else: - msg = "ASYNCIO_REACTOR is on but the Twisted asyncio reactor is not installed" + msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" with self.assertRaisesRegex(Exception, msg): - runner = CrawlerProcess(settings={'ASYNCIO_REACTOR': True}) + runner = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + }) @defer.inlineCallbacks def test_crawler_process_asyncio_enabled_false(self): - runner = CrawlerProcess(settings={'ASYNCIO_REACTOR': False}) + runner = CrawlerProcess(settings={"TWISTED_REACTOR": None}) with LogCapture(level=logging.DEBUG) as log: yield runner.crawl(NoRequestsSpider) - self.assertNotIn("Asyncio reactor is installed", str(log)) + self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log)) class CrawlerProcessSubprocess(unittest.TestCase): @@ -294,17 +302,17 @@ class CrawlerProcessSubprocess(unittest.TestCase): def test_simple(self): log = self.run_script('simple.py') self.assertIn('Spider closed (finished)', log) - self.assertNotIn("DEBUG: Asyncio reactor is installed", log) + self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) def test_asyncio_enabled_no_reactor(self): log = self.run_script('asyncio_enabled_no_reactor.py') self.assertIn('Spider closed (finished)', log) - self.assertIn("DEBUG: Asyncio reactor is installed", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) def test_asyncio_enabled_reactor(self): log = self.run_script('asyncio_enabled_reactor.py') self.assertIn('Spider closed (finished)', log) - self.assertIn("DEBUG: Asyncio reactor is installed", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) def test_ipv6_default_name_resolver(self): log = self.run_script('default_name_resolver.py') @@ -323,3 +331,18 @@ class CrawlerProcessSubprocess(unittest.TestCase): "'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log, "'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log, ])) + + def test_reactor_select(self): + log = self.run_script("twisted_reactor_select.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log) + + def test_reactor_poll(self): + log = self.run_script("twisted_reactor_poll.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log) + + def test_reactor_asyncio(self): + log = self.run_script("twisted_reactor_asyncio.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 44acc24af..295323e4d 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -2,7 +2,7 @@ from unittest import TestCase from pytest import mark -from scrapy.utils.asyncio import is_asyncio_reactor_installed, install_asyncio_reactor +from scrapy.utils.reactor import is_asyncio_reactor_installed, install_reactor @mark.usefixtures('reactor_pytest') @@ -14,4 +14,4 @@ class AsyncioTest(TestCase): def test_install_asyncio_reactor(self): # this should do nothing - install_asyncio_reactor() + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")