diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 038a459fd..bfb430d52 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -26,3 +26,15 @@ reactor manually. You can do that using :func:`~scrapy.utils.reactor.install_reactor`:: install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor') + +.. _using-custom-loops: + +Using custom asyncio loops +========================== + +You can also use custom asyncio event loops with the asyncio reactor. Set the +:setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to +use it instead of the default asyncio event loop. + + + diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 722ae4593..618b9989e 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -216,6 +216,26 @@ Default: ``None`` The name of the region associated with the AWS client. +.. setting:: ASYNCIO_EVENT_LOOP + +ASYNCIO_EVENT_LOOP +------------------ + +Default: ``None`` + +Import path of a given asyncio event loop class. + +If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the +asyncio event loop to be used with it. Set the setting to the import path of the +desired asyncio event loop class. If the setting is set to ``None`` the default asyncio +event loop will be used. + +If you are installing the asyncio reactor manually using the :func:`~scrapy.utils.reactor.install_reactor` +function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop +class to be used. + +Note that the event loop class must inherit from :class:`asyncio.AbstractEventLoop`. + .. setting:: BOT_NAME BOT_NAME diff --git a/scrapy/crawler.py b/scrapy/crawler.py index d028bea4d..4c6b0e496 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -340,5 +340,5 @@ class CrawlerProcess(CrawlerRunner): def _handle_twisted_reactor(self): if self.settings.get("TWISTED_REACTOR"): - install_reactor(self.settings["TWISTED_REACTOR"]) + install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"]) super()._handle_twisted_reactor() diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 896afa995..a0251394b 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -19,6 +19,8 @@ from os.path import join, abspath, dirname AJAXCRAWL_ENABLED = False +ASYNCIO_EVENT_LOOP = None + AUTOTHROTTLE_ENABLED = False AUTOTHROTTLE_DEBUG = False AUTOTHROTTLE_MAX_DELAY = 60.0 diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 1d6a2c39d..e41315738 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -150,6 +150,13 @@ def log_scrapy_info(settings): logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)}) from twisted.internet import reactor logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__) + from twisted.internet import asyncioreactor + if isinstance(reactor, asyncioreactor.AsyncioSelectorReactor): + logger.debug( + "Using asyncio event loop: %s.%s", + reactor._asyncioEventloop.__module__, + reactor._asyncioEventloop.__class__.__name__, + ) class StreamLogger: diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 3c705f69b..879d27907 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -50,13 +50,19 @@ class CallLaterOnce: return self._func(*self._a, **self._kw) -def install_reactor(reactor_path): +def install_reactor(reactor_path, event_loop_path=None): """Installs the :mod:`~twisted.internet.reactor` with the specified - import path.""" + import path. Also installs the asyncio event loop with the specified import + path if the asyncio reactor is enabled""" reactor_class = load_object(reactor_path) if reactor_class is asyncioreactor.AsyncioSelectorReactor: with suppress(error.ReactorAlreadyInstalledError): - asyncioreactor.install(asyncio.get_event_loop()) + if event_loop_path is not None: + event_loop_class = load_object(event_loop_path) + event_loop = event_loop_class() + else: + event_loop = asyncio.new_event_loop() + asyncioreactor.install(eventloop=event_loop) else: *module, _ = reactor_path.split(".") installer_path = module + ["install"] diff --git a/tests/CrawlerProcess/asyncio_custom_loop.py b/tests/CrawlerProcess/asyncio_custom_loop.py new file mode 100644 index 000000000..1e4ada722 --- /dev/null +++ b/tests/CrawlerProcess/asyncio_custom_loop.py @@ -0,0 +1,17 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +process = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop" +}) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index fe1cbc997..44ddcded8 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -13,6 +13,7 @@ pytest-twisted >= 1.11 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures +uvloop; platform_system != "Windows" # optional for shell wrapper tests bpython diff --git a/tests/test_commands.py b/tests/test_commands.py index f76f851e7..ee8a92604 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -16,6 +16,7 @@ from tempfile import mkdtemp from threading import Timer from unittest import skipIf +from pytest import mark from twisted.trial import unittest import scrapy @@ -570,6 +571,28 @@ class BadSpider(scrapy.Spider): log = self.get_log(self.debug_log_spider, args=[]) self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') + @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') + def test_custom_asyncio_loop_enabled_true(self): + log = self.get_log(self.debug_log_spider, args=[ + '-s', + 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor', + '-s', + 'ASYNCIO_EVENT_LOOP=uvloop.Loop', + ]) + self.assertIn("Using asyncio event loop: uvloop.Loop", log) + + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") + def test_custom_asyncio_loop_enabled_false(self): + log = self.get_log(self.debug_log_spider, args=[ + '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor' + ]) + import asyncio + loop = asyncio.new_event_loop() + self.assertIn("Using asyncio event loop: %s.%s" % (loop.__module__, loop.__class__.__name__), log) + def test_output(self): spider_code = """ import scrapy diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 1a4cfe813..7c2e251a9 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -345,6 +345,14 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') + @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') + def test_custom_loop_asyncio(self): + log = self.run_script("asyncio_custom_loop.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertIn("Using asyncio event loop: uvloop.Loop", log) + class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner')