From 5b5478ae9d6f8d5e0028fe47a63252679c457364 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 5 Nov 2020 14:01:34 -0300 Subject: [PATCH 1/5] Call asyncio.get_event_loop when installing the asyncio reactor --- scrapy/utils/reactor.py | 3 +- .../CrawlerProcess/asyncio_deferred_signal.py | 45 +++++++++++++++++++ tests/test_crawler.py | 16 +++++++ 3 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 tests/CrawlerProcess/asyncio_deferred_signal.py diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 831d29462..6723d9b37 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -60,8 +60,9 @@ def install_reactor(reactor_path, event_loop_path=None): if event_loop_path is not None: event_loop_class = load_object(event_loop_path) event_loop = event_loop_class() + asyncio.set_event_loop(event_loop) else: - event_loop = asyncio.new_event_loop() + event_loop = asyncio.get_event_loop() asyncioreactor.install(eventloop=event_loop) else: *module, _ = reactor_path.split(".") diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py new file mode 100644 index 000000000..bce300afe --- /dev/null +++ b/tests/CrawlerProcess/asyncio_deferred_signal.py @@ -0,0 +1,45 @@ +import asyncio +import sys + +import scrapy + +from scrapy.crawler import CrawlerProcess +from twisted.internet.defer import Deferred + + +class UppercasePipeline: + async def _open_spider(self, spider): + spider.logger.info("async pipeline opened!") + await asyncio.sleep(0.1) + + def open_spider(self, spider): + loop = asyncio.get_event_loop() + return Deferred.fromFuture(loop.create_task(self._open_spider(spider))) + + def process_item(self, item, spider): + return {"url": item["url"].upper()} + + +class UrlSpider(scrapy.Spider): + name = "url_spider" + start_urls = ["data:,"] + custom_settings = { + "ITEM_PIPELINES": {UppercasePipeline: 100}, + } + + def parse(self, response): + yield {"url": response.url} + + +if __name__ == "__main__": + try: + ASYNCIO_EVENT_LOOP = sys.argv[1] + except IndexError: + ASYNCIO_EVENT_LOOP = None + + process = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": ASYNCIO_EVENT_LOOP, + }) + process.crawl(UrlSpider) + process.start() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index b6de33189..0faaa79a3 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -364,6 +364,22 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) self.assertIn("Using asyncio event loop: uvloop.Loop", log) + @mark.skipif(sys.implementation.name == "pypy", reason="uvloop does not support pypy properly") + @mark.skipif(platform.system() == "Windows", reason="uvloop does not support Windows") + def test_custom_loop_asyncio_deferred_signal(self): + log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertIn("Using asyncio event loop: uvloop.Loop", log) + self.assertIn("async pipeline opened!", log) + + def test_default_loop_asyncio_deferred_signal(self): + log = self.run_script("asyncio_deferred_signal.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertNotIn("Using asyncio event loop: uvloop.Loop", log) + self.assertIn("async pipeline opened!", log) + class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner') From 3095d39740c4818d2c1c98d392255981ebed2a10 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 6 Nov 2020 12:16:10 -0300 Subject: [PATCH 2/5] Test: disable asyncio reactor on Windows for Py>=3.8 --- tests/test_crawler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 0faaa79a3..ab113710d 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -373,6 +373,9 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Using asyncio event loop: uvloop.Loop", log) self.assertIn("async pipeline opened!", log) + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_default_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py") self.assertIn("Spider closed (finished)", log) From 114229eb4a5e3e0289000500cf063518be908d40 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 6 Nov 2020 13:29:14 -0300 Subject: [PATCH 3/5] Docs: add a note about asyncio.set_event_loop --- docs/topics/settings.rst | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 912757850..0086a6c74 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -249,19 +249,25 @@ ASYNCIO_EVENT_LOOP Default: ``None`` -Import path of a given asyncio event loop class. +Import path of a given ``asyncio`` event loop class. -If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the -asyncio event loop to be used with it. Set the setting to the import path of the +If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the +asyncio event loop to be used with it. Set the setting to the import path of the desired asyncio event loop class. If the setting is set to ``None`` the default asyncio event loop will be used. If you are installing the asyncio reactor manually using the :func:`~scrapy.utils.reactor.install_reactor` -function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop -class to be used. +function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop +class to be used. Note that the event loop class must inherit from :class:`asyncio.AbstractEventLoop`. +.. caution:: Please be aware that, when using a non-default event loop + (either defined via :setting:`ASYNCIO_EVENT_LOOP` or installed with + :func:`~scrapy.utils.reactor.install_reactor`), Scrapy will call + :func:`asyncio.set_event_loop`, which will set the specified event loop + as the current loop for the current OS thread. + .. setting:: BOT_NAME BOT_NAME From 7e98a76ac455a8c69950104766719cde313bbb74 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 9 Nov 2020 12:17:15 -0300 Subject: [PATCH 4/5] Use deferred_from_coro in asyncio test --- tests/CrawlerProcess/asyncio_deferred_signal.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py index bce300afe..dd82aa2ff 100644 --- a/tests/CrawlerProcess/asyncio_deferred_signal.py +++ b/tests/CrawlerProcess/asyncio_deferred_signal.py @@ -1,9 +1,9 @@ import asyncio import sys -import scrapy - +from scrapy import Spider from scrapy.crawler import CrawlerProcess +from scrapy.utils.defer import deferred_from_coro from twisted.internet.defer import Deferred @@ -14,13 +14,13 @@ class UppercasePipeline: def open_spider(self, spider): loop = asyncio.get_event_loop() - return Deferred.fromFuture(loop.create_task(self._open_spider(spider))) + return deferred_from_coro(self._open_spider(spider)) def process_item(self, item, spider): return {"url": item["url"].upper()} -class UrlSpider(scrapy.Spider): +class UrlSpider(Spider): name = "url_spider" start_urls = ["data:,"] custom_settings = { From b20cfef1e54d9f22769f6d0ec6ae06031bf86ec3 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 9 Nov 2020 13:58:52 -0300 Subject: [PATCH 5/5] Remove unnecessary line from test --- tests/CrawlerProcess/asyncio_deferred_signal.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py index dd82aa2ff..46c2a12a4 100644 --- a/tests/CrawlerProcess/asyncio_deferred_signal.py +++ b/tests/CrawlerProcess/asyncio_deferred_signal.py @@ -13,7 +13,6 @@ class UppercasePipeline: await asyncio.sleep(0.1) def open_spider(self, spider): - loop = asyncio.get_event_loop() return deferred_from_coro(self._open_spider(spider)) def process_item(self, item, spider):