From c5ab58056c29c2c35b183572b0780acfbb15dfe8 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Sat, 1 Jan 2022 00:38:10 +0500 Subject: [PATCH] Set WindowsSelectorEventLoopPolicy on Windows (#5315) --- .github/workflows/tests-windows.yml | 3 ++ docs/topics/asyncio.rst | 28 +++++++++++++++++++ scrapy/utils/reactor.py | 5 ++++ .../CrawlerProcess/asyncio_enabled_reactor.py | 3 ++ tests/test_commands.py | 11 +++----- tests/test_crawler.py | 16 ----------- tests/test_downloader_handlers.py | 16 +++++++++++ tests/test_utils_asyncio.py | 7 +---- 8 files changed, 60 insertions(+), 29 deletions(-) diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 6fabf5cde..ab7385118 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -23,6 +23,9 @@ jobs: - python-version: "3.10" env: TOXENV: py + - python-version: "3.10" + env: + TOXENV: asyncio steps: - uses: actions/checkout@v2 diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 402352721..8712d4268 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -36,6 +36,34 @@ use it instead of the default asyncio event loop. .. _asyncio-await-dfd: +Windows-specific notes +====================== + +The Windows implementation of :mod:`asyncio` can use two event loop +implementations: :class:`~asyncio.SelectorEventLoop` (default before Python +3.8, required when using Twisted) and :class:`~asyncio.ProactorEventLoop` +(default since Python 3.8, cannot work with Twisted). So on Python 3.8+ the +event loop class needs to be changed. Scrapy since VERSION does this +automatically when you change the :setting:`TWISTED_REACTOR` setting or call +:func:`~scrapy.utils.reactor.install_reactor`, but if you install the reactor +by other means or use an older Scrapy version you need to call the following +code before installing the reactor:: + + import asyncio + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + +You can put this in the same function that installs the reactor, if you do that +yourself, or in some code that runs before the reactor is installed, e.g. +``settings.py``. + +.. note:: Other libraries you use may require + :class:`~asyncio.ProactorEventLoop`, e.g. because it supports + subprocesses (this is the case with `playwright`_), so you cannot use + them together with Scrapy on Windows (but you should be able to use + them on WSL or native Linux). + +.. _playwright: https://github.com/microsoft/playwright-python + Awaiting on Deferreds ===================== diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 6723d9b37..96395543c 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -1,4 +1,5 @@ import asyncio +import sys from contextlib import suppress from twisted.internet import asyncioreactor, error @@ -57,6 +58,10 @@ def install_reactor(reactor_path, event_loop_path=None): reactor_class = load_object(reactor_path) if reactor_class is asyncioreactor.AsyncioSelectorReactor: with suppress(error.ReactorAlreadyInstalledError): + if sys.version_info >= (3, 8) and sys.platform == "win32": + policy = asyncio.get_event_loop_policy() + if not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy): + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) if event_loop_path is not None: event_loop_class = load_object(event_loop_path) event_loop = event_loop_class() diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index 8568bd8b8..f2a93074b 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -1,6 +1,9 @@ import asyncio +import sys from twisted.internet import asyncioreactor +if sys.version_info >= (3, 8) and sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncioreactor.install(asyncio.get_event_loop()) import scrapy diff --git a/tests/test_commands.py b/tests/test_commands.py index 75098a77a..81d1a1cab 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -674,9 +674,6 @@ class MySpider(scrapy.Spider): self.assertIn("start_requests", log) self.assertIn("badspider.py", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_true(self): log = self.get_log(self.debug_log_spider, args=[ '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor' @@ -699,15 +696,15 @@ class MySpider(scrapy.Spider): ]) self.assertIn("Using asyncio event loop: uvloop.Loop", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_custom_asyncio_loop_enabled_false(self): log = self.get_log(self.debug_log_spider, args=[ '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor' ]) import asyncio - loop = asyncio.new_event_loop() + if sys.platform != 'win32': + loop = asyncio.new_event_loop() + else: + loop = asyncio.SelectorEventLoop() self.assertIn(f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}", log) def test_output(self): diff --git a/tests/test_crawler.py b/tests/test_crawler.py index be067155e..7bc4fba40 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -4,7 +4,6 @@ import platform import subprocess import sys import warnings -from unittest import skipIf from pytest import raises, mark from testfixtures import LogCapture @@ -284,9 +283,6 @@ class CrawlerRunnerHasSpider(unittest.TestCase): }) @defer.inlineCallbacks - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_crawler_process_asyncio_enabled_true(self): with LogCapture(level=logging.DEBUG) as log: if self.reactor_pytest == 'asyncio': @@ -328,17 +324,11 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn('Spider closed (finished)', log) self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_no_reactor(self): log = self.run_script('asyncio_enabled_no_reactor.py') self.assertIn('Spider closed (finished)', log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_reactor(self): log = self.run_script('asyncio_enabled_reactor.py') self.assertIn('Spider closed (finished)', log) @@ -377,9 +367,6 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_reactor_asyncio(self): log = self.run_script("twisted_reactor_asyncio.py") self.assertIn("Spider closed (finished)", log) @@ -404,9 +391,6 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Using asyncio event loop: uvloop.Loop", log) self.assertIn("async pipeline opened!", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_default_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py") self.assertIn("Spider closed (finished)", log) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 9c11820e5..a1ea4c679 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -1,6 +1,7 @@ import contextlib import os import shutil +import sys import tempfile from typing import Optional, Type from unittest import mock @@ -287,6 +288,12 @@ class HttpTestCase(unittest.TestCase): @defer.inlineCallbacks def test_timeout_download_from_spider_nodata_rcvd(self): + if self.reactor_pytest == "asyncio" and sys.platform == "win32": + # https://twistedmatrix.com/trac/ticket/10279 + raise unittest.SkipTest( + "This test produces DirtyReactorAggregateError on Windows with asyncio" + ) + # client connects but no data is received spider = Spider('foo') meta = {'download_timeout': 0.5} @@ -296,6 +303,11 @@ class HttpTestCase(unittest.TestCase): @defer.inlineCallbacks def test_timeout_download_from_spider_server_hangs(self): + if self.reactor_pytest == "asyncio" and sys.platform == "win32": + # https://twistedmatrix.com/trac/ticket/10279 + raise unittest.SkipTest( + "This test produces DirtyReactorAggregateError on Windows with asyncio" + ) # client connects, server send headers and some body bytes but hangs spider = Spider('foo') meta = {'download_timeout': 0.5} @@ -1055,6 +1067,10 @@ class BaseFTPTestCase(unittest.TestCase): class FTPTestCase(BaseFTPTestCase): def test_invalid_credentials(self): + if self.reactor_pytest == "asyncio" and sys.platform == "win32": + raise unittest.SkipTest( + "This test produces DirtyReactorAggregateError on Windows with asyncio" + ) from twisted.protocols.ftp import ConnectionLost meta = dict(self.req_meta) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index a2114bd18..295323e4d 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -1,6 +1,4 @@ -import platform -import sys -from unittest import skipIf, TestCase +from unittest import TestCase from pytest import mark @@ -14,9 +12,6 @@ class AsyncioTest(TestCase): # the result should depend only on the pytest --reactor argument self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio') - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_install_asyncio_reactor(self): # this should do nothing install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")