Merge remote-tracking branch 'origin/master' into crawler-process-reactor-later

.github/workflows/tests-windows.yml

@@ -23,6 +23,9 @@ jobs:
         - python-version: "3.10"
           env:
             TOXENV: py
+        - python-version: "3.10"
+          env:
+            TOXENV: asyncio
 
     steps:
     - uses: actions/checkout@v2

docs/topics/asyncio.rst

@@ -36,6 +36,34 @@ use it instead of the default asyncio event loop.
 
 .. _asyncio-await-dfd:
 
+Windows-specific notes
+======================
+
+The Windows implementation of :mod:`asyncio` can use two event loop
+implementations: :class:`~asyncio.SelectorEventLoop` (default before Python
+3.8, required when using Twisted) and :class:`~asyncio.ProactorEventLoop`
+(default since Python 3.8, cannot work with Twisted). So on Python 3.8+ the
+event loop class needs to be changed. Scrapy since VERSION does this
+automatically when you change the :setting:`TWISTED_REACTOR` setting or call
+:func:`~scrapy.utils.reactor.install_reactor`, but if you install the reactor
+by other means or use an older Scrapy version you need to call the following
+code before installing the reactor::
+
+    import asyncio
+    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+
+You can put this in the same function that installs the reactor, if you do that
+yourself, or in some code that runs before the reactor is installed, e.g.
+``settings.py``.
+
+.. note:: Other libraries you use may require
+          :class:`~asyncio.ProactorEventLoop`, e.g. because it supports
+          subprocesses (this is the case with `playwright`_), so you cannot use
+          them together with Scrapy on Windows (but you should be able to use
+          them on WSL or native Linux).
+
+.. _playwright: https://github.com/microsoft/playwright-python
+
 Awaiting on Deferreds
 =====================
 

scrapy/utils/reactor.py

@@ -1,4 +1,5 @@
 import asyncio
+import sys
 from contextlib import suppress
 
 from twisted.internet import asyncioreactor, error
@@ -57,6 +58,10 @@ def install_reactor(reactor_path, event_loop_path=None):
     reactor_class = load_object(reactor_path)
     if reactor_class is asyncioreactor.AsyncioSelectorReactor:
         with suppress(error.ReactorAlreadyInstalledError):
+            if sys.version_info >= (3, 8) and sys.platform == "win32":
+                policy = asyncio.get_event_loop_policy()
+                if not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy):
+                    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
             if event_loop_path is not None:
                 event_loop_class = load_object(event_loop_path)
                 event_loop = event_loop_class()

tests/CrawlerProcess/asyncio_enabled_reactor.py

@@ -1,6 +1,9 @@
 import asyncio
+import sys
 
 from twisted.internet import asyncioreactor
+if sys.version_info >= (3, 8) and sys.platform == "win32":
+    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
 asyncioreactor.install(asyncio.get_event_loop())
 
 import scrapy

tests/test_commands.py

@@ -674,9 +674,6 @@ class MySpider(scrapy.Spider):
         self.assertIn("start_requests", log)
         self.assertIn("badspider.py", log)
 
-    # https://twistedmatrix.com/trac/ticket/9766
-    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
-            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_asyncio_enabled_true(self):
         log = self.get_log(self.debug_log_spider, args=[
             '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor'
@@ -699,15 +696,15 @@ class MySpider(scrapy.Spider):
         ])
         self.assertIn("Using asyncio event loop: uvloop.Loop", log)
 
-    # https://twistedmatrix.com/trac/ticket/9766
-    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
-            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_custom_asyncio_loop_enabled_false(self):
         log = self.get_log(self.debug_log_spider, args=[
             '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor'
         ])
         import asyncio
-        loop = asyncio.new_event_loop()
+        if sys.platform != 'win32':
+            loop = asyncio.new_event_loop()
+        else:
+            loop = asyncio.SelectorEventLoop()
         self.assertIn(f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}", log)
 
     def test_output(self):

tests/test_crawler.py

@@ -4,7 +4,6 @@ import platform
 import subprocess
 import sys
 import warnings
-from unittest import skipIf
 
 from pytest import raises, mark
 from twisted import version as twisted_version
@@ -303,17 +302,11 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn('Spider closed (finished)', log)
         self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
-    # https://twistedmatrix.com/trac/ticket/9766
-    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
-            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_asyncio_enabled_no_reactor(self):
         log = self.run_script('asyncio_enabled_no_reactor.py')
         self.assertIn('Spider closed (finished)', log)
         self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
-    # https://twistedmatrix.com/trac/ticket/9766
-    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
-            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_asyncio_enabled_reactor(self):
         log = self.run_script('asyncio_enabled_reactor.py')
         self.assertIn('Spider closed (finished)', log)
@@ -352,33 +345,21 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn("Spider closed (finished)", log)
         self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log)
 
-    # https://twistedmatrix.com/trac/ticket/9766
-    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
-            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_reactor_asyncio(self):
         log = self.run_script("twisted_reactor_asyncio.py")
         self.assertIn("Spider closed (finished)", log)
         self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
-    # https://twistedmatrix.com/trac/ticket/9766
-    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
-            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_reactor_asyncio_custom_settings(self):
         log = self.run_script("twisted_reactor_custom_settings.py")
         self.assertIn("Spider closed (finished)", log)
         self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
-    # https://twistedmatrix.com/trac/ticket/9766
-    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
-            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_reactor_asyncio_custom_settings_same(self):
         log = self.run_script("twisted_reactor_custom_settings_same.py")
         self.assertIn("Spider closed (finished)", log)
         self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
-    # https://twistedmatrix.com/trac/ticket/9766
-    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
-            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_reactor_asyncio_custom_settings_conflict(self):
         log = self.run_script("twisted_reactor_custom_settings_conflict.py")
         self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log)
@@ -403,9 +384,6 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn("Using asyncio event loop: uvloop.Loop", log)
         self.assertIn("async pipeline opened!", log)
 
-    # https://twistedmatrix.com/trac/ticket/9766
-    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
-            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_default_loop_asyncio_deferred_signal(self):
         log = self.run_script("asyncio_deferred_signal.py")
         self.assertIn("Spider closed (finished)", log)

tests/test_downloader_handlers.py

@@ -1,6 +1,7 @@
 import contextlib
 import os
 import shutil
+import sys
 import tempfile
 from typing import Optional, Type
 from unittest import mock
@@ -287,6 +288,12 @@ class HttpTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_timeout_download_from_spider_nodata_rcvd(self):
+        if self.reactor_pytest == "asyncio" and sys.platform == "win32":
+            # https://twistedmatrix.com/trac/ticket/10279
+            raise unittest.SkipTest(
+                "This test produces DirtyReactorAggregateError on Windows with asyncio"
+            )
+
         # client connects but no data is received
         spider = Spider('foo')
         meta = {'download_timeout': 0.5}
@@ -296,6 +303,11 @@ class HttpTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_timeout_download_from_spider_server_hangs(self):
+        if self.reactor_pytest == "asyncio" and sys.platform == "win32":
+            # https://twistedmatrix.com/trac/ticket/10279
+            raise unittest.SkipTest(
+                "This test produces DirtyReactorAggregateError on Windows with asyncio"
+            )
         # client connects, server send headers and some body bytes but hangs
         spider = Spider('foo')
         meta = {'download_timeout': 0.5}
@@ -1055,6 +1067,10 @@ class BaseFTPTestCase(unittest.TestCase):
 class FTPTestCase(BaseFTPTestCase):
 
     def test_invalid_credentials(self):
+        if self.reactor_pytest == "asyncio" and sys.platform == "win32":
+            raise unittest.SkipTest(
+                "This test produces DirtyReactorAggregateError on Windows with asyncio"
+            )
         from twisted.protocols.ftp import ConnectionLost
 
         meta = dict(self.req_meta)

tests/test_utils_asyncio.py

@@ -1,6 +1,4 @@
-import platform
-import sys
-from unittest import skipIf, TestCase
+from unittest import TestCase
 
 from pytest import mark
 
@@ -14,9 +12,6 @@ class AsyncioTest(TestCase):
         # the result should depend only on the pytest --reactor argument
         self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio')
 
-    # https://twistedmatrix.com/trac/ticket/9766
-    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
-            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_install_asyncio_reactor(self):
         # this should do nothing
         install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")