Merge pull request #5984 from wRAR/ci-3.12

Python 3.12 support

.github/workflows/tests-ubuntu.yml

@@ -48,6 +48,16 @@ jobs:
           env:
             TOXENV: botocore
 
+        - python-version: "3.12.0-beta.4"
+          env:
+            TOXENV: py
+        - python-version: "3.12.0-beta.4"
+          env:
+            TOXENV: asyncio
+        - python-version: "3.12.0-beta.4"
+          env:
+            TOXENV: extra-deps
+
     steps:
     - uses: actions/checkout@v3
 
@@ -57,7 +67,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
 
     - name: Install system libraries
-      if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned')
+      if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned')  || contains(matrix.python-version, '3.12.0')
       run: |
         sudo apt-get update
         sudo apt-get install libxml2-dev libxslt-dev

conftest.py

@@ -1,6 +1,10 @@
+import platform
+import sys
 from pathlib import Path
 
 import pytest
+from twisted import version as twisted_version
+from twisted.python.versions import Version
 from twisted.web.http import H2_ENABLED
 
 from scrapy.utils.reactor import install_reactor
@@ -14,6 +18,10 @@ def _py_files(folder):
 collect_ignore = [
     # not a test, but looks like a test
     "scrapy/utils/testsite.py",
+    "tests/ftpserver.py",
+    "tests/mockserver.py",
+    "tests/pipelines.py",
+    "tests/spiders.py",
     # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
     *_py_files("tests/CrawlerProcess"),
     # contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess
@@ -73,6 +81,20 @@ def only_not_asyncio(request, reactor_pytest):
         pytest.skip("This test is only run without --reactor=asyncio")
 
 
+@pytest.fixture(autouse=True)
+def requires_uvloop(request):
+    if not request.node.get_closest_marker("requires_uvloop"):
+        return
+    if sys.implementation.name == "pypy":
+        pytest.skip("uvloop does not support pypy properly")
+    if platform.system() == "Windows":
+        pytest.skip("uvloop does not support Windows")
+    if twisted_version == Version("twisted", 21, 2, 0):
+        pytest.skip("https://twistedmatrix.com/trac/ticket/10106")
+    if sys.version_info >= (3, 12):
+        pytest.skip("uvloop doesn't support Python 3.12 yet")
+
+
 def pytest_configure(config):
     if config.getoption("--reactor") == "asyncio":
         install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")

pytest.ini

@@ -20,6 +20,7 @@ addopts =
 markers =
     only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
     only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed
+    requires_uvloop: marks tests as only enabled when uvloop is known to be working
 filterwarnings =
     ignore:scrapy.downloadermiddlewares.decompression is deprecated
     ignore:Module scrapy.utils.reqser is deprecated

scrapy/cmdline.py

@@ -48,7 +48,11 @@ def _get_commands_from_module(module, inproject):
 
 def _get_commands_from_entry_points(inproject, group="scrapy.commands"):
     cmds = {}
-    for entry_point in entry_points().get(group, {}):
+    if sys.version_info >= (3, 10):
+        eps = entry_points(group=group)
+    else:
+        eps = entry_points().get(group, ())
+    for entry_point in eps:
         obj = entry_point.load()
         if inspect.isclass(obj):
             cmds[entry_point.name] = obj()

setup.py

@@ -62,6 +62,7 @@ setup(
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
         "Programming Language :: Python :: Implementation :: CPython",
         "Programming Language :: Python :: Implementation :: PyPy",
         "Topic :: Internet :: WWW/HTTP",

tests/requirements.txt

@@ -1,15 +1,17 @@
 # Tests requirements
 attrs
-pyftpdlib
+# https://github.com/giampaolo/pyftpdlib/issues/560
+pyftpdlib; python_version < "3.12"
 pytest
 pytest-cov==4.0.0
 pytest-xdist
 sybil >= 1.3.0  # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
 testfixtures
-uvloop; platform_system != "Windows"
+# uvloop currently doesn't build on 3.12
+uvloop; platform_system != "Windows" and python_version < "3.12"
 
-# optional for shell wrapper tests
-bpython
+# bpython requires greenlet which currently doesn't build on 3.12
+bpython; python_version < "3.12"  # optional for shell wrapper tests
 brotli  # optional for HTTP compress downloader middleware tests
 zstandard; implementation_name != 'pypy'  # optional for HTTP compress downloader middleware tests
 ipython

tests/test_commands.py

@@ -18,8 +18,6 @@ from typing import Dict, Generator, Optional, Union
 from unittest import skipIf
 
 from pytest import mark
-from twisted import version as twisted_version
-from twisted.python.versions import Version
 from twisted.trial import unittest
 
 import scrapy
@@ -802,17 +800,7 @@ class MySpider(scrapy.Spider):
             "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log
         )
 
-    @mark.skipif(
-        sys.implementation.name == "pypy",
-        reason="uvloop does not support pypy properly",
-    )
-    @mark.skipif(
-        platform.system() == "Windows", reason="uvloop does not support Windows"
-    )
-    @mark.skipif(
-        twisted_version == Version("twisted", 21, 2, 0),
-        reason="https://twistedmatrix.com/trac/ticket/10106",
-    )
+    @mark.requires_uvloop
     def test_custom_asyncio_loop_enabled_true(self):
         log = self.get_log(
             self.debug_log_spider,

tests/test_crawler.py

@@ -8,9 +8,7 @@ from pathlib import Path
 
 from packaging.version import parse as parse_version
 from pytest import mark, raises
-from twisted import version as twisted_version
 from twisted.internet import defer
-from twisted.python.versions import Version
 from twisted.trial import unittest
 from w3lib import __version__ as w3lib_version
 
@@ -466,17 +464,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
             log,
         )
 
-    @mark.skipif(
-        sys.implementation.name == "pypy",
-        reason="uvloop does not support pypy properly",
-    )
-    @mark.skipif(
-        platform.system() == "Windows", reason="uvloop does not support Windows"
-    )
-    @mark.skipif(
-        twisted_version == Version("twisted", 21, 2, 0),
-        reason="https://twistedmatrix.com/trac/ticket/10106",
-    )
+    @mark.requires_uvloop
     def test_custom_loop_asyncio(self):
         log = self.run_script("asyncio_custom_loop.py")
         self.assertIn("Spider closed (finished)", log)
@@ -485,17 +473,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         )
         self.assertIn("Using asyncio event loop: uvloop.Loop", log)
 
-    @mark.skipif(
-        sys.implementation.name == "pypy",
-        reason="uvloop does not support pypy properly",
-    )
-    @mark.skipif(
-        platform.system() == "Windows", reason="uvloop does not support Windows"
-    )
-    @mark.skipif(
-        twisted_version == Version("twisted", 21, 2, 0),
-        reason="https://twistedmatrix.com/trac/ticket/10106",
-    )
+    @mark.requires_uvloop
     def test_custom_loop_asyncio_deferred_signal(self):
         log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop")
         self.assertIn("Spider closed (finished)", log)
@@ -505,17 +483,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn("Using asyncio event loop: uvloop.Loop", log)
         self.assertIn("async pipeline opened!", log)
 
-    @mark.skipif(
-        sys.implementation.name == "pypy",
-        reason="uvloop does not support pypy properly",
-    )
-    @mark.skipif(
-        platform.system() == "Windows", reason="uvloop does not support Windows"
-    )
-    @mark.skipif(
-        twisted_version == Version("twisted", 21, 2, 0),
-        reason="https://twistedmatrix.com/trac/ticket/10106",
-    )
+    @mark.requires_uvloop
     def test_asyncio_enabled_reactor_same_loop(self):
         log = self.run_script("asyncio_enabled_reactor_same_loop.py")
         self.assertIn("Spider closed (finished)", log)
@@ -524,17 +492,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         )
         self.assertIn("Using asyncio event loop: uvloop.Loop", log)
 
-    @mark.skipif(
-        sys.implementation.name == "pypy",
-        reason="uvloop does not support pypy properly",
-    )
-    @mark.skipif(
-        platform.system() == "Windows", reason="uvloop does not support Windows"
-    )
-    @mark.skipif(
-        twisted_version == Version("twisted", 21, 2, 0),
-        reason="https://twistedmatrix.com/trac/ticket/10106",
-    )
+    @mark.requires_uvloop
     def test_asyncio_enabled_reactor_different_loop(self):
         log = self.run_script("asyncio_enabled_reactor_different_loop.py")
         self.assertNotIn("Spider closed (finished)", log)

tests/test_feedexport.py

@@ -125,6 +125,9 @@ class FileFeedStorageTest(unittest.TestCase):
             path.unlink()
 
 
+@pytest.mark.skipif(
+    sys.version_info >= (3, 12), reason="pyftpdlib doesn't support Python 3.12 yet"
+)
 class FTPFeedStorageTest(unittest.TestCase):
     def get_test_spider(self, settings=None):
         class TestSpider(scrapy.Spider):

tests/test_pipeline_files.py

@@ -1,6 +1,7 @@
 import dataclasses
 import os
 import random
+import sys
 import time
 from datetime import datetime
 from io import BytesIO
@@ -11,6 +12,7 @@ from unittest import mock
 from urllib.parse import urlparse
 
 import attr
+import pytest
 from itemadapter import ItemAdapter
 from twisted.internet import defer
 from twisted.trial import unittest
@@ -641,6 +643,9 @@ class TestGCSFilesStore(unittest.TestCase):
                     store.bucket.get_blob.assert_called_with(expected_blob_path)
 
 
+@pytest.mark.skipif(
+    sys.version_info >= (3, 12), reason="pyftpdlib doesn't support Python 3.12 yet"
+)
 class TestFTPFileStore(unittest.TestCase):
     @defer.inlineCallbacks
     def test_persist(self):