Add Python 3.14 to CI. (#6604)

* Add Python 3.14 (alpha3) to CI.

* Disable mitmproxy on 3.14 for now.

* 3.14.0-alpha.4.

* 3.14.0-alpha.5

* 3.14.0-beta2.

* 3.14 release.

* Fix test_non_pickable_object.

* Fix handling of file:/path feed URIs.

* Better mocking of streams for TextTestResult.

* Do not use .php in test_file_path() as it's now a known extension.

* Fix the URL in TestFeedExporterSignals.

* Fix typing.

* Bump more envs to 3.14.

* Silence pylint.

* Fix another test for .php handling change.

* Remove test_install_asyncio_reactor.

* More bumps to 3.14.

* Revert docs-tests to use 3.13.

* Debug options for Windows.

* Re-enable xdist.

* Revert Windows PYTEST_ADDOPTS.

* Silence loop policy deprecation warnings.

* Restore a lost pylint suppression.

* Update asyncio_enabled_reactor_same_loop.py to new Twisted.

* Fix RobotFileParser tests for Python 3.14.5.

.github/workflows/checks.yml

@@ -17,7 +17,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - python-version: "3.13"
+        - python-version: "3.14"
           env:
             TOXENV: pylint
         - python-version: "3.10"
@@ -27,13 +27,13 @@ jobs:
           env:
             TOXENV: typing-tests
         # Keep in sync with pyproject.toml tool.sphinx-scrapy.python-version.
-        - python-version: "3.13"
+        - python-version: "3.14"
           env:
             TOXENV: docs
         - python-version: "3.13"
           env:
             TOXENV: docs-tests
-        - python-version: "3.13"
+        - python-version: "3.14"
           env:
             TOXENV: twinecheck
 

.github/workflows/publish.yml

@@ -21,7 +21,7 @@ jobs:
       - uses: actions/checkout@v6
       - uses: actions/setup-python@v6
         with:
-          python-version: "3.13"
+          python-version: "3.14"
       - run: |
           python -m pip install --upgrade build
           python -m build

.github/workflows/tests-macos.yml

@@ -18,11 +18,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10", "3.11", "3.12", "3.13"]
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
         env:
           - TOXENV: py
         include:
-          - python-version: '3.13'
+          - python-version: '3.14'
             env:
               TOXENV: no-reactor
 

.github/workflows/tests-ubuntu.yml

@@ -31,10 +31,13 @@ jobs:
         - python-version: "3.13"
           env:
             TOXENV: py
-        - python-version: "3.13"
+        - python-version: "3.14"
+          env:
+            TOXENV: py
+        - python-version: "3.14"
           env:
             TOXENV: default-reactor
-        - python-version: "3.13"
+        - python-version: "3.14"
           env:
             TOXENV: no-reactor
         # pinned due to https://github.com/pypy/pypy/issues/5388
@@ -63,20 +66,20 @@ jobs:
           env:
             TOXENV: botocore-pinned
 
-        - python-version: "3.13"
+        - python-version: "3.14"
           env:
             TOXENV: extra-deps
-        - python-version: "3.13"
+        - python-version: "3.14"
           env:
             TOXENV: no-reactor-extra-deps
         # pinned due to https://github.com/pypy/pypy/issues/5388
         - python-version: pypy3.11-7.3.20
           env:
             TOXENV: pypy3-extra-deps
-        - python-version: "3.13"
+        - python-version: "3.14"
           env:
             TOXENV: botocore
-        - python-version: "3.13"
+        - python-version: "3.14"
           env:
             TOXENV: mitmproxy
 

.github/workflows/tests-windows.yml

@@ -31,10 +31,13 @@ jobs:
         - python-version: "3.13"
           env:
             TOXENV: py
-        - python-version: "3.13"
+        - python-version: "3.14"
+          env:
+            TOXENV: py
+        - python-version: "3.14"
           env:
             TOXENV: default-reactor
-        - python-version: "3.13"
+        - python-version: "3.14"
           env:
             TOXENV: no-reactor
 
@@ -46,7 +49,7 @@ jobs:
           env:
             TOXENV: extra-deps-pinned
 
-        - python-version: "3.13"
+        - python-version: "3.14"
           env:
             TOXENV: extra-deps
 

.readthedocs.yml

@@ -2,7 +2,7 @@ version: 2
 build:
   os: ubuntu-24.04
   tools:
-    python: "3.13"
+    python: "3.14"
   commands:
     - pip install tox
     - tox -e docs

docs/topics/downloader-middleware.rst

@@ -1152,9 +1152,9 @@ Based on :class:`~urllib.robotparser.RobotFileParser`:
 * is compliant with `Martijn Koster's 1996 draft specification
   <https://www.robotstxt.org/norobots-rfc.txt>`_
 
-* lacks support for wildcard matching
+* lacks support for wildcard matching (before Python 3.14.5)
 
-* doesn't use the length based rule
+* doesn't use the length based rule (before Python 3.14.5)
 
 It is faster than Protego and backward-compatible with versions of Scrapy before 1.8.0.
 

pyproject.toml

@@ -39,6 +39,7 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
     "Topic :: Internet :: WWW/HTTP",
@@ -448,4 +449,4 @@ split-on-trailing-comma = false
 convention = "pep257"
 
 [tool.sphinx-scrapy]
-python-version = "3.13"  # Keep in sync with .github/workflows/checks.yml.
+python-version = "3.14"  # Keep in sync with .github/workflows/checks.yml.

scrapy/utils/_deps_compat.py

@@ -6,8 +6,6 @@ from twisted.python.versions import Version as TxVersion
 TWISTED_FAILURE_HAS_STACK = TWISTED_VERSION < TxVersion("twisted", 24, 10, 0)
 # changes to private _sslverify code, https://github.com/twisted/twisted/pull/12506
 TWISTED_TLS_NEW_IMPL = TWISTED_VERSION >= TxVersion("twisted", 26, 4, 0)
-# AsyncioSelectorReactor no longer calls get_event_loop(), https://github.com/twisted/twisted/pull/12508
-TWISTED_LOOP_314_CHANGES = TWISTED_VERSION >= TxVersion("twisted", 26, 4, 0)
 
 PYOPENSSL_VERSION = Version(PYOPENSSL_VERSION_STRING)
 # SSL.Context.use_certificate() wants an X509 object, SSL.Context.use_privatekey() wants a PKey object

scrapy/utils/reactor.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import asyncio
 import sys
+import warnings
 from contextlib import suppress
 from typing import TYPE_CHECKING, Any, Generic, ParamSpec, TypeVar
 from warnings import catch_warnings, filterwarnings
@@ -93,16 +94,19 @@ _asyncio_reactor_path = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
 
 
 def set_asyncio_event_loop_policy() -> None:
-    """The policy functions from asyncio often behave unexpectedly,
+    """Needed due to https://github.com/twisted/twisted/issues/12527."""
-    so we restrict their use to the absolutely essential case.
+    if sys.platform != "win32":
-    This should only be used to install the reactor.
+        return
-    """
+    with warnings.catch_warnings():
-    policy = asyncio.get_event_loop_policy()
+        warnings.filterwarnings(
-    if sys.platform == "win32" and not isinstance(
+            "ignore",
-        policy, asyncio.WindowsSelectorEventLoopPolicy
+            message=r"'asyncio\.(get_event_loop_policy|WindowsSelectorEventLoopPolicy)' is deprecated",
-    ):
+            category=DeprecationWarning,
-        policy = asyncio.WindowsSelectorEventLoopPolicy()
+        )
-        asyncio.set_event_loop_policy(policy)
+        policy = asyncio.get_event_loop_policy()
+        if not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy):
+            policy = asyncio.WindowsSelectorEventLoopPolicy()  # pylint: disable=deprecated-class
+            asyncio.set_event_loop_policy(policy)
 
 
 def install_reactor(reactor_path: str, event_loop_path: str | None = None) -> None:

tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py

@@ -9,8 +9,9 @@ from scrapy.crawler import CrawlerProcess
 
 if sys.platform == "win32":
     asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
-asyncio.set_event_loop(Loop())
+loop = Loop()
-asyncioreactor.install()
+asyncio.set_event_loop(loop)
+asyncioreactor.install(loop)
 
 
 class NoRequestsSpider(scrapy.Spider):

tests/test_crawler_subprocess.py

@@ -14,7 +14,6 @@ from packaging.version import parse as parse_version
 from pexpect.popen_spawn import PopenSpawn
 from w3lib import __version__ as w3lib_version
 
-from scrapy.utils._deps_compat import TWISTED_LOOP_314_CHANGES
 from tests.utils import async_sleep, get_script_run_env
 from tests.utils.decorators import coroutine_test
 
@@ -172,9 +171,6 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
         assert "Using asyncio event loop: uvloop.Loop" in log
         assert "async pipeline opened!" in log
 
-    @pytest.mark.xfail(
-        TWISTED_LOOP_314_CHANGES, reason="Breaks with Twisted changes for Python 3.14"
-    )
     @pytest.mark.requires_uvloop
     def test_asyncio_enabled_reactor_same_loop(self):
         log = self.run_script("asyncio_enabled_reactor_same_loop.py")

tests/test_robotstxt_interface.py

@@ -1,3 +1,5 @@
+import sys
+
 import pytest
 
 from scrapy.robotstxt import (
@@ -137,16 +139,32 @@ class TestDecodeRobotsTxt:
 
 
 class TestPythonRobotParser(BaseRobotParserTest):
+    # https://github.com/python/cpython/pull/149374 improves it
+    IMPROVED_ROBOTFILEPARSER = sys.version_info >= (3, 14, 5)
+
     def setup_method(self):
         super()._setUp(PythonRobotParser)
 
+    @pytest.mark.skipif(
+        not IMPROVED_ROBOTFILEPARSER,
+        reason="RobotFileParser from this Python version does not support length based directives precedence.",
+    )
     def test_length_based_precedence(self):
-        pytest.skip(
+        super().test_length_based_precedence()
-            "RobotFileParser does not support length based directives precedence."
-        )
 
+    @pytest.mark.skipif(
+        IMPROVED_ROBOTFILEPARSER,
+        reason="RobotFileParser from this Python version does not support order based directives precedence.",
+    )
+    def test_order_based_precedence(self):
+        super().test_order_based_precedence()
+
+    @pytest.mark.skipif(
+        not IMPROVED_ROBOTFILEPARSER,
+        reason="RobotFileParser from this Python version does not support wildcards.",
+    )
     def test_allowed_wildcards(self):
-        pytest.skip("RobotFileParser does not support wildcards.")
+        super().test_allowed_wildcards()
 
 
 @pytest.mark.skipif(not rerp_available(), reason="Rerp parser is not installed")