RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add Python 3.14 to CI. (#6604) 

* Add Python 3.14 (alpha3) to CI.

* Disable mitmproxy on 3.14 for now.

* 3.14.0-alpha.4.

* 3.14.0-alpha.5

* 3.14.0-beta2.

* 3.14 release.

* Fix test_non_pickable_object.

* Fix handling of file:/path feed URIs.

* Better mocking of streams for TextTestResult.

* Do not use .php in test_file_path() as it's now a known extension.

* Fix the URL in TestFeedExporterSignals.

* Fix typing.

* Bump more envs to 3.14.

* Silence pylint.

* Fix another test for .php handling change.

* Remove test_install_asyncio_reactor.

* More bumps to 3.14.

* Revert docs-tests to use 3.13.

* Debug options for Windows.

* Re-enable xdist.

* Revert Windows PYTEST_ADDOPTS.

* Silence loop policy deprecation warnings.

* Restore a lost pylint suppression.

* Update asyncio_enabled_reactor_same_loop.py to new Twisted.

* Fix RobotFileParser tests for Python 3.14.5.
This commit is contained in:
Andrey Rakhmatullin 2026-05-12 23:49:37 +05:00 committed by GitHub
parent 3b34ab88c0
commit 2798c03bb0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 65 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.github/workflows/checks.yml vendored
View File

@ -17,7 +17,7 @@ jobs:
fail-fast: false
matrix:
include:
- python-version: "3.13"
- python-version: "3.14"
env:
TOXENV: pylint
- python-version: "3.10"
@ -27,13 +27,13 @@ jobs:
env:
TOXENV: typing-tests
# Keep in sync with pyproject.toml tool.sphinx-scrapy.python-version.
- python-version: "3.13"
- python-version: "3.14"
env:
TOXENV: docs
- python-version: "3.13"
env:
TOXENV: docs-tests
- python-version: "3.13"
- python-version: "3.14"
env:
TOXENV: twinecheck

2
.github/workflows/publish.yml vendored
View File

@ -21,7 +21,7 @@ jobs:
- uses: actions/checkout@v6
- uses: actions/setup-python@v6
with:
python-version: "3.13"
python-version: "3.14"
- run: |
python -m pip install --upgrade build
python -m build

4
.github/workflows/tests-macos.yml vendored
View File

@ -18,11 +18,11 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: ["3.10", "3.11", "3.12", "3.13"]
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
env:
- TOXENV: py
include:
- python-version: '3.13'
- python-version: '3.14'
env:
TOXENV: no-reactor

15
.github/workflows/tests-ubuntu.yml vendored
View File

@ -31,10 +31,13 @@ jobs:
- python-version: "3.13"
env:
TOXENV: py
- python-version: "3.13"
- python-version: "3.14"
env:
TOXENV: py
- python-version: "3.14"
env:
TOXENV: default-reactor
- python-version: "3.13"
- python-version: "3.14"
env:
TOXENV: no-reactor
# pinned due to https://github.com/pypy/pypy/issues/5388
@ -63,20 +66,20 @@ jobs:
env:
TOXENV: botocore-pinned
- python-version: "3.13"
- python-version: "3.14"
env:
TOXENV: extra-deps
- python-version: "3.13"
- python-version: "3.14"
env:
TOXENV: no-reactor-extra-deps
# pinned due to https://github.com/pypy/pypy/issues/5388
- python-version: pypy3.11-7.3.20
env:
TOXENV: pypy3-extra-deps
- python-version: "3.13"
- python-version: "3.14"
env:
TOXENV: botocore
- python-version: "3.13"
- python-version: "3.14"
env:
TOXENV: mitmproxy

9
.github/workflows/tests-windows.yml vendored
View File

@ -31,10 +31,13 @@ jobs:
- python-version: "3.13"
env:
TOXENV: py
- python-version: "3.13"
- python-version: "3.14"
env:
TOXENV: py
- python-version: "3.14"
env:
TOXENV: default-reactor
- python-version: "3.13"
- python-version: "3.14"
env:
TOXENV: no-reactor
@ -46,7 +49,7 @@ jobs:
env:
TOXENV: extra-deps-pinned
- python-version: "3.13"
- python-version: "3.14"
env:
TOXENV: extra-deps

2
.readthedocs.yml
View File

@ -2,7 +2,7 @@ version: 2
build:
os: ubuntu-24.04
tools:
python: "3.13"
python: "3.14"
commands:
- pip install tox
- tox -e docs

4
docs/topics/downloader-middleware.rst
View File

@ -1152,9 +1152,9 @@ Based on :class:`~urllib.robotparser.RobotFileParser`:
* is compliant with `Martijn Koster's 1996 draft specification
<https://www.robotstxt.org/norobots-rfc.txt>`_
* lacks support for wildcard matching
* lacks support for wildcard matching (before Python 3.14.5)
* doesn't use the length based rule
* doesn't use the length based rule (before Python 3.14.5)
It is faster than Protego and backward-compatible with versions of Scrapy before 1.8.0.

3
pyproject.toml
View File

@ -39,6 +39,7 @@ classifiers = [
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
"Programming Language :: Python :: 3.14",
"Programming Language :: Python :: Implementation :: CPython",
"Programming Language :: Python :: Implementation :: PyPy",
"Topic :: Internet :: WWW/HTTP",
@ -448,4 +449,4 @@ split-on-trailing-comma = false
convention = "pep257"
[tool.sphinx-scrapy]
python-version = "3.13" # Keep in sync with .github/workflows/checks.yml.
python-version = "3.14" # Keep in sync with .github/workflows/checks.yml.

2
scrapy/utils/_deps_compat.py
View File

@ -6,8 +6,6 @@ from twisted.python.versions import Version as TxVersion
TWISTED_FAILURE_HAS_STACK = TWISTED_VERSION < TxVersion("twisted", 24, 10, 0)
# changes to private _sslverify code, https://github.com/twisted/twisted/pull/12506
TWISTED_TLS_NEW_IMPL = TWISTED_VERSION >= TxVersion("twisted", 26, 4, 0)
# AsyncioSelectorReactor no longer calls get_event_loop(), https://github.com/twisted/twisted/pull/12508
TWISTED_LOOP_314_CHANGES = TWISTED_VERSION >= TxVersion("twisted", 26, 4, 0)
PYOPENSSL_VERSION = Version(PYOPENSSL_VERSION_STRING)
# SSL.Context.use_certificate() wants an X509 object, SSL.Context.use_privatekey() wants a PKey object

20
scrapy/utils/reactor.py
View File

@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio
import sys
import warnings
from contextlib import suppress
from typing import TYPE_CHECKING, Any, Generic, ParamSpec, TypeVar
from warnings import catch_warnings, filterwarnings
@ -93,15 +94,18 @@ _asyncio_reactor_path = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
def set_asyncio_event_loop_policy() -> None:
"""The policy functions from asyncio often behave unexpectedly,
so we restrict their use to the absolutely essential case.
This should only be used to install the reactor.
"""
"""Needed due to https://github.com/twisted/twisted/issues/12527."""
if sys.platform != "win32":
return
with warnings.catch_warnings():
warnings.filterwarnings(
"ignore",
message=r"'asyncio\.(get_event_loop_policy|WindowsSelectorEventLoopPolicy)' is deprecated",
category=DeprecationWarning,
)
policy = asyncio.get_event_loop_policy()
if sys.platform == "win32" and not isinstance(
policy, asyncio.WindowsSelectorEventLoopPolicy
):
policy = asyncio.WindowsSelectorEventLoopPolicy()
if not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy):
policy = asyncio.WindowsSelectorEventLoopPolicy() # pylint: disable=deprecated-class
asyncio.set_event_loop_policy(policy)

5
tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py
View File

@ -9,8 +9,9 @@ from scrapy.crawler import CrawlerProcess
if sys.platform == "win32":
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
asyncio.set_event_loop(Loop())
asyncioreactor.install()
loop = Loop()
asyncio.set_event_loop(loop)
asyncioreactor.install(loop)
class NoRequestsSpider(scrapy.Spider):

4
tests/test_crawler_subprocess.py
View File

@ -14,7 +14,6 @@ from packaging.version import parse as parse_version
from pexpect.popen_spawn import PopenSpawn
from w3lib import __version__ as w3lib_version
from scrapy.utils._deps_compat import TWISTED_LOOP_314_CHANGES
from tests.utils import async_sleep, get_script_run_env
from tests.utils.decorators import coroutine_test
@ -172,9 +171,6 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
assert "Using asyncio event loop: uvloop.Loop" in log
assert "async pipeline opened!" in log
@pytest.mark.xfail(
TWISTED_LOOP_314_CHANGES, reason="Breaks with Twisted changes for Python 3.14"
)
@pytest.mark.requires_uvloop
def test_asyncio_enabled_reactor_same_loop(self):
log = self.run_script("asyncio_enabled_reactor_same_loop.py")

26
tests/test_robotstxt_interface.py
View File

@ -1,3 +1,5 @@
import sys
import pytest
from scrapy.robotstxt import (
@ -137,16 +139,32 @@ class TestDecodeRobotsTxt:
class TestPythonRobotParser(BaseRobotParserTest):
# https://github.com/python/cpython/pull/149374 improves it
IMPROVED_ROBOTFILEPARSER = sys.version_info >= (3, 14, 5)
def setup_method(self):
super()._setUp(PythonRobotParser)
def test_length_based_precedence(self):
pytest.skip(
"RobotFileParser does not support length based directives precedence."
@pytest.mark.skipif(
not IMPROVED_ROBOTFILEPARSER,
reason="RobotFileParser from this Python version does not support length based directives precedence.",
)
def test_length_based_precedence(self):
super().test_length_based_precedence()
@pytest.mark.skipif(
IMPROVED_ROBOTFILEPARSER,
reason="RobotFileParser from this Python version does not support order based directives precedence.",
)
def test_order_based_precedence(self):
super().test_order_based_precedence()
@pytest.mark.skipif(
not IMPROVED_ROBOTFILEPARSER,
reason="RobotFileParser from this Python version does not support wildcards.",
)
def test_allowed_wildcards(self):
pytest.skip("RobotFileParser does not support wildcards.")
super().test_allowed_wildcards()
@pytest.mark.skipif(not rerp_available(), reason="Rerp parser is not installed")