diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index cb05784fa..49ea3277a 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -17,7 +17,7 @@ jobs: fail-fast: false matrix: include: - - python-version: "3.13" + - python-version: "3.14" env: TOXENV: pylint - python-version: "3.10" @@ -27,13 +27,13 @@ jobs: env: TOXENV: typing-tests # Keep in sync with pyproject.toml tool.sphinx-scrapy.python-version. - - python-version: "3.13" + - python-version: "3.14" env: TOXENV: docs - python-version: "3.13" env: TOXENV: docs-tests - - python-version: "3.13" + - python-version: "3.14" env: TOXENV: twinecheck diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ad327e465..7779bbb6b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/checkout@v6 - uses: actions/setup-python@v6 with: - python-version: "3.13" + python-version: "3.14" - run: | python -m pip install --upgrade build python -m build diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 2a1c62833..0409b3ef2 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -18,11 +18,11 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] env: - TOXENV: py include: - - python-version: '3.13' + - python-version: '3.14' env: TOXENV: no-reactor diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index a193ca05f..524c79cdb 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -31,10 +31,13 @@ jobs: - python-version: "3.13" env: TOXENV: py - - python-version: "3.13" + - python-version: "3.14" + env: + TOXENV: py + - python-version: "3.14" env: TOXENV: default-reactor - - python-version: "3.13" + - python-version: "3.14" env: TOXENV: no-reactor # pinned due to https://github.com/pypy/pypy/issues/5388 @@ -63,20 +66,20 @@ jobs: env: TOXENV: botocore-pinned - - python-version: "3.13" + - python-version: "3.14" env: TOXENV: extra-deps - - python-version: "3.13" + - python-version: "3.14" env: TOXENV: no-reactor-extra-deps # pinned due to https://github.com/pypy/pypy/issues/5388 - python-version: pypy3.11-7.3.20 env: TOXENV: pypy3-extra-deps - - python-version: "3.13" + - python-version: "3.14" env: TOXENV: botocore - - python-version: "3.13" + - python-version: "3.14" env: TOXENV: mitmproxy diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 48aa56e15..840c7f68e 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -31,10 +31,13 @@ jobs: - python-version: "3.13" env: TOXENV: py - - python-version: "3.13" + - python-version: "3.14" + env: + TOXENV: py + - python-version: "3.14" env: TOXENV: default-reactor - - python-version: "3.13" + - python-version: "3.14" env: TOXENV: no-reactor @@ -46,7 +49,7 @@ jobs: env: TOXENV: extra-deps-pinned - - python-version: "3.13" + - python-version: "3.14" env: TOXENV: extra-deps diff --git a/.readthedocs.yml b/.readthedocs.yml index 6d1aeb507..a2773dcf2 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -2,7 +2,7 @@ version: 2 build: os: ubuntu-24.04 tools: - python: "3.13" + python: "3.14" commands: - pip install tox - tox -e docs diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 873c32c13..a51e431d2 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1152,9 +1152,9 @@ Based on :class:`~urllib.robotparser.RobotFileParser`: * is compliant with `Martijn Koster's 1996 draft specification `_ -* lacks support for wildcard matching +* lacks support for wildcard matching (before Python 3.14.5) -* doesn't use the length based rule +* doesn't use the length based rule (before Python 3.14.5) It is faster than Protego and backward-compatible with versions of Scrapy before 1.8.0. diff --git a/pyproject.toml b/pyproject.toml index 45e18ccc9..6c9ff3100 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Internet :: WWW/HTTP", @@ -448,4 +449,4 @@ split-on-trailing-comma = false convention = "pep257" [tool.sphinx-scrapy] -python-version = "3.13" # Keep in sync with .github/workflows/checks.yml. +python-version = "3.14" # Keep in sync with .github/workflows/checks.yml. diff --git a/scrapy/utils/_deps_compat.py b/scrapy/utils/_deps_compat.py index b7086bc98..cb5424476 100644 --- a/scrapy/utils/_deps_compat.py +++ b/scrapy/utils/_deps_compat.py @@ -6,8 +6,6 @@ from twisted.python.versions import Version as TxVersion TWISTED_FAILURE_HAS_STACK = TWISTED_VERSION < TxVersion("twisted", 24, 10, 0) # changes to private _sslverify code, https://github.com/twisted/twisted/pull/12506 TWISTED_TLS_NEW_IMPL = TWISTED_VERSION >= TxVersion("twisted", 26, 4, 0) -# AsyncioSelectorReactor no longer calls get_event_loop(), https://github.com/twisted/twisted/pull/12508 -TWISTED_LOOP_314_CHANGES = TWISTED_VERSION >= TxVersion("twisted", 26, 4, 0) PYOPENSSL_VERSION = Version(PYOPENSSL_VERSION_STRING) # SSL.Context.use_certificate() wants an X509 object, SSL.Context.use_privatekey() wants a PKey object diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index ddebd8ba2..37db6c25f 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -2,6 +2,7 @@ from __future__ import annotations import asyncio import sys +import warnings from contextlib import suppress from typing import TYPE_CHECKING, Any, Generic, ParamSpec, TypeVar from warnings import catch_warnings, filterwarnings @@ -93,16 +94,19 @@ _asyncio_reactor_path = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" def set_asyncio_event_loop_policy() -> None: - """The policy functions from asyncio often behave unexpectedly, - so we restrict their use to the absolutely essential case. - This should only be used to install the reactor. - """ - policy = asyncio.get_event_loop_policy() - if sys.platform == "win32" and not isinstance( - policy, asyncio.WindowsSelectorEventLoopPolicy - ): - policy = asyncio.WindowsSelectorEventLoopPolicy() - asyncio.set_event_loop_policy(policy) + """Needed due to https://github.com/twisted/twisted/issues/12527.""" + if sys.platform != "win32": + return + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message=r"'asyncio\.(get_event_loop_policy|WindowsSelectorEventLoopPolicy)' is deprecated", + category=DeprecationWarning, + ) + policy = asyncio.get_event_loop_policy() + if not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy): + policy = asyncio.WindowsSelectorEventLoopPolicy() # pylint: disable=deprecated-class + asyncio.set_event_loop_policy(policy) def install_reactor(reactor_path: str, event_loop_path: str | None = None) -> None: diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py index 578e0029d..a2e63a0d0 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py @@ -9,8 +9,9 @@ from scrapy.crawler import CrawlerProcess if sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) -asyncio.set_event_loop(Loop()) -asyncioreactor.install() +loop = Loop() +asyncio.set_event_loop(loop) +asyncioreactor.install(loop) class NoRequestsSpider(scrapy.Spider): diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py index dc2fdda4c..beae4f277 100644 --- a/tests/test_crawler_subprocess.py +++ b/tests/test_crawler_subprocess.py @@ -14,7 +14,6 @@ from packaging.version import parse as parse_version from pexpect.popen_spawn import PopenSpawn from w3lib import __version__ as w3lib_version -from scrapy.utils._deps_compat import TWISTED_LOOP_314_CHANGES from tests.utils import async_sleep, get_script_run_env from tests.utils.decorators import coroutine_test @@ -172,9 +171,6 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): assert "Using asyncio event loop: uvloop.Loop" in log assert "async pipeline opened!" in log - @pytest.mark.xfail( - TWISTED_LOOP_314_CHANGES, reason="Breaks with Twisted changes for Python 3.14" - ) @pytest.mark.requires_uvloop def test_asyncio_enabled_reactor_same_loop(self): log = self.run_script("asyncio_enabled_reactor_same_loop.py") diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 6c0dd9d2c..29b23496a 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -1,3 +1,5 @@ +import sys + import pytest from scrapy.robotstxt import ( @@ -137,16 +139,32 @@ class TestDecodeRobotsTxt: class TestPythonRobotParser(BaseRobotParserTest): + # https://github.com/python/cpython/pull/149374 improves it + IMPROVED_ROBOTFILEPARSER = sys.version_info >= (3, 14, 5) + def setup_method(self): super()._setUp(PythonRobotParser) + @pytest.mark.skipif( + not IMPROVED_ROBOTFILEPARSER, + reason="RobotFileParser from this Python version does not support length based directives precedence.", + ) def test_length_based_precedence(self): - pytest.skip( - "RobotFileParser does not support length based directives precedence." - ) + super().test_length_based_precedence() + @pytest.mark.skipif( + IMPROVED_ROBOTFILEPARSER, + reason="RobotFileParser from this Python version does not support order based directives precedence.", + ) + def test_order_based_precedence(self): + super().test_order_based_precedence() + + @pytest.mark.skipif( + not IMPROVED_ROBOTFILEPARSER, + reason="RobotFileParser from this Python version does not support wildcards.", + ) def test_allowed_wildcards(self): - pytest.skip("RobotFileParser does not support wildcards.") + super().test_allowed_wildcards() @pytest.mark.skipif(not rerp_available(), reason="Rerp parser is not installed")