From 8bb06bf00b4a098f9c8349743df2bf1539a3a3d3 Mon Sep 17 00:00:00 2001 From: Adrian Date: Wed, 12 Aug 2026 20:33:38 +0200 Subject: [PATCH] Stop marking the Twisted-based HTTP/2 download handler as experimental, and privatize its API (#7986) --- conftest.py | 2 +- docs/topics/download-handlers.rst | 6 ------ scrapy/core/{http2 => _http2}/__init__.py | 0 scrapy/core/{http2 => _http2}/agent.py | 2 +- scrapy/core/{http2 => _http2}/protocol.py | 2 +- scrapy/core/{http2 => _http2}/stream.py | 2 +- scrapy/core/downloader/handlers/http2.py | 2 +- tests/test_downloader_handler_twisted_http2.py | 2 +- tests/test_http2_client_protocol.py | 14 +++++++------- 9 files changed, 13 insertions(+), 19 deletions(-) rename scrapy/core/{http2 => _http2}/__init__.py (100%) rename scrapy/core/{http2 => _http2}/agent.py (98%) rename scrapy/core/{http2 => _http2}/protocol.py (99%) rename scrapy/core/{http2 => _http2}/stream.py (99%) diff --git a/conftest.py b/conftest.py index 5a535c168..7a5f5b85a 100644 --- a/conftest.py +++ b/conftest.py @@ -47,7 +47,7 @@ if not H2_ENABLED: collect_ignore.extend( ( "scrapy/core/downloader/handlers/http2.py", - *_py_files("scrapy/core/http2"), + *_py_files("scrapy/core/_http2"), ) ) diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst index 433a6d139..d95e7b52b 100644 --- a/docs/topics/download-handlers.rst +++ b/docs/topics/download-handlers.rst @@ -187,12 +187,6 @@ If you want to use this handler you need to replace the default one for the Features and limitations ^^^^^^^^^^^^^^^^^^^^^^^^ -.. warning:: - - This handler is experimental, and not yet recommended for production - environments. Future Scrapy versions may introduce related changes without - a deprecation period or warning. - =========================== ================================================ HTTP proxies No (not implemented) SOCKS proxies No (not supported by the library) diff --git a/scrapy/core/http2/__init__.py b/scrapy/core/_http2/__init__.py similarity index 100% rename from scrapy/core/http2/__init__.py rename to scrapy/core/_http2/__init__.py diff --git a/scrapy/core/http2/agent.py b/scrapy/core/_http2/agent.py similarity index 98% rename from scrapy/core/http2/agent.py rename to scrapy/core/_http2/agent.py index 042557208..38be86b22 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/_http2/agent.py @@ -14,8 +14,8 @@ from twisted.web.client import ( ) from twisted.web.error import SchemeNotSupported +from scrapy.core._http2.protocol import H2ClientFactory, H2ClientProtocol from scrapy.core.downloader.contextfactory import _AcceptableProtocolsContextFactory -from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol if TYPE_CHECKING: from twisted.internet.base import ReactorBase diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/_http2/protocol.py similarity index 99% rename from scrapy/core/http2/protocol.py rename to scrapy/core/_http2/protocol.py index 2d59aba31..aff985d94 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/_http2/protocol.py @@ -31,7 +31,7 @@ from twisted.internet.ssl import Certificate from twisted.protocols.policies import TimeoutMixin from zope.interface import implementer -from scrapy.core.http2.stream import Stream, StreamCloseReason +from scrapy.core._http2.stream import Stream, StreamCloseReason from scrapy.exceptions import DownloadTimeoutError from scrapy.http import Request, Response from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute diff --git a/scrapy/core/http2/stream.py b/scrapy/core/_http2/stream.py similarity index 99% rename from scrapy/core/http2/stream.py rename to scrapy/core/_http2/stream.py index 4fc300d90..4a07d198b 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/_http2/stream.py @@ -27,7 +27,7 @@ from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: from collections.abc import Sequence - from scrapy.core.http2.protocol import H2ClientProtocol + from scrapy.core._http2.protocol import H2ClientProtocol from scrapy.crawler import Crawler from scrapy.http import Request, Response diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 9b3d4fbd4..935a2ae4c 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -4,9 +4,9 @@ from time import monotonic from typing import TYPE_CHECKING from urllib.parse import urldefrag +from scrapy.core._http2.agent import H2Agent, H2ConnectionPool from scrapy.core.downloader.contextfactory import _load_context_factory_from_settings from scrapy.core.downloader.handlers.base import BaseDownloadHandler -from scrapy.core.http2.agent import H2Agent, H2ConnectionPool from scrapy.exceptions import ( DownloadTimeoutError, NotConfigured, diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index 2c3954b5e..95e1ff633 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -143,7 +143,7 @@ class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase): response = await download_handler.download_request(request) assert response.text == actual_content_length assert ( - "scrapy.core.http2.stream", + "scrapy.core._http2.stream", logging.WARNING, f"Ignoring bad Content-Length header " f"{bad_content_length!r} of request {request}, sending " diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 431b7a458..161015231 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -36,7 +36,7 @@ from tests.mockserver.utils import ssl_context_factory if TYPE_CHECKING: from collections.abc import AsyncGenerator, Callable, Coroutine, Generator - from scrapy.core.http2.protocol import H2ClientProtocol + from scrapy.core._http2.protocol import H2ClientProtocol pytestmark = [ @@ -242,7 +242,7 @@ class TestHttps2ClientProtocol: ) -> AsyncGenerator[H2ClientProtocol]: from twisted.internet import reactor - from scrapy.core.http2.protocol import H2ClientFactory # noqa: PLC0415 + from scrapy.core._http2.protocol import H2ClientFactory # noqa: PLC0415 client_options = optionsForClientTLS( hostname=self.host, @@ -460,7 +460,7 @@ class TestHttps2ClientProtocol: def test_invalid_negotiated_protocol( self, server_port: int, client: H2ClientProtocol ) -> Generator[Deferred[Any], Any, None]: - with mock.patch("scrapy.core.http2.protocol.PROTOCOL_NAME", new=b"not-h2"): + with mock.patch("scrapy.core._http2.protocol.PROTOCOL_NAME", new=b"not-h2"): request = Request(url=self.get_url(server_port, "/status?n=200")) with pytest.raises(ResponseFailed): yield make_request_dfd(client, request) @@ -527,7 +527,7 @@ class TestHttps2ClientProtocol: expected_body: bytes, caplog: pytest.LogCaptureFixture, ) -> None: - with caplog.at_level("WARNING", "scrapy.core.http2.stream"): + with caplog.at_level("WARNING", "scrapy.core._http2.stream"): response = await make_request(client, request) assert response.status == 200 assert response.body == expected_body @@ -605,7 +605,7 @@ class TestHttps2ClientProtocol: def assert_inactive_stream(failure): assert failure.check(ResponseFailed) is not None - from scrapy.core.http2.stream import InactiveStreamClosed # noqa: PLC0415 + from scrapy.core._http2.stream import InactiveStreamClosed # noqa: PLC0415 assert any( isinstance(e, InactiveStreamClosed) for e in failure.value.reasons @@ -692,7 +692,7 @@ class TestHttps2ClientProtocol: @staticmethod async def _check_invalid_netloc(client: H2ClientProtocol, url: str) -> None: - from scrapy.core.http2.stream import InvalidHostname # noqa: PLC0415 + from scrapy.core._http2.stream import InvalidHostname # noqa: PLC0415 request = Request(url) with pytest.raises(InvalidHostname) as exc_info: @@ -737,7 +737,7 @@ class TestHttps2ClientProtocol: yield make_request_dfd(client, request) for err in exc_info.value.reasons: - from scrapy.core.http2.protocol import H2ClientProtocol # noqa: PLC0415 + from scrapy.core._http2.protocol import H2ClientProtocol # noqa: PLC0415 if isinstance(err, DownloadTimeoutError): assert (