diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst index d95e7b52b..2d04b621c 100644 --- a/docs/topics/download-handlers.rst +++ b/docs/topics/download-handlers.rst @@ -209,12 +209,8 @@ Known limitations of the HTTP/2 support: - No support for HTTP/2 Cleartext (h2c), since no major browser supports HTTP/2 unencrypted (refer `http2 faq`_). -- No setting to specify a maximum `frame size`_ larger than the default - value, 16384. Connections to servers that send a larger frame will fail. - - No support for `server pushes`_, which are ignored. -.. _frame size: https://datatracker.ietf.org/doc/html/rfc7540#section-4.2 .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption .. _server pushes: https://datatracker.ietf.org/doc/html/rfc7540#section-8.2 diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 010c9d1c6..e58815fad 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1499,6 +1499,28 @@ Default: ``None`` The Project ID that will be used when storing data on `Google Cloud Storage`_. +.. setting:: HTTP2_MAX_FRAME_SIZE + +HTTP2_MAX_FRAME_SIZE +-------------------- + +.. versionadded:: VERSION + +Default: ``16384`` + +Maximum `frame size`_, in bytes, that servers may send, between ``16384`` and +``16777215``. Connections to servers that send a larger frame fail. + +Raise it for servers that send larger frames regardless of this value. Note +that :setting:`DOWNLOAD_MAXSIZE` and :setting:`DOWNLOAD_WARNSIZE` are checked +once per received frame, so a higher value allows a response to exceed them by +more before being caught. + +:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler` ignores +this setting, as ``httpx`` does not allow configuring the frame size. + +.. _frame size: https://datatracker.ietf.org/doc/html/rfc7540#section-4.2 + .. setting:: ITEM_PIPELINES ITEM_PIPELINES diff --git a/scrapy/core/_http2/protocol.py b/scrapy/core/_http2/protocol.py index aff985d94..1e673ce8b 100644 --- a/scrapy/core/_http2/protocol.py +++ b/scrapy/core/_http2/protocol.py @@ -21,6 +21,7 @@ from h2.events import ( WindowUpdated, ) from h2.exceptions import FrameTooLargeError, H2Error +from h2.settings import SettingCodes from twisted.internet.interfaces import ( IAddress, IHandshakeListener, @@ -261,6 +262,9 @@ class H2ClientProtocol(Protocol, TimeoutMixin): # Initiate H2 Connection self.conn.initiate_connection() + max_frame_size = self._crawler.settings.getint("HTTP2_MAX_FRAME_SIZE") + if max_frame_size != self.conn.local_settings.max_frame_size: + self.conn.update_settings({SettingCodes.MAX_FRAME_SIZE: max_frame_size}) self._write_to_transport() def _lose_connection_with_error(self, errors: list[BaseException]) -> None: diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index d7b30b849..b97844103 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -108,6 +108,7 @@ __all__ = [ "FTP_PASSWORD", "FTP_USER", "GCS_PROJECT_ID", + "HTTP2_MAX_FRAME_SIZE", "HTTPAUTH_DOMAIN", "HTTPAUTH_PASS", "HTTPAUTH_USER", @@ -402,6 +403,8 @@ FTP_PASSWORD = "guest" # noqa: S105 GCS_PROJECT_ID = None +HTTP2_MAX_FRAME_SIZE = 16384 + HTTPAUTH_USER = "" HTTPAUTH_PASS = "" HTTPAUTH_DOMAIN = None diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 161015231..075f1acb9 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -37,6 +37,7 @@ if TYPE_CHECKING: from collections.abc import AsyncGenerator, Callable, Coroutine, Generator from scrapy.core._http2.protocol import H2ClientProtocol + from scrapy.crawler import Crawler pytestmark = [ @@ -236,9 +237,16 @@ class TestHttps2ClientProtocol: ) + self.certificate_file.read_text(encoding="utf-8") return PrivateCertificate.loadPEM(pem) # type: ignore[no-any-return] + @pytest.fixture + def crawler(self, request: pytest.FixtureRequest) -> Crawler: + return get_crawler(settings_dict=getattr(request, "param", None)) + @async_yield_fixture # type: ignore[untyped-decorator] async def client( - self, server_port: int, client_certificate: PrivateCertificate + self, + server_port: int, + client_certificate: PrivateCertificate, + crawler: Crawler, ) -> AsyncGenerator[H2ClientProtocol]: from twisted.internet import reactor @@ -250,7 +258,7 @@ class TestHttps2ClientProtocol: acceptableProtocols=[b"h2"], ) uri = URI.fromBytes(bytes(self.get_url(server_port, "/"), "utf-8")) - h2_client_factory = H2ClientFactory(uri, get_crawler(), Deferred()) + h2_client_factory = H2ClientFactory(uri, crawler, Deferred()) client_endpoint = SSL4ClientEndpoint( reactor, self.host, server_port, client_options ) @@ -312,6 +320,17 @@ class TestHttps2ClientProtocol: request = Request(self.get_url(server_port, "/get-data-html-large")) await self._check_GET(client, request, Data.HTML_LARGE, 200) + @pytest.mark.parametrize( + "crawler", [{"HTTP2_MAX_FRAME_SIZE": 1024**2}], indirect=True + ) + @deferred_f_from_coro_f + async def test_GET_large_frames( + self, server_port: int, client: H2ClientProtocol + ) -> None: + request = Request(self.get_url(server_port, "/get-data-html-large")) + await self._check_GET(client, request, Data.HTML_LARGE, 200) + assert client.conn.local_settings.max_frame_size == 1024**2 + async def _check_GET_x10( self, client: H2ClientProtocol,