diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst index e3f7e16e2..f2832c4f1 100644 --- a/docs/topics/download-handlers.rst +++ b/docs/topics/download-handlers.rst @@ -243,12 +243,8 @@ Known limitations of the HTTP/2 support: - No support for HTTP/2 Cleartext (h2c), since no major browser supports HTTP/2 unencrypted (refer `http2 faq`_). -- No setting to specify a maximum `frame size`_ larger than the default - value, 16384. Connections to servers that send a larger frame will fail. - - No support for `server pushes`_, which are ignored. -.. _frame size: https://datatracker.ietf.org/doc/html/rfc7540#section-4.2 .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption .. _server pushes: https://datatracker.ietf.org/doc/html/rfc7540#section-8.2 diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 98859a01e..e99a6cb17 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1563,6 +1563,28 @@ Default: ``None`` The Project ID that will be used when storing data on `Google Cloud Storage`_. +.. setting:: HTTP2_MAX_FRAME_SIZE + +HTTP2_MAX_FRAME_SIZE +-------------------- + +.. versionadded:: VERSION + +Default: ``16384`` + +Maximum `frame size`_, in bytes, that servers may send, between ``16384`` and +``16777215``. Connections to servers that send a larger frame fail. + +Raise it for servers that send larger frames regardless of this value. Note +that :setting:`DOWNLOAD_MAXSIZE` and :setting:`DOWNLOAD_WARNSIZE` are checked +once per received frame, so a higher value allows a response to exceed them by +more before being caught. + +:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler` ignores +this setting, as ``httpx`` does not allow configuring the frame size. + +.. _frame size: https://datatracker.ietf.org/doc/html/rfc7540#section-4.2 + .. setting:: ITEM_PIPELINES ITEM_PIPELINES diff --git a/scrapy/core/_http2/protocol.py b/scrapy/core/_http2/protocol.py index 734e0820e..43e9ce0bb 100644 --- a/scrapy/core/_http2/protocol.py +++ b/scrapy/core/_http2/protocol.py @@ -21,6 +21,7 @@ from h2.events import ( WindowUpdated, ) from h2.exceptions import DenialOfServiceError, FrameTooLargeError, H2Error +from h2.settings import SettingCodes from twisted.internet.interfaces import ( IAddress, IHandshakeListener, @@ -282,6 +283,9 @@ class H2ClientProtocol(Protocol, TimeoutMixin): # Initiate H2 Connection self.conn.initiate_connection() + max_frame_size = self._crawler.settings.getint("HTTP2_MAX_FRAME_SIZE") + if max_frame_size != self.conn.local_settings.max_frame_size: + self.conn.update_settings({SettingCodes.MAX_FRAME_SIZE: max_frame_size}) self._write_to_transport() def _lose_connection_with_error(self, errors: list[BaseException]) -> None: diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index fbc6f2530..5d1ec246c 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -9,7 +9,7 @@ from __future__ import annotations import logging from collections.abc import AsyncIterator, Callable, Coroutine, Iterable from functools import wraps -from inspect import isasyncgenfunction +from inspect import isasyncgenfunction, iscoroutine from itertools import islice from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar from warnings import warn @@ -244,8 +244,20 @@ class SpiderMiddlewareManager(MiddlewareManager): warn(msg, category=ScrapyDeprecationWarning, stacklevel=2) self._set_compat_spider(spider) start = self._spider.start() + if not hasattr(start, "__aiter__"): + if iscoroutine(start): + start.close() + start = self._reject_start(start) return await self._process_chain("process_start", start) + async def _reject_start(self, start: Any) -> AsyncIterator[Any]: + raise TypeError( + f"{global_object_name(type(self._spider))}.start() must be an" + f" asynchronous generator, i.e. an async def method with yield" + f" statements, got {type(start)}" + ) + yield # pylint: disable=unreachable # makes this method an asynchronous generator + # This method is only needed until _async compatibility methods are removed. @staticmethod def _get_process_spider_output(mw: Any) -> Callable[..., Any] | None: diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 6639a18b6..13a5ad29d 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -110,6 +110,7 @@ __all__ = [ "FTP_PASSWORD", "FTP_USER", "GCS_PROJECT_ID", + "HTTP2_MAX_FRAME_SIZE", "HTTPAUTH_DOMAIN", "HTTPAUTH_PASS", "HTTPAUTH_USER", @@ -409,6 +410,8 @@ FTP_PASSWORD = "guest" # noqa: S105 GCS_PROJECT_ID = None +HTTP2_MAX_FRAME_SIZE = 16384 + HTTPAUTH_USER = "" HTTPAUTH_PASS = "" HTTPAUTH_DOMAIN = None diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index 1ecf8b8de..8e5197df7 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -4,6 +4,8 @@ from collections import deque from logging import ERROR from typing import TYPE_CHECKING, Any +import pytest + from scrapy import Request, Spider, signals from scrapy.core.scheduler import BaseScheduler from scrapy.exceptions import CloseSpider @@ -13,7 +15,7 @@ from tests.mockserver.http import MockServer from tests.utils.decorators import coroutine_test if TYPE_CHECKING: - import pytest + from collections.abc import Iterator from scrapy.http import Response @@ -50,6 +52,27 @@ class MemoryScheduler(BaseScheduler): self.paused = False +class NoneStartSpider(Spider): + name = "test" + + def start(self) -> None: # type: ignore[override] + return None + + +class CoroutineStartSpider(Spider): + name = "test" + + async def start(self) -> None: # type: ignore[override] + return None + + +class SyncStartSpider(Spider): + name = "test" + + def start(self) -> Iterator[Request]: # type: ignore[override] + yield Request("data:,a") + + class TestMain: @coroutine_test async def test_sleep(self): @@ -141,6 +164,34 @@ class TestMain: assert crawler.stats.get_value("finish_reason") == "shutdown" assert not actual_urls + @pytest.mark.parametrize( + ("spider_cls", "expected_type"), + [ + (NoneStartSpider, ""), + (CoroutineStartSpider, ""), + (SyncStartSpider, ""), + ], + ) + @coroutine_test + async def test_start_not_an_async_generator( + self, + spider_cls: type[Spider], + expected_type: str, + caplog: pytest.LogCaptureFixture, + ) -> None: + crawler = get_crawler(spider_cls) + + caplog.clear() + with caplog.at_level(ERROR): + await crawler.crawl_async() + + assert ( + f"{spider_cls.__name__}.start() must be an asynchronous generator," + f" i.e. an async def method with yield statements, got {expected_type}" + ) in caplog.text + assert crawler.stats + assert crawler.stats.get_value("finish_reason") == "start_error" + @coroutine_test async def test_start_error(self, caplog: pytest.LogCaptureFixture) -> None: class TestSpider(Spider): diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 161015231..e96a5484e 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -37,6 +37,7 @@ if TYPE_CHECKING: from collections.abc import AsyncGenerator, Callable, Coroutine, Generator from scrapy.core._http2.protocol import H2ClientProtocol + from scrapy.crawler import Crawler pytestmark = [ @@ -75,7 +76,7 @@ class Data: STR_LARGE = generate_random_string(LARGE_SIZE) EXTRA_SMALL = generate_random_string(1024 * 15) - EXTRA_LARGE = generate_random_string((1024**2) * 15) + EXTRA_LARGE = generate_random_string(LARGE_SIZE) HTML_SMALL = make_html_body(STR_SMALL) HTML_LARGE = make_html_body(STR_LARGE) @@ -236,9 +237,16 @@ class TestHttps2ClientProtocol: ) + self.certificate_file.read_text(encoding="utf-8") return PrivateCertificate.loadPEM(pem) # type: ignore[no-any-return] + @pytest.fixture + def crawler(self, request: pytest.FixtureRequest) -> Crawler: + return get_crawler(settings_dict=getattr(request, "param", None)) + @async_yield_fixture # type: ignore[untyped-decorator] async def client( - self, server_port: int, client_certificate: PrivateCertificate + self, + server_port: int, + client_certificate: PrivateCertificate, + crawler: Crawler, ) -> AsyncGenerator[H2ClientProtocol]: from twisted.internet import reactor @@ -250,7 +258,7 @@ class TestHttps2ClientProtocol: acceptableProtocols=[b"h2"], ) uri = URI.fromBytes(bytes(self.get_url(server_port, "/"), "utf-8")) - h2_client_factory = H2ClientFactory(uri, get_crawler(), Deferred()) + h2_client_factory = H2ClientFactory(uri, crawler, Deferred()) client_endpoint = SSL4ClientEndpoint( reactor, self.host, server_port, client_options ) @@ -312,6 +320,17 @@ class TestHttps2ClientProtocol: request = Request(self.get_url(server_port, "/get-data-html-large")) await self._check_GET(client, request, Data.HTML_LARGE, 200) + @pytest.mark.parametrize( + "crawler", [{"HTTP2_MAX_FRAME_SIZE": 1024**2}], indirect=True + ) + @deferred_f_from_coro_f + async def test_GET_large_frames( + self, server_port: int, client: H2ClientProtocol + ) -> None: + request = Request(self.get_url(server_port, "/get-data-html-large")) + await self._check_GET(client, request, Data.HTML_LARGE, 200) + assert client.conn.local_settings.max_frame_size == 1024**2 + async def _check_GET_x10( self, client: H2ClientProtocol,