Merge remote-tracking branch 'origin/master' into max-header-length

docs/topics/download-handlers.rst

@@ -243,12 +243,8 @@ Known limitations of the HTTP/2 support:
 -   No support for HTTP/2 Cleartext (h2c), since no major browser supports
     HTTP/2 unencrypted (refer `http2 faq`_).
 
--   No setting to specify a maximum `frame size`_ larger than the default
-    value, 16384. Connections to servers that send a larger frame will fail.
-
 -   No support for `server pushes`_, which are ignored.
 
-.. _frame size: https://datatracker.ietf.org/doc/html/rfc7540#section-4.2
 .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption
 .. _server pushes: https://datatracker.ietf.org/doc/html/rfc7540#section-8.2
 

docs/topics/settings.rst

@@ -1563,6 +1563,28 @@ Default: ``None``
 
 The Project ID that will be used when storing data on `Google Cloud Storage`_.
 
+.. setting:: HTTP2_MAX_FRAME_SIZE
+
+HTTP2_MAX_FRAME_SIZE
+--------------------
+
+.. versionadded:: VERSION
+
+Default: ``16384``
+
+Maximum `frame size`_, in bytes, that servers may send, between ``16384`` and
+``16777215``. Connections to servers that send a larger frame fail.
+
+Raise it for servers that send larger frames regardless of this value. Note
+that :setting:`DOWNLOAD_MAXSIZE` and :setting:`DOWNLOAD_WARNSIZE` are checked
+once per received frame, so a higher value allows a response to exceed them by
+more before being caught.
+
+:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler` ignores
+this setting, as ``httpx`` does not allow configuring the frame size.
+
+.. _frame size: https://datatracker.ietf.org/doc/html/rfc7540#section-4.2
+
 .. setting:: ITEM_PIPELINES
 
 ITEM_PIPELINES

scrapy/core/_http2/protocol.py

@@ -21,6 +21,7 @@ from h2.events import (
     WindowUpdated,
 )
 from h2.exceptions import DenialOfServiceError, FrameTooLargeError, H2Error
+from h2.settings import SettingCodes
 from twisted.internet.interfaces import (
     IAddress,
     IHandshakeListener,
@@ -282,6 +283,9 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
 
         # Initiate H2 Connection
         self.conn.initiate_connection()
+        max_frame_size = self._crawler.settings.getint("HTTP2_MAX_FRAME_SIZE")
+        if max_frame_size != self.conn.local_settings.max_frame_size:
+            self.conn.update_settings({SettingCodes.MAX_FRAME_SIZE: max_frame_size})
         self._write_to_transport()
 
     def _lose_connection_with_error(self, errors: list[BaseException]) -> None:

scrapy/core/spidermw.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 import logging
 from collections.abc import AsyncIterator, Callable, Coroutine, Iterable
 from functools import wraps
-from inspect import isasyncgenfunction
+from inspect import isasyncgenfunction, iscoroutine
 from itertools import islice
 from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar
 from warnings import warn
@@ -244,8 +244,20 @@ class SpiderMiddlewareManager(MiddlewareManager):
             warn(msg, category=ScrapyDeprecationWarning, stacklevel=2)
             self._set_compat_spider(spider)
         start = self._spider.start()
+        if not hasattr(start, "__aiter__"):
+            if iscoroutine(start):
+                start.close()
+            start = self._reject_start(start)
         return await self._process_chain("process_start", start)
 
+    async def _reject_start(self, start: Any) -> AsyncIterator[Any]:
+        raise TypeError(
+            f"{global_object_name(type(self._spider))}.start() must be an"
+            f" asynchronous generator, i.e. an async def method with yield"
+            f" statements, got {type(start)}"
+        )
+        yield  # pylint: disable=unreachable  # makes this method an asynchronous generator
+
     # This method is only needed until _async compatibility methods are removed.
     @staticmethod
     def _get_process_spider_output(mw: Any) -> Callable[..., Any] | None:

scrapy/settings/default_settings.py

@@ -110,6 +110,7 @@ __all__ = [
     "FTP_PASSWORD",
     "FTP_USER",
     "GCS_PROJECT_ID",
+    "HTTP2_MAX_FRAME_SIZE",
     "HTTPAUTH_DOMAIN",
     "HTTPAUTH_PASS",
     "HTTPAUTH_USER",
@@ -409,6 +410,8 @@ FTP_PASSWORD = "guest"  # noqa: S105
 
 GCS_PROJECT_ID = None
 
+HTTP2_MAX_FRAME_SIZE = 16384
+
 HTTPAUTH_USER = ""
 HTTPAUTH_PASS = ""
 HTTPAUTH_DOMAIN = None

tests/test_engine_loop.py

@@ -4,6 +4,8 @@ from collections import deque
 from logging import ERROR
 from typing import TYPE_CHECKING, Any
 
+import pytest
+
 from scrapy import Request, Spider, signals
 from scrapy.core.scheduler import BaseScheduler
 from scrapy.exceptions import CloseSpider
@@ -13,7 +15,7 @@ from tests.mockserver.http import MockServer
 from tests.utils.decorators import coroutine_test
 
 if TYPE_CHECKING:
-    import pytest
+    from collections.abc import Iterator
 
     from scrapy.http import Response
 
@@ -50,6 +52,27 @@ class MemoryScheduler(BaseScheduler):
         self.paused = False
 
 
+class NoneStartSpider(Spider):
+    name = "test"
+
+    def start(self) -> None:  # type: ignore[override]
+        return None
+
+
+class CoroutineStartSpider(Spider):
+    name = "test"
+
+    async def start(self) -> None:  # type: ignore[override]
+        return None
+
+
+class SyncStartSpider(Spider):
+    name = "test"
+
+    def start(self) -> Iterator[Request]:  # type: ignore[override]
+        yield Request("data:,a")
+
+
 class TestMain:
     @coroutine_test
     async def test_sleep(self):
@@ -141,6 +164,34 @@ class TestMain:
         assert crawler.stats.get_value("finish_reason") == "shutdown"
         assert not actual_urls
 
+    @pytest.mark.parametrize(
+        ("spider_cls", "expected_type"),
+        [
+            (NoneStartSpider, "<class 'NoneType'>"),
+            (CoroutineStartSpider, "<class 'coroutine'>"),
+            (SyncStartSpider, "<class 'generator'>"),
+        ],
+    )
+    @coroutine_test
+    async def test_start_not_an_async_generator(
+        self,
+        spider_cls: type[Spider],
+        expected_type: str,
+        caplog: pytest.LogCaptureFixture,
+    ) -> None:
+        crawler = get_crawler(spider_cls)
+
+        caplog.clear()
+        with caplog.at_level(ERROR):
+            await crawler.crawl_async()
+
+        assert (
+            f"{spider_cls.__name__}.start() must be an asynchronous generator,"
+            f" i.e. an async def method with yield statements, got {expected_type}"
+        ) in caplog.text
+        assert crawler.stats
+        assert crawler.stats.get_value("finish_reason") == "start_error"
+
     @coroutine_test
     async def test_start_error(self, caplog: pytest.LogCaptureFixture) -> None:
         class TestSpider(Spider):

tests/test_http2_client_protocol.py

@@ -37,6 +37,7 @@ if TYPE_CHECKING:
     from collections.abc import AsyncGenerator, Callable, Coroutine, Generator
 
     from scrapy.core._http2.protocol import H2ClientProtocol
+    from scrapy.crawler import Crawler
 
 
 pytestmark = [
@@ -75,7 +76,7 @@ class Data:
     STR_LARGE = generate_random_string(LARGE_SIZE)
 
     EXTRA_SMALL = generate_random_string(1024 * 15)
-    EXTRA_LARGE = generate_random_string((1024**2) * 15)
+    EXTRA_LARGE = generate_random_string(LARGE_SIZE)
 
     HTML_SMALL = make_html_body(STR_SMALL)
     HTML_LARGE = make_html_body(STR_LARGE)
@@ -236,9 +237,16 @@ class TestHttps2ClientProtocol:
         ) + self.certificate_file.read_text(encoding="utf-8")
         return PrivateCertificate.loadPEM(pem)  # type: ignore[no-any-return]
 
+    @pytest.fixture
+    def crawler(self, request: pytest.FixtureRequest) -> Crawler:
+        return get_crawler(settings_dict=getattr(request, "param", None))
+
     @async_yield_fixture  # type: ignore[untyped-decorator]
     async def client(
-        self, server_port: int, client_certificate: PrivateCertificate
+        self,
+        server_port: int,
+        client_certificate: PrivateCertificate,
+        crawler: Crawler,
     ) -> AsyncGenerator[H2ClientProtocol]:
         from twisted.internet import reactor
 
@@ -250,7 +258,7 @@ class TestHttps2ClientProtocol:
             acceptableProtocols=[b"h2"],
         )
         uri = URI.fromBytes(bytes(self.get_url(server_port, "/"), "utf-8"))
-        h2_client_factory = H2ClientFactory(uri, get_crawler(), Deferred())
+        h2_client_factory = H2ClientFactory(uri, crawler, Deferred())
         client_endpoint = SSL4ClientEndpoint(
             reactor, self.host, server_port, client_options
         )
@@ -312,6 +320,17 @@ class TestHttps2ClientProtocol:
         request = Request(self.get_url(server_port, "/get-data-html-large"))
         await self._check_GET(client, request, Data.HTML_LARGE, 200)
 
+    @pytest.mark.parametrize(
+        "crawler", [{"HTTP2_MAX_FRAME_SIZE": 1024**2}], indirect=True
+    )
+    @deferred_f_from_coro_f
+    async def test_GET_large_frames(
+        self, server_port: int, client: H2ClientProtocol
+    ) -> None:
+        request = Request(self.get_url(server_port, "/get-data-html-large"))
+        await self._check_GET(client, request, Data.HTML_LARGE, 200)
+        assert client.conn.local_settings.max_frame_size == 1024**2
+
     async def _check_GET_x10(
         self,
         client: H2ClientProtocol,