Merge remote-tracking branch 'origin/master' into todo

.pre-commit-config.yaml

@@ -27,7 +27,7 @@ repos:
   hooks:
     - id: sphinx-lint
 - repo: https://github.com/scrapy/sphinx-scrapy
-  rev: 0.8.10
+  rev: 0.8.11
   hooks:
     - id: sphinx-scrapy
 - repo: https://github.com/zizmorcore/zizmor-pre-commit

conftest.py

@@ -47,7 +47,7 @@ if not H2_ENABLED:
     collect_ignore.extend(
         (
             "scrapy/core/downloader/handlers/http2.py",
-            *_py_files("scrapy/core/http2"),
+            *_py_files("scrapy/core/_http2"),
         )
     )
 
@@ -107,6 +107,14 @@ def pytest_configure(config):
         install_reactor_import_hook()
 
 
+def pytest_collection_modifyitems(items):
+    for item in items:
+        if item.get_closest_marker("requires_internet"):
+            # Requests to real websites fail every now and then in CI for
+            # reasons unrelated to the code under test.
+            item.add_marker(pytest.mark.flaky(reruns=2, reruns_delay=5))
+
+
 def pytest_runtest_setup(item):
     # Skip tests based on reactor markers
     reactor = item.config.getoption("--reactor")

docs/_ext/scrapydocs.py

@@ -137,14 +137,6 @@ def source_role(
     return [node], []
 
 
-def issue_role(
-    name, rawtext, text: str, lineno, inliner, options=None, content=None
-) -> tuple[list[Any], list[Any]]:
-    ref = "https://github.com/scrapy/scrapy/issues/" + text
-    node = nodes.reference(rawtext, "issue " + text, refuri=ref)
-    return [node], []
-
-
 def commit_role(
     name, rawtext, text: str, lineno, inliner, options=None, content=None
 ) -> tuple[list[Any], list[Any]]:
@@ -164,7 +156,6 @@ def rev_role(
 def setup(app: Sphinx) -> dict[str, Any]:
     app.add_role("source", source_role)
     app.add_role("commit", commit_role)
-    app.add_role("issue", issue_role)
     app.add_role("rev", rev_role)
 
     app.add_node(

docs/faq.rst

@@ -141,7 +141,7 @@ middleware with a :ref:`custom downloader middleware
 
 -   If you can meet the installation requirements, use pyre2_ instead of
     Python’s re_ to compile your URL-filtering regular expression. See
-    :issue:`1908`.
+    :gh:`1908`.
 
 See also `other suggestions at StackOverflow
 <https://stackoverflow.com/q/36440681>`__.
@@ -419,7 +419,7 @@ Running ``runspider`` I get ``error: No spider found in file: <filename>``
 This may happen if your Scrapy project has a spider module with a name that
 conflicts with the name of one of the `Python standard library modules`_, such
 as ``csv.py`` or ``os.py``, or any `Python package`_ that you have installed.
-See :issue:`2680`.
+See :gh:`2680`.
 
 
 .. _has been reported: https://github.com/scrapy/scrapy/issues/2905

docs/requirements.in

@@ -6,4 +6,4 @@ sphinx-notfound-page
 sphinx-reredirects
 sphinx-rtd-theme
 sphinx-rtd-dark-mode
-sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.10
+sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.11

docs/requirements.txt

@@ -156,7 +156,7 @@ sphinx-rtd-theme==3.1.0
     # via
     #   -r docs/requirements.in
     #   sphinx-rtd-dark-mode
-sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@fe176adc1a8577601bc3fa39b590ebed71a7e9b8
+sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@6f8e5e0bbd171a857da480f7188f2a205041cb60
     # via -r docs/requirements.in
 sphinx-sitemap==2.9.0
     # via sphinx-scrapy

docs/topics/download-handlers.rst

@@ -187,12 +187,6 @@ If you want to use this handler you need to replace the default one for the
 Features and limitations
 ^^^^^^^^^^^^^^^^^^^^^^^^
 
-.. warning::
-
-    This handler is experimental, and not yet recommended for production
-    environments. Future Scrapy versions may introduce related changes without
-    a deprecation period or warning.
-
 =========================== ================================================
 HTTP proxies                No (not implemented)
 SOCKS proxies               No (not supported by the library)
@@ -215,12 +209,8 @@ Known limitations of the HTTP/2 support:
 -   No support for HTTP/2 Cleartext (h2c), since no major browser supports
     HTTP/2 unencrypted (refer `http2 faq`_).
 
--   No setting to specify a maximum `frame size`_ larger than the default
-    value, 16384. Connections to servers that send a larger frame will fail.
-
 -   No support for `server pushes`_, which are ignored.
 
-.. _frame size: https://datatracker.ietf.org/doc/html/rfc7540#section-4.2
 .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption
 .. _server pushes: https://datatracker.ietf.org/doc/html/rfc7540#section-8.2
 

docs/topics/settings.rst

@@ -1499,6 +1499,28 @@ Default: ``None``
 
 The Project ID that will be used when storing data on `Google Cloud Storage`_.
 
+.. setting:: HTTP2_MAX_FRAME_SIZE
+
+HTTP2_MAX_FRAME_SIZE
+--------------------
+
+.. versionadded:: VERSION
+
+Default: ``16384``
+
+Maximum `frame size`_, in bytes, that servers may send, between ``16384`` and
+``16777215``. Connections to servers that send a larger frame fail.
+
+Raise it for servers that send larger frames regardless of this value. Note
+that :setting:`DOWNLOAD_MAXSIZE` and :setting:`DOWNLOAD_WARNSIZE` are checked
+once per received frame, so a higher value allows a response to exceed them by
+more before being caught.
+
+:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler` ignores
+this setting, as ``httpx`` does not allow configuring the frame size.
+
+.. _frame size: https://datatracker.ietf.org/doc/html/rfc7540#section-4.2
+
 .. setting:: ITEM_PIPELINES
 
 ITEM_PIPELINES
@@ -1529,6 +1551,20 @@ Default: ``{}``
 A dict containing the pipelines enabled by default in Scrapy. You should never
 modify this setting in your project, modify :setting:`ITEM_PIPELINES` instead.
 
+.. setting:: ITEM_PROCESSOR
+
+ITEM_PROCESSOR
+--------------
+
+Default: ``"scrapy.pipelines.ItemPipelineManager"``
+
+The :ref:`component <topics-components>` that builds the :ref:`item pipeline
+<topics-item-pipeline>` from :setting:`ITEM_PIPELINES` and runs scraped items
+through it. It must implement :class:`~scrapy.pipelines.ItemProcessorProtocol`.
+
+.. autoclass:: scrapy.pipelines.ItemProcessorProtocol
+    :members:
+
 
 .. setting:: JOBDIR
 

scrapy/core/_http2/agent.py

@@ -14,8 +14,8 @@ from twisted.web.client import (
 )
 from twisted.web.error import SchemeNotSupported
 
+from scrapy.core._http2.protocol import H2ClientFactory, H2ClientProtocol
 from scrapy.core.downloader.contextfactory import _AcceptableProtocolsContextFactory
-from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol
 
 if TYPE_CHECKING:
     from twisted.internet.base import ReactorBase

scrapy/core/_http2/protocol.py

@@ -21,6 +21,7 @@ from h2.events import (
     WindowUpdated,
 )
 from h2.exceptions import FrameTooLargeError, H2Error
+from h2.settings import SettingCodes
 from twisted.internet.interfaces import (
     IAddress,
     IHandshakeListener,
@@ -31,7 +32,7 @@ from twisted.internet.ssl import Certificate
 from twisted.protocols.policies import TimeoutMixin
 from zope.interface import implementer
 
-from scrapy.core.http2.stream import Stream, StreamCloseReason
+from scrapy.core._http2.stream import Stream, StreamCloseReason
 from scrapy.exceptions import DownloadTimeoutError
 from scrapy.http import Request, Response
 from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute
@@ -261,6 +262,9 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
 
         # Initiate H2 Connection
         self.conn.initiate_connection()
+        max_frame_size = self._crawler.settings.getint("HTTP2_MAX_FRAME_SIZE")
+        if max_frame_size != self.conn.local_settings.max_frame_size:
+            self.conn.update_settings({SettingCodes.MAX_FRAME_SIZE: max_frame_size})
         self._write_to_transport()
 
     def _lose_connection_with_error(self, errors: list[BaseException]) -> None:

scrapy/core/_http2/stream.py

@@ -27,7 +27,7 @@ from scrapy.utils.httpobj import urlparse_cached
 if TYPE_CHECKING:
     from collections.abc import Sequence
 
-    from scrapy.core.http2.protocol import H2ClientProtocol
+    from scrapy.core._http2.protocol import H2ClientProtocol
     from scrapy.crawler import Crawler
     from scrapy.http import Request, Response
 

scrapy/core/downloader/handlers/http2.py

@@ -4,9 +4,9 @@ from time import monotonic
 from typing import TYPE_CHECKING
 from urllib.parse import urldefrag
 
+from scrapy.core._http2.agent import H2Agent, H2ConnectionPool
 from scrapy.core.downloader.contextfactory import _load_context_factory_from_settings
 from scrapy.core.downloader.handlers.base import BaseDownloadHandler
-from scrapy.core.http2.agent import H2Agent, H2ConnectionPool
 from scrapy.exceptions import (
     DownloadTimeoutError,
     NotConfigured,

scrapy/core/spidermw.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 import logging
 from collections.abc import AsyncIterator, Callable, Coroutine, Iterable
 from functools import wraps
-from inspect import isasyncgenfunction
+from inspect import isasyncgenfunction, iscoroutine
 from itertools import islice
 from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar
 from warnings import warn
@@ -244,8 +244,20 @@ class SpiderMiddlewareManager(MiddlewareManager):
             warn(msg, category=ScrapyDeprecationWarning, stacklevel=2)
             self._set_compat_spider(spider)
         start = self._spider.start()
+        if not hasattr(start, "__aiter__"):
+            if iscoroutine(start):
+                start.close()
+            start = self._reject_start(start)
         return await self._process_chain("process_start", start)
 
+    async def _reject_start(self, start: Any) -> AsyncIterator[Any]:
+        raise TypeError(
+            f"{global_object_name(type(self._spider))}.start() must be an"
+            f" asynchronous generator, i.e. an async def method with yield"
+            f" statements, got {type(start)}"
+        )
+        yield  # pylint: disable=unreachable  # makes this method an asynchronous generator
+
     # This method is only needed until _async compatibility methods are removed.
     @staticmethod
     def _get_process_spider_output(mw: Any) -> Callable[..., Any] | None:

scrapy/extensions/periodic_log.py

@@ -1,11 +1,12 @@
 from __future__ import annotations
 
 import logging
+import warnings
 from datetime import datetime, timezone
 from typing import TYPE_CHECKING, Any
 
 from scrapy import Spider, signals
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call
 from scrapy.utils.serialize import ScrapyJSONEncoder
 
@@ -38,6 +39,7 @@ class PeriodicLog:
     ):
         self.stats: StatsCollector = stats
         self.interval: float = interval
+        self._multiplier: float = 60.0 / interval
         self.task: AsyncioLoopingCall | LoopingCall | None = None
         self.encoder: JSONEncoder = ScrapyJSONEncoder(sort_keys=True, indent=4)
         self.ext_stats_enabled: bool = bool(ext_stats)
@@ -56,6 +58,24 @@ class PeriodicLog:
         )
         self.ext_timing_enabled: bool = ext_timing_enabled
 
+    @property
+    def multiplier(self) -> float:
+        warnings.warn(
+            "The PeriodicLog.multiplier attribute is deprecated.",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
+        return self._multiplier
+
+    @multiplier.setter
+    def multiplier(self, value: float) -> None:
+        warnings.warn(
+            "The PeriodicLog.multiplier attribute is deprecated.",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
+        self._multiplier = value
+
     @classmethod
     def from_crawler(cls, crawler: Crawler) -> Self:
         interval: float = crawler.settings.getfloat("LOGSTATS_INTERVAL")

scrapy/pipelines/__init__.py

@@ -8,7 +8,7 @@ from __future__ import annotations
 
 import asyncio
 import warnings
-from typing import TYPE_CHECKING, Any, cast
+from typing import TYPE_CHECKING, Any, Protocol, cast
 
 from twisted.internet.defer import Deferred, DeferredList, FirstError
 
@@ -28,6 +28,23 @@ if TYPE_CHECKING:
     from scrapy.settings import Settings
 
 
+class ItemProcessorProtocol(Protocol):
+    """Protocol for item processor implementations.
+
+    See :setting:`ITEM_PROCESSOR`.
+    """
+
+    async def open_spider_async(self) -> None:
+        """Get the item processor ready to process items."""
+
+    async def process_item_async(self, item: Any) -> Any:
+        """Return the processed *item*, or raise
+        :exc:`~scrapy.exceptions.DropItem` to drop it."""
+
+    async def close_spider_async(self) -> None:
+        """Release any resource that the item processor is using."""
+
+
 class ItemPipelineManager(MiddlewareManager):
     component_name = "item pipeline"
 

scrapy/pipelines/files.py

@@ -27,9 +27,7 @@ from twisted.internet.defer import Deferred, maybeDeferred
 from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import Request, Response
 from scrapy.http.request import NO_CALLBACK
-from scrapy.pipelines.media import (
-    FileException as FileException,  # noqa: PLC0414  # re-exported for backward compatibility
-)
+from scrapy.pipelines.media import FileException as _FileException
 from scrapy.pipelines.media import (
     FileInfo,
     FileInfoOrError,
@@ -626,7 +624,7 @@ class FilesPipeline(MediaPipeline):
             f"{request} referred in <{referer}>: {failure.value}",
             extra={"spider": info.spider},
         )
-        raise FileException
+        raise _FileException
 
     async def media_downloaded(
         self,
@@ -645,7 +643,7 @@ class FilesPipeline(MediaPipeline):
                 {"status": response.status, "request": request, "referer": referer},
                 extra={"spider": info.spider},
             )
-            raise FileException("download-error")
+            raise _FileException("download-error")
 
         if not response.body:
             logger.warning(
@@ -654,7 +652,7 @@ class FilesPipeline(MediaPipeline):
                 {"request": request, "referer": referer},
                 extra={"spider": info.spider},
             )
-            raise FileException("empty-content")
+            raise _FileException("empty-content")
 
         status = "cached" if "cached" in response.flags else "downloaded"
         logger.debug(
@@ -670,7 +668,7 @@ class FilesPipeline(MediaPipeline):
             checksum: str = await ensure_awaitable(
                 self.file_downloaded(response, request, info, item=item)
             )
-        except FileException as exc:
+        except _FileException as exc:
             logger.warning(
                 "File (error): Error processing file from %(request)s "
                 "referred in <%(referer)s>: %(errormsg)s",
@@ -687,7 +685,7 @@ class FilesPipeline(MediaPipeline):
                 exc_info=True,
                 extra={"spider": info.spider},
             )
-            raise FileException(str(exc)) from exc
+            raise _FileException(str(exc)) from exc
 
         return {
             "url": request.url,
@@ -770,3 +768,15 @@ class FilesPipeline(MediaPipeline):
             if media_type:
                 media_ext = cast("str", mimetypes.guess_extension(media_type))
         return f"full/{media_guid}{media_ext}"
+
+
+def __getattr__(name: str) -> Any:
+    if name == "FileException":
+        warnings.warn(
+            "scrapy.pipelines.files.FileException is deprecated, use "
+            "scrapy.pipelines.media.FileException instead.",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
+        return _FileException
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

scrapy/pipelines/images.py

@@ -18,13 +18,8 @@ from itemadapter import ItemAdapter
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import Request, Response
 from scrapy.http.request import NO_CALLBACK
-from scrapy.pipelines.files import (
-    FileException,
-    FilesPipeline,
-    GCSFilesStore,
-    S3FilesStore,
-    _md5sum,
-)
+from scrapy.pipelines.files import FilesPipeline, GCSFilesStore, S3FilesStore, _md5sum
+from scrapy.pipelines.media import FileException
 from scrapy.utils.defer import ensure_awaitable
 from scrapy.utils.python import to_bytes
 

scrapy/settings/default_settings.py

@@ -108,6 +108,7 @@ __all__ = [
     "FTP_PASSWORD",
     "FTP_USER",
     "GCS_PROJECT_ID",
+    "HTTP2_MAX_FRAME_SIZE",
     "HTTPAUTH_DOMAIN",
     "HTTPAUTH_PASS",
     "HTTPAUTH_USER",
@@ -402,6 +403,8 @@ FTP_PASSWORD = "guest"  # noqa: S105
 
 GCS_PROJECT_ID = None
 
+HTTP2_MAX_FRAME_SIZE = 16384
+
 HTTPAUTH_USER = ""
 HTTPAUTH_PASS = ""
 HTTPAUTH_DOMAIN = None

scrapy/utils/log.py

@@ -2,7 +2,9 @@ from __future__ import annotations
 
 import logging
 import pprint
+import re
 import sys
+import warnings
 from collections.abc import MutableMapping
 from logging.config import dictConfig
 from typing import TYPE_CHECKING, Any, cast
@@ -12,6 +14,7 @@ from twisted.python import log as twisted_log
 from twisted.python.failure import Failure
 
 import scrapy
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.settings import Settings
 from scrapy.utils.versions import get_versions
 
@@ -242,6 +245,9 @@ class LogCounterHandler(logging.Handler):
         self.crawler.stats.inc_value(sname)
 
 
+_MSG_MAPPING_PLACEHOLDER = re.compile(r"%\(\w+\)")
+
+
 def logformatter_adapter(
     logkws: LogFormatterResult,
 ) -> tuple[Any, ...]:
@@ -257,6 +263,20 @@ def logformatter_adapter(
     # argument, so empty args are left out. Tuple args become one positional
     # argument each, while a dict is a single positional argument.
     if not args:
+        if _MSG_MAPPING_PLACEHOLDER.search(message):
+            # The log formatter method has already returned, so there is no
+            # frame of it left in the stack to point at. msg is part of the
+            # warning message instead, so that each offending method gets its
+            # own warning.
+            warnings.warn(
+                f"A log formatter method returned msg {message!r} with "
+                f"%(name)s placeholders and no args. Interpolating msg with "
+                f"the returned dict is deprecated, return those values under "
+                f"args instead.",
+                ScrapyDeprecationWarning,
+                stacklevel=1,
+            )
+            return (level, message, logkws)
         return (level, message)
     if isinstance(args, tuple):
         return (level, message, *args)

tests/test_downloader_handler_twisted_http2.py

@@ -143,7 +143,7 @@ class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase):
                 response = await download_handler.download_request(request)
         assert response.text == actual_content_length
         assert (
-            "scrapy.core.http2.stream",
+            "scrapy.core._http2.stream",
             logging.WARNING,
             f"Ignoring bad Content-Length header "
             f"{bad_content_length!r} of request {request}, sending "

tests/test_downloadermiddleware_httpcache.py

@@ -111,7 +111,7 @@ class StorageTestMixin(TestBase):
         raise NotImplementedError
 
     def test_storage(self):
-        with self._storage(HTTPCACHE_EXPIRATION_SECS=1) as (storage, crawler):
+        with self._storage(HTTPCACHE_EXPIRATION_SECS=100) as (storage, crawler):
             request2 = self.request.copy()
             assert storage.retrieve_response(crawler.spider, request2) is None
 

tests/test_engine_loop.py

@@ -4,6 +4,8 @@ from collections import deque
 from logging import ERROR
 from typing import TYPE_CHECKING, Any
 
+import pytest
+
 from scrapy import Request, Spider, signals
 from scrapy.core.scheduler import BaseScheduler
 from scrapy.exceptions import CloseSpider
@@ -15,8 +17,6 @@ from tests.utils.decorators import coroutine_test
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator, Callable, Iterator
 
-    import pytest
-
     from scrapy.http import Response
 
 
@@ -52,6 +52,27 @@ class MemoryScheduler(BaseScheduler):
         self.paused = False
 
 
+class NoneStartSpider(Spider):
+    name = "test"
+
+    def start(self) -> None:  # type: ignore[override]
+        return None
+
+
+class CoroutineStartSpider(Spider):
+    name = "test"
+
+    async def start(self) -> None:  # type: ignore[override]
+        return None
+
+
+class SyncStartSpider(Spider):
+    name = "test"
+
+    def start(self) -> Iterator[Request]:  # type: ignore[override]
+        yield Request("data:,a")
+
+
 class TestMain:
     @coroutine_test
     async def test_sleep(self):
@@ -145,6 +166,34 @@ class TestMain:
         assert crawler.stats.get_value("finish_reason") == "shutdown"
         assert not actual_urls
 
+    @pytest.mark.parametrize(
+        ("spider_cls", "expected_type"),
+        [
+            (NoneStartSpider, "<class 'NoneType'>"),
+            (CoroutineStartSpider, "<class 'coroutine'>"),
+            (SyncStartSpider, "<class 'generator'>"),
+        ],
+    )
+    @coroutine_test
+    async def test_start_not_an_async_generator(
+        self,
+        spider_cls: type[Spider],
+        expected_type: str,
+        caplog: pytest.LogCaptureFixture,
+    ) -> None:
+        crawler = get_crawler(spider_cls)
+
+        caplog.clear()
+        with caplog.at_level(ERROR):
+            await crawler.crawl_async()
+
+        assert (
+            f"{spider_cls.__name__}.start() must be an asynchronous generator,"
+            f" i.e. an async def method with yield statements, got {expected_type}"
+        ) in caplog.text
+        assert crawler.stats
+        assert crawler.stats.get_value("finish_reason") == "start_error"
+
     @coroutine_test
     async def test_start_error(self, caplog: pytest.LogCaptureFixture) -> None:
         class TestSpider(Spider):

tests/test_extension_periodic_log.py

@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Any
 
 import pytest
 
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.extensions.periodic_log import PeriodicLog
 from scrapy.utils.misc import build_from_crawler
 from scrapy.utils.test import get_crawler
@@ -249,3 +249,17 @@ class TestPeriodicLog:
         assert data["time"]["log_interval_real"] >= 0
         assert data["time"]["elapsed"] >= 0
         assert data["time"]["start_time"] <= data["time"]["utcnow"]
+
+    def test_multiplier_deprecated(self) -> None:
+        crawler = get_crawler(
+            MetaSpider,
+            {"PERIODIC_LOG_TIMING_ENABLED": True, "LOGSTATS_INTERVAL": 30},
+        )
+        crawler._apply_settings()
+        ext = build_from_crawler(PeriodicLog, crawler)
+        with pytest.warns(ScrapyDeprecationWarning):
+            assert ext.multiplier == 2.0
+        with pytest.warns(ScrapyDeprecationWarning):
+            ext.multiplier = 3.0
+        with pytest.warns(ScrapyDeprecationWarning):
+            assert ext.multiplier == 3.0

tests/test_http2_client_protocol.py

@@ -36,7 +36,8 @@ from tests.mockserver.utils import ssl_context_factory
 if TYPE_CHECKING:
     from collections.abc import AsyncGenerator, Callable, Coroutine, Generator
 
-    from scrapy.core.http2.protocol import H2ClientProtocol
+    from scrapy.core._http2.protocol import H2ClientProtocol
+    from scrapy.crawler import Crawler
 
 
 pytestmark = [
@@ -75,7 +76,7 @@ class Data:
     STR_LARGE = generate_random_string(LARGE_SIZE)
 
     EXTRA_SMALL = generate_random_string(1024 * 15)
-    EXTRA_LARGE = generate_random_string((1024**2) * 15)
+    EXTRA_LARGE = generate_random_string(LARGE_SIZE)
 
     HTML_SMALL = make_html_body(STR_SMALL)
     HTML_LARGE = make_html_body(STR_LARGE)
@@ -236,13 +237,20 @@ class TestHttps2ClientProtocol:
         ) + self.certificate_file.read_text(encoding="utf-8")
         return PrivateCertificate.loadPEM(pem)  # type: ignore[no-any-return]
 
+    @pytest.fixture
+    def crawler(self, request: pytest.FixtureRequest) -> Crawler:
+        return get_crawler(settings_dict=getattr(request, "param", None))
+
     @async_yield_fixture  # type: ignore[untyped-decorator]
     async def client(
-        self, server_port: int, client_certificate: PrivateCertificate
+        self,
+        server_port: int,
+        client_certificate: PrivateCertificate,
+        crawler: Crawler,
     ) -> AsyncGenerator[H2ClientProtocol]:
         from twisted.internet import reactor
 
-        from scrapy.core.http2.protocol import H2ClientFactory  # noqa: PLC0415
+        from scrapy.core._http2.protocol import H2ClientFactory  # noqa: PLC0415
 
         client_options = optionsForClientTLS(
             hostname=self.host,
@@ -250,7 +258,7 @@ class TestHttps2ClientProtocol:
             acceptableProtocols=[b"h2"],
         )
         uri = URI.fromBytes(bytes(self.get_url(server_port, "/"), "utf-8"))
-        h2_client_factory = H2ClientFactory(uri, get_crawler(), Deferred())
+        h2_client_factory = H2ClientFactory(uri, crawler, Deferred())
         client_endpoint = SSL4ClientEndpoint(
             reactor, self.host, server_port, client_options
         )
@@ -312,6 +320,17 @@ class TestHttps2ClientProtocol:
         request = Request(self.get_url(server_port, "/get-data-html-large"))
         await self._check_GET(client, request, Data.HTML_LARGE, 200)
 
+    @pytest.mark.parametrize(
+        "crawler", [{"HTTP2_MAX_FRAME_SIZE": 1024**2}], indirect=True
+    )
+    @deferred_f_from_coro_f
+    async def test_GET_large_frames(
+        self, server_port: int, client: H2ClientProtocol
+    ) -> None:
+        request = Request(self.get_url(server_port, "/get-data-html-large"))
+        await self._check_GET(client, request, Data.HTML_LARGE, 200)
+        assert client.conn.local_settings.max_frame_size == 1024**2
+
     async def _check_GET_x10(
         self,
         client: H2ClientProtocol,
@@ -460,7 +479,7 @@ class TestHttps2ClientProtocol:
     def test_invalid_negotiated_protocol(
         self, server_port: int, client: H2ClientProtocol
     ) -> Generator[Deferred[Any], Any, None]:
-        with mock.patch("scrapy.core.http2.protocol.PROTOCOL_NAME", new=b"not-h2"):
+        with mock.patch("scrapy.core._http2.protocol.PROTOCOL_NAME", new=b"not-h2"):
             request = Request(url=self.get_url(server_port, "/status?n=200"))
             with pytest.raises(ResponseFailed):
                 yield make_request_dfd(client, request)
@@ -527,7 +546,7 @@ class TestHttps2ClientProtocol:
         expected_body: bytes,
         caplog: pytest.LogCaptureFixture,
     ) -> None:
-        with caplog.at_level("WARNING", "scrapy.core.http2.stream"):
+        with caplog.at_level("WARNING", "scrapy.core._http2.stream"):
             response = await make_request(client, request)
         assert response.status == 200
         assert response.body == expected_body
@@ -605,7 +624,7 @@ class TestHttps2ClientProtocol:
         def assert_inactive_stream(failure):
             assert failure.check(ResponseFailed) is not None
 
-            from scrapy.core.http2.stream import InactiveStreamClosed  # noqa: PLC0415
+            from scrapy.core._http2.stream import InactiveStreamClosed  # noqa: PLC0415
 
             assert any(
                 isinstance(e, InactiveStreamClosed) for e in failure.value.reasons
@@ -692,7 +711,7 @@ class TestHttps2ClientProtocol:
 
     @staticmethod
     async def _check_invalid_netloc(client: H2ClientProtocol, url: str) -> None:
-        from scrapy.core.http2.stream import InvalidHostname  # noqa: PLC0415
+        from scrapy.core._http2.stream import InvalidHostname  # noqa: PLC0415
 
         request = Request(url)
         with pytest.raises(InvalidHostname) as exc_info:
@@ -737,7 +756,7 @@ class TestHttps2ClientProtocol:
             yield make_request_dfd(client, request)
 
         for err in exc_info.value.reasons:
-            from scrapy.core.http2.protocol import H2ClientProtocol  # noqa: PLC0415
+            from scrapy.core._http2.protocol import H2ClientProtocol  # noqa: PLC0415
 
             if isinstance(err, DownloadTimeoutError):
                 assert (

tests/test_pipeline_files.py

@@ -24,18 +24,18 @@ from twisted.internet.defer import Deferred
 from twisted.python.failure import Failure
 
 from scrapy.crawler import Crawler
-from scrapy.exceptions import IgnoreRequest, NotConfigured
+from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import Request, Response
 from scrapy.item import Field, Item
+from scrapy.pipelines import files
 from scrapy.pipelines.files import (
-    FileException,
     FilesPipeline,
     FSFilesStore,
     FTPFilesStore,
     GCSFilesStore,
     S3FilesStore,
 )
-from scrapy.pipelines.media import _MediaRequestFiltered
+from scrapy.pipelines.media import FileException, _MediaRequestFiltered
 from scrapy.settings import Settings
 from scrapy.utils.asyncio import call_later
 from scrapy.utils.defer import maybe_deferred_to_future
@@ -1249,3 +1249,11 @@ def test_files_pipeline_raises_notconfigured_when_files_store_invalid(store):
 
     with pytest.raises(NotConfigured):
         build_from_crawler(FilesPipeline, crawler)
+
+
+def test_file_exception_deprecated_import():
+    with pytest.warns(ScrapyDeprecationWarning, match="FileException"):
+        assert files.FileException is FileException
+
+    with pytest.raises(AttributeError):
+        files.nonexistent

tests/test_pipeline_media.py

@@ -10,8 +10,8 @@ from twisted.python.failure import Failure
 from scrapy import signals
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request, Response
-from scrapy.pipelines.files import FileException
 from scrapy.pipelines.media import (
+    FileException,
     FileInfo,
     FileInfoOrError,
     MediaPipeline,

tests/test_utils_log.py

@@ -4,12 +4,14 @@ import json
 import logging
 import re
 import sys
+import warnings
 from io import StringIO
 from typing import TYPE_CHECKING, Any, cast
 
 import pytest
 from twisted.python.failure import Failure
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.log import (
     LogCounterHandler,
     SpiderLoggerAdapter,
@@ -329,7 +331,9 @@ class TestLogformatterAdapter:
             "LogFormatterResult",
             {"level": logging.INFO, "msg": "90% done", "args": args},
         )
-        assert self._log(caplog, logkws) == "90% done"
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", ScrapyDeprecationWarning)
+            assert self._log(caplog, logkws) == "90% done"
 
     @pytest.mark.parametrize(
         ("msg", "args"),
@@ -342,4 +346,16 @@ class TestLogformatterAdapter:
         args: dict[str, Any] | tuple[Any, ...],
     ) -> None:
         logkws: LogFormatterResult = {"level": logging.INFO, "msg": msg, "args": args}
-        assert self._log(caplog, logkws) == "90% done"
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", ScrapyDeprecationWarning)
+            assert self._log(caplog, logkws) == "90% done"
+
+    def test_msg_mapping_placeholders_without_args(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        logkws = cast(
+            "LogFormatterResult",
+            {"level": logging.INFO, "msg": "%(pct)d%% done", "pct": 90},
+        )
+        with pytest.warns(ScrapyDeprecationWarning, match="no args"):
+            assert self._log(caplog, logkws) == "90% done"

tests/test_utils_response.py

@@ -184,32 +184,31 @@ def test_inject_base_url(body: bytes) -> None:
     assert open_in_browser(resp, _openfunc=check_base_url)
 
 
-def test_open_in_browser_redos_comment():
-    MAX_CPU_TIME = 0.02
+def _assert_open_in_browser_is_fast(body: bytes) -> None:
+    # The exploit inputs are large enough that a vulnerable implementation
+    # needs seconds to go through them, while a safe one stays in the low
+    # milliseconds even on a slow interpreter.
+    max_cpu_time = 0.2
 
+    response = HtmlResponse("https://example.com", body=body)
+    start_time = process_time()
+    open_in_browser(response, lambda url: True)
+    end_time = process_time()
+    assert end_time - start_time < max_cpu_time
+
+
+def test_open_in_browser_redos_comment():
     # Exploit input from
     # https://makenowjust-labs.github.io/recheck/playground/
     # for /<!--.*?-->/ (old pattern to remove comments).
-    body = b"-><!--\x00" * 25_000 + b"->\n<!---->"
-    response = HtmlResponse("https://example.com", body=body)
-    start_time = process_time()
-    open_in_browser(response, lambda url: True)
-    end_time = process_time()
-    assert end_time - start_time < MAX_CPU_TIME
+    _assert_open_in_browser_is_fast(b"-><!--\x00" * 250_000 + b"->\n<!---->")
 
 
 def test_open_in_browser_redos_head():
-    MAX_CPU_TIME = 0.02
-
     # Exploit input from
     # https://makenowjust-labs.github.io/recheck/playground/
     # for /(<head(?:>|\s.*?>))/ (old pattern to find the head element).
-    body = b"<head\t" * 8_000
-    response = HtmlResponse("https://example.com", body=body)
-    start_time = process_time()
-    open_in_browser(response, lambda url: True)
-    end_time = process_time()
-    assert end_time - start_time < MAX_CPU_TIME
+    _assert_open_in_browser_is_fast(b"<head\t" * 80_000)
 
 
 @pytest.mark.parametrize(

tests/utils/bases/download_handlers_http.py

@@ -1513,6 +1513,12 @@ class TestMitmProxyBase(ABC):
         assert "Proxy Authentication Required" in log or "407" in log
 
 
+# Tests below are rerun on failure (see pytest_collection_modifyitems() in the
+# root conftest.py), so an attempt must give up soon enough for a rerun to be
+# cheap.
+REAL_WEBSITE_SETTINGS = {"DOWNLOAD_TIMEOUT": 30}
+
+
 class TestRealWebsiteBase(ABC):
     @property
     @abstractmethod
@@ -1537,7 +1543,9 @@ class TestRealWebsiteBase(ABC):
     async def get_dh(
         self, settings_dict: dict[str, Any] | None = None
     ) -> AsyncGenerator[DownloadHandlerProtocol]:
-        crawler = get_crawler(DefaultSpider, settings_dict)
+        crawler = get_crawler(
+            DefaultSpider, {**REAL_WEBSITE_SETTINGS, **(settings_dict or {})}
+        )
         crawler.spider = crawler._create_spider()
         dh = build_from_crawler(self.download_handler_cls, crawler)
         try:
@@ -1555,7 +1563,9 @@ class TestRealWebsiteBase(ABC):
 
     @coroutine_test
     async def test_download_with_spider(self) -> None:
-        crawler = get_crawler(SingleRequestSpider, self.settings_dict)
+        crawler = get_crawler(
+            SingleRequestSpider, {**REAL_WEBSITE_SETTINGS, **(self.settings_dict or {})}
+        )
         await maybe_deferred_to_future(
             crawler.crawl(seed=Request("https://books.toscrape.com/"))
         )

tox.ini

@@ -5,7 +5,7 @@
 
 [tox]
 requires =
-    sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.10
+    sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.11
     tox-uv
 envlist =
     pre-commit
@@ -44,6 +44,7 @@ deps =
     pygments
     pytest
     pytest-cov >= 7.0.0
+    pytest-rerunfailures
     pytest-timeout
     pytest-xdist
     sybil >= 1.3.0  # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
@@ -124,7 +125,7 @@ commands =
 [testenv:twinecheck]
 basepython = python3
 deps =
-    twine==6.2.0
+    twine==7.0.0
     build==1.5.0
 commands =
     python -m build --sdist