From 95fe7acc86f83643bfa98d5a0da97b8457a61774 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 18:14:20 +0200 Subject: [PATCH] Warn only once per request about the download warn size (#7963) --- scrapy/core/downloader/handlers/http11.py | 7 +++++-- tests/utils/bases/download_handlers_http.py | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 17dca5acb..3b5c08666 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -548,7 +548,8 @@ class _ScrapyAgent: txresponse._transport._producer.abortConnection() raise DownloadCancelledError(warning_msg) - if warnsize and expected_size > warnsize: + reached_warnsize = bool(warnsize and expected_size > warnsize) + if reached_warnsize: logger.warning( get_warnsize_msg(expected_size, warnsize, request, expected=True) ) @@ -561,6 +562,7 @@ class _ScrapyAgent: request=request, maxsize=maxsize, warnsize=warnsize, + reached_warnsize=reached_warnsize, fail_on_dataloss=fail_on_dataloss, crawler=self._crawler, tls_verbose_logging=self._tls_verbose_logging, @@ -625,6 +627,7 @@ class _ResponseReader(Protocol): fail_on_dataloss: bool, crawler: Crawler, *, + reached_warnsize: bool = False, tls_verbose_logging: bool = False, ): self._finished: Deferred[_ResultT] = finished @@ -634,7 +637,7 @@ class _ResponseReader(Protocol): self._maxsize: int = maxsize self._warnsize: int = warnsize self._fail_on_dataloss: bool = fail_on_dataloss - self._reached_warnsize: bool = False + self._reached_warnsize: bool = reached_warnsize self._bytes_received: int = 0 self._certificate: ssl.Certificate | None = None self._ip_address: ipaddress.IPv4Address | ipaddress.IPv6Address | None = None diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.py index e44f9bcb8..65244b938 100644 --- a/tests/utils/bases/download_handlers_http.py +++ b/tests/utils/bases/download_handlers_http.py @@ -628,6 +628,7 @@ class TestHttpBase(ABC): "Expected to receive 5 bytes which is larger than download warn size (4)" in caplog.text ) + assert caplog.text.count("download warn size (4)") == 1 @coroutine_test async def test_download_with_warnsize_no_content_length(