Warn only once per request about the download warn size (#7963)

This commit is contained in:
Adrian 2026-08-09 18:14:20 +02:00 committed by GitHub
parent a6c017c2ca
commit 95fe7acc86
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 6 additions and 2 deletions

View File

@ -548,7 +548,8 @@ class _ScrapyAgent:
txresponse._transport._producer.abortConnection()
raise DownloadCancelledError(warning_msg)
if warnsize and expected_size > warnsize:
reached_warnsize = bool(warnsize and expected_size > warnsize)
if reached_warnsize:
logger.warning(
get_warnsize_msg(expected_size, warnsize, request, expected=True)
)
@ -561,6 +562,7 @@ class _ScrapyAgent:
request=request,
maxsize=maxsize,
warnsize=warnsize,
reached_warnsize=reached_warnsize,
fail_on_dataloss=fail_on_dataloss,
crawler=self._crawler,
tls_verbose_logging=self._tls_verbose_logging,
@ -625,6 +627,7 @@ class _ResponseReader(Protocol):
fail_on_dataloss: bool,
crawler: Crawler,
*,
reached_warnsize: bool = False,
tls_verbose_logging: bool = False,
):
self._finished: Deferred[_ResultT] = finished
@ -634,7 +637,7 @@ class _ResponseReader(Protocol):
self._maxsize: int = maxsize
self._warnsize: int = warnsize
self._fail_on_dataloss: bool = fail_on_dataloss
self._reached_warnsize: bool = False
self._reached_warnsize: bool = reached_warnsize
self._bytes_received: int = 0
self._certificate: ssl.Certificate | None = None
self._ip_address: ipaddress.IPv4Address | ipaddress.IPv6Address | None = None

View File

@ -628,6 +628,7 @@ class TestHttpBase(ABC):
"Expected to receive 5 bytes which is larger than download warn size (4)"
in caplog.text
)
assert caplog.text.count("download warn size (4)") == 1
@coroutine_test
async def test_download_with_warnsize_no_content_length(