diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 312ecf229..21def678a 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -93,6 +93,7 @@ class HTTP11DownloadHandler(BaseDownloadHandler): "DOWNLOAD_FAIL_ON_DATALOSS" ) self._disconnect_timeout: int = 1 + self._fail_on_dataloss_warned: bool = False async def download_request(self, request: Request) -> Response: """Return a deferred for the HTTP download""" @@ -115,8 +116,19 @@ class HTTP11DownloadHandler(BaseDownloadHandler): fail_on_dataloss=self._fail_on_dataloss, crawler=self._crawler, ) - with wrap_twisted_exceptions(): - return await maybe_deferred_to_future(agent.download_request(request)) + try: + with wrap_twisted_exceptions(): + return await maybe_deferred_to_future(agent.download_request(request)) + except ResponseDataLossError: + if not self._fail_on_dataloss_warned: + logger.warning( + "Got data loss in %s. If you want to process broken " + "responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False" + " -- This message won't be shown in further requests", + request.url, + ) + self._fail_on_dataloss_warned = True + raise async def close(self) -> None: from twisted.internet import reactor @@ -633,7 +645,6 @@ class _ResponseReader(Protocol): self._maxsize: int = maxsize self._warnsize: int = warnsize self._fail_on_dataloss: bool = fail_on_dataloss - self._fail_on_dataloss_warned: bool = False self._reached_warnsize: bool = False self._bytes_received: int = 0 self._certificate: ssl.Certificate | None = None @@ -738,15 +749,6 @@ class _ResponseReader(Protocol): self._finish_response(flags=["dataloss"]) return - if not self._fail_on_dataloss_warned: - logger.warning( - "Got data loss in %s. If you want to process broken " - "responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False" - " -- This message won't be shown in further requests", - self._txresponse.request.absoluteURI.decode(), - ) - self._fail_on_dataloss_warned = True - exc = ResponseDataLossError() exc.__cause__ = reason.value reason = Failure(exc) diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index a9197645a..3fbc8d416 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -86,6 +86,9 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): def test_download_cause_data_loss(self) -> None: # type: ignore[override] pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) + def test_download_cause_data_loss_double_warning(self) -> None: # type: ignore[override] + pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) + def test_download_allow_data_loss(self) -> None: # type: ignore[override] pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index d33de8881..5e3b49548 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -516,6 +516,21 @@ class TestHttp11Base(TestHttpBase): with pytest.raises(ResponseDataLossError): await download_handler.download_request(request) + @deferred_f_from_coro_f + async def test_download_cause_data_loss_double_warning( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + request = Request(mockserver.url("/broken", is_secure=self.is_secure)) + async with self.get_dh() as download_handler: + with pytest.raises(ResponseDataLossError): + await download_handler.download_request(request) + assert "Got data loss" in caplog.text + caplog.clear() + with pytest.raises(ResponseDataLossError): + await download_handler.download_request(request) + # no repeated warning + assert "Got data loss" not in caplog.text + @pytest.mark.parametrize("url", ["broken", "broken-chunked"]) @deferred_f_from_coro_f async def test_download_allow_data_loss(