diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 6ca04a50e..13a432476 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -123,12 +123,14 @@ class HttpCompressionMiddleware: response.body, content_encoding, max_size ) except _DecompressionMaxSizeExceeded as e: - raise IgnoreRequest( + msg = ( f"Ignored response {response} because its body " f"({len(response.body)} B compressed, " f"{e.decompressed_size} B decompressed so far) exceeded " f"DOWNLOAD_MAXSIZE ({max_size} B) during decompression." - ) from e + ) + logger.warning(msg) + raise IgnoreRequest(msg) from e if len(response.body) < warn_size <= len(decoded_body): logger.warning( f"{response} body size after decompression " diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 5c4085657..311aff7df 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -558,6 +558,30 @@ class TestHttpCompression: self._test_compression_bomb_setting("zstd") + def test_compression_bomb_setting_logs_warning(self, caplog): + settings = {"DOWNLOAD_MAXSIZE": 1_000_000} + crawler = get_crawler(Spider, settings_dict=settings) + spider = crawler._create_spider("scrapytest.org") + mw = HttpCompressionMiddleware.from_crawler(crawler) + mw.open_spider(spider) + + response = self._getresponse("bomb-gzip") # 11_511_612 B + caplog.clear() + with ( + caplog.at_level( + WARNING, logger="scrapy.downloadermiddlewares.httpcompression" + ), + pytest.raises(IgnoreRequest) as exc_info, + ): + mw.process_response(response.request, response) + assert caplog.record_tuples == [ + ( + "scrapy.downloadermiddlewares.httpcompression", + WARNING, + str(exc_info.value), + ) + ] + def _test_compression_bomb_spider_attr(self, compression_id): class DownloadMaxSizeSpider(Spider): download_maxsize = 1_000_000