From 2fc74b8c9dc16ba25087f4495d3877fb4fd3dc31 Mon Sep 17 00:00:00 2001 From: Mridankan Mandal Date: Mon, 13 Jul 2026 01:45:10 +0530 Subject: [PATCH 1/3] Log ignored responses exceeding DOWNLOAD_MAXSIZE Signed-off-by: Mridankan Mandal --- scrapy/downloadermiddlewares/httpcompression.py | 6 ++++-- .../test_downloadermiddleware_httpcompression.py | 16 +++++++++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 6ca04a50e..13a432476 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -123,12 +123,14 @@ class HttpCompressionMiddleware: response.body, content_encoding, max_size ) except _DecompressionMaxSizeExceeded as e: - raise IgnoreRequest( + msg = ( f"Ignored response {response} because its body " f"({len(response.body)} B compressed, " f"{e.decompressed_size} B decompressed so far) exceeded " f"DOWNLOAD_MAXSIZE ({max_size} B) during decompression." - ) from e + ) + logger.warning(msg) + raise IgnoreRequest(msg) from e if len(response.body) < warn_size <= len(decoded_body): logger.warning( f"{response} body size after decompression " diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 5c4085657..efc9caf53 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -538,9 +538,23 @@ class TestHttpCompression: mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") # 11_511_612 B - with pytest.raises(IgnoreRequest) as exc_info: + with ( + LogCapture( + "scrapy.downloadermiddlewares.httpcompression", + propagate=False, + level=WARNING, + ) as log, + pytest.raises(IgnoreRequest) as exc_info, + ): mw.process_response(response.request, response) assert exc_info.value.__cause__.decompressed_size < 1_100_000 + log.check( + ( + "scrapy.downloadermiddlewares.httpcompression", + "WARNING", + str(exc_info.value), + ), + ) def test_compression_bomb_setting_br(self): _skip_if_no_br() From 0dfad68136f3556fa052d0eb4ed556de5b1e53ca Mon Sep 17 00:00:00 2001 From: Mridankan Mandal Date: Thu, 16 Jul 2026 17:13:40 +0530 Subject: [PATCH 2/3] Use caplog in compression warning test Signed-off-by: Mridankan Mandal --- ...st_downloadermiddleware_httpcompression.py | 42 ++++++++++++------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index efc9caf53..35af15484 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -538,23 +538,9 @@ class TestHttpCompression: mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") # 11_511_612 B - with ( - LogCapture( - "scrapy.downloadermiddlewares.httpcompression", - propagate=False, - level=WARNING, - ) as log, - pytest.raises(IgnoreRequest) as exc_info, - ): + with pytest.raises(IgnoreRequest) as exc_info: mw.process_response(response.request, response) assert exc_info.value.__cause__.decompressed_size < 1_100_000 - log.check( - ( - "scrapy.downloadermiddlewares.httpcompression", - "WARNING", - str(exc_info.value), - ), - ) def test_compression_bomb_setting_br(self): _skip_if_no_br() @@ -572,6 +558,32 @@ class TestHttpCompression: self._test_compression_bomb_setting("zstd") + def test_compression_bomb_setting_logs_warning( + self, caplog: pytest.LogCaptureFixture + ): + settings = {"DOWNLOAD_MAXSIZE": 1_000_000} + crawler = get_crawler(Spider, settings_dict=settings) + spider = crawler._create_spider("scrapytest.org") + mw = HttpCompressionMiddleware.from_crawler(crawler) + mw.open_spider(spider) + + response = self._getresponse("bomb-gzip") # 11_511_612 B + with ( + caplog.at_level( + WARNING, logger="scrapy.downloadermiddlewares.httpcompression" + ), + pytest.raises(IgnoreRequest) as exc_info, + ): + mw.process_response(response.request, response) + assert exc_info.value.__cause__.decompressed_size < 1_100_000 + assert caplog.record_tuples == [ + ( + "scrapy.downloadermiddlewares.httpcompression", + WARNING, + str(exc_info.value), + ) + ] + def _test_compression_bomb_spider_attr(self, compression_id): class DownloadMaxSizeSpider(Spider): download_maxsize = 1_000_000 From 3caf8d922c4396b45cbf56ff33760151410ad7f5 Mon Sep 17 00:00:00 2001 From: Mridankan Mandal Date: Thu, 16 Jul 2026 18:32:37 +0530 Subject: [PATCH 3/3] Make compression warning test robust Signed-off-by: Mridankan Mandal --- tests/test_downloadermiddleware_httpcompression.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 35af15484..311aff7df 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -558,9 +558,7 @@ class TestHttpCompression: self._test_compression_bomb_setting("zstd") - def test_compression_bomb_setting_logs_warning( - self, caplog: pytest.LogCaptureFixture - ): + def test_compression_bomb_setting_logs_warning(self, caplog): settings = {"DOWNLOAD_MAXSIZE": 1_000_000} crawler = get_crawler(Spider, settings_dict=settings) spider = crawler._create_spider("scrapytest.org") @@ -568,6 +566,7 @@ class TestHttpCompression: mw.open_spider(spider) response = self._getresponse("bomb-gzip") # 11_511_612 B + caplog.clear() with ( caplog.at_level( WARNING, logger="scrapy.downloadermiddlewares.httpcompression" @@ -575,7 +574,6 @@ class TestHttpCompression: pytest.raises(IgnoreRequest) as exc_info, ): mw.process_response(response.request, response) - assert exc_info.value.__cause__.decompressed_size < 1_100_000 assert caplog.record_tuples == [ ( "scrapy.downloadermiddlewares.httpcompression",