Merge 3caf8d922c into d8ba1571e7

scrapy/downloadermiddlewares/httpcompression.py

@@ -123,12 +123,14 @@ class HttpCompressionMiddleware:
                     response.body, content_encoding, max_size
                 )
             except _DecompressionMaxSizeExceeded as e:
-                raise IgnoreRequest(
+                msg = (
                     f"Ignored response {response} because its body "
                     f"({len(response.body)} B compressed, "
                     f"{e.decompressed_size} B decompressed so far) exceeded "
                     f"DOWNLOAD_MAXSIZE ({max_size} B) during decompression."
-                ) from e
+                )
+                logger.warning(msg)
+                raise IgnoreRequest(msg) from e
             if len(response.body) < warn_size <= len(decoded_body):
                 logger.warning(
                     f"{response} body size after decompression "

tests/test_downloadermiddleware_httpcompression.py

@@ -558,6 +558,30 @@ class TestHttpCompression:
 
         self._test_compression_bomb_setting("zstd")
 
+    def test_compression_bomb_setting_logs_warning(self, caplog):
+        settings = {"DOWNLOAD_MAXSIZE": 1_000_000}
+        crawler = get_crawler(Spider, settings_dict=settings)
+        spider = crawler._create_spider("scrapytest.org")
+        mw = HttpCompressionMiddleware.from_crawler(crawler)
+        mw.open_spider(spider)
+
+        response = self._getresponse("bomb-gzip")  # 11_511_612 B
+        caplog.clear()
+        with (
+            caplog.at_level(
+                WARNING, logger="scrapy.downloadermiddlewares.httpcompression"
+            ),
+            pytest.raises(IgnoreRequest) as exc_info,
+        ):
+            mw.process_response(response.request, response)
+        assert caplog.record_tuples == [
+            (
+                "scrapy.downloadermiddlewares.httpcompression",
+                WARNING,
+                str(exc_info.value),
+            )
+        ]
+
     def _test_compression_bomb_spider_attr(self, compression_id):
         class DownloadMaxSizeSpider(Spider):
             download_maxsize = 1_000_000