Protect against deflate bombs

scrapy/downloadermiddlewares/httpcompression.py

@@ -1,11 +1,10 @@
 import io
 import warnings
-import zlib
 
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Response, TextResponse
 from scrapy.responsetypes import responsetypes
-from scrapy.utils._compression import _DecompressionMaxSizeExceeded
+from scrapy.utils._compression import _DecompressionMaxSizeExceeded, _inflate
 from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.gz import gunzip
 
@@ -97,23 +96,14 @@ class HttpCompressionMiddleware:
 
     def _decode(self, body, encoding):
         if encoding == b"gzip" or encoding == b"x-gzip":
-            body = gunzip(body, max_size=self._max_size)
-
+            return gunzip(body, max_size=self._max_size)
         if encoding == b"deflate":
-            try:
-                body = zlib.decompress(body)
-            except zlib.error:
-                # ugly hack to work with raw deflate content that may
-                # be sent by microsoft servers. For more information, see:
-                # http://carsten.codimi.de/gzip.yaws/
-                # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx
-                # http://www.gzip.org/zlib/zlib_faq.html#faq38
-                body = zlib.decompress(body, -15)
+            return _inflate(body, max_size=self._max_size)
         if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS:
-            body = brotli.decompress(body)
+            return brotli.decompress(body)
         if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS:
             # Using its streaming API since its simple API could handle only cases
             # where there is content size data embedded in the frame
             reader = zstandard.ZstdDecompressor().stream_reader(io.BytesIO(body))
-            body = reader.read()
+            return reader.read()
         return body

scrapy/utils/_compression.py

@@ -1,2 +1,41 @@
+import zlib
+from io import BytesIO
+from typing import List
+
+
 class _DecompressionMaxSizeExceeded(ValueError):
     pass
+
+
+def _inflate(data: bytes, *, max_size: int = 0) -> bytes:
+    decompressor = zlib.decompressobj()
+    raw_decompressor = zlib.decompressobj(wbits=-15)
+    input_stream = BytesIO(data)
+    output_list: List[bytes] = []
+    output_chunk = b"."
+    decompressed_size = 0
+    CHUNK_SIZE = 8196
+    while output_chunk:
+        input_chunk = input_stream.read(CHUNK_SIZE)
+        try:
+            output_chunk = decompressor.decompress(input_chunk)
+        except zlib.error:
+            if decompressor != raw_decompressor:
+                # ugly hack to work with raw deflate content that may
+                # be sent by microsoft servers. For more information, see:
+                # http://carsten.codimi.de/gzip.yaws/
+                # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx
+                # http://www.gzip.org/zlib/zlib_faq.html#faq38
+                decompressor = raw_decompressor
+                output_chunk = decompressor.decompress(input_chunk)
+            else:
+                raise
+        decompressed_size += len(output_chunk)
+        if max_size and decompressed_size > max_size:
+            raise _DecompressionMaxSizeExceeded(
+                f"The number of bytes decompressed so far "
+                f"({decompressed_size}B) exceed the specified maximum "
+                f"({max_size}B)."
+            )
+        output_list.append(output_chunk)
+    return b"".join(output_list)

scrapy/utils/gz.py

@@ -8,7 +8,7 @@ from scrapy.http import Response
 from ._compression import _DecompressionMaxSizeExceeded
 
 
-def gunzip(data: bytes, max_size: int = 0) -> bytes:
+def gunzip(data: bytes, *, max_size: int = 0) -> bytes:
     """Gunzip the given data and return as much data as possible.
 
     This is resilient to CRC checksum errors.

tests/test_downloadermiddleware_httpcompression.py

@@ -39,8 +39,8 @@ FORMAT = {
         f"bomb-{format_id}": (f"bomb-{format_id}.bin", format_id)
         for format_id in (
             # "br",
+            "deflate",  # 27 968 → 11 511 612
             "gzip",  # 27 988 → 11 511 612
-            # "deflate",
             # "zstd",
         )
     },
@@ -383,6 +383,9 @@ class HttpCompressionTest(TestCase):
             self.spider,
         )
 
+    def test_compression_bomb_deflate(self):
+        self._test_compression_bomb("deflate")
+
     def test_compression_bomb_gzip(self):
         self._test_compression_bomb("gzip")