Protect against brotli bombs

scrapy/downloadermiddlewares/httpcompression.py

@@ -4,25 +4,25 @@ import warnings
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Response, TextResponse
 from scrapy.responsetypes import responsetypes
-from scrapy.utils._compression import _DecompressionMaxSizeExceeded, _inflate
+from scrapy.utils._compression import _DecompressionMaxSizeExceeded, _inflate, _unbrotli
 from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.gz import gunzip
 
 ACCEPTED_ENCODINGS = [b"gzip", b"deflate"]
 
 try:
-    import brotli
-
-    ACCEPTED_ENCODINGS.append(b"br")
+    import brotli  # noqa: F401
 except ImportError:
     pass
+else:
+    ACCEPTED_ENCODINGS.append(b"br")
 
 try:
     import zstandard
-
-    ACCEPTED_ENCODINGS.append(b"zstd")
 except ImportError:
     pass
+else:
+    ACCEPTED_ENCODINGS.append(b"zstd")
 
 
 class HttpCompressionMiddleware:
@@ -100,7 +100,7 @@ class HttpCompressionMiddleware:
         if encoding == b"deflate":
             return _inflate(body, max_size=self._max_size)
         if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS:
-            return brotli.decompress(body)
+            return _unbrotli(body, max_size=self._max_size)
         if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS:
             # Using its streaming API since its simple API could handle only cases
             # where there is content size data embedded in the frame

scrapy/utils/_compression.py

@@ -2,6 +2,11 @@ import zlib
 from io import BytesIO
 from typing import List
 
+try:
+    import brotli
+except ImportError:
+    pass
+
 
 class _DecompressionMaxSizeExceeded(ValueError):
     pass
@@ -39,3 +44,24 @@ def _inflate(data: bytes, *, max_size: int = 0) -> bytes:
             )
         output_list.append(output_chunk)
     return b"".join(output_list)
+
+
+def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes:
+    decompressor = brotli.Decompressor()
+    input_stream = BytesIO(data)
+    output_list: List[bytes] = []
+    output_chunk = b"."
+    decompressed_size = 0
+    CHUNK_SIZE = 8196
+    while output_chunk:
+        input_chunk = input_stream.read(CHUNK_SIZE)
+        output_chunk = decompressor.process(input_chunk)
+        decompressed_size += len(output_chunk)
+        if max_size and decompressed_size > max_size:
+            raise _DecompressionMaxSizeExceeded(
+                f"The number of bytes decompressed so far "
+                f"({decompressed_size}B) exceed the specified maximum "
+                f"({max_size}B)."
+            )
+        output_list.append(output_chunk)
+    return b"".join(output_list)

tests/sample_data/compressed/bomb-br.bin

@@ -0,0 +1,2 @@
+<EFBFBD>;§¯ר<C2AF>”n<E2809D>×Vp	SmoY2·ו
+ן(ה)-׀´=_o

tests/test_downloadermiddleware_httpcompression.py

@@ -38,7 +38,7 @@ FORMAT = {
     **{
         f"bomb-{format_id}": (f"bomb-{format_id}.bin", format_id)
         for format_id in (
-            # "br",
+            "br",  # 34 → 11 511 612
             "deflate",  # 27 968 → 11 511 612
             "gzip",  # 27 988 → 11 511 612
             # "zstd",
@@ -383,6 +383,13 @@ class HttpCompressionTest(TestCase):
             self.spider,
         )
 
+    def test_compression_bomb_br(self):
+        try:
+            import brotli  # noqa: F401
+        except ImportError:
+            raise SkipTest("no brotli")
+        self._test_compression_bomb("br")
+
     def test_compression_bomb_deflate(self):
         self._test_compression_bomb("deflate")