Protect against gzip bombs

scrapy/downloadermiddlewares/httpcompression.py

@@ -2,9 +2,10 @@ import io
 import warnings
 import zlib
 
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Response, TextResponse
 from scrapy.responsetypes import responsetypes
+from scrapy.utils._compression import _DecompressionMaxSizeExceeded
 from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.gz import gunzip
 
@@ -29,24 +30,26 @@ class HttpCompressionMiddleware:
     """This middleware allows compressed (gzip, deflate) traffic to be
     sent/received from web sites"""
 
-    def __init__(self, stats=None):
-        self.stats = stats
+    def __init__(self, crawler=None):
+        self.stats = crawler.stats
+        self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")
 
     @classmethod
     def from_crawler(cls, crawler):
         if not crawler.settings.getbool("COMPRESSION_ENABLED"):
             raise NotConfigured
         try:
-            return cls(stats=crawler.stats)
+            return cls(crawler=crawler)
         except TypeError:
             warnings.warn(
                 "HttpCompressionMiddleware subclasses must either modify "
-                "their '__init__' method to support a 'stats' parameter or "
-                "reimplement the 'from_crawler' method.",
+                "their '__init__' method to support a 'crawler' parameter or "
+                "reimplement their 'from_crawler' method.",
                 ScrapyDeprecationWarning,
             )
             result = cls()
             result.stats = crawler.stats
+            result._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")
             return result
 
     def process_request(self, request, spider):
@@ -59,7 +62,14 @@ class HttpCompressionMiddleware:
             content_encoding = response.headers.getlist("Content-Encoding")
             if content_encoding:
                 encoding = content_encoding.pop()
-                decoded_body = self._decode(response.body, encoding.lower())
+                try:
+                    decoded_body = self._decode(response.body, encoding.lower())
+                except _DecompressionMaxSizeExceeded:
+                    raise IgnoreRequest(
+                        f"Ignored response {response} because its body "
+                        f"({len(response.body)}B) exceeded DOWNLOAD_MAXSIZE "
+                        f"({self._max_size}B) during decompression."
+                    )
                 if self.stats:
                     self.stats.inc_value(
                         "httpcompression/response_bytes",
@@ -85,7 +95,7 @@ class HttpCompressionMiddleware:
 
     def _decode(self, body, encoding):
         if encoding == b"gzip" or encoding == b"x-gzip":
-            body = gunzip(body)
+            body = gunzip(body, max_size=self._max_size)
 
         if encoding == b"deflate":
             try:

scrapy/utils/_compression.py

@@ -0,0 +1,2 @@
+class _DecompressionMaxSizeExceeded(ValueError):
+    pass

scrapy/utils/gz.py

@@ -5,8 +5,10 @@ from typing import List
 
 from scrapy.http import Response
 
+from ._compression import _DecompressionMaxSizeExceeded
 
-def gunzip(data: bytes) -> bytes:
+
+def gunzip(data: bytes, max_size: int = 0) -> bytes:
     """Gunzip the given data and return as much data as possible.
 
     This is resilient to CRC checksum errors.
@@ -14,10 +16,10 @@ def gunzip(data: bytes) -> bytes:
     f = GzipFile(fileobj=BytesIO(data))
     output_list: List[bytes] = []
     chunk = b"."
+    decompressed_size = 0
     while chunk:
         try:
             chunk = f.read1(8196)
-            output_list.append(chunk)
         except (OSError, EOFError, struct.error):
             # complete only if there is some data, otherwise re-raise
             # see issue 87 about catching struct.error
@@ -25,6 +27,14 @@ def gunzip(data: bytes) -> bytes:
             if output_list:
                 break
             raise
+        decompressed_size += len(chunk)
+        if max_size and decompressed_size > max_size:
+            raise _DecompressionMaxSizeExceeded(
+                f"The number of bytes decompressed so far "
+                f"({decompressed_size}B) exceed the specified maximum "
+                f"({max_size}B)."
+            )
+        output_list.append(chunk)
     return b"".join(output_list)
 
 

tests/test_downloadermiddleware_httpcompression.py

@@ -10,7 +10,7 @@ from scrapy.downloadermiddlewares.httpcompression import (
     ACCEPTED_ENCODINGS,
     HttpCompressionMiddleware,
 )
-from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
+from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import HtmlResponse, Request, Response
 from scrapy.responsetypes import responsetypes
 from scrapy.spiders import Spider
@@ -35,12 +35,24 @@ FORMAT = {
         "html-zstd-streaming-no-content-size.bin",
         "zstd",
     ),
+    **{
+        f"bomb-{format_id}": (f"bomb-{format_id}.bin", format_id)
+        for format_id in (
+            # "br",
+            "gzip",  # 27 988 → 11 511 612
+            # "deflate",
+            # "zstd",
+        )
+    },
 }
 
 
 class HttpCompressionTest(TestCase):
     def setUp(self):
-        self.crawler = get_crawler(Spider)
+        settings = {
+            "DOWNLOAD_MAXSIZE": 10_000_000,  # For compression bomb tests.
+        }
+        self.crawler = get_crawler(Spider, settings_dict=settings)
         self.spider = self.crawler._create_spider("scrapytest.org")
         self.mw = HttpCompressionMiddleware.from_crawler(self.crawler)
         self.crawler.stats.open_spider(self.spider)
@@ -373,6 +385,19 @@ class HttpCompressionTest(TestCase):
         self.assertStatsEqual("httpcompression/response_count", None)
         self.assertStatsEqual("httpcompression/response_bytes", None)
 
+    def _test_compression_bomb(self, compression_id):
+        response = self._getresponse(f"bomb-{compression_id}")
+        self.assertRaises(
+            IgnoreRequest,
+            self.mw.process_response,
+            response.request,
+            response,
+            self.spider,
+        )
+
+    def test_compression_bomb_gzip(self):
+        self._test_compression_bomb("gzip")
+
 
 class HttpCompressionSubclassTest(TestCase):
     def test_init_missing_stats(self):