From 6969041c5f6891a0298d7e68ece762adee1bb222 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 22 Nov 2023 15:52:00 +0100 Subject: [PATCH] Protect against gzip bombs --- .../downloadermiddlewares/httpcompression.py | 26 +++++++++++----- scrapy/utils/_compression.py | 2 ++ scrapy/utils/gz.py | 14 +++++++-- tests/sample_data/compressed/bomb-gzip.bin | Bin 0 -> 27988 bytes ...st_downloadermiddleware_httpcompression.py | 29 ++++++++++++++++-- 5 files changed, 59 insertions(+), 12 deletions(-) create mode 100644 scrapy/utils/_compression.py create mode 100644 tests/sample_data/compressed/bomb-gzip.bin diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index ead426951..5dd67ea87 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -2,9 +2,10 @@ import io import warnings import zlib -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes +from scrapy.utils._compression import _DecompressionMaxSizeExceeded from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.gz import gunzip @@ -29,24 +30,26 @@ class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" - def __init__(self, stats=None): - self.stats = stats + def __init__(self, crawler=None): + self.stats = crawler.stats + self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool("COMPRESSION_ENABLED"): raise NotConfigured try: - return cls(stats=crawler.stats) + return cls(crawler=crawler) except TypeError: warnings.warn( "HttpCompressionMiddleware subclasses must either modify " - "their '__init__' method to support a 'stats' parameter or " - "reimplement the 'from_crawler' method.", + "their '__init__' method to support a 'crawler' parameter or " + "reimplement their 'from_crawler' method.", ScrapyDeprecationWarning, ) result = cls() result.stats = crawler.stats + result._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") return result def process_request(self, request, spider): @@ -59,7 +62,14 @@ class HttpCompressionMiddleware: content_encoding = response.headers.getlist("Content-Encoding") if content_encoding: encoding = content_encoding.pop() - decoded_body = self._decode(response.body, encoding.lower()) + try: + decoded_body = self._decode(response.body, encoding.lower()) + except _DecompressionMaxSizeExceeded: + raise IgnoreRequest( + f"Ignored response {response} because its body " + f"({len(response.body)}B) exceeded DOWNLOAD_MAXSIZE " + f"({self._max_size}B) during decompression." + ) if self.stats: self.stats.inc_value( "httpcompression/response_bytes", @@ -85,7 +95,7 @@ class HttpCompressionMiddleware: def _decode(self, body, encoding): if encoding == b"gzip" or encoding == b"x-gzip": - body = gunzip(body) + body = gunzip(body, max_size=self._max_size) if encoding == b"deflate": try: diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py new file mode 100644 index 000000000..e726a70f5 --- /dev/null +++ b/scrapy/utils/_compression.py @@ -0,0 +1,2 @@ +class _DecompressionMaxSizeExceeded(ValueError): + pass diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index c7f74030e..cd5059a5c 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -5,8 +5,10 @@ from typing import List from scrapy.http import Response +from ._compression import _DecompressionMaxSizeExceeded -def gunzip(data: bytes) -> bytes: + +def gunzip(data: bytes, max_size: int = 0) -> bytes: """Gunzip the given data and return as much data as possible. This is resilient to CRC checksum errors. @@ -14,10 +16,10 @@ def gunzip(data: bytes) -> bytes: f = GzipFile(fileobj=BytesIO(data)) output_list: List[bytes] = [] chunk = b"." + decompressed_size = 0 while chunk: try: chunk = f.read1(8196) - output_list.append(chunk) except (OSError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error @@ -25,6 +27,14 @@ def gunzip(data: bytes) -> bytes: if output_list: break raise + decompressed_size += len(chunk) + if max_size and decompressed_size > max_size: + raise _DecompressionMaxSizeExceeded( + f"The number of bytes decompressed so far " + f"({decompressed_size}B) exceed the specified maximum " + f"({max_size}B)." + ) + output_list.append(chunk) return b"".join(output_list) diff --git a/tests/sample_data/compressed/bomb-gzip.bin b/tests/sample_data/compressed/bomb-gzip.bin new file mode 100644 index 0000000000000000000000000000000000000000..64aa0c3696cc1c6d86d218c70635d88f2035ec9e GIT binary patch literal 27988 zcmeIyElY!86b9f&oiK|G(Y#;~27Xl0-yq1UE0YN_<|{r6TM$G+ga1HfWkC=@i}}T- zAQQ0;tA;J=f*|@M2A1oDJDzYj_mw}*X4m_rN*LQrYP)-t7`Kz1`EpV#FVq|L(0ja} zI9SSs+qBrtti6t3Pxsob#`n?W)Wc%`Y$l!UY&@@AZN0t3&;4p=`TZgaH}D5)fC3Vd zkc1>8Aqh!HLK2dYgd`*(2}wvo5|WUFBqSjTNk~Exl8}TXBq0e&NJ0{lkc1>8Aqh!H zLK2dYgd`*(2}wvo5|WUFBqSjTNk~Exl8}TXBq0e&NJ0{lkc1>8Aqh!HLK2d2h!PI& z=1wx8 S;eSfl{TO{ZZ^qTjoA3)j<4WiN literal 0 HcmV?d00001 diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 9dad056de..f834e78f5 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -10,7 +10,7 @@ from scrapy.downloadermiddlewares.httpcompression import ( ACCEPTED_ENCODINGS, HttpCompressionMiddleware, ) -from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Request, Response from scrapy.responsetypes import responsetypes from scrapy.spiders import Spider @@ -35,12 +35,24 @@ FORMAT = { "html-zstd-streaming-no-content-size.bin", "zstd", ), + **{ + f"bomb-{format_id}": (f"bomb-{format_id}.bin", format_id) + for format_id in ( + # "br", + "gzip", # 27 988 → 11 511 612 + # "deflate", + # "zstd", + ) + }, } class HttpCompressionTest(TestCase): def setUp(self): - self.crawler = get_crawler(Spider) + settings = { + "DOWNLOAD_MAXSIZE": 10_000_000, # For compression bomb tests. + } + self.crawler = get_crawler(Spider, settings_dict=settings) self.spider = self.crawler._create_spider("scrapytest.org") self.mw = HttpCompressionMiddleware.from_crawler(self.crawler) self.crawler.stats.open_spider(self.spider) @@ -373,6 +385,19 @@ class HttpCompressionTest(TestCase): self.assertStatsEqual("httpcompression/response_count", None) self.assertStatsEqual("httpcompression/response_bytes", None) + def _test_compression_bomb(self, compression_id): + response = self._getresponse(f"bomb-{compression_id}") + self.assertRaises( + IgnoreRequest, + self.mw.process_response, + response.request, + response, + self.spider, + ) + + def test_compression_bomb_gzip(self): + self._test_compression_bomb("gzip") + class HttpCompressionSubclassTest(TestCase): def test_init_missing_stats(self):