RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Log ignored responses exceeding DOWNLOAD_MAXSIZE 

Signed-off-by: Mridankan Mandal <xerontitan90@gmail.com>
This commit is contained in:
Mridankan Mandal 2026-07-13 01:45:10 +05:30
parent d8ba1571e7
commit 2fc74b8c9d
No known key found for this signature in database
GPG Key ID: C24AD565E950FF59
2 changed files with 19 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scrapy/downloadermiddlewares/httpcompression.py
View File

@ -123,12 +123,14 @@ class HttpCompressionMiddleware:
response.body, content_encoding, max_size response.body, content_encoding, max_size
) )
except _DecompressionMaxSizeExceeded as e: except _DecompressionMaxSizeExceeded as e:
raise IgnoreRequest( msg = (
f"Ignored response {response} because its body " f"Ignored response {response} because its body "
f"({len(response.body)} B compressed, " f"({len(response.body)} B compressed, "
f"{e.decompressed_size} B decompressed so far) exceeded " f"{e.decompressed_size} B decompressed so far) exceeded "
f"DOWNLOAD_MAXSIZE ({max_size} B) during decompression." f"DOWNLOAD_MAXSIZE ({max_size} B) during decompression."
) from e )
logger.warning(msg)
raise IgnoreRequest(msg) from e
if len(response.body) < warn_size <= len(decoded_body): if len(response.body) < warn_size <= len(decoded_body):
logger.warning( logger.warning(
f"{response} body size after decompression " f"{response} body size after decompression "

16
tests/test_downloadermiddleware_httpcompression.py
View File

@ -538,9 +538,23 @@ class TestHttpCompression:
mw.open_spider(spider) mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}") # 11_511_612 B response = self._getresponse(f"bomb-{compression_id}") # 11_511_612 B
with pytest.raises(IgnoreRequest) as exc_info: with (
LogCapture(
"scrapy.downloadermiddlewares.httpcompression",
propagate=False,
level=WARNING,
) as log,
pytest.raises(IgnoreRequest) as exc_info,
):
mw.process_response(response.request, response) mw.process_response(response.request, response)
assert exc_info.value.__cause__.decompressed_size < 1_100_000 assert exc_info.value.__cause__.decompressed_size < 1_100_000
log.check(
(
"scrapy.downloadermiddlewares.httpcompression",
"WARNING",
str(exc_info.value),
),
)
def test_compression_bomb_setting_br(self): def test_compression_bomb_setting_br(self):
_skip_if_no_br() _skip_if_no_br()