RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use caplog in compression warning test 

Signed-off-by: Mridankan Mandal <xerontitan90@gmail.com>
This commit is contained in:
Mridankan Mandal 2026-07-16 17:13:40 +05:30
parent 2fc74b8c9d
commit 0dfad68136
No known key found for this signature in database
GPG Key ID: C24AD565E950FF59
1 changed files with 27 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
tests/test_downloadermiddleware_httpcompression.py
View File

@ -538,23 +538,9 @@ class TestHttpCompression:
mw.open_spider(spider) mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}") # 11_511_612 B response = self._getresponse(f"bomb-{compression_id}") # 11_511_612 B
with ( with pytest.raises(IgnoreRequest) as exc_info:
LogCapture(
"scrapy.downloadermiddlewares.httpcompression",
propagate=False,
level=WARNING,
) as log,
pytest.raises(IgnoreRequest) as exc_info,
):
mw.process_response(response.request, response) mw.process_response(response.request, response)
assert exc_info.value.__cause__.decompressed_size < 1_100_000 assert exc_info.value.__cause__.decompressed_size < 1_100_000
log.check(
(
"scrapy.downloadermiddlewares.httpcompression",
"WARNING",
str(exc_info.value),
),
)
def test_compression_bomb_setting_br(self): def test_compression_bomb_setting_br(self):
_skip_if_no_br() _skip_if_no_br()
@ -572,6 +558,32 @@ class TestHttpCompression:
self._test_compression_bomb_setting("zstd") self._test_compression_bomb_setting("zstd")
def test_compression_bomb_setting_logs_warning(
self, caplog: pytest.LogCaptureFixture
):
settings = {"DOWNLOAD_MAXSIZE": 1_000_000}
crawler = get_crawler(Spider, settings_dict=settings)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
mw.open_spider(spider)
response = self._getresponse("bomb-gzip") # 11_511_612 B
with (
caplog.at_level(
WARNING, logger="scrapy.downloadermiddlewares.httpcompression"
),
pytest.raises(IgnoreRequest) as exc_info,
):
mw.process_response(response.request, response)
assert exc_info.value.__cause__.decompressed_size < 1_100_000
assert caplog.record_tuples == [
(
"scrapy.downloadermiddlewares.httpcompression",
WARNING,
str(exc_info.value),
)
]
def _test_compression_bomb_spider_attr(self, compression_id): def _test_compression_bomb_spider_attr(self, compression_id):
class DownloadMaxSizeSpider(Spider): class DownloadMaxSizeSpider(Spider):
download_maxsize = 1_000_000 download_maxsize = 1_000_000