From 391af6afcca232aed82eda516b035dbbd39cdcb1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 27 Feb 2025 22:37:01 +0500 Subject: [PATCH] Unknown encoding handling in HttpCompressionMiddleware, restore x-gzip support (#6618) * Unknown encoding handling in HttpCompressionMiddleware. * Implement the changes for unknown encoding handling. * Restore support for Content-Encoding: x-gzip. * Simplify the decoding logic. * Add tests for the unsupported encoding warning. * Add a test for the "no zstandard" warning. --- .../downloadermiddlewares/httpcompression.py | 32 +++++-- ...st_downloadermiddleware_httpcompression.py | 84 ++++++++++++++++++- 2 files changed, 108 insertions(+), 8 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index a65757972..58891b952 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -111,6 +111,8 @@ class HttpCompressionMiddleware: f"({len(decoded_body)} B) is larger than the " f"download warning size ({warn_size} B)." ) + if content_encoding: + self._warn_unknown_encoding(response, content_encoding) response.headers["Content-Encoding"] = content_encoding if self.stats: self.stats.inc_value( @@ -143,9 +145,11 @@ class HttpCompressionMiddleware: body = self._decode(body, encoding, max_size) return body, to_keep + @staticmethod def _split_encodings( - self, content_encoding: list[bytes] + content_encoding: list[bytes], ) -> tuple[list[bytes], list[bytes]]: + supported_encodings = {*ACCEPTED_ENCODINGS, b"x-gzip"} to_keep: list[bytes] = [ encoding.strip().lower() for encoding in chain.from_iterable( @@ -155,19 +159,35 @@ class HttpCompressionMiddleware: to_decode: list[bytes] = [] while to_keep: encoding = to_keep.pop() - if encoding not in ACCEPTED_ENCODINGS: + if encoding not in supported_encodings: to_keep.append(encoding) return to_decode, to_keep to_decode.append(encoding) return to_decode, to_keep - def _decode(self, body: bytes, encoding: bytes, max_size: int) -> bytes: + @staticmethod + def _decode(body: bytes, encoding: bytes, max_size: int) -> bytes: if encoding in {b"gzip", b"x-gzip"}: return gunzip(body, max_size=max_size) if encoding == b"deflate": return _inflate(body, max_size=max_size) - if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS: + if encoding == b"br": return _unbrotli(body, max_size=max_size) - if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS: + if encoding == b"zstd": return _unzstd(body, max_size=max_size) - return body + # shouldn't be reached + return body # pragma: no cover + + def _warn_unknown_encoding( + self, response: Response, encodings: list[bytes] + ) -> None: + encodings_str = b",".join(encodings).decode() + msg = ( + f"{self.__class__.__name__} cannot decode the response for {response.url} " + f"from unsupported encoding(s) '{encodings_str}'." + ) + if b"br" in encodings: + msg += " You need to install brotli or brotlicffi to decode 'br'." + if b"zstd" in encodings: + msg += " You need to install zstandard to decode 'zstd'." + logger.warning(msg) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 78d0dd99d..a1c5883ec 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -23,7 +23,7 @@ SAMPLEDIR = Path(tests_datadir, "compressed") FORMAT = { "gzip": ("html-gzip.bin", "gzip"), - "x-gzip": ("html-gzip.bin", "gzip"), + "x-gzip": ("html-gzip.bin", "x-gzip"), "rawdeflate": ("html-rawdeflate.bin", "deflate"), "zlibdeflate": ("html-zlibdeflate.bin", "deflate"), "gzip-deflate": ("html-gzip-deflate.bin", "gzip, deflate"), @@ -145,6 +145,41 @@ class HttpCompressionTest(TestCase): self.assertStatsEqual("httpcompression/response_count", 1) self.assertStatsEqual("httpcompression/response_bytes", 74837) + def test_process_response_br_unsupported(self): + try: + try: + import brotli # noqa: F401 + + raise SkipTest("Requires not having brotli support") + except ImportError: + import brotlicffi # noqa: F401 + + raise SkipTest("Requires not having brotli support") + except ImportError: + pass + response = self._getresponse("br") + request = response.request + self.assertEqual(response.headers["Content-Encoding"], b"br") + with LogCapture( + "scrapy.downloadermiddlewares.httpcompression", + propagate=False, + level=WARNING, + ) as log: + newresponse = self.mw.process_response(request, response, self.spider) + log.check( + ( + "scrapy.downloadermiddlewares.httpcompression", + "WARNING", + ( + "HttpCompressionMiddleware cannot decode the response for" + " http://scrapytest.org/ from unsupported encoding(s) 'br'." + " You need to install brotli or brotlicffi to decode 'br'." + ), + ), + ) + assert newresponse is not response + self.assertEqual(newresponse.headers.getlist("Content-Encoding"), [b"br"]) + def test_process_response_zstd(self): try: import zstandard # noqa: F401 @@ -166,6 +201,36 @@ class HttpCompressionTest(TestCase): assert newresponse.body.startswith(b"