Unknown encoding handling in HttpCompressionMiddleware, restore x-gzip support (#6618)

* Unknown encoding handling in HttpCompressionMiddleware.

* Implement the changes for unknown encoding handling.

* Restore support for Content-Encoding: x-gzip.

* Simplify the decoding logic.

* Add tests for the unsupported encoding warning.

* Add a test for the "no zstandard" warning.

scrapy/downloadermiddlewares/httpcompression.py

@@ -111,6 +111,8 @@ class HttpCompressionMiddleware:
                         f"({len(decoded_body)} B) is larger than the "
                         f"download warning size ({warn_size} B)."
                     )
+                if content_encoding:
+                    self._warn_unknown_encoding(response, content_encoding)
                 response.headers["Content-Encoding"] = content_encoding
                 if self.stats:
                     self.stats.inc_value(
@@ -143,9 +145,11 @@ class HttpCompressionMiddleware:
             body = self._decode(body, encoding, max_size)
         return body, to_keep
 
+    @staticmethod
     def _split_encodings(
-        self, content_encoding: list[bytes]
+        content_encoding: list[bytes],
     ) -> tuple[list[bytes], list[bytes]]:
+        supported_encodings = {*ACCEPTED_ENCODINGS, b"x-gzip"}
         to_keep: list[bytes] = [
             encoding.strip().lower()
             for encoding in chain.from_iterable(
@@ -155,19 +159,35 @@ class HttpCompressionMiddleware:
         to_decode: list[bytes] = []
         while to_keep:
             encoding = to_keep.pop()
-            if encoding not in ACCEPTED_ENCODINGS:
+            if encoding not in supported_encodings:
                 to_keep.append(encoding)
                 return to_decode, to_keep
             to_decode.append(encoding)
         return to_decode, to_keep
 
-    def _decode(self, body: bytes, encoding: bytes, max_size: int) -> bytes:
+    @staticmethod
+    def _decode(body: bytes, encoding: bytes, max_size: int) -> bytes:
         if encoding in {b"gzip", b"x-gzip"}:
             return gunzip(body, max_size=max_size)
         if encoding == b"deflate":
             return _inflate(body, max_size=max_size)
-        if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS:
+        if encoding == b"br":
             return _unbrotli(body, max_size=max_size)
-        if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS:
+        if encoding == b"zstd":
             return _unzstd(body, max_size=max_size)
-        return body
+        # shouldn't be reached
+        return body  # pragma: no cover
+
+    def _warn_unknown_encoding(
+        self, response: Response, encodings: list[bytes]
+    ) -> None:
+        encodings_str = b",".join(encodings).decode()
+        msg = (
+            f"{self.__class__.__name__} cannot decode the response for {response.url} "
+            f"from unsupported encoding(s) '{encodings_str}'."
+        )
+        if b"br" in encodings:
+            msg += " You need to install brotli or brotlicffi to decode 'br'."
+        if b"zstd" in encodings:
+            msg += " You need to install zstandard to decode 'zstd'."
+        logger.warning(msg)

tests/test_downloadermiddleware_httpcompression.py

@@ -23,7 +23,7 @@ SAMPLEDIR = Path(tests_datadir, "compressed")
 
 FORMAT = {
     "gzip": ("html-gzip.bin", "gzip"),
-    "x-gzip": ("html-gzip.bin", "gzip"),
+    "x-gzip": ("html-gzip.bin", "x-gzip"),
     "rawdeflate": ("html-rawdeflate.bin", "deflate"),
     "zlibdeflate": ("html-zlibdeflate.bin", "deflate"),
     "gzip-deflate": ("html-gzip-deflate.bin", "gzip, deflate"),
@@ -145,6 +145,41 @@ class HttpCompressionTest(TestCase):
         self.assertStatsEqual("httpcompression/response_count", 1)
         self.assertStatsEqual("httpcompression/response_bytes", 74837)
 
+    def test_process_response_br_unsupported(self):
+        try:
+            try:
+                import brotli  # noqa: F401
+
+                raise SkipTest("Requires not having brotli support")
+            except ImportError:
+                import brotlicffi  # noqa: F401
+
+                raise SkipTest("Requires not having brotli support")
+        except ImportError:
+            pass
+        response = self._getresponse("br")
+        request = response.request
+        self.assertEqual(response.headers["Content-Encoding"], b"br")
+        with LogCapture(
+            "scrapy.downloadermiddlewares.httpcompression",
+            propagate=False,
+            level=WARNING,
+        ) as log:
+            newresponse = self.mw.process_response(request, response, self.spider)
+        log.check(
+            (
+                "scrapy.downloadermiddlewares.httpcompression",
+                "WARNING",
+                (
+                    "HttpCompressionMiddleware cannot decode the response for"
+                    " http://scrapytest.org/ from unsupported encoding(s) 'br'."
+                    " You need to install brotli or brotlicffi to decode 'br'."
+                ),
+            ),
+        )
+        assert newresponse is not response
+        self.assertEqual(newresponse.headers.getlist("Content-Encoding"), [b"br"])
+
     def test_process_response_zstd(self):
         try:
             import zstandard  # noqa: F401
@@ -166,6 +201,36 @@ class HttpCompressionTest(TestCase):
             assert newresponse.body.startswith(b"<!DOCTYPE")
             assert "Content-Encoding" not in newresponse.headers
 
+    def test_process_response_zstd_unsupported(self):
+        try:
+            import zstandard  # noqa: F401
+
+            raise SkipTest("Requires not having zstandard support")
+        except ImportError:
+            pass
+        response = self._getresponse("zstd-static-content-size")
+        request = response.request
+        self.assertEqual(response.headers["Content-Encoding"], b"zstd")
+        with LogCapture(
+            "scrapy.downloadermiddlewares.httpcompression",
+            propagate=False,
+            level=WARNING,
+        ) as log:
+            newresponse = self.mw.process_response(request, response, self.spider)
+        log.check(
+            (
+                "scrapy.downloadermiddlewares.httpcompression",
+                "WARNING",
+                (
+                    "HttpCompressionMiddleware cannot decode the response for"
+                    " http://scrapytest.org/ from unsupported encoding(s) 'zstd'."
+                    " You need to install zstandard to decode 'zstd'."
+                ),
+            ),
+        )
+        assert newresponse is not response
+        self.assertEqual(newresponse.headers.getlist("Content-Encoding"), [b"zstd"])
+
     def test_process_response_rawdeflate(self):
         response = self._getresponse("rawdeflate")
         request = response.request
@@ -221,7 +286,22 @@ class HttpCompressionTest(TestCase):
         response = self._getresponse("gzip-deflate")
         response.headers["Content-Encoding"] = [b"gzip, foo, deflate"]
         request = response.request
-        newresponse = self.mw.process_response(request, response, self.spider)
+        with LogCapture(
+            "scrapy.downloadermiddlewares.httpcompression",
+            propagate=False,
+            level=WARNING,
+        ) as log:
+            newresponse = self.mw.process_response(request, response, self.spider)
+        log.check(
+            (
+                "scrapy.downloadermiddlewares.httpcompression",
+                "WARNING",
+                (
+                    "HttpCompressionMiddleware cannot decode the response for"
+                    " http://scrapytest.org/ from unsupported encoding(s) 'gzip,foo'."
+                ),
+            ),
+        )
         assert newresponse is not response
         self.assertEqual(
             newresponse.headers.getlist("Content-Encoding"), [b"gzip", b"foo"]