Support multi-encoding Content-Encoding headers

scrapy/downloadermiddlewares/httpcompression.py

@@ -52,30 +52,39 @@ class HttpCompressionMiddleware:
 
     def process_response(self, request, response, spider):
 
-        if request.method == 'HEAD':
+        if (
+            request.method == 'HEAD'
+            or not isinstance(response, Response)
+            or 'Content-Encoding' not in response.headers
+        ):
             return response
-        if isinstance(response, Response):
-            content_encoding = response.headers.getlist('Content-Encoding')
-            if content_encoding:
-                encoding = content_encoding.pop()
-                decoded_body = self._decode(response.body, encoding.lower())
-                if self.stats:
-                    self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider)
-                    self.stats.inc_value('httpcompression/response_count', spider=spider)
-                respcls = get_response_class(
-                    http_headers=response.headers,
-                    url=response.url,
-                    body=decoded_body,
-                )
-                kwargs = dict(cls=respcls, body=decoded_body)
-                if issubclass(respcls, TextResponse):
-                    # Force recalculating the encoding based on the new,
-                    # decoded (uncompressed) body.
-                    kwargs['encoding'] = None
-                response = response.replace(**kwargs)
-                if not content_encoding:
-                    del response.headers['Content-Encoding']
-
+        header_list = response.headers.getlist('Content-Encoding')
+        encodings = [
+            item.strip() for item in b",".join(header_list).split(b",")
+        ]
+        if not encodings:
+            return response
+        while encodings:
+            encoding = encodings.pop()
+            decoded_body = self._decode(response.body, encoding.lower())
+            if encodings:
+                response.headers['Content-Encoding'] = b",".join(encodings)
+            else:
+                del response.headers['Content-Encoding']
+            respcls = get_response_class(
+                http_headers=response.headers,
+                url=response.url,
+                body=decoded_body,
+            )
+            kwargs = dict(cls=respcls, body=decoded_body)
+            if issubclass(respcls, TextResponse):
+                # Force recalculating the encoding based on the new,
+                # decoded (uncompressed) body.
+                kwargs['encoding'] = None
+            response = response.replace(**kwargs)
+        if self.stats:
+            self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider)
+            self.stats.inc_value('httpcompression/response_count', spider=spider)
         return response
 
     def _decode(self, body, encoding):

tests/test_downloadermiddleware_httpcompression.py

@@ -22,6 +22,7 @@ FORMAT = {
     'rawdeflate': ('html-rawdeflate.bin', 'deflate'),
     'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'),
     'br': ('html-br.bin', 'br'),
+    'br,gzip': ('html-br-gzip.bin', 'br,gzip'),
     # $ zstd raw.html --content-size -o html-zstd-static-content-size.bin
     'zstd-static-content-size': ('html-zstd-static-content-size.bin', 'zstd'),
     # $ zstd raw.html --no-content-size -o html-zstd-static-no-content-size.bin
@@ -133,6 +134,37 @@ class HttpCompressionTest(TestCase):
         self.assertStatsEqual('httpcompression/response_count', 1)
         self.assertStatsEqual('httpcompression/response_bytes', 74837)
 
+    def test_process_response_br_gzip(self):
+        try:
+            import brotli  # noqa: F401
+        except ImportError:
+            raise SkipTest("no brotli")
+        response = self._getresponse('br,gzip')
+        request = response.request
+        self.assertEqual(response.headers['Content-Encoding'], b'br,gzip')
+        newresponse = self.mw.process_response(request, response, self.spider)
+        assert newresponse is not response
+        assert newresponse.body.startswith(b"<!DOCTYPE")
+        assert 'Content-Encoding' not in newresponse.headers
+        self.assertStatsEqual('httpcompression/response_count', 1)
+        self.assertStatsEqual('httpcompression/response_bytes', 74837)
+
+    def test_process_response_br_gzip_header_list(self):
+        try:
+            import brotli  # noqa: F401
+        except ImportError:
+            raise SkipTest("no brotli")
+        response = self._getresponse('br,gzip')
+        request = response.request
+        self.assertEqual(response.headers['Content-Encoding'], b'br,gzip')
+        response.headers.setlist('Content-Encoding', [b"br", b"gzip"])
+        newresponse = self.mw.process_response(request, response, self.spider)
+        assert newresponse is not response
+        assert newresponse.body.startswith(b"<!DOCTYPE")
+        assert 'Content-Encoding' not in newresponse.headers
+        self.assertStatsEqual('httpcompression/response_count', 1)
+        self.assertStatsEqual('httpcompression/response_bytes', 74837)
+
     def test_process_response_zstd(self):
         try:
             import zstandard  # noqa: F401
@@ -189,14 +221,6 @@ class HttpCompressionTest(TestCase):
         self.assertStatsEqual('httpcompression/response_count', None)
         self.assertStatsEqual('httpcompression/response_bytes', None)
 
-    def test_multipleencodings(self):
-        response = self._getresponse('gzip')
-        response.headers['Content-Encoding'] = ['uuencode', 'gzip']
-        request = response.request
-        newresponse = self.mw.process_response(request, response, self.spider)
-        assert newresponse is not response
-        self.assertEqual(newresponse.headers.getlist('Content-Encoding'), [b'uuencode'])
-
     def test_process_response_encoding_inside_body(self):
         headers = {
             'Content-Type': 'text/html',