Update

docs/topics/downloader-middleware.rst

@@ -754,6 +754,30 @@ Default: ``True``
 
 Whether the Compression middleware will be enabled.
 
+.. setting:: COMPRESSION_KEEP_ENCODING_HEADER
+
+COMPRESSION_KEEP_ENCODING_HEADER
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. versionadded:: VERSION
+
+Default: ``False``
+
+Whether to keep the original ``Content-Encoding`` header of a response after
+:class:`HttpCompressionMiddleware` has decompressed its body.
+
+When ``True``, the original ``Content-Encoding`` header is kept, so that you
+can tell how the response body was encoded before Scrapy decompressed it. When
+``False``, that header is removed from decompressed responses.
+
+In both cases, a ``decoded`` flag (see :attr:`Response.flags`) is added to
+decompressed responses, and responses that already have that flag are not
+decompressed again.
+
+.. note:: ``False`` is the current default for backward compatibility, but it
+   is deprecated. Set this setting to ``True`` to keep the header; ``True``
+   will be the only supported behavior in a future version of Scrapy. New
+   projects created with :command:`startproject` set it to ``True``.
 
 HttpProxyMiddleware
 -------------------

scrapy/downloadermiddlewares/httpcompression.py

@@ -89,7 +89,12 @@ class HttpCompressionMiddleware:
         )
         if not self.keep_encoding_header:
             warnings.warn(
-                "Setting COMPRESSION_KEEP_ENCODING_HEADER=False is deprecated",
+                "COMPRESSION_KEEP_ENCODING_HEADER is False (its current default "
+                "value), so HttpCompressionMiddleware removes the "
+                "Content-Encoding header from decoded responses. This is "
+                "deprecated; set COMPRESSION_KEEP_ENCODING_HEADER to True to "
+                "keep that header, which will be the only supported behavior "
+                "in a future Scrapy version, and to silence this warning.",
                 ScrapyDeprecationWarning,
                 stacklevel=2,
             )
@@ -124,7 +129,7 @@ class HttpCompressionMiddleware:
     ) -> Request | Response:
         if request.method == "HEAD":
             return response
-        if b"decoded" in response.flags:
+        if "decoded" in response.flags:
             return response
         content_encoding = response.headers.getlist("Content-Encoding")
         if not content_encoding:
@@ -132,7 +137,7 @@ class HttpCompressionMiddleware:
         max_size = request.meta.get("download_maxsize", self._max_size)
         warn_size = request.meta.get("download_warnsize", self._warn_size)
         try:
-            decoded_body, content_encoding = self._handle_encoding(
+            decoded_body, to_keep = self._handle_encoding(
                 response.body, content_encoding, max_size
             )
         except _DecompressionMaxSizeExceeded as e:
@@ -148,9 +153,11 @@ class HttpCompressionMiddleware:
                 f"({len(decoded_body)} B) is larger than the "
                 f"download warning size ({warn_size} B)."
             )
-        if content_encoding:
-            self._warn_unknown_encoding(response, content_encoding)
-        response.headers["Content-Encoding"] = content_encoding
+        if to_keep:
+            self._warn_unknown_encoding(response, to_keep)
+        # Drop the encodings that have been decoded so that the response class
+        # is guessed from the decoded body rather than from the compressed one.
+        response.headers["Content-Encoding"] = to_keep
         if self.stats:
             self.stats.inc_value(
                 "httpcompression/response_bytes",
@@ -165,9 +172,13 @@ class HttpCompressionMiddleware:
             # force recalculating the encoding until we make sure the
             # responsetypes guessing is reliable
             kwargs["encoding"] = None
-        kwargs["flags"] = [*response.flags, b"decoded"]
+        kwargs["flags"] = [*response.flags, "decoded"]
         response = response.replace(cls=respcls, **kwargs)
-        if not self.keep_encoding_header and not content_encoding:
+        if self.keep_encoding_header:
+            # Restore the original Content-Encoding header so that the spider
+            # can tell how the response body was encoded before decoding.
+            response.headers["Content-Encoding"] = content_encoding
+        elif not to_keep:
             del response.headers["Content-Encoding"]
         return response
 

scrapy/templates/project/module/settings.py.tmpl

@@ -83,8 +83,6 @@ DOWNLOAD_DELAY = 1
 #HTTPCACHE_IGNORE_HTTP_CODES = []
 #HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
 
-# Keep original Content-Encoding header
-COMPRESSION_KEEP_ENCODING_HEADER = True
-
 # Set settings whose default value is deprecated to a future-proof value
+COMPRESSION_KEEP_ENCODING_HEADER = True
 FEED_EXPORT_ENCODING = "utf-8"

tests/test_downloadermiddleware_httpcompression.py

@@ -796,8 +796,29 @@ class TestHttpCompression:
         newresponse = mw.process_response(request, response)
         assert newresponse is not response
         assert newresponse.body.startswith(b"<!DOCTYPE")
-        assert "Content-Encoding" in newresponse.headers
-        assert b"decoded" in newresponse.flags
+        # The original Content-Encoding header is preserved.
+        assert newresponse.headers.getlist("Content-Encoding") == [b"gzip"]
+        assert "decoded" in newresponse.flags
+
+    def test_process_response_keep_encoding_header_partial(self):
+        crawler = get_crawler(
+            Spider, settings_dict={"COMPRESSION_KEEP_ENCODING_HEADER": True}
+        )
+        mw = HttpCompressionMiddleware.from_crawler(crawler)
+        crawler.stats.open_spider()
+        response = self._getresponse("gzip")
+        response.headers["Content-Encoding"] = ["uuencode", "gzip"]
+        request = response.request
+
+        newresponse = mw.process_response(request, response)
+        assert newresponse is not response
+        # The full original Content-Encoding header is preserved, including
+        # the encoding that could not be decoded.
+        assert newresponse.headers.getlist("Content-Encoding") == [
+            b"uuencode",
+            b"gzip",
+        ]
+        assert "decoded" in newresponse.flags
 
     def test_process_response_drop_encoding_header(self):
         crawler = get_crawler(
@@ -814,7 +835,17 @@ class TestHttpCompression:
         assert newresponse is not response
         assert newresponse.body.startswith(b"<!DOCTYPE")
         assert "Content-Encoding" not in newresponse.headers
-        assert b"decoded" in newresponse.flags
+        assert "decoded" in newresponse.flags
+
+    def test_process_response_already_decoded(self):
+        response = self._getresponse("gzip")
+        response.flags.append("decoded")
+        request = response.request
+
+        newresponse = self.mw.process_response(request, response)
+        assert newresponse is response
+        assert newresponse.headers["Content-Encoding"] == b"gzip"
+        self.assertStatsEqual("httpcompression/response_count", None)
 
     def test_keep_encoding_header_deprecation_warning(self):
         crawler = get_crawler(
@@ -822,6 +853,6 @@ class TestHttpCompression:
         )
         with pytest.warns(
             ScrapyDeprecationWarning,
-            match="Setting COMPRESSION_KEEP_ENCODING_HEADER=False is deprecated",
+            match="COMPRESSION_KEEP_ENCODING_HEADER is False",
         ):
             HttpCompressionMiddleware.from_crawler(crawler)