Merge 1bda79a521 into d8ba1571e7

docs/topics/downloader-middleware.rst

@@ -760,6 +760,30 @@ Default: ``True``
 
 Whether the Compression middleware will be enabled.
 
+.. setting:: COMPRESSION_KEEP_ENCODING_HEADER
+
+COMPRESSION_KEEP_ENCODING_HEADER
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. versionadded:: VERSION
+
+Default: ``False``
+
+Whether to keep the original ``Content-Encoding`` header of a response after
+:class:`HttpCompressionMiddleware` has decompressed its body.
+
+When ``True``, the original ``Content-Encoding`` header is kept, so that you
+can tell how the response body was encoded before Scrapy decompressed it. When
+``False``, that header is removed from decompressed responses.
+
+In both cases, a ``decoded`` flag (see :attr:`Response.flags`) is added to
+decompressed responses, and responses that already have that flag are not
+decompressed again.
+
+.. note:: ``False`` is the current default for backward compatibility, but it
+   is deprecated. Set this setting to ``True`` to keep the header; ``True``
+   will be the only supported behavior in a future version of Scrapy. New
+   projects created with :command:`startproject` set it to ``True``.
 
 HttpProxyMiddleware
 -------------------

scrapy/downloadermiddlewares/httpcompression.py

@@ -79,10 +79,25 @@ class HttpCompressionMiddleware:
             self.stats = stats
             self._max_size = 1073741824
             self._warn_size = 33554432
+            self.keep_encoding_header = False
             return
         self.stats = crawler.stats
         self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")
         self._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE")
+        self.keep_encoding_header = crawler.settings.getbool(
+            "COMPRESSION_KEEP_ENCODING_HEADER"
+        )
+        if not self.keep_encoding_header:
+            warnings.warn(
+                "COMPRESSION_KEEP_ENCODING_HEADER is False (its current default "
+                "value), so HttpCompressionMiddleware removes the "
+                "Content-Encoding header from decoded responses. This is "
+                "deprecated; set COMPRESSION_KEEP_ENCODING_HEADER to True to "
+                "keep that header, which will be the only supported behavior "
+                "in a future Scrapy version, and to silence this warning.",
+                ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
         crawler.signals.connect(self.open_spider, signals.spider_opened)
 
     @classmethod
@@ -114,47 +129,57 @@ class HttpCompressionMiddleware:
     ) -> Request | Response:
         if request.method == "HEAD":
             return response
+        if "decoded" in response.flags:
+            return response
         content_encoding = response.headers.getlist("Content-Encoding")
-        if content_encoding:
-            max_size = request.meta.get("download_maxsize", self._max_size)
-            warn_size = request.meta.get("download_warnsize", self._warn_size)
-            try:
-                decoded_body, content_encoding = self._handle_encoding(
-                    response.body, content_encoding, max_size
-                )
-            except _DecompressionMaxSizeExceeded as e:
-                raise IgnoreRequest(
-                    f"Ignored response {response} because its body "
-                    f"({len(response.body)} B compressed, "
-                    f"{e.decompressed_size} B decompressed so far) exceeded "
-                    f"DOWNLOAD_MAXSIZE ({max_size} B) during decompression."
-                ) from e
-            if len(response.body) < warn_size <= len(decoded_body):
-                logger.warning(
-                    f"{response} body size after decompression "
-                    f"({len(decoded_body)} B) is larger than the "
-                    f"download warning size ({warn_size} B)."
-                )
-            if content_encoding:
-                self._warn_unknown_encoding(response, content_encoding)
-            response.headers["Content-Encoding"] = content_encoding
-            if self.stats:
-                self.stats.inc_value(
-                    "httpcompression/response_bytes",
-                    len(decoded_body),
-                )
-                self.stats.inc_value("httpcompression/response_count")
-            respcls = responsetypes.from_args(
-                headers=response.headers, url=response.url, body=decoded_body
+        if not content_encoding:
+            return response
+        max_size = request.meta.get("download_maxsize", self._max_size)
+        warn_size = request.meta.get("download_warnsize", self._warn_size)
+        try:
+            decoded_body, to_keep = self._handle_encoding(
+                response.body, content_encoding, max_size
             )
-            kwargs: dict[str, Any] = {"body": decoded_body}
-            if issubclass(respcls, TextResponse):
-                # force recalculating the encoding until we make sure the
-                # responsetypes guessing is reliable
-                kwargs["encoding"] = None
-            response = response.replace(cls=respcls, **kwargs)
-            if not content_encoding:
-                del response.headers["Content-Encoding"]
+        except _DecompressionMaxSizeExceeded as e:
+            raise IgnoreRequest(
+                f"Ignored response {response} because its body "
+                f"({len(response.body)} B compressed, "
+                f"{e.decompressed_size} B decompressed so far) exceeded "
+                f"DOWNLOAD_MAXSIZE ({max_size} B) during decompression."
+            ) from e
+        if len(response.body) < warn_size <= len(decoded_body):
+            logger.warning(
+                f"{response} body size after decompression "
+                f"({len(decoded_body)} B) is larger than the "
+                f"download warning size ({warn_size} B)."
+            )
+        if to_keep:
+            self._warn_unknown_encoding(response, to_keep)
+        # Drop the encodings that have been decoded so that the response class
+        # is guessed from the decoded body rather than from the compressed one.
+        response.headers["Content-Encoding"] = to_keep
+        if self.stats:
+            self.stats.inc_value(
+                "httpcompression/response_bytes",
+                len(decoded_body),
+            )
+            self.stats.inc_value("httpcompression/response_count")
+        respcls = responsetypes.from_args(
+            headers=response.headers, url=response.url, body=decoded_body
+        )
+        kwargs: dict[str, Any] = {"body": decoded_body}
+        if issubclass(respcls, TextResponse):
+            # force recalculating the encoding until we make sure the
+            # responsetypes guessing is reliable
+            kwargs["encoding"] = None
+        kwargs["flags"] = [*response.flags, "decoded"]
+        response = response.replace(cls=respcls, **kwargs)
+        if self.keep_encoding_header:
+            # Restore the original Content-Encoding header so that the spider
+            # can tell how the response body was encoded before decoding.
+            response.headers["Content-Encoding"] = content_encoding
+        elif not to_keep:
+            del response.headers["Content-Encoding"]
         return response
 
     def _handle_encoding(

scrapy/settings/default_settings.py

@@ -42,6 +42,7 @@ __all__ = [
     "CLOSESPIDER_TIMEOUT_NO_ITEM",
     "COMMANDS_MODULE",
     "COMPRESSION_ENABLED",
+    "COMPRESSION_KEEP_ENCODING_HEADER",
     "CONCURRENT_ITEMS",
     "CONCURRENT_REQUESTS",
     "CONCURRENT_REQUESTS_PER_DOMAIN",
@@ -247,6 +248,7 @@ CLOSESPIDER_TIMEOUT_NO_ITEM = 0
 COMMANDS_MODULE = ""
 
 COMPRESSION_ENABLED = True
+COMPRESSION_KEEP_ENCODING_HEADER = False
 
 CONCURRENT_ITEMS = 100
 

scrapy/templates/project/module/settings.py.tmpl

@@ -84,4 +84,5 @@ DOWNLOAD_DELAY = 1
 #HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
 
 # Set settings whose default value is deprecated to a future-proof value
+COMPRESSION_KEEP_ENCODING_HEADER = True
 FEED_EXPORT_ENCODING = "utf-8"

tests/AsyncCrawlerProcess/reactorless_datauri.py

@@ -12,7 +12,12 @@ class DataSpider(Spider):
         return {"data": response.text}
 
 
-process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False})
+process = AsyncCrawlerProcess(
+    settings={
+        "TWISTED_REACTOR_ENABLED": False,
+        "COMPRESSION_KEEP_ENCODING_HEADER": True,
+    }
+)
 
 process.crawl(DataSpider)
 process.start()

tests/AsyncCrawlerProcess/reactorless_simple.py

@@ -12,7 +12,12 @@ class NoRequestsSpider(scrapy.Spider):
         yield
 
 
-process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False})
+process = AsyncCrawlerProcess(
+    settings={
+        "TWISTED_REACTOR_ENABLED": False,
+        "COMPRESSION_KEEP_ENCODING_HEADER": True,
+    }
+)
 
 process.crawl(NoRequestsSpider)
 process.start()

tests/AsyncCrawlerRunner/reactorless_datauri.py

@@ -17,7 +17,12 @@ class DataSpider(Spider):
 
 async def main() -> None:
     configure_logging()
-    runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False})
+    runner = AsyncCrawlerRunner(
+        settings={
+            "TWISTED_REACTOR_ENABLED": False,
+            "COMPRESSION_KEEP_ENCODING_HEADER": True,
+        }
+    )
     await runner.crawl(DataSpider)
 
 

tests/AsyncCrawlerRunner/reactorless_simple.py

@@ -17,7 +17,12 @@ class NoRequestsSpider(Spider):
 
 async def main() -> None:
     configure_logging()
-    runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False})
+    runner = AsyncCrawlerRunner(
+        settings={
+            "TWISTED_REACTOR_ENABLED": False,
+            "COMPRESSION_KEEP_ENCODING_HEADER": True,
+        }
+    )
     await runner.crawl(NoRequestsSpider)
 
 

tests/test_downloadermiddleware_httpcompression.py

@@ -782,3 +782,77 @@ class TestHttpCompression:
                 continue
             resp = self._get_truncated_response(check_key)
             assert len(resp.body) == 0
+
+    def test_process_response_keep_encoding_header(self):
+        crawler = get_crawler(
+            Spider, settings_dict={"COMPRESSION_KEEP_ENCODING_HEADER": True}
+        )
+        mw = HttpCompressionMiddleware.from_crawler(crawler)
+        crawler.stats.open_spider()
+        response = self._getresponse("gzip")
+        request = response.request
+
+        assert response.headers["Content-Encoding"] == b"gzip"
+        newresponse = mw.process_response(request, response)
+        assert newresponse is not response
+        assert newresponse.body.startswith(b"<!DOCTYPE")
+        # The original Content-Encoding header is preserved.
+        assert newresponse.headers.getlist("Content-Encoding") == [b"gzip"]
+        assert "decoded" in newresponse.flags
+
+    def test_process_response_keep_encoding_header_partial(self):
+        crawler = get_crawler(
+            Spider, settings_dict={"COMPRESSION_KEEP_ENCODING_HEADER": True}
+        )
+        mw = HttpCompressionMiddleware.from_crawler(crawler)
+        crawler.stats.open_spider()
+        response = self._getresponse("gzip")
+        response.headers["Content-Encoding"] = ["uuencode", "gzip"]
+        request = response.request
+
+        newresponse = mw.process_response(request, response)
+        assert newresponse is not response
+        # The full original Content-Encoding header is preserved, including
+        # the encoding that could not be decoded.
+        assert newresponse.headers.getlist("Content-Encoding") == [
+            b"uuencode",
+            b"gzip",
+        ]
+        assert "decoded" in newresponse.flags
+
+    def test_process_response_drop_encoding_header(self):
+        crawler = get_crawler(
+            Spider, settings_dict={"COMPRESSION_KEEP_ENCODING_HEADER": False}
+        )
+        with pytest.warns(ScrapyDeprecationWarning):
+            mw = HttpCompressionMiddleware.from_crawler(crawler)
+        crawler.stats.open_spider()
+        response = self._getresponse("gzip")
+        request = response.request
+
+        assert response.headers["Content-Encoding"] == b"gzip"
+        newresponse = mw.process_response(request, response)
+        assert newresponse is not response
+        assert newresponse.body.startswith(b"<!DOCTYPE")
+        assert "Content-Encoding" not in newresponse.headers
+        assert "decoded" in newresponse.flags
+
+    def test_process_response_already_decoded(self):
+        response = self._getresponse("gzip")
+        response.flags.append("decoded")
+        request = response.request
+
+        newresponse = self.mw.process_response(request, response)
+        assert newresponse is response
+        assert newresponse.headers["Content-Encoding"] == b"gzip"
+        self.assertStatsEqual("httpcompression/response_count", None)
+
+    def test_keep_encoding_header_deprecation_warning(self):
+        crawler = get_crawler(
+            Spider, settings_dict={"COMPRESSION_KEEP_ENCODING_HEADER": False}
+        )
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="COMPRESSION_KEEP_ENCODING_HEADER is False",
+        ):
+            HttpCompressionMiddleware.from_crawler(crawler)