diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 096f87edd..a7c04b582 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -760,6 +760,30 @@ Default: ``True`` Whether the Compression middleware will be enabled. +.. setting:: COMPRESSION_KEEP_ENCODING_HEADER + +COMPRESSION_KEEP_ENCODING_HEADER +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: VERSION + +Default: ``False`` + +Whether to keep the original ``Content-Encoding`` header of a response after +:class:`HttpCompressionMiddleware` has decompressed its body. + +When ``True``, the original ``Content-Encoding`` header is kept, so that you +can tell how the response body was encoded before Scrapy decompressed it. When +``False``, that header is removed from decompressed responses. + +In both cases, a ``decoded`` flag (see :attr:`Response.flags`) is added to +decompressed responses, and responses that already have that flag are not +decompressed again. + +.. note:: ``False`` is the current default for backward compatibility, but it + is deprecated. Set this setting to ``True`` to keep the header; ``True`` + will be the only supported behavior in a future version of Scrapy. New + projects created with :command:`startproject` set it to ``True``. HttpProxyMiddleware ------------------- diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 6ca04a50e..60984d5af 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -79,10 +79,25 @@ class HttpCompressionMiddleware: self.stats = stats self._max_size = 1073741824 self._warn_size = 33554432 + self.keep_encoding_header = False return self.stats = crawler.stats self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") self._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE") + self.keep_encoding_header = crawler.settings.getbool( + "COMPRESSION_KEEP_ENCODING_HEADER" + ) + if not self.keep_encoding_header: + warnings.warn( + "COMPRESSION_KEEP_ENCODING_HEADER is False (its current default " + "value), so HttpCompressionMiddleware removes the " + "Content-Encoding header from decoded responses. This is " + "deprecated; set COMPRESSION_KEEP_ENCODING_HEADER to True to " + "keep that header, which will be the only supported behavior " + "in a future Scrapy version, and to silence this warning.", + ScrapyDeprecationWarning, + stacklevel=2, + ) crawler.signals.connect(self.open_spider, signals.spider_opened) @classmethod @@ -114,47 +129,57 @@ class HttpCompressionMiddleware: ) -> Request | Response: if request.method == "HEAD": return response + if "decoded" in response.flags: + return response content_encoding = response.headers.getlist("Content-Encoding") - if content_encoding: - max_size = request.meta.get("download_maxsize", self._max_size) - warn_size = request.meta.get("download_warnsize", self._warn_size) - try: - decoded_body, content_encoding = self._handle_encoding( - response.body, content_encoding, max_size - ) - except _DecompressionMaxSizeExceeded as e: - raise IgnoreRequest( - f"Ignored response {response} because its body " - f"({len(response.body)} B compressed, " - f"{e.decompressed_size} B decompressed so far) exceeded " - f"DOWNLOAD_MAXSIZE ({max_size} B) during decompression." - ) from e - if len(response.body) < warn_size <= len(decoded_body): - logger.warning( - f"{response} body size after decompression " - f"({len(decoded_body)} B) is larger than the " - f"download warning size ({warn_size} B)." - ) - if content_encoding: - self._warn_unknown_encoding(response, content_encoding) - response.headers["Content-Encoding"] = content_encoding - if self.stats: - self.stats.inc_value( - "httpcompression/response_bytes", - len(decoded_body), - ) - self.stats.inc_value("httpcompression/response_count") - respcls = responsetypes.from_args( - headers=response.headers, url=response.url, body=decoded_body + if not content_encoding: + return response + max_size = request.meta.get("download_maxsize", self._max_size) + warn_size = request.meta.get("download_warnsize", self._warn_size) + try: + decoded_body, to_keep = self._handle_encoding( + response.body, content_encoding, max_size ) - kwargs: dict[str, Any] = {"body": decoded_body} - if issubclass(respcls, TextResponse): - # force recalculating the encoding until we make sure the - # responsetypes guessing is reliable - kwargs["encoding"] = None - response = response.replace(cls=respcls, **kwargs) - if not content_encoding: - del response.headers["Content-Encoding"] + except _DecompressionMaxSizeExceeded as e: + raise IgnoreRequest( + f"Ignored response {response} because its body " + f"({len(response.body)} B compressed, " + f"{e.decompressed_size} B decompressed so far) exceeded " + f"DOWNLOAD_MAXSIZE ({max_size} B) during decompression." + ) from e + if len(response.body) < warn_size <= len(decoded_body): + logger.warning( + f"{response} body size after decompression " + f"({len(decoded_body)} B) is larger than the " + f"download warning size ({warn_size} B)." + ) + if to_keep: + self._warn_unknown_encoding(response, to_keep) + # Drop the encodings that have been decoded so that the response class + # is guessed from the decoded body rather than from the compressed one. + response.headers["Content-Encoding"] = to_keep + if self.stats: + self.stats.inc_value( + "httpcompression/response_bytes", + len(decoded_body), + ) + self.stats.inc_value("httpcompression/response_count") + respcls = responsetypes.from_args( + headers=response.headers, url=response.url, body=decoded_body + ) + kwargs: dict[str, Any] = {"body": decoded_body} + if issubclass(respcls, TextResponse): + # force recalculating the encoding until we make sure the + # responsetypes guessing is reliable + kwargs["encoding"] = None + kwargs["flags"] = [*response.flags, "decoded"] + response = response.replace(cls=respcls, **kwargs) + if self.keep_encoding_header: + # Restore the original Content-Encoding header so that the spider + # can tell how the response body was encoded before decoding. + response.headers["Content-Encoding"] = content_encoding + elif not to_keep: + del response.headers["Content-Encoding"] return response def _handle_encoding( diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 3a7dfd018..87237ca82 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -42,6 +42,7 @@ __all__ = [ "CLOSESPIDER_TIMEOUT_NO_ITEM", "COMMANDS_MODULE", "COMPRESSION_ENABLED", + "COMPRESSION_KEEP_ENCODING_HEADER", "CONCURRENT_ITEMS", "CONCURRENT_REQUESTS", "CONCURRENT_REQUESTS_PER_DOMAIN", @@ -247,6 +248,7 @@ CLOSESPIDER_TIMEOUT_NO_ITEM = 0 COMMANDS_MODULE = "" COMPRESSION_ENABLED = True +COMPRESSION_KEEP_ENCODING_HEADER = False CONCURRENT_ITEMS = 100 diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 0432a7231..d8c60f405 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -84,4 +84,5 @@ DOWNLOAD_DELAY = 1 #HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" # Set settings whose default value is deprecated to a future-proof value +COMPRESSION_KEEP_ENCODING_HEADER = True FEED_EXPORT_ENCODING = "utf-8" diff --git a/tests/AsyncCrawlerProcess/reactorless_datauri.py b/tests/AsyncCrawlerProcess/reactorless_datauri.py index 5270b7b83..c31554cd4 100644 --- a/tests/AsyncCrawlerProcess/reactorless_datauri.py +++ b/tests/AsyncCrawlerProcess/reactorless_datauri.py @@ -12,7 +12,12 @@ class DataSpider(Spider): return {"data": response.text} -process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False}) +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR_ENABLED": False, + "COMPRESSION_KEEP_ENCODING_HEADER": True, + } +) process.crawl(DataSpider) process.start() diff --git a/tests/AsyncCrawlerProcess/reactorless_simple.py b/tests/AsyncCrawlerProcess/reactorless_simple.py index 33b4e8cb2..99f397365 100644 --- a/tests/AsyncCrawlerProcess/reactorless_simple.py +++ b/tests/AsyncCrawlerProcess/reactorless_simple.py @@ -12,7 +12,12 @@ class NoRequestsSpider(scrapy.Spider): yield -process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False}) +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR_ENABLED": False, + "COMPRESSION_KEEP_ENCODING_HEADER": True, + } +) process.crawl(NoRequestsSpider) process.start() diff --git a/tests/AsyncCrawlerRunner/reactorless_datauri.py b/tests/AsyncCrawlerRunner/reactorless_datauri.py index d964f9c1c..19f8a05ad 100644 --- a/tests/AsyncCrawlerRunner/reactorless_datauri.py +++ b/tests/AsyncCrawlerRunner/reactorless_datauri.py @@ -17,7 +17,12 @@ class DataSpider(Spider): async def main() -> None: configure_logging() - runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False}) + runner = AsyncCrawlerRunner( + settings={ + "TWISTED_REACTOR_ENABLED": False, + "COMPRESSION_KEEP_ENCODING_HEADER": True, + } + ) await runner.crawl(DataSpider) diff --git a/tests/AsyncCrawlerRunner/reactorless_simple.py b/tests/AsyncCrawlerRunner/reactorless_simple.py index e91c7c89b..99f364d23 100644 --- a/tests/AsyncCrawlerRunner/reactorless_simple.py +++ b/tests/AsyncCrawlerRunner/reactorless_simple.py @@ -17,7 +17,12 @@ class NoRequestsSpider(Spider): async def main() -> None: configure_logging() - runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False}) + runner = AsyncCrawlerRunner( + settings={ + "TWISTED_REACTOR_ENABLED": False, + "COMPRESSION_KEEP_ENCODING_HEADER": True, + } + ) await runner.crawl(NoRequestsSpider) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 5c4085657..493146b84 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -782,3 +782,77 @@ class TestHttpCompression: continue resp = self._get_truncated_response(check_key) assert len(resp.body) == 0 + + def test_process_response_keep_encoding_header(self): + crawler = get_crawler( + Spider, settings_dict={"COMPRESSION_KEEP_ENCODING_HEADER": True} + ) + mw = HttpCompressionMiddleware.from_crawler(crawler) + crawler.stats.open_spider() + response = self._getresponse("gzip") + request = response.request + + assert response.headers["Content-Encoding"] == b"gzip" + newresponse = mw.process_response(request, response) + assert newresponse is not response + assert newresponse.body.startswith(b"