From db12600cffada9971a9df5107f437129e0b511b8 Mon Sep 17 00:00:00 2001
From: Hassaan Naushahi <hassaan.naushahi@zyte.com>
Date: Fri, 2 Jun 2023 15:24:58 +0500
Subject: [PATCH 1/7] testing first commit

---
 scrapy/downloadermiddlewares/httpcompression.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index ead426951..344efd0ea 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -29,7 +29,7 @@ class HttpCompressionMiddleware:
     """This middleware allows compressed (gzip, deflate) traffic to be
     sent/received from web sites"""
 
-    def __init__(self, stats=None):
+    def __init__(self, stats=None, settings=None):
         self.stats = stats
 
     @classmethod

From 4d2e3c354cd2ac48a9d37908a3ded055cfd4ce0c Mon Sep 17 00:00:00 2001
From: Hassaan Naushahi <hassaan.naushahi@zyte.com>
Date: Fri, 2 Jun 2023 15:53:13 +0500
Subject: [PATCH 2/7] Add Content-Encoding header in response flag

---
 .../downloadermiddlewares/httpcompression.py  | 81 ++++++++++++-------
 1 file changed, 54 insertions(+), 27 deletions(-)

diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 344efd0ea..30a6952df 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -31,22 +31,51 @@ class HttpCompressionMiddleware:
 
     def __init__(self, stats=None, settings=None):
         self.stats = stats
+        if not stats:
+            warnings.warn(
+                "The default value of COMPRESSION_KEEP_ENCODING_HEADER, "
+                "False, is deprecated, and will stop working and stop "
+                "being its default value in a future version of Scrapy. "
+                "Set COMPRESSION_KEEP_ENCODING_HEADER=True in your "
+                "settings to remove this warning.",
+                ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
+        if settings:
+            self.keep_encoding_header = settings.getbool('COMPRESSION_KEEP_ENCODING_HEADER')
+            if not self.keep_encoding_header:
+                warnings.warn(
+                    "Setting COMPRESSION_KEEP_ENCODING_HEADER=False is deprecated",
+                    ScrapyDeprecationWarning,
+                )
+        else:
+            self.keep_encoding_header = False
+            warnings.warn(
+                "The default value of COMPRESSION_KEEP_ENCODING_HEADER, "
+                "False, is deprecated, and will stop working and stop "
+                "being its default value in a future version of Scrapy. "
+                "Set COMPRESSION_KEEP_ENCODING_HEADER=True in your "
+                "settings to remove this warning.",
+                ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
 
     @classmethod
     def from_crawler(cls, crawler):
         if not crawler.settings.getbool("COMPRESSION_ENABLED"):
             raise NotConfigured
         try:
-            return cls(stats=crawler.stats)
+            return cls(stats=crawler.stats, settings=crawler.settings)
         except TypeError:
             warnings.warn(
                 "HttpCompressionMiddleware subclasses must either modify "
-                "their '__init__' method to support a 'stats' parameter or "
+                "their '__init__' method to support 'stats' and 'settings' parameters or "
                 "reimplement the 'from_crawler' method.",
                 ScrapyDeprecationWarning,
             )
             result = cls()
             result.stats = crawler.stats
+            result.keep_encoding_header = False
             return result
 
     def process_request(self, request, spider):
@@ -55,32 +84,30 @@ class HttpCompressionMiddleware:
     def process_response(self, request, response, spider):
         if request.method == "HEAD":
             return response
-        if isinstance(response, Response):
-            content_encoding = response.headers.getlist("Content-Encoding")
-            if content_encoding:
-                encoding = content_encoding.pop()
-                decoded_body = self._decode(response.body, encoding.lower())
-                if self.stats:
-                    self.stats.inc_value(
-                        "httpcompression/response_bytes",
-                        len(decoded_body),
-                        spider=spider,
-                    )
-                    self.stats.inc_value(
-                        "httpcompression/response_count", spider=spider
-                    )
-                respcls = responsetypes.from_args(
-                    headers=response.headers, url=response.url, body=decoded_body
-                )
-                kwargs = dict(cls=respcls, body=decoded_body)
-                if issubclass(respcls, TextResponse):
-                    # force recalculating the encoding until we make sure the
-                    # responsetypes guessing is reliable
-                    kwargs["encoding"] = None
-                response = response.replace(**kwargs)
-                if not content_encoding:
-                    del response.headers["Content-Encoding"]
+        if b'decoded' in response.flags:
+            return response
+        content_encoding = response.headers.getlist('Content-Encoding')
+        if not content_encoding:
+            return response
 
+        encoding = content_encoding[0]
+        decoded_body = self._decode(response.body, encoding.lower())
+        if self.stats:
+            self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider)
+            self.stats.inc_value('httpcompression/response_count', spider=spider)
+        respcls = responsetypes.from_args(
+            headers=response.headers, url=response.url, body=decoded_body
+        )
+        kwargs = dict(cls=respcls, body=decoded_body)
+        if issubclass(respcls, TextResponse):
+            # force recalculating the encoding until we make sure the
+            # responsetypes guessing is reliable
+            kwargs['encoding'] = None
+
+        kwargs['flags'] = response.flags + [b'decoded']
+        response = response.replace(**kwargs)
+        if not self.keep_encoding_header:
+            del response.headers['Content-Encoding']
         return response
 
     def _decode(self, body, encoding):

From 918c8ffd867599dc57bff25ba78d313a48ead1cb Mon Sep 17 00:00:00 2001
From: Hassaan Naushahi <hassaan.naushahi@zyte.com>
Date: Fri, 8 Sep 2023 14:39:30 +0500
Subject: [PATCH 3/7] Added default setting values for successful tests

---
 scrapy/settings/default_settings.py              | 2 +-
 scrapy/templates/project/module/settings.py.tmpl | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 260ec1701..8400fc057 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -37,7 +37,7 @@ CLOSESPIDER_ERRORCOUNT = 0
 COMMANDS_MODULE = ""
 
 COMPRESSION_ENABLED = True
-
+COMPRESSION_KEEP_ENCODING_HEADER = False
 CONCURRENT_ITEMS = 100
 
 CONCURRENT_REQUESTS = 16
diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index ecb1e5e5c..9d337c530 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -86,6 +86,7 @@ ROBOTSTXT_OBEY = True
 #HTTPCACHE_DIR = "httpcache"
 #HTTPCACHE_IGNORE_HTTP_CODES = []
 #HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
+COMPRESSION_KEEP_ENCODING_HEADER = True
 
 # Set settings whose default value is deprecated to a future-proof value
 REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7"

From 72962d6f5bb972e9a49cbe0960ed0903ade98c33 Mon Sep 17 00:00:00 2001
From: Hassaan Naushahi <hassaan.naushahi@zyte.com>
Date: Fri, 8 Sep 2023 17:30:03 +0500
Subject: [PATCH 4/7] Added test case and code review suggestions

---
 scrapy/settings/default_settings.py           |   1 +
 .../templates/project/module/settings.py.tmpl |   2 +
 ...st_downloadermiddleware_httpcompression.py | 173 ++++++++++++++----
 3 files changed, 139 insertions(+), 37 deletions(-)

diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 8400fc057..04bb5a696 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -38,6 +38,7 @@ COMMANDS_MODULE = ""
 
 COMPRESSION_ENABLED = True
 COMPRESSION_KEEP_ENCODING_HEADER = False
+
 CONCURRENT_ITEMS = 100
 
 CONCURRENT_REQUESTS = 16
diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index 9d337c530..856b9240b 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -86,6 +86,8 @@ ROBOTSTXT_OBEY = True
 #HTTPCACHE_DIR = "httpcache"
 #HTTPCACHE_IGNORE_HTTP_CODES = []
 #HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
+
+# Keep original Content-Encoding header
 COMPRESSION_KEEP_ENCODING_HEADER = True
 
 # Set settings whose default value is deprecated to a future-proof value
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index fac5588ff..f9e9b4661 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -13,6 +13,7 @@ from scrapy.downloadermiddlewares.httpcompression import (
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import HtmlResponse, Request, Response
 from scrapy.responsetypes import responsetypes
+from scrapy.settings import Settings
 from scrapy.spiders import Spider
 from scrapy.utils.gz import gunzip
 from scrapy.utils.test import get_crawler
@@ -109,23 +110,25 @@ class HttpCompressionTest(TestCase):
 
         self.assertEqual(response.headers["Content-Encoding"], b"gzip")
         newresponse = self.mw.process_response(request, response, self.spider)
-        assert newresponse is not response
-        assert newresponse.body.startswith(b"<!DOCTYPE")
-        assert "Content-Encoding" not in newresponse.headers
+        self.assertNotEqual(newresponse, response)
+        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
+        self.assertIn('Content-Encoding', newresponse.headers)
+        self.assertNotIn(b'gzip', newresponse.headers['Content-Encoding'])
         self.assertStatsEqual("httpcompression/response_count", 1)
         self.assertStatsEqual("httpcompression/response_bytes", 74837)
 
     def test_process_response_gzip_no_stats(self):
-        mw = HttpCompressionMiddleware()
+        mw = HttpCompressionMiddleware(settings=Settings({'COMPRESSION_KEEP_ENCODING_HEADER': True}))
         response = self._getresponse("gzip")
         request = response.request
 
         self.assertEqual(response.headers["Content-Encoding"], b"gzip")
         newresponse = mw.process_response(request, response, self.spider)
         self.assertEqual(mw.stats, None)
-        assert newresponse is not response
-        assert newresponse.body.startswith(b"<!DOCTYPE")
-        assert "Content-Encoding" not in newresponse.headers
+        self.assertNotEqual(newresponse, response)
+        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
+        self.assertIn('Content-Encoding', newresponse.headers)
+        self.assertNotIn(b'gzip', newresponse.headers['Content-Encoding'])
 
     def test_process_response_br(self):
         try:
@@ -136,9 +139,10 @@ class HttpCompressionTest(TestCase):
         request = response.request
         self.assertEqual(response.headers["Content-Encoding"], b"br")
         newresponse = self.mw.process_response(request, response, self.spider)
-        assert newresponse is not response
-        assert newresponse.body.startswith(b"<!DOCTYPE")
-        assert "Content-Encoding" not in newresponse.headers
+        self.assertNotEqual(newresponse, response)
+        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
+        self.assertIn('Content-Encoding', newresponse.headers)
+        self.assertEqual(b'br', newresponse.headers['Content-Encoding'])
         self.assertStatsEqual("httpcompression/response_count", 1)
         self.assertStatsEqual("httpcompression/response_bytes", 74837)
 
@@ -158,10 +162,11 @@ class HttpCompressionTest(TestCase):
             if raw_content is None:
                 raw_content = newresponse.body
             else:
-                assert raw_content == newresponse.body
-            assert newresponse is not response
-            assert newresponse.body.startswith(b"<!DOCTYPE")
-            assert "Content-Encoding" not in newresponse.headers
+                self.assertEqual(raw_content, newresponse.body)
+            self.assertNotEqual(newresponse, response)
+            self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
+            self.assertIn('Content-Encoding', newresponse.headers)
+            self.assertEqual(b'zstd', newresponse.headers['Content-Encoding'])
 
     def test_process_response_rawdeflate(self):
         response = self._getresponse("rawdeflate")
@@ -169,11 +174,13 @@ class HttpCompressionTest(TestCase):
 
         self.assertEqual(response.headers["Content-Encoding"], b"deflate")
         newresponse = self.mw.process_response(request, response, self.spider)
-        assert newresponse is not response
-        assert newresponse.body.startswith(b"<!DOCTYPE")
-        assert "Content-Encoding" not in newresponse.headers
+        self.assertNotEqual(newresponse, response)
+        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
+        self.assertIn('Content-Encoding', newresponse.headers)
+        self.assertIn(b'deflate', newresponse.headers['Content-Encoding'])
         self.assertStatsEqual("httpcompression/response_count", 1)
         self.assertStatsEqual("httpcompression/response_bytes", 74840)
+        self.assertIn(b'decoded', newresponse.flags)
 
     def test_process_response_zlibdelate(self):
         response = self._getresponse("zlibdeflate")
@@ -181,20 +188,23 @@ class HttpCompressionTest(TestCase):
 
         self.assertEqual(response.headers["Content-Encoding"], b"deflate")
         newresponse = self.mw.process_response(request, response, self.spider)
-        assert newresponse is not response
-        assert newresponse.body.startswith(b"<!DOCTYPE")
-        assert "Content-Encoding" not in newresponse.headers
+        self.assertNotEqual(newresponse, response)
+        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
+        self.assertIn('Content-Encoding', newresponse.headers)
+        self.assertIn(b'deflate', newresponse.headers['Content-Encoding'])
         self.assertStatsEqual("httpcompression/response_count", 1)
         self.assertStatsEqual("httpcompression/response_bytes", 74840)
+        self.assertIn(b'decoded', newresponse.flags)
 
     def test_process_response_plain(self):
         response = Response("http://scrapytest.org", body=b"<!DOCTYPE...")
         request = Request("http://scrapytest.org")
 
-        assert not response.headers.get("Content-Encoding")
+        self.assertTrue(not response.headers.get('Content-Encoding'))
         newresponse = self.mw.process_response(request, response, self.spider)
-        assert newresponse is response
-        assert newresponse.body.startswith(b"<!DOCTYPE")
+        self.assertEqual(newresponse, response)
+        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
+        self.assertNotIn('Content-Encoding', newresponse.headers)
         self.assertStatsEqual("httpcompression/response_count", None)
         self.assertStatsEqual("httpcompression/response_bytes", None)
 
@@ -203,8 +213,10 @@ class HttpCompressionTest(TestCase):
         response.headers["Content-Encoding"] = ["uuencode", "gzip"]
         request = response.request
         newresponse = self.mw.process_response(request, response, self.spider)
-        assert newresponse is not response
-        self.assertEqual(newresponse.headers.getlist("Content-Encoding"), [b"uuencode"])
+        self.assertEqual(newresponse.headers.getlist('Content-Encoding'), [b'uuencode'])
+        self.assertNotEqual(newresponse, response)
+        self.assertEqual(newresponse.headers.getlist('Content-Encoding'), [b'uuencode', b'gzip'])
+        self.assertIn(b'decoded', newresponse.flags)
 
     def test_process_response_encoding_inside_body(self):
         headers = {
@@ -225,11 +237,14 @@ class HttpCompressionTest(TestCase):
         request = Request("http://www.example.com/")
 
         newresponse = self.mw.process_response(request, response, self.spider)
-        assert isinstance(newresponse, HtmlResponse)
+        self.assertIsInstance(newresponse, HtmlResponse)
         self.assertEqual(newresponse.body, plainbody)
         self.assertEqual(newresponse.encoding, resolve_encoding("gb2312"))
         self.assertStatsEqual("httpcompression/response_count", 1)
         self.assertStatsEqual("httpcompression/response_bytes", 104)
+        self.assertIn('Content-Encoding', newresponse.headers)
+        self.assertEqual(b'gzip', newresponse.headers['Content-Encoding'])
+        self.assertIn(b'decoded', newresponse.flags)
 
     def test_process_response_force_recalculate_encoding(self):
         headers = {
@@ -250,11 +265,14 @@ class HttpCompressionTest(TestCase):
         request = Request("http://www.example.com/")
 
         newresponse = self.mw.process_response(request, response, self.spider)
-        assert isinstance(newresponse, HtmlResponse)
+        self.assertIsInstance(newresponse, HtmlResponse)
         self.assertEqual(newresponse.body, plainbody)
         self.assertEqual(newresponse.encoding, resolve_encoding("gb2312"))
+        self.assertIn('Content-Encoding', newresponse.headers)
+        self.assertEqual(b'gzip', newresponse.headers['Content-Encoding'])
         self.assertStatsEqual("httpcompression/response_count", 1)
         self.assertStatsEqual("httpcompression/response_bytes", 104)
+        self.assertIn(b'decoded', newresponse.flags)
 
     def test_process_response_no_content_type_header(self):
         headers = {
@@ -273,11 +291,14 @@ class HttpCompressionTest(TestCase):
         request = Request("http://www.example.com/index")
 
         newresponse = self.mw.process_response(request, response, self.spider)
-        assert isinstance(newresponse, respcls)
+        self.assertIsInstance(newresponse, respcls)
         self.assertEqual(newresponse.body, plainbody)
         self.assertEqual(newresponse.encoding, resolve_encoding("gb2312"))
+        self.assertIn('Content-Encoding', newresponse.headers)
+        self.assertEqual(b'identity', newresponse.headers['Content-Encoding'])
         self.assertStatsEqual("httpcompression/response_count", 1)
         self.assertStatsEqual("httpcompression/response_bytes", 104)
+        self.assertIn(b'decode', newresponse.flags)
 
     def test_process_response_gzipped_contenttype(self):
         response = self._getresponse("gzip")
@@ -287,9 +308,11 @@ class HttpCompressionTest(TestCase):
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertIsNot(newresponse, response)
         self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
-        self.assertNotIn("Content-Encoding", newresponse.headers)
+        self.assertIn('Content-Encoding', newresponse.headers)
+        self.assertEqual(b'gzip', newresponse.headers['Content-Encoding'])
         self.assertStatsEqual("httpcompression/response_count", 1)
         self.assertStatsEqual("httpcompression/response_bytes", 74837)
+        self.assertIn(b'decoded', newresponse.flags)
 
     def test_process_response_gzip_app_octetstream_contenttype(self):
         response = self._getresponse("gzip")
@@ -299,9 +322,11 @@ class HttpCompressionTest(TestCase):
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertIsNot(newresponse, response)
         self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
-        self.assertNotIn("Content-Encoding", newresponse.headers)
+        self.assertIn('Content-Encoding', newresponse.headers)
+        self.assertEqual(b'gzip', newresponse.headers['Content-Encoding'])
         self.assertStatsEqual("httpcompression/response_count", 1)
         self.assertStatsEqual("httpcompression/response_bytes", 74837)
+        self.assertIn(b'decoded', newresponse.flags)
 
     def test_process_response_gzip_binary_octetstream_contenttype(self):
         response = self._getresponse("x-gzip")
@@ -311,9 +336,11 @@ class HttpCompressionTest(TestCase):
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertIsNot(newresponse, response)
         self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
-        self.assertNotIn("Content-Encoding", newresponse.headers)
+        self.assertIn('Content-Encoding', newresponse.headers)
+        self.assertEqual(b'gzip', newresponse.headers['Content-Encoding'])
         self.assertStatsEqual("httpcompression/response_count", 1)
         self.assertStatsEqual("httpcompression/response_bytes", 74837)
+        self.assertIn(b'decoded', newresponse.flags)
 
     def test_process_response_gzipped_gzip_file(self):
         """Test that a gzip Content-Encoded .gz file is gunzipped
@@ -360,6 +387,9 @@ class HttpCompressionTest(TestCase):
         self.assertEqual(gunzip(newresponse.body), plainbody)
         self.assertStatsEqual("httpcompression/response_count", 1)
         self.assertStatsEqual("httpcompression/response_bytes", 230)
+        self.assertIn('Content-Encoding', newresponse.headers)
+        self.assertEqual(b'gzip', newresponse.headers['Content-Encoding'])
+        self.assertIn(b'decoded', newresponse.flags
 
     def test_process_response_head_request_no_decode_required(self):
         response = self._getresponse("gzip")
@@ -370,12 +400,46 @@ class HttpCompressionTest(TestCase):
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertIs(newresponse, response)
         self.assertEqual(response.body, b"")
+        self.assertIn('Content-Encoding', newresponse.headers)
+        self.assertEqual(b'gzip', newresponse.headers['Content-Encoding'])
         self.assertStatsEqual("httpcompression/response_count", None)
         self.assertStatsEqual("httpcompression/response_bytes", None)
 
+    def test_process_response_keeps_content_encoding_header(self):
+        settings = {'COMPRESSION_ENABLED': True,
+                    'COMPRESSION_KEEP_ENCODING_HEADER': True}
+        crawler = get_crawler(Spider, settings)
+        spider = crawler._create_spider('example.com')
+        mw = HttpCompressionMiddleware.from_crawler(crawler)
+        response = self._getresponse('gzip')
+        request = response.request
+
+        self.assertEqual(response.headers['Content-Encoding'], b'gzip')
+        newresponse = mw.process_response(request, response, spider)
+        self.assertIsNot(newresponse, response)
+        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
+        self.assertIn('Content-Encoding', newresponse.headers)
+        self.assertNotIn(b'gzip', newresponse.headers['Content-Encoding'])
+        self.assertIn(b'decoded', newresponse.flags)
+
+    def test_process_response_doesnt_keep_content_encoding_header(self):
+        settings = {'COMPRESSION_ENABLED': True,
+                    'COMPRESSION_KEEP_ENCODING_HEADER': False}
+        crawler = get_crawler(Spider, settings)
+        spider = crawler._create_spider('example.com')
+        mw = HttpCompressionMiddleware.from_crawler(crawler)
+        response = self._getresponse('gzip')
+        request = response.request
+
+        self.assertEqual(response.headers['Content-Encoding'], b'gzip')
+        newresponse = mw.process_response(request, response, spider)
+        self.assertIsNot(newresponse, response)
+        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
+        self.assertNotIn('Content-Encoding', newresponse.headers)
+        self.assertIn(b'decoded', newresponse.flags)
 
 class HttpCompressionSubclassTest(TestCase):
-    def test_init_missing_stats(self):
+    def test_from_crawler_missing_args(self):
         class HttpCompressionMiddlewareSubclass(HttpCompressionMiddleware):
             def __init__(self):
                 super().__init__()
@@ -391,10 +455,45 @@ class HttpCompressionSubclassTest(TestCase):
         self.assertEqual(
             messages,
             (
-                (
-                    "HttpCompressionMiddleware subclasses must either modify "
-                    "their '__init__' method to support a 'stats' parameter "
-                    "or reimplement the 'from_crawler' method."
-                ),
+                "HttpCompressionMiddleware subclasses must either modify "
+                "their '__init__' method to support 'stats' and 'settings' parameters "
+                "or reimplement the 'from_crawler' method.",
+                "HttpCompressionMiddleware now accepts a 'stats' parameter which should be specified.",
+                "HttpCompressionMiddleware now accepts a 'settings' parameter which should be specified.",
+            ),
+        )
+
+    def test_init_missing_args(self):
+        with catch_warnings(record=True) as caught_warnings:
+            self.assertIsNotNone(HttpCompressionMiddleware(stats=None, settings=None))
+        messages = tuple(
+            str(warning.message) for warning in caught_warnings
+            if warning.category is ScrapyDeprecationWarning
+        )
+
+        self.assertEqual(
+            messages,
+            (
+                "HttpCompressionMiddleware now accepts a 'stats' parameter which should be specified.",
+                "HttpCompressionMiddleware now accepts a 'settings' parameter which should be specified.",
+            ),
+        )
+
+    def test_init_keep_encoding_header_deprecation_warning(self):
+        from scrapy.settings import Settings
+        settings = Settings({'COMPRESSION_KEEP_ENCODING_HEADER': False})
+
+        with catch_warnings(record=True) as caught_warnings:
+            mw = HttpCompressionMiddleware(stats={'foo': 'bar'}, settings=settings)
+            self.assertIsNotNone(mw)
+        messages = tuple(
+            str(warning.message) for warning in caught_warnings
+            if warning.category is ScrapyDeprecationWarning
+        )
+
+        self.assertEqual(
+            messages,
+            (
+                "Setting COMPRESSION_KEEP_ENCODING_HEADER=False is deprecated",
             ),
         )

From 50e5897429cd3a5b12c1aa0de6db8ab4dac656d4 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 30 Dec 2024 19:30:31 +0500
Subject: [PATCH 5/7] Fix merge errors and other simple mistakes.

---
 .../downloadermiddlewares/httpcompression.py  |  35 ++--
 ...st_downloadermiddleware_httpcompression.py | 161 ++++++++++--------
 2 files changed, 101 insertions(+), 95 deletions(-)

diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index dfdad9864..95d5dba66 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -1,12 +1,13 @@
 from __future__ import annotations
 
 import io
+import warnings
 import zlib
 from typing import TYPE_CHECKING, List, Optional, Union
 
 from scrapy import Request, Spider
 from scrapy.crawler import Crawler
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import Response, TextResponse
 from scrapy.responsetypes import responsetypes
 from scrapy.statscollectors import StatsCollector
@@ -37,20 +38,12 @@ class HttpCompressionMiddleware:
     """This middleware allows compressed (gzip, deflate) traffic to be
     sent/received from web sites"""
 
-    def __init__(self, stats: Optional[StatsCollector] = None):
+    def __init__(self, stats: Optional[StatsCollector] = None, settings=None):
         self.stats = stats
-        if not stats:
-            warnings.warn(
-                "The default value of COMPRESSION_KEEP_ENCODING_HEADER, "
-                "False, is deprecated, and will stop working and stop "
-                "being its default value in a future version of Scrapy. "
-                "Set COMPRESSION_KEEP_ENCODING_HEADER=True in your "
-                "settings to remove this warning.",
-                ScrapyDeprecationWarning,
-                stacklevel=2,
-            )
         if settings:
-            self.keep_encoding_header = settings.getbool('COMPRESSION_KEEP_ENCODING_HEADER')
+            self.keep_encoding_header = settings.getbool(
+                "COMPRESSION_KEEP_ENCODING_HEADER"
+            )
             if not self.keep_encoding_header:
                 warnings.warn(
                     "Setting COMPRESSION_KEEP_ENCODING_HEADER=False is deprecated",
@@ -97,17 +90,19 @@ class HttpCompressionMiddleware:
     ) -> Union[Request, Response]:
         if request.method == "HEAD":
             return response
-        if b'decoded' in response.flags:
+        if b"decoded" in response.flags:
             return response
-        content_encoding = response.headers.getlist('Content-Encoding')
+        content_encoding = response.headers.getlist("Content-Encoding")
         if not content_encoding:
             return response
 
         encoding = content_encoding[0]
         decoded_body = self._decode(response.body, encoding.lower())
         if self.stats:
-            self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider)
-            self.stats.inc_value('httpcompression/response_count', spider=spider)
+            self.stats.inc_value(
+                "httpcompression/response_bytes", len(decoded_body), spider=spider
+            )
+            self.stats.inc_value("httpcompression/response_count", spider=spider)
         respcls = responsetypes.from_args(
             headers=response.headers, url=response.url, body=decoded_body
         )
@@ -115,12 +110,12 @@ class HttpCompressionMiddleware:
         if issubclass(respcls, TextResponse):
             # force recalculating the encoding until we make sure the
             # responsetypes guessing is reliable
-            kwargs['encoding'] = None
+            kwargs["encoding"] = None
 
-        kwargs['flags'] = response.flags + [b'decoded']
+        kwargs["flags"] = response.flags + [b"decoded"]
         response = response.replace(**kwargs)
         if not self.keep_encoding_header:
-            del response.headers['Content-Encoding']
+            del response.headers["Content-Encoding"]
         return response
 
     def _decode(self, body: bytes, encoding: bytes) -> bytes:
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 0c4fff629..b52882795 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -2,6 +2,7 @@ from gzip import GzipFile
 from io import BytesIO
 from pathlib import Path
 from unittest import SkipTest, TestCase
+from warnings import catch_warnings
 
 from w3lib.encoding import resolve_encoding
 
@@ -9,7 +10,7 @@ from scrapy.downloadermiddlewares.httpcompression import (
     ACCEPTED_ENCODINGS,
     HttpCompressionMiddleware,
 )
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import HtmlResponse, Request, Response
 from scrapy.responsetypes import responsetypes
 from scrapy.settings import Settings
@@ -110,14 +111,16 @@ class HttpCompressionTest(TestCase):
         self.assertEqual(response.headers["Content-Encoding"], b"gzip")
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertNotEqual(newresponse, response)
-        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
-        self.assertIn('Content-Encoding', newresponse.headers)
-        self.assertNotIn(b'gzip', newresponse.headers['Content-Encoding'])
+        self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
+        self.assertIn("Content-Encoding", newresponse.headers)
+        self.assertNotIn(b"gzip", newresponse.headers["Content-Encoding"])
         self.assertStatsEqual("httpcompression/response_count", 1)
         self.assertStatsEqual("httpcompression/response_bytes", 74837)
 
     def test_process_response_gzip_no_stats(self):
-        mw = HttpCompressionMiddleware(settings=Settings({'COMPRESSION_KEEP_ENCODING_HEADER': True}))
+        mw = HttpCompressionMiddleware(
+            settings=Settings({"COMPRESSION_KEEP_ENCODING_HEADER": True})
+        )
         response = self._getresponse("gzip")
         request = response.request
 
@@ -125,9 +128,9 @@ class HttpCompressionTest(TestCase):
         newresponse = mw.process_response(request, response, self.spider)
         self.assertEqual(mw.stats, None)
         self.assertNotEqual(newresponse, response)
-        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
-        self.assertIn('Content-Encoding', newresponse.headers)
-        self.assertNotIn(b'gzip', newresponse.headers['Content-Encoding'])
+        self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
+        self.assertIn("Content-Encoding", newresponse.headers)
+        self.assertNotIn(b"gzip", newresponse.headers["Content-Encoding"])
 
     def test_process_response_br(self):
         try:
@@ -139,9 +142,9 @@ class HttpCompressionTest(TestCase):
         self.assertEqual(response.headers["Content-Encoding"], b"br")
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertNotEqual(newresponse, response)
-        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
-        self.assertIn('Content-Encoding', newresponse.headers)
-        self.assertEqual(b'br', newresponse.headers['Content-Encoding'])
+        self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
+        self.assertIn("Content-Encoding", newresponse.headers)
+        self.assertEqual(b"br", newresponse.headers["Content-Encoding"])
         self.assertStatsEqual("httpcompression/response_count", 1)
         self.assertStatsEqual("httpcompression/response_bytes", 74837)
 
@@ -163,9 +166,9 @@ class HttpCompressionTest(TestCase):
             else:
                 self.assertEqual(raw_content, newresponse.body)
             self.assertNotEqual(newresponse, response)
-            self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
-            self.assertIn('Content-Encoding', newresponse.headers)
-            self.assertEqual(b'zstd', newresponse.headers['Content-Encoding'])
+            self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
+            self.assertIn("Content-Encoding", newresponse.headers)
+            self.assertEqual(b"zstd", newresponse.headers["Content-Encoding"])
 
     def test_process_response_rawdeflate(self):
         response = self._getresponse("rawdeflate")
@@ -174,12 +177,12 @@ class HttpCompressionTest(TestCase):
         self.assertEqual(response.headers["Content-Encoding"], b"deflate")
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertNotEqual(newresponse, response)
-        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
-        self.assertIn('Content-Encoding', newresponse.headers)
-        self.assertIn(b'deflate', newresponse.headers['Content-Encoding'])
+        self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
+        self.assertIn("Content-Encoding", newresponse.headers)
+        self.assertIn(b"deflate", newresponse.headers["Content-Encoding"])
         self.assertStatsEqual("httpcompression/response_count", 1)
         self.assertStatsEqual("httpcompression/response_bytes", 74840)
-        self.assertIn(b'decoded', newresponse.flags)
+        self.assertIn(b"decoded", newresponse.flags)
 
     def test_process_response_zlibdelate(self):
         response = self._getresponse("zlibdeflate")
@@ -188,22 +191,22 @@ class HttpCompressionTest(TestCase):
         self.assertEqual(response.headers["Content-Encoding"], b"deflate")
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertNotEqual(newresponse, response)
-        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
-        self.assertIn('Content-Encoding', newresponse.headers)
-        self.assertIn(b'deflate', newresponse.headers['Content-Encoding'])
+        self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
+        self.assertIn("Content-Encoding", newresponse.headers)
+        self.assertIn(b"deflate", newresponse.headers["Content-Encoding"])
         self.assertStatsEqual("httpcompression/response_count", 1)
         self.assertStatsEqual("httpcompression/response_bytes", 74840)
-        self.assertIn(b'decoded', newresponse.flags)
+        self.assertIn(b"decoded", newresponse.flags)
 
     def test_process_response_plain(self):
         response = Response("http://scrapytest.org", body=b"<!DOCTYPE...")
         request = Request("http://scrapytest.org")
 
-        self.assertTrue(not response.headers.get('Content-Encoding'))
+        self.assertTrue(not response.headers.get("Content-Encoding"))
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertEqual(newresponse, response)
-        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
-        self.assertNotIn('Content-Encoding', newresponse.headers)
+        self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
+        self.assertNotIn("Content-Encoding", newresponse.headers)
         self.assertStatsEqual("httpcompression/response_count", None)
         self.assertStatsEqual("httpcompression/response_bytes", None)
 
@@ -212,10 +215,12 @@ class HttpCompressionTest(TestCase):
         response.headers["Content-Encoding"] = ["uuencode", "gzip"]
         request = response.request
         newresponse = self.mw.process_response(request, response, self.spider)
-        self.assertEqual(newresponse.headers.getlist('Content-Encoding'), [b'uuencode'])
+        self.assertEqual(newresponse.headers.getlist("Content-Encoding"), [b"uuencode"])
         self.assertNotEqual(newresponse, response)
-        self.assertEqual(newresponse.headers.getlist('Content-Encoding'), [b'uuencode', b'gzip'])
-        self.assertIn(b'decoded', newresponse.flags)
+        self.assertEqual(
+            newresponse.headers.getlist("Content-Encoding"), [b"uuencode", b"gzip"]
+        )
+        self.assertIn(b"decoded", newresponse.flags)
 
     def test_process_response_encoding_inside_body(self):
         headers = {
@@ -240,9 +245,9 @@ class HttpCompressionTest(TestCase):
         self.assertEqual(newresponse.body, plainbody)
         self.assertEqual(newresponse.encoding, resolve_encoding("gb2312"))
         self.assertStatsEqual("httpcompression/response_count", 1)
-        self.assertIn('Content-Encoding', newresponse.headers)
-        self.assertEqual(b'gzip', newresponse.headers['Content-Encoding'])
-        self.assertIn(b'decoded', newresponse.flags)
+        self.assertIn("Content-Encoding", newresponse.headers)
+        self.assertEqual(b"gzip", newresponse.headers["Content-Encoding"])
+        self.assertIn(b"decoded", newresponse.flags)
         self.assertStatsEqual("httpcompression/response_bytes", len(plainbody))
 
     def test_process_response_force_recalculate_encoding(self):
@@ -267,10 +272,10 @@ class HttpCompressionTest(TestCase):
         self.assertIsInstance(newresponse, HtmlResponse)
         self.assertEqual(newresponse.body, plainbody)
         self.assertEqual(newresponse.encoding, resolve_encoding("gb2312"))
-        self.assertIn('Content-Encoding', newresponse.headers)
-        self.assertEqual(b'gzip', newresponse.headers['Content-Encoding'])
+        self.assertIn("Content-Encoding", newresponse.headers)
+        self.assertEqual(b"gzip", newresponse.headers["Content-Encoding"])
         self.assertStatsEqual("httpcompression/response_count", 1)
-        self.assertIn(b'decoded', newresponse.flags)
+        self.assertIn(b"decoded", newresponse.flags)
         self.assertStatsEqual("httpcompression/response_bytes", len(plainbody))
 
     def test_process_response_no_content_type_header(self):
@@ -293,10 +298,10 @@ class HttpCompressionTest(TestCase):
         self.assertIsInstance(newresponse, respcls)
         self.assertEqual(newresponse.body, plainbody)
         self.assertEqual(newresponse.encoding, resolve_encoding("gb2312"))
-        self.assertIn('Content-Encoding', newresponse.headers)
-        self.assertEqual(b'identity', newresponse.headers['Content-Encoding'])
+        self.assertIn("Content-Encoding", newresponse.headers)
+        self.assertEqual(b"identity", newresponse.headers["Content-Encoding"])
         self.assertStatsEqual("httpcompression/response_count", 1)
-        self.assertIn(b'decode', newresponse.flags)
+        self.assertIn(b"decode", newresponse.flags)
         self.assertStatsEqual("httpcompression/response_bytes", len(plainbody))
 
     def test_process_response_gzipped_contenttype(self):
@@ -307,11 +312,11 @@ class HttpCompressionTest(TestCase):
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertIsNot(newresponse, response)
         self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
-        self.assertIn('Content-Encoding', newresponse.headers)
-        self.assertEqual(b'gzip', newresponse.headers['Content-Encoding'])
+        self.assertIn("Content-Encoding", newresponse.headers)
+        self.assertEqual(b"gzip", newresponse.headers["Content-Encoding"])
         self.assertStatsEqual("httpcompression/response_count", 1)
         self.assertStatsEqual("httpcompression/response_bytes", 74837)
-        self.assertIn(b'decoded', newresponse.flags)
+        self.assertIn(b"decoded", newresponse.flags)
 
     def test_process_response_gzip_app_octetstream_contenttype(self):
         response = self._getresponse("gzip")
@@ -321,11 +326,11 @@ class HttpCompressionTest(TestCase):
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertIsNot(newresponse, response)
         self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
-        self.assertIn('Content-Encoding', newresponse.headers)
-        self.assertEqual(b'gzip', newresponse.headers['Content-Encoding'])
+        self.assertIn("Content-Encoding", newresponse.headers)
+        self.assertEqual(b"gzip", newresponse.headers["Content-Encoding"])
         self.assertStatsEqual("httpcompression/response_count", 1)
         self.assertStatsEqual("httpcompression/response_bytes", 74837)
-        self.assertIn(b'decoded', newresponse.flags)
+        self.assertIn(b"decoded", newresponse.flags)
 
     def test_process_response_gzip_binary_octetstream_contenttype(self):
         response = self._getresponse("x-gzip")
@@ -335,11 +340,11 @@ class HttpCompressionTest(TestCase):
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertIsNot(newresponse, response)
         self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
-        self.assertIn('Content-Encoding', newresponse.headers)
-        self.assertEqual(b'gzip', newresponse.headers['Content-Encoding'])
+        self.assertIn("Content-Encoding", newresponse.headers)
+        self.assertEqual(b"gzip", newresponse.headers["Content-Encoding"])
         self.assertStatsEqual("httpcompression/response_count", 1)
         self.assertStatsEqual("httpcompression/response_bytes", 74837)
-        self.assertIn(b'decoded', newresponse.flags)
+        self.assertIn(b"decoded", newresponse.flags)
 
     def test_process_response_gzipped_gzip_file(self):
         """Test that a gzip Content-Encoded .gz file is gunzipped
@@ -386,9 +391,9 @@ class HttpCompressionTest(TestCase):
         self.assertEqual(gunzip(newresponse.body), plainbody)
         self.assertStatsEqual("httpcompression/response_count", 1)
         self.assertStatsEqual("httpcompression/response_bytes", 230)
-        self.assertIn('Content-Encoding', newresponse.headers)
-        self.assertEqual(b'gzip', newresponse.headers['Content-Encoding'])
-        self.assertIn(b'decoded', newresponse.flags
+        self.assertIn("Content-Encoding", newresponse.headers)
+        self.assertEqual(b"gzip", newresponse.headers["Content-Encoding"])
+        self.assertIn(b"decoded", newresponse.flags)
 
     def test_process_response_head_request_no_decode_required(self):
         response = self._getresponse("gzip")
@@ -399,43 +404,48 @@ class HttpCompressionTest(TestCase):
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertIs(newresponse, response)
         self.assertEqual(response.body, b"")
-        self.assertIn('Content-Encoding', newresponse.headers)
-        self.assertEqual(b'gzip', newresponse.headers['Content-Encoding'])
+        self.assertIn("Content-Encoding", newresponse.headers)
+        self.assertEqual(b"gzip", newresponse.headers["Content-Encoding"])
         self.assertStatsEqual("httpcompression/response_count", None)
         self.assertStatsEqual("httpcompression/response_bytes", None)
 
     def test_process_response_keeps_content_encoding_header(self):
-        settings = {'COMPRESSION_ENABLED': True,
-                    'COMPRESSION_KEEP_ENCODING_HEADER': True}
+        settings = {
+            "COMPRESSION_ENABLED": True,
+            "COMPRESSION_KEEP_ENCODING_HEADER": True,
+        }
         crawler = get_crawler(Spider, settings)
-        spider = crawler._create_spider('example.com')
+        spider = crawler._create_spider("example.com")
         mw = HttpCompressionMiddleware.from_crawler(crawler)
-        response = self._getresponse('gzip')
+        response = self._getresponse("gzip")
         request = response.request
 
-        self.assertEqual(response.headers['Content-Encoding'], b'gzip')
+        self.assertEqual(response.headers["Content-Encoding"], b"gzip")
         newresponse = mw.process_response(request, response, spider)
         self.assertIsNot(newresponse, response)
-        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
-        self.assertIn('Content-Encoding', newresponse.headers)
-        self.assertNotIn(b'gzip', newresponse.headers['Content-Encoding'])
-        self.assertIn(b'decoded', newresponse.flags)
+        self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
+        self.assertIn("Content-Encoding", newresponse.headers)
+        self.assertNotIn(b"gzip", newresponse.headers["Content-Encoding"])
+        self.assertIn(b"decoded", newresponse.flags)
 
     def test_process_response_doesnt_keep_content_encoding_header(self):
-        settings = {'COMPRESSION_ENABLED': True,
-                    'COMPRESSION_KEEP_ENCODING_HEADER': False}
+        settings = {
+            "COMPRESSION_ENABLED": True,
+            "COMPRESSION_KEEP_ENCODING_HEADER": False,
+        }
         crawler = get_crawler(Spider, settings)
-        spider = crawler._create_spider('example.com')
+        spider = crawler._create_spider("example.com")
         mw = HttpCompressionMiddleware.from_crawler(crawler)
-        response = self._getresponse('gzip')
+        response = self._getresponse("gzip")
         request = response.request
 
-        self.assertEqual(response.headers['Content-Encoding'], b'gzip')
+        self.assertEqual(response.headers["Content-Encoding"], b"gzip")
         newresponse = mw.process_response(request, response, spider)
         self.assertIsNot(newresponse, response)
-        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
-        self.assertNotIn('Content-Encoding', newresponse.headers)
-        self.assertIn(b'decoded', newresponse.flags)
+        self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
+        self.assertNotIn("Content-Encoding", newresponse.headers)
+        self.assertIn(b"decoded", newresponse.flags)
+
 
 class HttpCompressionSubclassTest(TestCase):
     def test_from_crawler_missing_args(self):
@@ -466,7 +476,8 @@ class HttpCompressionSubclassTest(TestCase):
         with catch_warnings(record=True) as caught_warnings:
             self.assertIsNotNone(HttpCompressionMiddleware(stats=None, settings=None))
         messages = tuple(
-            str(warning.message) for warning in caught_warnings
+            str(warning.message)
+            for warning in caught_warnings
             if warning.category is ScrapyDeprecationWarning
         )
 
@@ -480,19 +491,19 @@ class HttpCompressionSubclassTest(TestCase):
 
     def test_init_keep_encoding_header_deprecation_warning(self):
         from scrapy.settings import Settings
-        settings = Settings({'COMPRESSION_KEEP_ENCODING_HEADER': False})
+
+        settings = Settings({"COMPRESSION_KEEP_ENCODING_HEADER": False})
 
         with catch_warnings(record=True) as caught_warnings:
-            mw = HttpCompressionMiddleware(stats={'foo': 'bar'}, settings=settings)
+            mw = HttpCompressionMiddleware(stats={"foo": "bar"}, settings=settings)
             self.assertIsNotNone(mw)
         messages = tuple(
-            str(warning.message) for warning in caught_warnings
+            str(warning.message)
+            for warning in caught_warnings
             if warning.category is ScrapyDeprecationWarning
         )
 
         self.assertEqual(
             messages,
-            (
-                "Setting COMPRESSION_KEEP_ENCODING_HEADER=False is deprecated",
-            ),
+            ("Setting COMPRESSION_KEEP_ENCODING_HEADER=False is deprecated",),
         )

From 44de6ae3bf62c82c97ac4482eb233911dddfa5aa Mon Sep 17 00:00:00 2001
From: Adrian Chaves <adrian@zyte.com>
Date: Fri, 26 Jun 2026 22:30:51 +0200
Subject: [PATCH 6/7] Update

---
 docs/topics/downloader-middleware.rst         | 24 ++++++++++++
 .../downloadermiddlewares/httpcompression.py  | 27 +++++++++----
 .../templates/project/module/settings.py.tmpl |  4 +-
 ...st_downloadermiddleware_httpcompression.py | 39 +++++++++++++++++--
 4 files changed, 79 insertions(+), 15 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 5649453b1..e8cab6a20 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -754,6 +754,30 @@ Default: ``True``
 
 Whether the Compression middleware will be enabled.
 
+.. setting:: COMPRESSION_KEEP_ENCODING_HEADER
+
+COMPRESSION_KEEP_ENCODING_HEADER
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. versionadded:: VERSION
+
+Default: ``False``
+
+Whether to keep the original ``Content-Encoding`` header of a response after
+:class:`HttpCompressionMiddleware` has decompressed its body.
+
+When ``True``, the original ``Content-Encoding`` header is kept, so that you
+can tell how the response body was encoded before Scrapy decompressed it. When
+``False``, that header is removed from decompressed responses.
+
+In both cases, a ``decoded`` flag (see :attr:`Response.flags`) is added to
+decompressed responses, and responses that already have that flag are not
+decompressed again.
+
+.. note:: ``False`` is the current default for backward compatibility, but it
+   is deprecated. Set this setting to ``True`` to keep the header; ``True``
+   will be the only supported behavior in a future version of Scrapy. New
+   projects created with :command:`startproject` set it to ``True``.
 
 HttpProxyMiddleware
 -------------------
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index b2c944482..26f1f01f8 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -89,7 +89,12 @@ class HttpCompressionMiddleware:
         )
         if not self.keep_encoding_header:
             warnings.warn(
-                "Setting COMPRESSION_KEEP_ENCODING_HEADER=False is deprecated",
+                "COMPRESSION_KEEP_ENCODING_HEADER is False (its current default "
+                "value), so HttpCompressionMiddleware removes the "
+                "Content-Encoding header from decoded responses. This is "
+                "deprecated; set COMPRESSION_KEEP_ENCODING_HEADER to True to "
+                "keep that header, which will be the only supported behavior "
+                "in a future Scrapy version, and to silence this warning.",
                 ScrapyDeprecationWarning,
                 stacklevel=2,
             )
@@ -124,7 +129,7 @@ class HttpCompressionMiddleware:
     ) -> Request | Response:
         if request.method == "HEAD":
             return response
-        if b"decoded" in response.flags:
+        if "decoded" in response.flags:
             return response
         content_encoding = response.headers.getlist("Content-Encoding")
         if not content_encoding:
@@ -132,7 +137,7 @@ class HttpCompressionMiddleware:
         max_size = request.meta.get("download_maxsize", self._max_size)
         warn_size = request.meta.get("download_warnsize", self._warn_size)
         try:
-            decoded_body, content_encoding = self._handle_encoding(
+            decoded_body, to_keep = self._handle_encoding(
                 response.body, content_encoding, max_size
             )
         except _DecompressionMaxSizeExceeded as e:
@@ -148,9 +153,11 @@ class HttpCompressionMiddleware:
                 f"({len(decoded_body)} B) is larger than the "
                 f"download warning size ({warn_size} B)."
             )
-        if content_encoding:
-            self._warn_unknown_encoding(response, content_encoding)
-        response.headers["Content-Encoding"] = content_encoding
+        if to_keep:
+            self._warn_unknown_encoding(response, to_keep)
+        # Drop the encodings that have been decoded so that the response class
+        # is guessed from the decoded body rather than from the compressed one.
+        response.headers["Content-Encoding"] = to_keep
         if self.stats:
             self.stats.inc_value(
                 "httpcompression/response_bytes",
@@ -165,9 +172,13 @@ class HttpCompressionMiddleware:
             # force recalculating the encoding until we make sure the
             # responsetypes guessing is reliable
             kwargs["encoding"] = None
-        kwargs["flags"] = [*response.flags, b"decoded"]
+        kwargs["flags"] = [*response.flags, "decoded"]
         response = response.replace(cls=respcls, **kwargs)
-        if not self.keep_encoding_header and not content_encoding:
+        if self.keep_encoding_header:
+            # Restore the original Content-Encoding header so that the spider
+            # can tell how the response body was encoded before decoding.
+            response.headers["Content-Encoding"] = content_encoding
+        elif not to_keep:
             del response.headers["Content-Encoding"]
         return response
 
diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index 34ce6cb0e..d8c60f405 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -83,8 +83,6 @@ DOWNLOAD_DELAY = 1
 #HTTPCACHE_IGNORE_HTTP_CODES = []
 #HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
 
-# Keep original Content-Encoding header
-COMPRESSION_KEEP_ENCODING_HEADER = True
-
 # Set settings whose default value is deprecated to a future-proof value
+COMPRESSION_KEEP_ENCODING_HEADER = True
 FEED_EXPORT_ENCODING = "utf-8"
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 039bd2dae..493146b84 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -796,8 +796,29 @@ class TestHttpCompression:
         newresponse = mw.process_response(request, response)
         assert newresponse is not response
         assert newresponse.body.startswith(b"<!DOCTYPE")
-        assert "Content-Encoding" in newresponse.headers
-        assert b"decoded" in newresponse.flags
+        # The original Content-Encoding header is preserved.
+        assert newresponse.headers.getlist("Content-Encoding") == [b"gzip"]
+        assert "decoded" in newresponse.flags
+
+    def test_process_response_keep_encoding_header_partial(self):
+        crawler = get_crawler(
+            Spider, settings_dict={"COMPRESSION_KEEP_ENCODING_HEADER": True}
+        )
+        mw = HttpCompressionMiddleware.from_crawler(crawler)
+        crawler.stats.open_spider()
+        response = self._getresponse("gzip")
+        response.headers["Content-Encoding"] = ["uuencode", "gzip"]
+        request = response.request
+
+        newresponse = mw.process_response(request, response)
+        assert newresponse is not response
+        # The full original Content-Encoding header is preserved, including
+        # the encoding that could not be decoded.
+        assert newresponse.headers.getlist("Content-Encoding") == [
+            b"uuencode",
+            b"gzip",
+        ]
+        assert "decoded" in newresponse.flags
 
     def test_process_response_drop_encoding_header(self):
         crawler = get_crawler(
@@ -814,7 +835,17 @@ class TestHttpCompression:
         assert newresponse is not response
         assert newresponse.body.startswith(b"<!DOCTYPE")
         assert "Content-Encoding" not in newresponse.headers
-        assert b"decoded" in newresponse.flags
+        assert "decoded" in newresponse.flags
+
+    def test_process_response_already_decoded(self):
+        response = self._getresponse("gzip")
+        response.flags.append("decoded")
+        request = response.request
+
+        newresponse = self.mw.process_response(request, response)
+        assert newresponse is response
+        assert newresponse.headers["Content-Encoding"] == b"gzip"
+        self.assertStatsEqual("httpcompression/response_count", None)
 
     def test_keep_encoding_header_deprecation_warning(self):
         crawler = get_crawler(
@@ -822,6 +853,6 @@ class TestHttpCompression:
         )
         with pytest.warns(
             ScrapyDeprecationWarning,
-            match="Setting COMPRESSION_KEEP_ENCODING_HEADER=False is deprecated",
+            match="COMPRESSION_KEEP_ENCODING_HEADER is False",
         ):
             HttpCompressionMiddleware.from_crawler(crawler)

From 1bda79a5214c201fdfb26f6a5688b8676eb35584 Mon Sep 17 00:00:00 2001
From: Adrian Chaves <adrian@zyte.com>
Date: Fri, 26 Jun 2026 22:56:58 +0200
Subject: [PATCH 7/7] Address test issues

---
 tests/AsyncCrawlerProcess/reactorless_datauri.py | 7 ++++++-
 tests/AsyncCrawlerProcess/reactorless_simple.py  | 7 ++++++-
 tests/AsyncCrawlerRunner/reactorless_datauri.py  | 7 ++++++-
 tests/AsyncCrawlerRunner/reactorless_simple.py   | 7 ++++++-
 4 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/tests/AsyncCrawlerProcess/reactorless_datauri.py b/tests/AsyncCrawlerProcess/reactorless_datauri.py
index 5270b7b83..c31554cd4 100644
--- a/tests/AsyncCrawlerProcess/reactorless_datauri.py
+++ b/tests/AsyncCrawlerProcess/reactorless_datauri.py
@@ -12,7 +12,12 @@ class DataSpider(Spider):
         return {"data": response.text}
 
 
-process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False})
+process = AsyncCrawlerProcess(
+    settings={
+        "TWISTED_REACTOR_ENABLED": False,
+        "COMPRESSION_KEEP_ENCODING_HEADER": True,
+    }
+)
 
 process.crawl(DataSpider)
 process.start()
diff --git a/tests/AsyncCrawlerProcess/reactorless_simple.py b/tests/AsyncCrawlerProcess/reactorless_simple.py
index 33b4e8cb2..99f397365 100644
--- a/tests/AsyncCrawlerProcess/reactorless_simple.py
+++ b/tests/AsyncCrawlerProcess/reactorless_simple.py
@@ -12,7 +12,12 @@ class NoRequestsSpider(scrapy.Spider):
         yield
 
 
-process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False})
+process = AsyncCrawlerProcess(
+    settings={
+        "TWISTED_REACTOR_ENABLED": False,
+        "COMPRESSION_KEEP_ENCODING_HEADER": True,
+    }
+)
 
 process.crawl(NoRequestsSpider)
 process.start()
diff --git a/tests/AsyncCrawlerRunner/reactorless_datauri.py b/tests/AsyncCrawlerRunner/reactorless_datauri.py
index d964f9c1c..19f8a05ad 100644
--- a/tests/AsyncCrawlerRunner/reactorless_datauri.py
+++ b/tests/AsyncCrawlerRunner/reactorless_datauri.py
@@ -17,7 +17,12 @@ class DataSpider(Spider):
 
 async def main() -> None:
     configure_logging()
-    runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False})
+    runner = AsyncCrawlerRunner(
+        settings={
+            "TWISTED_REACTOR_ENABLED": False,
+            "COMPRESSION_KEEP_ENCODING_HEADER": True,
+        }
+    )
     await runner.crawl(DataSpider)
 
 
diff --git a/tests/AsyncCrawlerRunner/reactorless_simple.py b/tests/AsyncCrawlerRunner/reactorless_simple.py
index e91c7c89b..99f364d23 100644
--- a/tests/AsyncCrawlerRunner/reactorless_simple.py
+++ b/tests/AsyncCrawlerRunner/reactorless_simple.py
@@ -17,7 +17,12 @@ class NoRequestsSpider(Spider):
 
 async def main() -> None:
     configure_logging()
-    runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False})
+    runner = AsyncCrawlerRunner(
+        settings={
+            "TWISTED_REACTOR_ENABLED": False,
+            "COMPRESSION_KEEP_ENCODING_HEADER": True,
+        }
+    )
     await runner.crawl(NoRequestsSpider)