From 50e5897429cd3a5b12c1aa0de6db8ab4dac656d4 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 30 Dec 2024 19:30:31 +0500
Subject: [PATCH] Fix merge errors and other simple mistakes.

---
 .../downloadermiddlewares/httpcompression.py  |  35 ++--
 ...st_downloadermiddleware_httpcompression.py | 161 ++++++++++--------
 2 files changed, 101 insertions(+), 95 deletions(-)

diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index dfdad9864..95d5dba66 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -1,12 +1,13 @@
 from __future__ import annotations
 
 import io
+import warnings
 import zlib
 from typing import TYPE_CHECKING, List, Optional, Union
 
 from scrapy import Request, Spider
 from scrapy.crawler import Crawler
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import Response, TextResponse
 from scrapy.responsetypes import responsetypes
 from scrapy.statscollectors import StatsCollector
@@ -37,20 +38,12 @@ class HttpCompressionMiddleware:
     """This middleware allows compressed (gzip, deflate) traffic to be
     sent/received from web sites"""
 
-    def __init__(self, stats: Optional[StatsCollector] = None):
+    def __init__(self, stats: Optional[StatsCollector] = None, settings=None):
         self.stats = stats
-        if not stats:
-            warnings.warn(
-                "The default value of COMPRESSION_KEEP_ENCODING_HEADER, "
-                "False, is deprecated, and will stop working and stop "
-                "being its default value in a future version of Scrapy. "
-                "Set COMPRESSION_KEEP_ENCODING_HEADER=True in your "
-                "settings to remove this warning.",
-                ScrapyDeprecationWarning,
-                stacklevel=2,
-            )
         if settings:
-            self.keep_encoding_header = settings.getbool('COMPRESSION_KEEP_ENCODING_HEADER')
+            self.keep_encoding_header = settings.getbool(
+                "COMPRESSION_KEEP_ENCODING_HEADER"
+            )
             if not self.keep_encoding_header:
                 warnings.warn(
                     "Setting COMPRESSION_KEEP_ENCODING_HEADER=False is deprecated",
@@ -97,17 +90,19 @@ class HttpCompressionMiddleware:
     ) -> Union[Request, Response]:
         if request.method == "HEAD":
             return response
-        if b'decoded' in response.flags:
+        if b"decoded" in response.flags:
             return response
-        content_encoding = response.headers.getlist('Content-Encoding')
+        content_encoding = response.headers.getlist("Content-Encoding")
         if not content_encoding:
             return response
 
         encoding = content_encoding[0]
         decoded_body = self._decode(response.body, encoding.lower())
         if self.stats:
-            self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider)
-            self.stats.inc_value('httpcompression/response_count', spider=spider)
+            self.stats.inc_value(
+                "httpcompression/response_bytes", len(decoded_body), spider=spider
+            )
+            self.stats.inc_value("httpcompression/response_count", spider=spider)
         respcls = responsetypes.from_args(
             headers=response.headers, url=response.url, body=decoded_body
         )
@@ -115,12 +110,12 @@ class HttpCompressionMiddleware:
         if issubclass(respcls, TextResponse):
             # force recalculating the encoding until we make sure the
             # responsetypes guessing is reliable
-            kwargs['encoding'] = None
+            kwargs["encoding"] = None
 
-        kwargs['flags'] = response.flags + [b'decoded']
+        kwargs["flags"] = response.flags + [b"decoded"]
         response = response.replace(**kwargs)
         if not self.keep_encoding_header:
-            del response.headers['Content-Encoding']
+            del response.headers["Content-Encoding"]
         return response
 
     def _decode(self, body: bytes, encoding: bytes) -> bytes:
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 0c4fff629..b52882795 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -2,6 +2,7 @@ from gzip import GzipFile
 from io import BytesIO
 from pathlib import Path
 from unittest import SkipTest, TestCase
+from warnings import catch_warnings
 
 from w3lib.encoding import resolve_encoding
 
@@ -9,7 +10,7 @@ from scrapy.downloadermiddlewares.httpcompression import (
     ACCEPTED_ENCODINGS,
     HttpCompressionMiddleware,
 )
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import HtmlResponse, Request, Response
 from scrapy.responsetypes import responsetypes
 from scrapy.settings import Settings
@@ -110,14 +111,16 @@ class HttpCompressionTest(TestCase):
         self.assertEqual(response.headers["Content-Encoding"], b"gzip")
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertNotEqual(newresponse, response)
-        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
-        self.assertIn('Content-Encoding', newresponse.headers)
-        self.assertNotIn(b'gzip', newresponse.headers['Content-Encoding'])
+        self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
+        self.assertIn("Content-Encoding", newresponse.headers)
+        self.assertNotIn(b"gzip", newresponse.headers["Content-Encoding"])
         self.assertStatsEqual("httpcompression/response_count", 1)
         self.assertStatsEqual("httpcompression/response_bytes", 74837)
 
     def test_process_response_gzip_no_stats(self):
-        mw = HttpCompressionMiddleware(settings=Settings({'COMPRESSION_KEEP_ENCODING_HEADER': True}))
+        mw = HttpCompressionMiddleware(
+            settings=Settings({"COMPRESSION_KEEP_ENCODING_HEADER": True})
+        )
         response = self._getresponse("gzip")
         request = response.request
 
@@ -125,9 +128,9 @@ class HttpCompressionTest(TestCase):
         newresponse = mw.process_response(request, response, self.spider)
         self.assertEqual(mw.stats, None)
         self.assertNotEqual(newresponse, response)
-        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
-        self.assertIn('Content-Encoding', newresponse.headers)
-        self.assertNotIn(b'gzip', newresponse.headers['Content-Encoding'])
+        self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
+        self.assertIn("Content-Encoding", newresponse.headers)
+        self.assertNotIn(b"gzip", newresponse.headers["Content-Encoding"])
 
     def test_process_response_br(self):
         try:
@@ -139,9 +142,9 @@ class HttpCompressionTest(TestCase):
         self.assertEqual(response.headers["Content-Encoding"], b"br")
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertNotEqual(newresponse, response)
-        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
-        self.assertIn('Content-Encoding', newresponse.headers)
-        self.assertEqual(b'br', newresponse.headers['Content-Encoding'])
+        self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
+        self.assertIn("Content-Encoding", newresponse.headers)
+        self.assertEqual(b"br", newresponse.headers["Content-Encoding"])
         self.assertStatsEqual("httpcompression/response_count", 1)
         self.assertStatsEqual("httpcompression/response_bytes", 74837)
 
@@ -163,9 +166,9 @@ class HttpCompressionTest(TestCase):
             else:
                 self.assertEqual(raw_content, newresponse.body)
             self.assertNotEqual(newresponse, response)
-            self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
-            self.assertIn('Content-Encoding', newresponse.headers)
-            self.assertEqual(b'zstd', newresponse.headers['Content-Encoding'])
+            self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
+            self.assertIn("Content-Encoding", newresponse.headers)
+            self.assertEqual(b"zstd", newresponse.headers["Content-Encoding"])
 
     def test_process_response_rawdeflate(self):
         response = self._getresponse("rawdeflate")
@@ -174,12 +177,12 @@ class HttpCompressionTest(TestCase):
         self.assertEqual(response.headers["Content-Encoding"], b"deflate")
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertNotEqual(newresponse, response)
-        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
-        self.assertIn('Content-Encoding', newresponse.headers)
-        self.assertIn(b'deflate', newresponse.headers['Content-Encoding'])
+        self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
+        self.assertIn("Content-Encoding", newresponse.headers)
+        self.assertIn(b"deflate", newresponse.headers["Content-Encoding"])
         self.assertStatsEqual("httpcompression/response_count", 1)
         self.assertStatsEqual("httpcompression/response_bytes", 74840)
-        self.assertIn(b'decoded', newresponse.flags)
+        self.assertIn(b"decoded", newresponse.flags)
 
     def test_process_response_zlibdelate(self):
         response = self._getresponse("zlibdeflate")
@@ -188,22 +191,22 @@ class HttpCompressionTest(TestCase):
         self.assertEqual(response.headers["Content-Encoding"], b"deflate")
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertNotEqual(newresponse, response)
-        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
-        self.assertIn('Content-Encoding', newresponse.headers)
-        self.assertIn(b'deflate', newresponse.headers['Content-Encoding'])
+        self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
+        self.assertIn("Content-Encoding", newresponse.headers)
+        self.assertIn(b"deflate", newresponse.headers["Content-Encoding"])
         self.assertStatsEqual("httpcompression/response_count", 1)
         self.assertStatsEqual("httpcompression/response_bytes", 74840)
-        self.assertIn(b'decoded', newresponse.flags)
+        self.assertIn(b"decoded", newresponse.flags)
 
     def test_process_response_plain(self):
         response = Response("http://scrapytest.org", body=b"<!DOCTYPE...")
         request = Request("http://scrapytest.org")
 
-        self.assertTrue(not response.headers.get('Content-Encoding'))
+        self.assertTrue(not response.headers.get("Content-Encoding"))
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertEqual(newresponse, response)
-        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
-        self.assertNotIn('Content-Encoding', newresponse.headers)
+        self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
+        self.assertNotIn("Content-Encoding", newresponse.headers)
         self.assertStatsEqual("httpcompression/response_count", None)
         self.assertStatsEqual("httpcompression/response_bytes", None)
 
@@ -212,10 +215,12 @@ class HttpCompressionTest(TestCase):
         response.headers["Content-Encoding"] = ["uuencode", "gzip"]
         request = response.request
         newresponse = self.mw.process_response(request, response, self.spider)
-        self.assertEqual(newresponse.headers.getlist('Content-Encoding'), [b'uuencode'])
+        self.assertEqual(newresponse.headers.getlist("Content-Encoding"), [b"uuencode"])
         self.assertNotEqual(newresponse, response)
-        self.assertEqual(newresponse.headers.getlist('Content-Encoding'), [b'uuencode', b'gzip'])
-        self.assertIn(b'decoded', newresponse.flags)
+        self.assertEqual(
+            newresponse.headers.getlist("Content-Encoding"), [b"uuencode", b"gzip"]
+        )
+        self.assertIn(b"decoded", newresponse.flags)
 
     def test_process_response_encoding_inside_body(self):
         headers = {
@@ -240,9 +245,9 @@ class HttpCompressionTest(TestCase):
         self.assertEqual(newresponse.body, plainbody)
         self.assertEqual(newresponse.encoding, resolve_encoding("gb2312"))
         self.assertStatsEqual("httpcompression/response_count", 1)
-        self.assertIn('Content-Encoding', newresponse.headers)
-        self.assertEqual(b'gzip', newresponse.headers['Content-Encoding'])
-        self.assertIn(b'decoded', newresponse.flags)
+        self.assertIn("Content-Encoding", newresponse.headers)
+        self.assertEqual(b"gzip", newresponse.headers["Content-Encoding"])
+        self.assertIn(b"decoded", newresponse.flags)
         self.assertStatsEqual("httpcompression/response_bytes", len(plainbody))
 
     def test_process_response_force_recalculate_encoding(self):
@@ -267,10 +272,10 @@ class HttpCompressionTest(TestCase):
         self.assertIsInstance(newresponse, HtmlResponse)
         self.assertEqual(newresponse.body, plainbody)
         self.assertEqual(newresponse.encoding, resolve_encoding("gb2312"))
-        self.assertIn('Content-Encoding', newresponse.headers)
-        self.assertEqual(b'gzip', newresponse.headers['Content-Encoding'])
+        self.assertIn("Content-Encoding", newresponse.headers)
+        self.assertEqual(b"gzip", newresponse.headers["Content-Encoding"])
         self.assertStatsEqual("httpcompression/response_count", 1)
-        self.assertIn(b'decoded', newresponse.flags)
+        self.assertIn(b"decoded", newresponse.flags)
         self.assertStatsEqual("httpcompression/response_bytes", len(plainbody))
 
     def test_process_response_no_content_type_header(self):
@@ -293,10 +298,10 @@ class HttpCompressionTest(TestCase):
         self.assertIsInstance(newresponse, respcls)
         self.assertEqual(newresponse.body, plainbody)
         self.assertEqual(newresponse.encoding, resolve_encoding("gb2312"))
-        self.assertIn('Content-Encoding', newresponse.headers)
-        self.assertEqual(b'identity', newresponse.headers['Content-Encoding'])
+        self.assertIn("Content-Encoding", newresponse.headers)
+        self.assertEqual(b"identity", newresponse.headers["Content-Encoding"])
         self.assertStatsEqual("httpcompression/response_count", 1)
-        self.assertIn(b'decode', newresponse.flags)
+        self.assertIn(b"decode", newresponse.flags)
         self.assertStatsEqual("httpcompression/response_bytes", len(plainbody))
 
     def test_process_response_gzipped_contenttype(self):
@@ -307,11 +312,11 @@ class HttpCompressionTest(TestCase):
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertIsNot(newresponse, response)
         self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
-        self.assertIn('Content-Encoding', newresponse.headers)
-        self.assertEqual(b'gzip', newresponse.headers['Content-Encoding'])
+        self.assertIn("Content-Encoding", newresponse.headers)
+        self.assertEqual(b"gzip", newresponse.headers["Content-Encoding"])
         self.assertStatsEqual("httpcompression/response_count", 1)
         self.assertStatsEqual("httpcompression/response_bytes", 74837)
-        self.assertIn(b'decoded', newresponse.flags)
+        self.assertIn(b"decoded", newresponse.flags)
 
     def test_process_response_gzip_app_octetstream_contenttype(self):
         response = self._getresponse("gzip")
@@ -321,11 +326,11 @@ class HttpCompressionTest(TestCase):
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertIsNot(newresponse, response)
         self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
-        self.assertIn('Content-Encoding', newresponse.headers)
-        self.assertEqual(b'gzip', newresponse.headers['Content-Encoding'])
+        self.assertIn("Content-Encoding", newresponse.headers)
+        self.assertEqual(b"gzip", newresponse.headers["Content-Encoding"])
         self.assertStatsEqual("httpcompression/response_count", 1)
         self.assertStatsEqual("httpcompression/response_bytes", 74837)
-        self.assertIn(b'decoded', newresponse.flags)
+        self.assertIn(b"decoded", newresponse.flags)
 
     def test_process_response_gzip_binary_octetstream_contenttype(self):
         response = self._getresponse("x-gzip")
@@ -335,11 +340,11 @@ class HttpCompressionTest(TestCase):
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertIsNot(newresponse, response)
         self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
-        self.assertIn('Content-Encoding', newresponse.headers)
-        self.assertEqual(b'gzip', newresponse.headers['Content-Encoding'])
+        self.assertIn("Content-Encoding", newresponse.headers)
+        self.assertEqual(b"gzip", newresponse.headers["Content-Encoding"])
         self.assertStatsEqual("httpcompression/response_count", 1)
         self.assertStatsEqual("httpcompression/response_bytes", 74837)
-        self.assertIn(b'decoded', newresponse.flags)
+        self.assertIn(b"decoded", newresponse.flags)
 
     def test_process_response_gzipped_gzip_file(self):
         """Test that a gzip Content-Encoded .gz file is gunzipped
@@ -386,9 +391,9 @@ class HttpCompressionTest(TestCase):
         self.assertEqual(gunzip(newresponse.body), plainbody)
         self.assertStatsEqual("httpcompression/response_count", 1)
         self.assertStatsEqual("httpcompression/response_bytes", 230)
-        self.assertIn('Content-Encoding', newresponse.headers)
-        self.assertEqual(b'gzip', newresponse.headers['Content-Encoding'])
-        self.assertIn(b'decoded', newresponse.flags
+        self.assertIn("Content-Encoding", newresponse.headers)
+        self.assertEqual(b"gzip", newresponse.headers["Content-Encoding"])
+        self.assertIn(b"decoded", newresponse.flags)
 
     def test_process_response_head_request_no_decode_required(self):
         response = self._getresponse("gzip")
@@ -399,43 +404,48 @@ class HttpCompressionTest(TestCase):
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertIs(newresponse, response)
         self.assertEqual(response.body, b"")
-        self.assertIn('Content-Encoding', newresponse.headers)
-        self.assertEqual(b'gzip', newresponse.headers['Content-Encoding'])
+        self.assertIn("Content-Encoding", newresponse.headers)
+        self.assertEqual(b"gzip", newresponse.headers["Content-Encoding"])
         self.assertStatsEqual("httpcompression/response_count", None)
         self.assertStatsEqual("httpcompression/response_bytes", None)
 
     def test_process_response_keeps_content_encoding_header(self):
-        settings = {'COMPRESSION_ENABLED': True,
-                    'COMPRESSION_KEEP_ENCODING_HEADER': True}
+        settings = {
+            "COMPRESSION_ENABLED": True,
+            "COMPRESSION_KEEP_ENCODING_HEADER": True,
+        }
         crawler = get_crawler(Spider, settings)
-        spider = crawler._create_spider('example.com')
+        spider = crawler._create_spider("example.com")
         mw = HttpCompressionMiddleware.from_crawler(crawler)
-        response = self._getresponse('gzip')
+        response = self._getresponse("gzip")
         request = response.request
 
-        self.assertEqual(response.headers['Content-Encoding'], b'gzip')
+        self.assertEqual(response.headers["Content-Encoding"], b"gzip")
         newresponse = mw.process_response(request, response, spider)
         self.assertIsNot(newresponse, response)
-        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
-        self.assertIn('Content-Encoding', newresponse.headers)
-        self.assertNotIn(b'gzip', newresponse.headers['Content-Encoding'])
-        self.assertIn(b'decoded', newresponse.flags)
+        self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
+        self.assertIn("Content-Encoding", newresponse.headers)
+        self.assertNotIn(b"gzip", newresponse.headers["Content-Encoding"])
+        self.assertIn(b"decoded", newresponse.flags)
 
     def test_process_response_doesnt_keep_content_encoding_header(self):
-        settings = {'COMPRESSION_ENABLED': True,
-                    'COMPRESSION_KEEP_ENCODING_HEADER': False}
+        settings = {
+            "COMPRESSION_ENABLED": True,
+            "COMPRESSION_KEEP_ENCODING_HEADER": False,
+        }
         crawler = get_crawler(Spider, settings)
-        spider = crawler._create_spider('example.com')
+        spider = crawler._create_spider("example.com")
         mw = HttpCompressionMiddleware.from_crawler(crawler)
-        response = self._getresponse('gzip')
+        response = self._getresponse("gzip")
         request = response.request
 
-        self.assertEqual(response.headers['Content-Encoding'], b'gzip')
+        self.assertEqual(response.headers["Content-Encoding"], b"gzip")
         newresponse = mw.process_response(request, response, spider)
         self.assertIsNot(newresponse, response)
-        self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
-        self.assertNotIn('Content-Encoding', newresponse.headers)
-        self.assertIn(b'decoded', newresponse.flags)
+        self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
+        self.assertNotIn("Content-Encoding", newresponse.headers)
+        self.assertIn(b"decoded", newresponse.flags)
+
 
 class HttpCompressionSubclassTest(TestCase):
     def test_from_crawler_missing_args(self):
@@ -466,7 +476,8 @@ class HttpCompressionSubclassTest(TestCase):
         with catch_warnings(record=True) as caught_warnings:
             self.assertIsNotNone(HttpCompressionMiddleware(stats=None, settings=None))
         messages = tuple(
-            str(warning.message) for warning in caught_warnings
+            str(warning.message)
+            for warning in caught_warnings
             if warning.category is ScrapyDeprecationWarning
         )
 
@@ -480,19 +491,19 @@ class HttpCompressionSubclassTest(TestCase):
 
     def test_init_keep_encoding_header_deprecation_warning(self):
         from scrapy.settings import Settings
-        settings = Settings({'COMPRESSION_KEEP_ENCODING_HEADER': False})
+
+        settings = Settings({"COMPRESSION_KEEP_ENCODING_HEADER": False})
 
         with catch_warnings(record=True) as caught_warnings:
-            mw = HttpCompressionMiddleware(stats={'foo': 'bar'}, settings=settings)
+            mw = HttpCompressionMiddleware(stats={"foo": "bar"}, settings=settings)
             self.assertIsNotNone(mw)
         messages = tuple(
-            str(warning.message) for warning in caught_warnings
+            str(warning.message)
+            for warning in caught_warnings
             if warning.category is ScrapyDeprecationWarning
         )
 
         self.assertEqual(
             messages,
-            (
-                "Setting COMPRESSION_KEEP_ENCODING_HEADER=False is deprecated",
-            ),
+            ("Setting COMPRESSION_KEEP_ENCODING_HEADER=False is deprecated",),
         )