From 2178521a79a49207f7a1be32ddc653164654d9c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 10 Jan 2023 13:41:36 +0100
Subject: [PATCH] =?UTF-8?q?Make=20sure=20that=20supplied=20=E2=80=9Cunknow?=
 =?UTF-8?q?n=E2=80=9D=20MIME=20types=20from=20Content-Type=20are=20ignored?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 scrapy/utils/response.py     | 10 +++++++-
 tests/test_utils_response.py | 45 ++++++++++++++++++++++++++++++++++++
 2 files changed, 54 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index 35bf8baf6..62accaf17 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -86,7 +86,15 @@ def _get_encoding_or_mime_type_from_headers(
         encodings = headers.getlist(b'Content-Encoding')
         if encodings:
             return encodings[-1], None
-    if b'Content-Type' in headers:
+    if (
+        b'Content-Type' in headers
+        and headers[b'Content-Type'].split(b";")[0].strip().lower() not in (
+            b"",
+            b"unknown/unknown",
+            b"application/unknown",
+            b"*/*",
+        )
+    ):
         return None, headers[b'Content-Type']
     if b'Content-Disposition' in headers:
         path = (
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index 0ef775ff4..171ada62d 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -338,6 +338,51 @@ PRE_XTRACTMIME_SCENARIOS = (
             (b"a"*RESOURCE_HEADER_BUFFER_LENGTH + BINARY_BYTES[0], TextResponse),
         )
     ),
+
+    # A Content-Type whose essence is "unknown/unknown", "application/unknown",
+    # or "*/*" has the same effect as no Content-Type being defined.
+    #
+    # https://mimesniff.spec.whatwg.org/#mime-type-sniffing-algorithm
+    *(
+        (
+            {
+                'body': b'<?xml',
+                'headers': Headers(
+                    {'Content-Type': [content_type + content_type_suffix]}
+                ),
+            },
+            XmlResponse,
+        )
+        for content_type_suffix in ("", "; foo=bar")
+        for content_type in (
+            "unknown/unknown",
+            "application/unknown",
+            "*/*",
+        )
+    ),
+    *(
+        (
+            {
+                "url": f"{protocol}://example.com/a",
+                'headers': Headers(
+                    {
+                        'Content-Disposition': [
+                            'attachment; filename="a.xml"',
+                        ],
+                        "Content-Type": [content_type + content_type_suffix],
+                    }
+                ),
+            },
+            XmlResponse,
+        )
+        for protocol in ("http", "https")
+        for content_type_suffix in ("", "; foo=bar")
+        for content_type in (
+            "unknown/unknown",
+            "application/unknown",
+            "*/*",
+        )
+    ),
 )
 
 # Scenarios that work differently with the previously-used, deprecated