Make sure that supplied “unknown” MIME types from Content-Type are ignored

scrapy/utils/response.py

@@ -86,7 +86,15 @@ def _get_encoding_or_mime_type_from_headers(
         encodings = headers.getlist(b'Content-Encoding')
         if encodings:
             return encodings[-1], None
-    if b'Content-Type' in headers:
+    if (
+        b'Content-Type' in headers
+        and headers[b'Content-Type'].split(b";")[0].strip().lower() not in (
+            b"",
+            b"unknown/unknown",
+            b"application/unknown",
+            b"*/*",
+        )
+    ):
         return None, headers[b'Content-Type']
     if b'Content-Disposition' in headers:
         path = (

tests/test_utils_response.py

@@ -338,6 +338,51 @@ PRE_XTRACTMIME_SCENARIOS = (
             (b"a"*RESOURCE_HEADER_BUFFER_LENGTH + BINARY_BYTES[0], TextResponse),
         )
     ),
+
+    # A Content-Type whose essence is "unknown/unknown", "application/unknown",
+    # or "*/*" has the same effect as no Content-Type being defined.
+    #
+    # https://mimesniff.spec.whatwg.org/#mime-type-sniffing-algorithm
+    *(
+        (
+            {
+                'body': b'<?xml',
+                'headers': Headers(
+                    {'Content-Type': [content_type + content_type_suffix]}
+                ),
+            },
+            XmlResponse,
+        )
+        for content_type_suffix in ("", "; foo=bar")
+        for content_type in (
+            "unknown/unknown",
+            "application/unknown",
+            "*/*",
+        )
+    ),
+    *(
+        (
+            {
+                "url": f"{protocol}://example.com/a",
+                'headers': Headers(
+                    {
+                        'Content-Disposition': [
+                            'attachment; filename="a.xml"',
+                        ],
+                        "Content-Type": [content_type + content_type_suffix],
+                    }
+                ),
+            },
+            XmlResponse,
+        )
+        for protocol in ("http", "https")
+        for content_type_suffix in ("", "; foo=bar")
+        for content_type in (
+            "unknown/unknown",
+            "application/unknown",
+            "*/*",
+        )
+    ),
 )
 
 # Scenarios that work differently with the previously-used, deprecated