From fcc9e9055399d12804fc5e673d00020a7f9ac21b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 10 Jan 2023 07:30:40 +0100
Subject: [PATCH] Wrap XHTML with XmlResponse

---
 scrapy/utils/response.py     |  11 +---
 tests/test_utils_response.py | 109 +++++++++++++++++++----------------
 2 files changed, 61 insertions(+), 59 deletions(-)

diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index 36aedf1b1..35bf8baf6 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -66,15 +66,6 @@ def _is_compressed_mime_type(mime_type):
     return mime_type in _ENCODING_MIME_TYPES
 
 
-def _is_html_mime_type(mime_type):
-    if mime_type in {
-        b'application/xhtml+xml',
-        b'application/vnd.wap.xhtml+xml',
-    }:
-        return True
-    return is_html_mime_type(mime_type)
-
-
 def _is_other_text_mime_type(mime_type):
     return (
         mime_type.startswith(b'text/')
@@ -131,7 +122,7 @@ def _get_encoding_or_mime_type_from_path(path):
 def _get_response_class_from_mime_type(mime_type):
     if not mime_type:
         return Response
-    if _is_html_mime_type(mime_type):
+    if is_html_mime_type(mime_type):
         return HtmlResponse
     if is_xml_mime_type(mime_type):
         return XmlResponse
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index eb6cb6b75..cbbf201c4 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -57,6 +57,12 @@ PRE_XTRACTMIME_SCENARIOS = (
             *(
                 (mime_type, load_object(class_path))
                 for mime_type, class_path in ResponseTypes.CLASSES.items()
+                if mime_type not in (
+                    # “Note that XHTML is best parsed as XML”
+                    # https://lxml.de/parsing.html
+                    "application/xhtml+xml",
+                    "application/vnd.wap.xhtml+xml",
+                )
             ),
         )
     ),
@@ -143,23 +149,16 @@ PRE_XTRACTMIME_SCENARIOS = (
     ),
 
     # Compressed content should be of type Response until uncompressed.
-    *(
-        (
-            {
-                'headers': Headers(
-                    {
-                        'Content-Encoding': ['zip'],
-                        'Content-Type': [content_type],
-                    }
-                )
-            },
-            Response,
-        )
-        for content_type in (
-            'text/html',
-            'text/xml',
-            'text/plain',
-        )
+    (
+        {
+            'headers': Headers(
+                {
+                    'Content-Encoding': ['zip'],
+                    'Content-Type': ['text/html'],
+                }
+            )
+        },
+        Response,
     ),
 
     # We take the file extension of URL paths into account, except for HTTP
@@ -168,11 +167,17 @@ PRE_XTRACTMIME_SCENARIOS = (
     # https://mimesniff.spec.whatwg.org/#interpreting-the-resource-metadata
     *(
         (
-            {'url': f'{protocol}://example.com/a.html'},
+            {'url': f'{protocol}://example.com/a.{extension}'},
             response_class,
         )
-        for protocol, response_class in (
-            *((protocol, HtmlResponse) for protocol in ("file", "ftp")),
+        for protocol in ("file", "ftp")
+        for extension, response_class in (
+            ("gz", Response),
+            ("html", HtmlResponse),
+            ("json", TextResponse),
+            ("pdf", Response),
+            ("txt", TextResponse),
+            ("xml", XmlResponse),
         )
     ),
 
@@ -188,20 +193,14 @@ PRE_XTRACTMIME_SCENARIOS = (
                 'headers': Headers(
                     {
                         'Content-Disposition': [
-                            f'attachment; filename="a.{file_extension}"',
+                            f'attachment; filename="a.xml"',
                         ]
                     }
                 ),
             },
-            response_class,
+            XmlResponse,
         )
         for protocol in ("http", "https")
-        for file_extension, response_class in (
-            ("gz", Response),
-            ("txt", TextResponse),
-            ("html", HtmlResponse),
-            ("xml", XmlResponse),
-        )
     ),
     *(
         (
@@ -210,33 +209,15 @@ PRE_XTRACTMIME_SCENARIOS = (
                 'headers': Headers(
                     {
                         'Content-Disposition': [
-                            f'attachment; filename="a.{file_extension}"',
+                            f'attachment; filename="a.html"',
                         ],
-                        "Content-Type": {content_type},
+                        "Content-Type": "text/xml",
                     }
                 ),
             },
-            response_class,
+            XmlResponse,
         )
         for protocol in ("http", "https")
-        for file_extension, content_type, response_class in (
-            ("xml", "text/plain", TextResponse),
-            ("xml", "text/html", HtmlResponse),
-            ("html", "text/xml", XmlResponse),
-        )
-    ),
-
-    # Binary file extensions should trigger a Response.
-    *(
-        (
-            {
-                "url": f"file:///a.{extension}",
-            },
-            Response,
-        )
-        for extension in (
-            'pdf',
-        )
     ),
 
     # Without anything else, the body determines the response class.
@@ -269,6 +250,24 @@ PRE_XTRACTMIME_SCENARIOS = (
 # Scenarios that work differently with the previously-used, deprecated
 # scrapy.responsetypes.responsetypes.from_args
 POST_XTRACTMIME_SCENARIOS = (
+    # Content-Type determines the type for the HTTP protocol.
+    *(
+        (
+            {
+                "url": f"{protocol}://example.com/foo",
+                "headers": Headers({"Content-Type": content_type}),
+            },
+            response_class,
+        )
+        for protocol in ("http", "https")
+        for content_type, response_class in (
+            # “Note that XHTML is best parsed as XML”
+            # https://lxml.de/parsing.html
+            ("application/xhtml+xml", XmlResponse),
+            ("application/vnd.wap.xhtml+xml", XmlResponse),
+        )
+    ),
+
     # Content-Type triumphs body, except for the Apache bug special case.
     (
         {
@@ -346,6 +345,18 @@ POST_XTRACTMIME_SCENARIOS = (
     # responses, because “they are unreliable and easily spoofed”.
     #
     # https://mimesniff.spec.whatwg.org/#interpreting-the-resource-metadata
+    *(
+        (
+            {'url': f'{protocol}://example.com/a.{extension}'},
+            response_class,
+        )
+        for protocol in ("file", "ftp")
+        for extension, response_class in (
+            # “Note that XHTML is best parsed as XML”
+            # https://lxml.de/parsing.html
+            ("xhtml", XmlResponse),
+        )
+    ),
     *(
         (
             {'url': f'{protocol}://example.com/a.html'},