Test ResponseTypes MIME types with the new implementation

scrapy/utils/response.py

@@ -61,6 +61,16 @@ _mime_overrides = get_data('scrapy', 'mime.types') or b''
 _MIME_TYPES.readfp(StringIO(_mime_overrides.decode()))
 
 
+
+def _is_html_mime_type(mime_type):
+    if mime_type in {
+        b'application/xhtml+xml',
+        b'application/vnd.wap.xhtml+xml',
+    }:
+        return True
+    return is_html_mime_type(mime_type)
+
+
 def _is_other_text_mime_type(mime_type):
     return (
         mime_type.startswith(b'text/')
@@ -75,7 +85,7 @@ def _is_other_text_mime_type(mime_type):
 
 
 _PRIORITIZED_MIME_TYPE_CHECKERS = (
-    is_html_mime_type,
+    _is_html_mime_type,
     is_xml_mime_type,
     _is_other_text_mime_type,
 )
@@ -123,7 +133,7 @@ def _get_mime_type_from_path(path):
 def _get_response_class_from_mime_type(mime_type):
     if not mime_type:
         return Response
-    if is_html_mime_type(mime_type):
+    if _is_html_mime_type(mime_type):
         return HtmlResponse
     if is_xml_mime_type(mime_type):
         return XmlResponse

tests/test_utils_response.py

@@ -6,6 +6,8 @@ import pytest
 
 from scrapy.http import HtmlResponse, Response, TextResponse, XmlResponse
 from scrapy.http.headers import Headers
+from scrapy.responsetypes import ResponseTypes
+from scrapy.utils.misc import load_object
 from scrapy.utils.python import to_bytes
 from scrapy.utils.response import (
     get_meta_refresh,
@@ -22,6 +24,17 @@ __doctests__ = ['scrapy.utils.response']
 # Scenarios that work the same with the previously-used, deprecated
 # scrapy.responsetypes.responsetypes.from_args
 PRE_XTRACTMIME_SCENARIOS = (
+    *(
+        (
+            {
+                'headers': Headers(
+                    {'Content-Type': [mime_type]}
+                ),
+            },
+            load_object(class_path),
+        )
+        for mime_type, class_path in ResponseTypes.CLASSES.items()
+    ),
     (
         {
             'url': 'http://www.example.com/data.csv',