Merge pull request #5917 from Laerte/master

fix: non-UTF-8 content-type headers

scrapy/http/response/text.py

@@ -100,11 +100,13 @@ class TextResponse(Response):
     @memoizemethod_noargs
     def _headers_encoding(self):
         content_type = self.headers.get(b"Content-Type", b"")
-        return http_content_type_encoding(to_unicode(content_type))
+        return http_content_type_encoding(to_unicode(content_type, encoding="latin-1"))
 
     def _body_inferred_encoding(self):
         if self._cached_benc is None:
-            content_type = to_unicode(self.headers.get(b"Content-Type", b""))
+            content_type = to_unicode(
+                self.headers.get(b"Content-Type", b""), encoding="latin-1"
+            )
             benc, ubody = html_to_unicode(
                 content_type,
                 self.body,

scrapy/responsetypes.py

@@ -51,7 +51,9 @@ class ResponseTypes:
         header"""
         if content_encoding:
             return Response
-        mimetype = to_unicode(content_type).split(";")[0].strip().lower()
+        mimetype = (
+            to_unicode(content_type, encoding="latin-1").split(";")[0].strip().lower()
+        )
         return self.from_mimetype(mimetype)
 
     def from_content_disposition(self, content_disposition):

tests/test_http_response.py

@@ -448,6 +448,13 @@ class TextResponseTest(BaseResponseTest):
             body=codecs.BOM_UTF8 + b"\xc2\xa3",
             headers={"Content-type": ["text/html; charset=cp1251"]},
         )
+        r9 = self.response_class(
+            "http://www.example.com",
+            body=b"\x80",
+            headers={
+                "Content-type": [b"application/x-download; filename=\x80dummy.txt"]
+            },
+        )
 
         self.assertEqual(r1._headers_encoding(), "utf-8")
         self.assertEqual(r2._headers_encoding(), None)
@@ -458,9 +465,12 @@ class TextResponseTest(BaseResponseTest):
         self.assertEqual(r4._headers_encoding(), None)
         self.assertEqual(r5._headers_encoding(), None)
         self.assertEqual(r8._headers_encoding(), "cp1251")
+        self.assertEqual(r9._headers_encoding(), None)
         self.assertEqual(r8._declared_encoding(), "utf-8")
+        self.assertEqual(r9._declared_encoding(), None)
         self._assert_response_encoding(r5, "utf-8")
         self._assert_response_encoding(r8, "utf-8")
+        self._assert_response_encoding(r9, "cp1252")
         assert (
             r4._body_inferred_encoding() is not None
             and r4._body_inferred_encoding() != "ascii"
@@ -470,6 +480,7 @@ class TextResponseTest(BaseResponseTest):
         self._assert_response_values(r3, "iso-8859-1", "\xa3")
         self._assert_response_values(r6, "gb18030", "\u2015")
         self._assert_response_values(r7, "gb18030", "\u2015")
+        self._assert_response_values(r9, "cp1252", "€")
 
         # TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies
         self.assertRaises(

tests/test_responsetypes.py

@@ -42,6 +42,7 @@ class ResponseTypesTest(unittest.TestCase):
             ("application/octet-stream", Response),
             ("application/x-json; encoding=UTF8;charset=UTF-8", TextResponse),
             ("application/json-amazonui-streaming;charset=UTF-8", TextResponse),
+            (b"application/x-download; filename=\x80dummy.txt", Response),
         ]
         for source, cls in mappings:
             retcls = responsetypes.from_content_type(source)