Merge pull request #5611 from mtabbasi/issue-5601

BOM should take precedence over Content-Type header when detecting the encoding

Fixes GH-5601.

scrapy/http/response/text.py

@@ -11,8 +11,13 @@ from typing import Generator, Tuple
 from urllib.parse import urljoin
 
 import parsel
-from w3lib.encoding import (html_body_declared_encoding, html_to_unicode,
-                            http_content_type_encoding, resolve_encoding)
+from w3lib.encoding import (
+    html_body_declared_encoding,
+    html_to_unicode,
+    http_content_type_encoding,
+    resolve_encoding,
+    read_bom,
+)
 from w3lib.html import strip_html5_whitespace
 
 from scrapy.http import Request
@@ -60,6 +65,7 @@ class TextResponse(Response):
     def _declared_encoding(self):
         return (
             self._encoding
+            or self._bom_encoding()
             or self._headers_encoding()
             or self._body_declared_encoding()
         )
@@ -117,6 +123,10 @@ class TextResponse(Response):
     def _body_declared_encoding(self):
         return html_body_declared_encoding(self.body)
 
+    @memoizemethod_noargs
+    def _bom_encoding(self):
+        return read_bom(self.body)[0]
+
     @property
     def selector(self):
         from scrapy.selector import Selector

tests/test_http_response.py

@@ -1,3 +1,4 @@
+import codecs
 import unittest
 from unittest import mock
 
@@ -358,6 +359,8 @@ class TextResponseTest(BaseResponseTest):
                                  headers={"Content-type": ["text/html; charset=gb2312"]})
         r7 = self.response_class("http://www.example.com", body=b"\xa8D",
                                  headers={"Content-type": ["text/html; charset=gbk"]})
+        r8 = self.response_class("http://www.example.com", body=codecs.BOM_UTF8 + b"\xc2\xa3",
+                                 headers={"Content-type": ["text/html; charset=cp1251"]})
 
         self.assertEqual(r1._headers_encoding(), "utf-8")
         self.assertEqual(r2._headers_encoding(), None)
@@ -367,7 +370,10 @@ class TextResponseTest(BaseResponseTest):
         self.assertEqual(r3._declared_encoding(), "cp1252")
         self.assertEqual(r4._headers_encoding(), None)
         self.assertEqual(r5._headers_encoding(), None)
+        self.assertEqual(r8._headers_encoding(), "cp1251")
+        self.assertEqual(r8._declared_encoding(), "utf-8")
         self._assert_response_encoding(r5, "utf-8")
+        self._assert_response_encoding(r8, "utf-8")
         assert r4._body_inferred_encoding() is not None and r4._body_inferred_encoding() != 'ascii'
         self._assert_response_values(r1, 'utf-8', "\xa3")
         self._assert_response_values(r2, 'utf-8', "\xa3")