diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 89516b9b6..bfcde878d 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -11,8 +11,13 @@ from typing import Generator, Tuple from urllib.parse import urljoin import parsel -from w3lib.encoding import (html_body_declared_encoding, html_to_unicode, - http_content_type_encoding, resolve_encoding) +from w3lib.encoding import ( + html_body_declared_encoding, + html_to_unicode, + http_content_type_encoding, + resolve_encoding, + read_bom, +) from w3lib.html import strip_html5_whitespace from scrapy.http import Request @@ -60,6 +65,7 @@ class TextResponse(Response): def _declared_encoding(self): return ( self._encoding + or self._bom_encoding() or self._headers_encoding() or self._body_declared_encoding() ) @@ -117,6 +123,10 @@ class TextResponse(Response): def _body_declared_encoding(self): return html_body_declared_encoding(self.body) + @memoizemethod_noargs + def _bom_encoding(self): + return read_bom(self.body)[0] + @property def selector(self): from scrapy.selector import Selector diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 2986f884f..5d67a5e74 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,3 +1,4 @@ +import codecs import unittest from unittest import mock @@ -358,6 +359,8 @@ class TextResponseTest(BaseResponseTest): headers={"Content-type": ["text/html; charset=gb2312"]}) r7 = self.response_class("http://www.example.com", body=b"\xa8D", headers={"Content-type": ["text/html; charset=gbk"]}) + r8 = self.response_class("http://www.example.com", body=codecs.BOM_UTF8 + b"\xc2\xa3", + headers={"Content-type": ["text/html; charset=cp1251"]}) self.assertEqual(r1._headers_encoding(), "utf-8") self.assertEqual(r2._headers_encoding(), None) @@ -367,7 +370,10 @@ class TextResponseTest(BaseResponseTest): self.assertEqual(r3._declared_encoding(), "cp1252") self.assertEqual(r4._headers_encoding(), None) self.assertEqual(r5._headers_encoding(), None) + self.assertEqual(r8._headers_encoding(), "cp1251") + self.assertEqual(r8._declared_encoding(), "utf-8") self._assert_response_encoding(r5, "utf-8") + self._assert_response_encoding(r8, "utf-8") assert r4._body_inferred_encoding() is not None and r4._body_inferred_encoding() != 'ascii' self._assert_response_values(r1, 'utf-8', "\xa3") self._assert_response_values(r2, 'utf-8', "\xa3")