From e411ea94eb853eedeee111469c6442d275864a09 Mon Sep 17 00:00:00 2001 From: Mohammadtaher Abbasi Date: Sun, 28 Aug 2022 20:28:13 +0430 Subject: [PATCH 1/2] BOM should take precedence over Content-Type header when detecting the encoding closes #5601 --- scrapy/http/response/text.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 89516b9b6..bfcde878d 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -11,8 +11,13 @@ from typing import Generator, Tuple from urllib.parse import urljoin import parsel -from w3lib.encoding import (html_body_declared_encoding, html_to_unicode, - http_content_type_encoding, resolve_encoding) +from w3lib.encoding import ( + html_body_declared_encoding, + html_to_unicode, + http_content_type_encoding, + resolve_encoding, + read_bom, +) from w3lib.html import strip_html5_whitespace from scrapy.http import Request @@ -60,6 +65,7 @@ class TextResponse(Response): def _declared_encoding(self): return ( self._encoding + or self._bom_encoding() or self._headers_encoding() or self._body_declared_encoding() ) @@ -117,6 +123,10 @@ class TextResponse(Response): def _body_declared_encoding(self): return html_body_declared_encoding(self.body) + @memoizemethod_noargs + def _bom_encoding(self): + return read_bom(self.body)[0] + @property def selector(self): from scrapy.selector import Selector From a988c4b78b9ca104b74cff60c46c1842e3f25652 Mon Sep 17 00:00:00 2001 From: Mohammadtaher Abbasi Date: Mon, 29 Aug 2022 17:08:30 +0430 Subject: [PATCH 2/2] add test --- tests/test_http_response.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 2986f884f..5d67a5e74 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,3 +1,4 @@ +import codecs import unittest from unittest import mock @@ -358,6 +359,8 @@ class TextResponseTest(BaseResponseTest): headers={"Content-type": ["text/html; charset=gb2312"]}) r7 = self.response_class("http://www.example.com", body=b"\xa8D", headers={"Content-type": ["text/html; charset=gbk"]}) + r8 = self.response_class("http://www.example.com", body=codecs.BOM_UTF8 + b"\xc2\xa3", + headers={"Content-type": ["text/html; charset=cp1251"]}) self.assertEqual(r1._headers_encoding(), "utf-8") self.assertEqual(r2._headers_encoding(), None) @@ -367,7 +370,10 @@ class TextResponseTest(BaseResponseTest): self.assertEqual(r3._declared_encoding(), "cp1252") self.assertEqual(r4._headers_encoding(), None) self.assertEqual(r5._headers_encoding(), None) + self.assertEqual(r8._headers_encoding(), "cp1251") + self.assertEqual(r8._declared_encoding(), "utf-8") self._assert_response_encoding(r5, "utf-8") + self._assert_response_encoding(r8, "utf-8") assert r4._body_inferred_encoding() is not None and r4._body_inferred_encoding() != 'ascii' self._assert_response_values(r1, 'utf-8', "\xa3") self._assert_response_values(r2, 'utf-8', "\xa3")