RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix: non-UTF-8 content-type headers

This commit is contained in:
Laerte Pereira 2023-05-02 12:11:23 -03:00
parent b50c032ee9
commit 8acde511a9
2 changed files with 7 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scrapy/http/response/text.py
View File

@ -100,11 +100,13 @@ class TextResponse(Response):
@memoizemethod_noargs
def _headers_encoding(self):
content_type = self.headers.get(b"Content-Type", b"")
return http_content_type_encoding(to_unicode(content_type))
return http_content_type_encoding(to_unicode(content_type, encoding="latin-1"))
def _body_inferred_encoding(self):
if self._cached_benc is None:
content_type = to_unicode(self.headers.get(b"Content-Type", b""))
content_type = to_unicode(
self.headers.get(b"Content-Type", b""), encoding="latin-1"
)
benc, ubody = html_to_unicode(
content_type,
self.body,

4
scrapy/responsetypes.py
View File

@ -51,7 +51,9 @@ class ResponseTypes:
header"""
if content_encoding:
return Response
mimetype = to_unicode(content_type).split(";")[0].strip().lower()
mimetype = (
to_unicode(content_type, encoding="latin-1").split(";")[0].strip().lower()
)
return self.from_mimetype(mimetype)
def from_content_disposition(self, content_disposition):