From 8acde511a902515c56f7452f950221657953b92e Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 2 May 2023 12:11:23 -0300 Subject: [PATCH] fix: non-UTF-8 content-type headers --- scrapy/http/response/text.py | 6 ++++-- scrapy/responsetypes.py | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 73bb811de..d580a7876 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -100,11 +100,13 @@ class TextResponse(Response): @memoizemethod_noargs def _headers_encoding(self): content_type = self.headers.get(b"Content-Type", b"") - return http_content_type_encoding(to_unicode(content_type)) + return http_content_type_encoding(to_unicode(content_type, encoding="latin-1")) def _body_inferred_encoding(self): if self._cached_benc is None: - content_type = to_unicode(self.headers.get(b"Content-Type", b"")) + content_type = to_unicode( + self.headers.get(b"Content-Type", b""), encoding="latin-1" + ) benc, ubody = html_to_unicode( content_type, self.body, diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index f01e9096c..58884f21a 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -51,7 +51,9 @@ class ResponseTypes: header""" if content_encoding: return Response - mimetype = to_unicode(content_type).split(";")[0].strip().lower() + mimetype = ( + to_unicode(content_type, encoding="latin-1").split(";")[0].strip().lower() + ) return self.from_mimetype(mimetype) def from_content_disposition(self, content_disposition):