From 180c091fb2a281ab1f79f9897b32687a877582e6 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Wed, 24 Feb 2010 14:01:29 -0200 Subject: [PATCH] Fixed encoding issue (reported in #135) when the encoding declared in the HTTP header is unknown. This is the patch proposed by Rolando, with an update to the Request/Response documentation. --- docs/topics/request-response.rst | 8 +++++--- scrapy/http/response/text.py | 8 +++++++- scrapy/tests/test_http_response.py | 4 ++++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 3472b1937..9f4dcdc46 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -466,12 +466,14 @@ TextResponse objects .. attribute:: TextResponse.encoding - A string with the encoding of this response. The encoding is resolved in the - following order: + A string with the encoding of this response. The encoding is resolved by + trying the following mechanisms, in order: 1. the encoding passed in the constructor `encoding` argument - 2. the encoding declared in the Content-Type HTTP header + 2. the encoding declared in the Content-Type HTTP header. If this + encoding is not valid (ie. unknown), it is ignored and the next + resolution mechanism is tried. 3. the encoding declared in the response body. The TextResponse class doesn't provide any special functionality for this. However, the diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 1c13e729c..d42a89197 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -5,6 +5,7 @@ discovering (through HTTP headers) to base Response class. See documentation in docs/topics/request-response.rst """ +import codecs import re from scrapy.xlib.BeautifulSoup import UnicodeDammit @@ -64,7 +65,12 @@ class TextResponse(Response): if content_type: encoding = self._ENCODING_RE.search(content_type) if encoding: - return encoding.group(1) + enc = encoding.group(1) + try: + codecs.lookup(enc) # check if the encoding is valid + return enc + except LookupError: + pass @memoizemethod_noargs def body_as_unicode(self): diff --git a/scrapy/tests/test_http_response.py b/scrapy/tests/test_http_response.py index 3b0a144d2..5851572ac 100644 --- a/scrapy/tests/test_http_response.py +++ b/scrapy/tests/test_http_response.py @@ -175,6 +175,8 @@ class TextResponseTest(BaseResponseTest): r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3") r3 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, body="\xa3") r4 = self.response_class("http://www.example.com", body="\xa2\xa3") + r5 = self.response_class("http://www.example.com", + headers={"Content-type": ["text/html; charset=None"]}, body="\xc2\xa3") self.assertEqual(r1.headers_encoding(), "utf-8") self.assertEqual(r2.headers_encoding(), None) @@ -182,6 +184,8 @@ class TextResponseTest(BaseResponseTest): self.assertEqual(r3.headers_encoding(), "iso-8859-1") self.assertEqual(r3.encoding, 'iso-8859-1') self.assertEqual(r4.headers_encoding(), None) + self.assertEqual(r5.headers_encoding(), None) + self.assertEqual(r5.encoding, "utf-8") assert r4.body_encoding() is not None and r4.body_encoding() != 'ascii' self._assert_response_values(r1, 'utf-8', u"\xa3") self._assert_response_values(r2, 'utf-8', u"\xa3")