From 4bb409923cc3d9a9a9de194e4c50e927fb420c4a Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Tue, 12 Jul 2011 09:52:50 -0300 Subject: [PATCH] improved encoding detection by adding support for HTML5 meta charset --- scrapy/http/response/html.py | 6 +++--- scrapy/tests/test_http_response.py | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/scrapy/http/response/html.py b/scrapy/http/response/html.py index dc812ac0e..1ef52490c 100644 --- a/scrapy/http/response/html.py +++ b/scrapy/http/response/html.py @@ -18,15 +18,15 @@ class HtmlResponse(TextResponse): _httpequiv_re = _template % ('http-equiv', 'Content-Type') _content_re = _template % ('content', r'(?P[^;]+);\s*charset=(?P[\w-]+)') - _encoding_re = _template % ('encoding', r'(?P[\w-]+)') + _content2_re = _template % ('charset', r'(?P[\w-]+)') METATAG_RE = re.compile(r'Some pagebla bla""" + r1 = self.response_class("http://www.example.com", body=body) + self._assert_response_values(r1, 'gb2312', body) class XmlResponseTest(TextResponseTest):