From 1330697c3dc9cc64189b0da9506767bbbe6d70f4 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Thu, 25 Mar 2010 15:47:10 -0300 Subject: [PATCH] Some improvements to Response encoding support: * added encoding aliases, configurable through a new ENCODING_ALIASES setting * Response.encoding now returns the real encoding detected for the body * simplified TextResponse API by removing body_encoding() and headers_encoding() methods * Response.encoding now tries to infer the encoding from the body always (it was done before only on HtmlResponse and TextResponse) * removed scrapy.utils.encoding.add_encoding_alias() function * updated implementation of scrapy.utils.response function to reflect these API changes * updated documentation to reflect API changes --- docs/topics/request-response.rst | 14 +---- docs/topics/settings.rst | 46 ++++++++++++++++ scrapy/__init__.py | 5 -- scrapy/conf/default_settings.py | 17 ++++++ .../contrib/downloadermiddleware/redirect.py | 10 ++-- scrapy/http/response/html.py | 3 - scrapy/http/response/text.py | 55 ++++++++++--------- scrapy/http/response/xml.py | 3 - .../test_downloadermiddleware_redirect.py | 8 +-- scrapy/tests/test_encoding_aliases.py | 21 ------- scrapy/tests/test_http_response.py | 34 +++++++----- scrapy/tests/test_utils_encoding.py | 25 +++++++++ scrapy/tests/test_utils_response.py | 23 +++----- scrapy/utils/encoding.py | 23 +++++--- scrapy/utils/response.py | 14 ++--- 15 files changed, 179 insertions(+), 122 deletions(-) delete mode 100644 scrapy/tests/test_encoding_aliases.py create mode 100644 scrapy/tests/test_utils_encoding.py diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 9f4dcdc46..63d289d09 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -485,23 +485,11 @@ TextResponse objects :class:`TextResponse` objects support the following methods in addition to the standard :class:`Response` ones: - .. method:: TextResponse.headers_encoding() - - Returns a string with the encoding declared in the headers (ie. the - Content-Type HTTP header). - - .. method:: TextResponse.body_encoding() - - Returns a string with the encoding of the body, either declared or inferred - from its contents. The body encoding declaration is implemented in - :class:`TextResponse` subclasses such as: :class:`HtmlResponse` or - :class:`XmlResponse`. - .. method:: TextResponse.body_as_unicode() Returns the body of the response as unicode. This is equivalent to:: - response.body.encode(response.encoding) + response.body.decode(response.encoding) But **not** equivalent to:: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index d2a409abb..6e5949eb5 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -438,6 +438,52 @@ The class used to detect and filter duplicate requests. The default (``RequestFingerprintDupeFilter``) filters based on request fingerprint (using ``scrapy.utils.request.request_fingerprint``) and grouping per domain. +.. setting:: ENCODING_ALIASES + +ENCODING_ALIASES +---------------- + +Default: ``{}`` + +A mapping of custom encoding aliases for your project, where the keys are the +aliases (and must be lower case) and the values are the encodings they map to. + +This setting extends the :setting:`ENCODING_ALIASES_BASE` setting which +contains some default mappings. + +.. setting:: ENCODING_ALIASES_BASE + +ENCODING_ALIASES_BASE +--------------------- + +Default:: + + { + 'gb2312': 'zh-cn', + 'cp1251': 'win-1251', + 'macintosh' : 'mac-roman', + 'x-sjis': 'shift-jis', + 'iso-8859-1': 'cp1252', + 'iso8859-1': 'cp1252', + '8859': 'cp1252', + 'cp819': 'cp1252', + 'latin': 'cp1252', + 'latin1': 'cp1252', + 'latin_1': 'cp1252', + 'l1': 'cp1252', + } + +The default encoding aliases defined in Scrapy. Don't override this setting in +your project, override :setting:`ENCODING_ALIASES` instead. + +The reason why `ISO-8859-1`_ (and all its aliases) are mapped to `CP1252`_ is +due to a well known browser hack. For more information see: `Character +encodings in HTML`_. + +.. _ISO-8859-1: http://en.wikipedia.org/wiki/ISO/IEC_8859-1 +.. _CP1252: http://en.wikipedia.org/wiki/Windows-1252 +.. _Character encodings in HTML: http://www.gnu.org/software/wget/manual/wget.html + .. setting:: EXTENSIONS EXTENSIONS diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 6f8ca15e6..fffaed1dc 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -14,11 +14,6 @@ if sys.version_info < (2,5): # monkey patches to fix external library issues from scrapy.xlib import twisted_250_monkeypatches -# add some common encoding aliases not included by default in Python -from scrapy.utils.encoding import add_encoding_alias -add_encoding_alias('gb2312', 'zh-cn') -add_encoding_alias('cp1251', 'win-1251') - # optional_features is a set containing Scrapy optional features optional_features = set() diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index 5550e75ba..5e7bea3cd 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -71,6 +71,23 @@ DOWNLOADER_STATS = True DUPEFILTER_CLASS = 'scrapy.contrib.dupefilter.RequestFingerprintDupeFilter' +ENCODING_ALIASES = {} + +ENCODING_ALIASES_BASE = { + 'zh-cn': 'gb2312', + 'win-1251': 'cp1251', + 'macintosh' : 'mac-roman', + 'x-sjis': 'shift-jis', + 'iso-8859-1': 'cp1252', + 'iso8859-1': 'cp1252', + '8859': 'cp1252', + 'cp819': 'cp1252', + 'latin': 'cp1252', + 'latin1': 'cp1252', + 'latin_1': 'cp1252', + 'l1': 'cp1252', +} + EXTENSIONS = {} EXTENSIONS_BASE = { diff --git a/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/contrib/downloadermiddleware/redirect.py index 249862824..1c6297b49 100644 --- a/scrapy/contrib/downloadermiddleware/redirect.py +++ b/scrapy/contrib/downloadermiddleware/redirect.py @@ -1,4 +1,5 @@ from scrapy import log +from scrapy.http import HtmlResponse from scrapy.utils.url import urljoin_rfc from scrapy.utils.response import get_meta_refresh from scrapy.core.exceptions import IgnoreRequest @@ -24,10 +25,11 @@ class RedirectMiddleware(object): redirected = request.replace(url=redirected_url) return self._redirect(redirected, request, spider, response.status) - interval, url = get_meta_refresh(response) - if url and interval < self.max_metarefresh_delay: - redirected = self._redirect_request_using_get(request, url) - return self._redirect(redirected, request, spider, 'meta refresh') + if isinstance(response, HtmlResponse): + interval, url = get_meta_refresh(response) + if url and interval < self.max_metarefresh_delay: + redirected = self._redirect_request_using_get(request, url) + return self._redirect(redirected, request, spider, 'meta refresh') return response diff --git a/scrapy/http/response/html.py b/scrapy/http/response/html.py index f1557e6f7..dc812ac0e 100644 --- a/scrapy/http/response/html.py +++ b/scrapy/http/response/html.py @@ -23,9 +23,6 @@ class HtmlResponse(TextResponse): METATAG_RE = re.compile(r'[\w-]+)') XMLDECL_RE = re.compile(r'<\?xml\s.*?%s' % _encoding_re, re.I) - def body_encoding(self): - return self._body_declared_encoding() or super(XmlResponse, self).body_encoding() - @memoizemethod_noargs def _body_declared_encoding(self): chunk = self.body[:5000] diff --git a/scrapy/tests/test_downloadermiddleware_redirect.py b/scrapy/tests/test_downloadermiddleware_redirect.py index ba7daece8..f9a93b910 100644 --- a/scrapy/tests/test_downloadermiddleware_redirect.py +++ b/scrapy/tests/test_downloadermiddleware_redirect.py @@ -3,7 +3,7 @@ import unittest from scrapy.contrib.downloadermiddleware.redirect import RedirectMiddleware from scrapy.spider import BaseSpider from scrapy.core.exceptions import IgnoreRequest -from scrapy.http import Request, Response, Headers +from scrapy.http import Request, Response, HtmlResponse, Headers class RedirectMiddlewareTest(unittest.TestCase): @@ -58,7 +58,7 @@ class RedirectMiddlewareTest(unittest.TestCase): """ req = Request(url='http://example.org') - rsp = Response(url='http://example.org', body=body) + rsp = HtmlResponse(url='http://example.org', body=body) req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) @@ -70,7 +70,7 @@ class RedirectMiddlewareTest(unittest.TestCase): """ req = Request(url='http://example.org') - rsp = Response(url='http://example.org', body=body) + rsp = HtmlResponse(url='http://example.org', body=body) rsp2 = self.mw.process_response(req, rsp, self.spider) assert rsp is rsp2 @@ -81,7 +81,7 @@ class RedirectMiddlewareTest(unittest.TestCase): """ req = Request(url='http://example.org', method='POST', body='test', headers={'Content-Type': 'text/plain', 'Content-length': '4'}) - rsp = Response(url='http://example.org', body=body) + rsp = HtmlResponse(url='http://example.org', body=body) req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) diff --git a/scrapy/tests/test_encoding_aliases.py b/scrapy/tests/test_encoding_aliases.py deleted file mode 100644 index cc3a0089f..000000000 --- a/scrapy/tests/test_encoding_aliases.py +++ /dev/null @@ -1,21 +0,0 @@ -import unittest - -import scrapy # adds encoding aliases (if not added before) - -class EncodingAliasesTestCase(unittest.TestCase): - - def test_encoding_aliases(self): - """Test common encdoing aliases not included in Python""" - - uni = u'\u041c\u041e\u0421K\u0412\u0410' - str = uni.encode('windows-1251') - self.assertEqual(uni.encode('windows-1251'), uni.encode('win-1251')) - self.assertEqual(str.decode('windows-1251'), str.decode('win-1251')) - - text = u'\u8f6f\u4ef6\u540d\u79f0' - str = uni.encode('gb2312') - self.assertEqual(uni.encode('gb2312'), uni.encode('zh-cn')) - self.assertEqual(str.decode('gb2312'), str.decode('zh-cn')) - -if __name__ == "__main__": - unittest.main() diff --git a/scrapy/tests/test_http_response.py b/scrapy/tests/test_http_response.py index 5851572ac..081d4c864 100644 --- a/scrapy/tests/test_http_response.py +++ b/scrapy/tests/test_http_response.py @@ -2,6 +2,7 @@ import unittest import weakref from scrapy.http import Response, TextResponse, HtmlResponse, XmlResponse, Headers +from scrapy.utils.encoding import resolve_encoding from scrapy.conf import settings @@ -112,10 +113,13 @@ class BaseResponseTest(unittest.TestCase): body_str = body assert isinstance(response.body, str) - self.assertEqual(response.encoding, encoding) + self._assert_response_encoding(response, encoding) self.assertEqual(response.body, body_str) self.assertEqual(response.body_as_unicode(), body_unicode) + def _assert_response_encoding(self, response, encoding): + self.assertEqual(response.encoding, resolve_encoding(encoding)) + class ResponseText(BaseResponseTest): def test_no_unicode_url(self): @@ -134,14 +138,14 @@ class TextResponseTest(BaseResponseTest): assert isinstance(r2, self.response_class) self.assertEqual(r2.url, "http://www.example.com/other") - self.assertEqual(r2.encoding, "cp852") + self._assert_response_encoding(r2, "cp852") self.assertEqual(r3.url, "http://www.example.com/other") - self.assertEqual(r3.encoding, "latin1") + self.assertEqual(r3._declared_encoding(), "latin1") def test_unicode_url(self): # instantiate with unicode url without encoding (should set default encoding) resp = self.response_class(u"http://www.example.com/") - self.assertEqual(resp.encoding, settings['DEFAULT_RESPONSE_ENCODING']) + self._assert_response_encoding(resp, settings['DEFAULT_RESPONSE_ENCODING']) # make sure urls are converted to str resp = self.response_class(url=u"http://www.example.com/", encoding='utf-8') @@ -175,18 +179,18 @@ class TextResponseTest(BaseResponseTest): r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3") r3 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, body="\xa3") r4 = self.response_class("http://www.example.com", body="\xa2\xa3") - r5 = self.response_class("http://www.example.com", - headers={"Content-type": ["text/html; charset=None"]}, body="\xc2\xa3") + r5 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=None"]}, body="\xc2\xa3") - self.assertEqual(r1.headers_encoding(), "utf-8") - self.assertEqual(r2.headers_encoding(), None) - self.assertEqual(r2.encoding, 'utf-8') - self.assertEqual(r3.headers_encoding(), "iso-8859-1") - self.assertEqual(r3.encoding, 'iso-8859-1') - self.assertEqual(r4.headers_encoding(), None) - self.assertEqual(r5.headers_encoding(), None) - self.assertEqual(r5.encoding, "utf-8") - assert r4.body_encoding() is not None and r4.body_encoding() != 'ascii' + self.assertEqual(r1._headers_encoding(), "utf-8") + self.assertEqual(r2._headers_encoding(), None) + self.assertEqual(r2._declared_encoding(), 'utf-8') + self._assert_response_encoding(r2, 'utf-8') + self.assertEqual(r3._headers_encoding(), "iso-8859-1") + self.assertEqual(r3._declared_encoding(), "iso-8859-1") + self.assertEqual(r4._headers_encoding(), None) + self.assertEqual(r5._headers_encoding(), None) + self._assert_response_encoding(r5, "utf-8") + assert r4._body_inferred_encoding() is not None and r4._body_inferred_encoding() != 'ascii' self._assert_response_values(r1, 'utf-8', u"\xa3") self._assert_response_values(r2, 'utf-8', u"\xa3") self._assert_response_values(r3, 'iso-8859-1', u"\xa3") diff --git a/scrapy/tests/test_utils_encoding.py b/scrapy/tests/test_utils_encoding.py new file mode 100644 index 000000000..b6800cd75 --- /dev/null +++ b/scrapy/tests/test_utils_encoding.py @@ -0,0 +1,25 @@ +import unittest + +from scrapy.utils.encoding import encoding_exists, resolve_encoding + +class UtilsEncodingTestCase(unittest.TestCase): + + _ENCODING_ALIASES = { + 'foo': 'cp1252', + 'bar': 'none', + } + + def test_resolve_encoding(self): + self.assertEqual(resolve_encoding('latin1', self._ENCODING_ALIASES), + 'latin1') + self.assertEqual(resolve_encoding('foo', self._ENCODING_ALIASES), + 'cp1252') + + def test_encoding_exists(self): + assert encoding_exists('latin1', self._ENCODING_ALIASES) + assert encoding_exists('foo', self._ENCODING_ALIASES) + assert not encoding_exists('bar', self._ENCODING_ALIASES) + assert not encoding_exists('none', self._ENCODING_ALIASES) + +if __name__ == "__main__": + unittest.main() diff --git a/scrapy/tests/test_utils_response.py b/scrapy/tests/test_utils_response.py index de506b639..a317cd732 100644 --- a/scrapy/tests/test_utils_response.py +++ b/scrapy/tests/test_utils_response.py @@ -29,7 +29,7 @@ class ResponseUtilsTest(unittest.TestCase): self.assertTrue(isinstance(body_or_str(u'text', unicode=True), unicode)) def test_get_base_url(self): - response = Response(url='http://example.org', body="""\ + response = HtmlResponse(url='http://example.org', body="""\ \ Dummy\ blahablsdfsal&\ @@ -42,17 +42,17 @@ class ResponseUtilsTest(unittest.TestCase): Dummy blahablsdfsal& """ - response = Response(url='http://example.org', body=body) + response = TextResponse(url='http://example.org', body=body) self.assertEqual(get_meta_refresh(response), (5, 'http://example.org/newpage')) # refresh without url should return (None, None) body = """""" - response = Response(url='http://example.org', body=body) + response = TextResponse(url='http://example.org', body=body) self.assertEqual(get_meta_refresh(response), (None, None)) body = """""" - response = Response(url='http://example.org', body=body) + response = TextResponse(url='http://example.org', body=body) self.assertEqual(get_meta_refresh(response), (5, 'http://example.org/newpage')) # meta refresh in multiple lines @@ -60,17 +60,17 @@ class ResponseUtilsTest(unittest.TestCase): """ - response = Response(url='http://example.org', body=body) + response = TextResponse(url='http://example.org', body=body) self.assertEqual(get_meta_refresh(response), (1, 'http://example.org/newpage')) # entities in the redirect url body = """""" - response = Response(url='http://example.com', body=body) + response = TextResponse(url='http://example.com', body=body) self.assertEqual(get_meta_refresh(response), (3, 'http://www.example.com/other')) # relative redirects body = """""" - response = Response(url='http://example.com/page/this.html', body=body) + response = TextResponse(url='http://example.com/page/this.html', body=body) self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/page/other.html')) # non-standard encodings (utf-16) @@ -81,7 +81,7 @@ class ResponseUtilsTest(unittest.TestCase): # non-ascii chars in the url (default encoding - utf8) body = """""" - response = Response(url='http://example.com', body=body) + response = TextResponse(url='http://example.com', body=body) self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/to%C2%A3')) # non-ascii chars in the url (custom encoding - latin1) @@ -89,13 +89,8 @@ class ResponseUtilsTest(unittest.TestCase): response = TextResponse(url='http://example.com', body=body, encoding='latin1') self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/to%C2%A3')) - # wrong encodings (possibly caused by truncated chunks) - body = """""" - response = Response(url='http://example.com', body=body) - self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/thisTHAT')) - # responses without refresh tag should return None None - response = Response(url='http://example.org') + response = TextResponse(url='http://example.org') self.assertEqual(get_meta_refresh(response), (None, None)) response = TextResponse(url='http://example.org') self.assertEqual(get_meta_refresh(response), (None, None)) diff --git a/scrapy/utils/encoding.py b/scrapy/utils/encoding.py index 9eb06d942..c7d645041 100644 --- a/scrapy/utils/encoding.py +++ b/scrapy/utils/encoding.py @@ -1,11 +1,20 @@ import codecs -def add_encoding_alias(encoding, alias, overwrite=False): +from scrapy.conf import settings + +_ENCODING_ALIASES = dict(settings['ENCODING_ALIASES_BASE']) +_ENCODING_ALIASES.update(settings['ENCODING_ALIASES']) + +def encoding_exists(encoding, _aliases=_ENCODING_ALIASES): + """Returns ``True`` if encoding is valid, otherwise returns ``False``""" try: - codecs.lookup(alias) - alias_exists = True + codecs.lookup(resolve_encoding(encoding, _aliases)) except LookupError: - alias_exists = False - if overwrite or not alias_exists: - codec = codecs.lookup(encoding) - codecs.register(lambda x: codec if x == alias else None) + return False + return True + +def resolve_encoding(alias, _aliases=_ENCODING_ALIASES): + """Return the encoding the given alias maps to, or the alias as passed if + no mapping is found. + """ + return _aliases.get(alias.lower(), alias) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 56c26c2f0..15b2afbab 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -18,7 +18,8 @@ from scrapy.xlib.BeautifulSoup import BeautifulSoup from scrapy.http import Response, HtmlResponse def body_or_str(obj, unicode=True): - assert isinstance(obj, (Response, basestring)), "obj must be Response or basestring, not %s" % type(obj).__name__ + assert isinstance(obj, (Response, basestring)), \ + "obj must be Response or basestring, not %s" % type(obj).__name__ if isinstance(obj, Response): return obj.body_as_unicode() if unicode else obj.body elif isinstance(obj, str): @@ -26,16 +27,17 @@ def body_or_str(obj, unicode=True): else: return obj if unicode else obj.encode('utf-8') -BASEURL_RE = re.compile(r']*http-equiv[^>]*refresh[^>]*content\s*=\s*(?P["\'])(?P\d+)\s*;\s*url=(?P.*?)(?P=quote)', re.DOTALL | re.IGNORECASE) +META_REFRESH_RE = re.compile(ur']*http-equiv[^>]*refresh[^>]*content\s*=\s*(?P["\'])(?P\d+)\s*;\s*url=(?P.*?)(?P=quote)', \ + re.DOTALL | re.IGNORECASE) _metaref_cache = weakref.WeakKeyDictionary() def get_meta_refresh(response): """Parse the http-equiv parameter of the HTML meta element from the given @@ -46,9 +48,7 @@ def get_meta_refresh(response): If no meta redirect is found, (None, None) is returned. """ if response not in _metaref_cache: - encoding = getattr(response, 'encoding', None) or 'utf-8' - body_chunk = remove_entities(unicode(response.body[0:4096], encoding, \ - errors='ignore')) + body_chunk = remove_entities(response.body_as_unicode()[0:4096]) match = META_REFRESH_RE.search(body_chunk) if match: interval = int(match.group('int'))