diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 9f4dcdc46..63d289d09 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -485,23 +485,11 @@ TextResponse objects :class:`TextResponse` objects support the following methods in addition to the standard :class:`Response` ones: - .. method:: TextResponse.headers_encoding() - - Returns a string with the encoding declared in the headers (ie. the - Content-Type HTTP header). - - .. method:: TextResponse.body_encoding() - - Returns a string with the encoding of the body, either declared or inferred - from its contents. The body encoding declaration is implemented in - :class:`TextResponse` subclasses such as: :class:`HtmlResponse` or - :class:`XmlResponse`. - .. method:: TextResponse.body_as_unicode() Returns the body of the response as unicode. This is equivalent to:: - response.body.encode(response.encoding) + response.body.decode(response.encoding) But **not** equivalent to:: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index d2a409abb..6e5949eb5 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -438,6 +438,52 @@ The class used to detect and filter duplicate requests. The default (``RequestFingerprintDupeFilter``) filters based on request fingerprint (using ``scrapy.utils.request.request_fingerprint``) and grouping per domain. +.. setting:: ENCODING_ALIASES + +ENCODING_ALIASES +---------------- + +Default: ``{}`` + +A mapping of custom encoding aliases for your project, where the keys are the +aliases (and must be lower case) and the values are the encodings they map to. + +This setting extends the :setting:`ENCODING_ALIASES_BASE` setting which +contains some default mappings. + +.. setting:: ENCODING_ALIASES_BASE + +ENCODING_ALIASES_BASE +--------------------- + +Default:: + + { + 'gb2312': 'zh-cn', + 'cp1251': 'win-1251', + 'macintosh' : 'mac-roman', + 'x-sjis': 'shift-jis', + 'iso-8859-1': 'cp1252', + 'iso8859-1': 'cp1252', + '8859': 'cp1252', + 'cp819': 'cp1252', + 'latin': 'cp1252', + 'latin1': 'cp1252', + 'latin_1': 'cp1252', + 'l1': 'cp1252', + } + +The default encoding aliases defined in Scrapy. Don't override this setting in +your project, override :setting:`ENCODING_ALIASES` instead. + +The reason why `ISO-8859-1`_ (and all its aliases) are mapped to `CP1252`_ is +due to a well known browser hack. For more information see: `Character +encodings in HTML`_. + +.. _ISO-8859-1: http://en.wikipedia.org/wiki/ISO/IEC_8859-1 +.. _CP1252: http://en.wikipedia.org/wiki/Windows-1252 +.. _Character encodings in HTML: http://www.gnu.org/software/wget/manual/wget.html + .. setting:: EXTENSIONS EXTENSIONS diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 6f8ca15e6..fffaed1dc 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -14,11 +14,6 @@ if sys.version_info < (2,5): # monkey patches to fix external library issues from scrapy.xlib import twisted_250_monkeypatches -# add some common encoding aliases not included by default in Python -from scrapy.utils.encoding import add_encoding_alias -add_encoding_alias('gb2312', 'zh-cn') -add_encoding_alias('cp1251', 'win-1251') - # optional_features is a set containing Scrapy optional features optional_features = set() diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index 5550e75ba..5e7bea3cd 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -71,6 +71,23 @@ DOWNLOADER_STATS = True DUPEFILTER_CLASS = 'scrapy.contrib.dupefilter.RequestFingerprintDupeFilter' +ENCODING_ALIASES = {} + +ENCODING_ALIASES_BASE = { + 'zh-cn': 'gb2312', + 'win-1251': 'cp1251', + 'macintosh' : 'mac-roman', + 'x-sjis': 'shift-jis', + 'iso-8859-1': 'cp1252', + 'iso8859-1': 'cp1252', + '8859': 'cp1252', + 'cp819': 'cp1252', + 'latin': 'cp1252', + 'latin1': 'cp1252', + 'latin_1': 'cp1252', + 'l1': 'cp1252', +} + EXTENSIONS = {} EXTENSIONS_BASE = { diff --git a/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/contrib/downloadermiddleware/redirect.py index 249862824..1c6297b49 100644 --- a/scrapy/contrib/downloadermiddleware/redirect.py +++ b/scrapy/contrib/downloadermiddleware/redirect.py @@ -1,4 +1,5 @@ from scrapy import log +from scrapy.http import HtmlResponse from scrapy.utils.url import urljoin_rfc from scrapy.utils.response import get_meta_refresh from scrapy.core.exceptions import IgnoreRequest @@ -24,10 +25,11 @@ class RedirectMiddleware(object): redirected = request.replace(url=redirected_url) return self._redirect(redirected, request, spider, response.status) - interval, url = get_meta_refresh(response) - if url and interval < self.max_metarefresh_delay: - redirected = self._redirect_request_using_get(request, url) - return self._redirect(redirected, request, spider, 'meta refresh') + if isinstance(response, HtmlResponse): + interval, url = get_meta_refresh(response) + if url and interval < self.max_metarefresh_delay: + redirected = self._redirect_request_using_get(request, url) + return self._redirect(redirected, request, spider, 'meta refresh') return response diff --git a/scrapy/http/response/html.py b/scrapy/http/response/html.py index f1557e6f7..dc812ac0e 100644 --- a/scrapy/http/response/html.py +++ b/scrapy/http/response/html.py @@ -23,9 +23,6 @@ class HtmlResponse(TextResponse): METATAG_RE = re.compile(r'[\w-]+)') XMLDECL_RE = re.compile(r'<\?xml\s.*?%s' % _encoding_re, re.I) - def body_encoding(self): - return self._body_declared_encoding() or super(XmlResponse, self).body_encoding() - @memoizemethod_noargs def _body_declared_encoding(self): chunk = self.body[:5000] diff --git a/scrapy/tests/test_downloadermiddleware_redirect.py b/scrapy/tests/test_downloadermiddleware_redirect.py index ba7daece8..f9a93b910 100644 --- a/scrapy/tests/test_downloadermiddleware_redirect.py +++ b/scrapy/tests/test_downloadermiddleware_redirect.py @@ -3,7 +3,7 @@ import unittest from scrapy.contrib.downloadermiddleware.redirect import RedirectMiddleware from scrapy.spider import BaseSpider from scrapy.core.exceptions import IgnoreRequest -from scrapy.http import Request, Response, Headers +from scrapy.http import Request, Response, HtmlResponse, Headers class RedirectMiddlewareTest(unittest.TestCase): @@ -58,7 +58,7 @@ class RedirectMiddlewareTest(unittest.TestCase):