RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Some improvements to Response encoding support: 

* added encoding aliases, configurable through a new ENCODING_ALIASES setting
* Response.encoding now returns the real encoding detected for the body
* simplified TextResponse API by removing body_encoding() and
  headers_encoding() methods
* Response.encoding now tries to infer the encoding from the body always (it
  was done before only on HtmlResponse and TextResponse)
* removed scrapy.utils.encoding.add_encoding_alias() function
* updated implementation of scrapy.utils.response function to reflect these API
  changes
* updated documentation to reflect API changes
This commit is contained in:
Pablo Hoffman 2010-03-25 15:47:10 -03:00
parent 173e94386b
commit 1330697c3d
15 changed files with 179 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
docs/topics/request-response.rst
View File

@ -485,23 +485,11 @@ TextResponse objects
:class:`TextResponse` objects support the following methods in addition to
the standard :class:`Response` ones:
.. method:: TextResponse.headers_encoding()
Returns a string with the encoding declared in the headers (ie. the
Content-Type HTTP header).
.. method:: TextResponse.body_encoding()
Returns a string with the encoding of the body, either declared or inferred
from its contents. The body encoding declaration is implemented in
:class:`TextResponse` subclasses such as: :class:`HtmlResponse` or
:class:`XmlResponse`.
.. method:: TextResponse.body_as_unicode()
Returns the body of the response as unicode. This is equivalent to::
response.body.encode(response.encoding)
response.body.decode(response.encoding)
But **not** equivalent to::

46
docs/topics/settings.rst
View File

@ -438,6 +438,52 @@ The class used to detect and filter duplicate requests.
The default (``RequestFingerprintDupeFilter``) filters based on request fingerprint
(using ``scrapy.utils.request.request_fingerprint``) and grouping per domain.
.. setting:: ENCODING_ALIASES
ENCODING_ALIASES
----------------
Default: ``{}``
A mapping of custom encoding aliases for your project, where the keys are the
aliases (and must be lower case) and the values are the encodings they map to.
This setting extends the :setting:`ENCODING_ALIASES_BASE` setting which
contains some default mappings.
.. setting:: ENCODING_ALIASES_BASE
ENCODING_ALIASES_BASE
---------------------
Default::
{
'gb2312': 'zh-cn',
'cp1251': 'win-1251',
'macintosh' : 'mac-roman',
'x-sjis': 'shift-jis',
'iso-8859-1': 'cp1252',
'iso8859-1': 'cp1252',
'8859': 'cp1252',
'cp819': 'cp1252',
'latin': 'cp1252',
'latin1': 'cp1252',
'latin_1': 'cp1252',
'l1': 'cp1252',
}
The default encoding aliases defined in Scrapy. Don't override this setting in
your project, override :setting:`ENCODING_ALIASES` instead.
The reason why `ISO-8859-1`_ (and all its aliases) are mapped to `CP1252`_ is
due to a well known browser hack. For more information see: `Character
encodings in HTML`_.
.. _ISO-8859-1: http://en.wikipedia.org/wiki/ISO/IEC_8859-1
.. _CP1252: http://en.wikipedia.org/wiki/Windows-1252
.. _Character encodings in HTML: http://www.gnu.org/software/wget/manual/wget.html
.. setting:: EXTENSIONS
EXTENSIONS

5
scrapy/__init__.py
View File

@ -14,11 +14,6 @@ if sys.version_info < (2,5):
# monkey patches to fix external library issues
from scrapy.xlib import twisted_250_monkeypatches
# add some common encoding aliases not included by default in Python
from scrapy.utils.encoding import add_encoding_alias
add_encoding_alias('gb2312', 'zh-cn')
add_encoding_alias('cp1251', 'win-1251')
# optional_features is a set containing Scrapy optional features
optional_features = set()

17
scrapy/conf/default_settings.py
View File

@ -71,6 +71,23 @@ DOWNLOADER_STATS = True
DUPEFILTER_CLASS = 'scrapy.contrib.dupefilter.RequestFingerprintDupeFilter'
ENCODING_ALIASES = {}
ENCODING_ALIASES_BASE = {
'zh-cn': 'gb2312',
'win-1251': 'cp1251',
'macintosh' : 'mac-roman',
'x-sjis': 'shift-jis',
'iso-8859-1': 'cp1252',
'iso8859-1': 'cp1252',
'8859': 'cp1252',
'cp819': 'cp1252',
'latin': 'cp1252',
'latin1': 'cp1252',
'latin_1': 'cp1252',
'l1': 'cp1252',
}
EXTENSIONS = {}
EXTENSIONS_BASE = {

10
scrapy/contrib/downloadermiddleware/redirect.py
View File

@ -1,4 +1,5 @@
from scrapy import log
from scrapy.http import HtmlResponse
from scrapy.utils.url import urljoin_rfc
from scrapy.utils.response import get_meta_refresh
from scrapy.core.exceptions import IgnoreRequest
@ -24,10 +25,11 @@ class RedirectMiddleware(object):
redirected = request.replace(url=redirected_url)
return self._redirect(redirected, request, spider, response.status)
interval, url = get_meta_refresh(response)
if url and interval < self.max_metarefresh_delay:
redirected = self._redirect_request_using_get(request, url)
return self._redirect(redirected, request, spider, 'meta refresh')
if isinstance(response, HtmlResponse):
interval, url = get_meta_refresh(response)
if url and interval < self.max_metarefresh_delay:
redirected = self._redirect_request_using_get(request, url)
return self._redirect(redirected, request, spider, 'meta refresh')
return response

3
scrapy/http/response/html.py
View File

@ -23,9 +23,6 @@ class HtmlResponse(TextResponse):
METATAG_RE = re.compile(r'<meta\s+%s\s+%s' % (_httpequiv_re, _content_re), re.I)
METATAG_RE2 = re.compile(r'<meta\s+%s\s+%s' % (_content_re, _httpequiv_re), re.I)
def body_encoding(self):
return self._body_declared_encoding() or super(HtmlResponse, self).body_encoding()
@memoizemethod_noargs
def _body_declared_encoding(self):
chunk = self.body[:5000]

55
scrapy/http/response/text.py
View File

@ -5,13 +5,13 @@ discovering (through HTTP headers) to base Response class.
See documentation in docs/topics/request-response.rst
"""
import codecs
import re
from scrapy.xlib.BeautifulSoup import UnicodeDammit
from scrapy.http.response import Response
from scrapy.utils.python import memoizemethod_noargs
from scrapy.utils.encoding import encoding_exists, resolve_encoding
from scrapy.conf import settings
class TextResponse(Response):
@ -19,12 +19,12 @@ class TextResponse(Response):
_DEFAULT_ENCODING = settings['DEFAULT_RESPONSE_ENCODING']
_ENCODING_RE = re.compile(r'charset=([\w-]+)', re.I)
__slots__ = ['_encoding', '_body_inferred_encoding']
__slots__ = ['_encoding', '_cached_benc']
def __init__(self, url, status=200, headers=None, body=None, meta=None, \
flags=None, encoding=None):
self._encoding = encoding
self._body_inferred_encoding = None
self._cached_benc = None
super(TextResponse, self).__init__(url, status, headers, body, meta, flags)
def _get_url(self):
@ -57,36 +57,39 @@ class TextResponse(Response):
@property
def encoding(self):
return self._encoding or self.headers_encoding() or self.body_encoding()
enc = self._declared_encoding()
if not (enc and encoding_exists(enc)):
enc = self._body_inferred_encoding() or self._DEFAULT_ENCODING
return resolve_encoding(enc)
@memoizemethod_noargs
def headers_encoding(self):
content_type = self.headers.get('Content-Type')
if content_type:
encoding = self._ENCODING_RE.search(content_type)
if encoding:
enc = encoding.group(1)
try:
codecs.lookup(enc) # check if the encoding is valid
return enc
except LookupError:
pass
def _declared_encoding(self):
return self._encoding or self._headers_encoding() \
or self._body_declared_encoding()
@memoizemethod_noargs
def body_as_unicode(self):
"""Return body as unicode"""
possible_encodings = (self._encoding, self.headers_encoding(), \
self._body_declared_encoding())
dammit = UnicodeDammit(self.body, possible_encodings)
self._body_inferred_encoding = dammit.originalEncoding
if self._body_inferred_encoding in ('ascii', None):
self._body_inferred_encoding = self._DEFAULT_ENCODING
return dammit.unicode
denc = self._declared_encoding()
dencs = [resolve_encoding(denc)] if denc else []
dammit = UnicodeDammit(self.body, dencs)
benc = dammit.originalEncoding
self._cached_benc = benc if benc != 'ascii' else None
return self.body.decode(benc) if benc == 'utf-16' else dammit.unicode
def body_encoding(self):
if self._body_inferred_encoding is None:
@memoizemethod_noargs
def _headers_encoding(self):
content_type = self.headers.get('Content-Type')
if content_type:
m = self._ENCODING_RE.search(content_type)
if m:
encoding = m.group(1)
if encoding_exists(encoding):
return encoding
def _body_inferred_encoding(self):
if self._cached_benc is None:
self.body_as_unicode()
return self._body_inferred_encoding
return self._cached_benc
def _body_declared_encoding(self):
# implemented in subclasses (XmlResponse, HtmlResponse)

3
scrapy/http/response/xml.py
View File

@ -18,9 +18,6 @@ class XmlResponse(TextResponse):
_encoding_re = _template % ('encoding', r'(?P<charset>[\w-]+)')
XMLDECL_RE = re.compile(r'<\?xml\s.*?%s' % _encoding_re, re.I)
def body_encoding(self):
return self._body_declared_encoding() or super(XmlResponse, self).body_encoding()
@memoizemethod_noargs
def _body_declared_encoding(self):
chunk = self.body[:5000]

8
scrapy/tests/test_downloadermiddleware_redirect.py
View File

@ -3,7 +3,7 @@ import unittest
from scrapy.contrib.downloadermiddleware.redirect import RedirectMiddleware
from scrapy.spider import BaseSpider
from scrapy.core.exceptions import IgnoreRequest
from scrapy.http import Request, Response, Headers
from scrapy.http import Request, Response, HtmlResponse, Headers
class RedirectMiddlewareTest(unittest.TestCase):
@ -58,7 +58,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
<head><meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>
</html>"""
req = Request(url='http://example.org')
rsp = Response(url='http://example.org', body=body)
rsp = HtmlResponse(url='http://example.org', body=body)
req2 = self.mw.process_response(req, rsp, self.spider)
assert isinstance(req2, Request)
@ -70,7 +70,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
<head><meta http-equiv="refresh" content="1000;url=http://example.org/newpage" /></head>
</html>"""
req = Request(url='http://example.org')
rsp = Response(url='http://example.org', body=body)
rsp = HtmlResponse(url='http://example.org', body=body)
rsp2 = self.mw.process_response(req, rsp, self.spider)
assert rsp is rsp2
@ -81,7 +81,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
</html>"""
req = Request(url='http://example.org', method='POST', body='test',
headers={'Content-Type': 'text/plain', 'Content-length': '4'})
rsp = Response(url='http://example.org', body=body)
rsp = HtmlResponse(url='http://example.org', body=body)
req2 = self.mw.process_response(req, rsp, self.spider)
assert isinstance(req2, Request)

21
scrapy/tests/test_encoding_aliases.py
View File

@ -1,21 +0,0 @@
import unittest
import scrapy # adds encoding aliases (if not added before)
class EncodingAliasesTestCase(unittest.TestCase):
def test_encoding_aliases(self):
"""Test common encdoing aliases not included in Python"""
uni = u'\u041c\u041e\u0421K\u0412\u0410'
str = uni.encode('windows-1251')
self.assertEqual(uni.encode('windows-1251'), uni.encode('win-1251'))
self.assertEqual(str.decode('windows-1251'), str.decode('win-1251'))
text = u'\u8f6f\u4ef6\u540d\u79f0'
str = uni.encode('gb2312')
self.assertEqual(uni.encode('gb2312'), uni.encode('zh-cn'))
self.assertEqual(str.decode('gb2312'), str.decode('zh-cn'))
if __name__ == "__main__":
unittest.main()

34
scrapy/tests/test_http_response.py
View File

@ -2,6 +2,7 @@ import unittest
import weakref
from scrapy.http import Response, TextResponse, HtmlResponse, XmlResponse, Headers
from scrapy.utils.encoding import resolve_encoding
from scrapy.conf import settings
@ -112,10 +113,13 @@ class BaseResponseTest(unittest.TestCase):
body_str = body
assert isinstance(response.body, str)
self.assertEqual(response.encoding, encoding)
self._assert_response_encoding(response, encoding)
self.assertEqual(response.body, body_str)
self.assertEqual(response.body_as_unicode(), body_unicode)
def _assert_response_encoding(self, response, encoding):
self.assertEqual(response.encoding, resolve_encoding(encoding))
class ResponseText(BaseResponseTest):
def test_no_unicode_url(self):
@ -134,14 +138,14 @@ class TextResponseTest(BaseResponseTest):
assert isinstance(r2, self.response_class)
self.assertEqual(r2.url, "http://www.example.com/other")
self.assertEqual(r2.encoding, "cp852")
self._assert_response_encoding(r2, "cp852")
self.assertEqual(r3.url, "http://www.example.com/other")
self.assertEqual(r3.encoding, "latin1")
self.assertEqual(r3._declared_encoding(), "latin1")
def test_unicode_url(self):
# instantiate with unicode url without encoding (should set default encoding)
resp = self.response_class(u"http://www.example.com/")
self.assertEqual(resp.encoding, settings['DEFAULT_RESPONSE_ENCODING'])
self._assert_response_encoding(resp, settings['DEFAULT_RESPONSE_ENCODING'])
# make sure urls are converted to str
resp = self.response_class(url=u"http://www.example.com/", encoding='utf-8')
@ -175,18 +179,18 @@ class TextResponseTest(BaseResponseTest):
r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3")
r3 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, body="\xa3")
r4 = self.response_class("http://www.example.com", body="\xa2\xa3")
r5 = self.response_class("http://www.example.com",
headers={"Content-type": ["text/html; charset=None"]}, body="\xc2\xa3")
r5 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=None"]}, body="\xc2\xa3")
self.assertEqual(r1.headers_encoding(), "utf-8")
self.assertEqual(r2.headers_encoding(), None)
self.assertEqual(r2.encoding, 'utf-8')
self.assertEqual(r3.headers_encoding(), "iso-8859-1")
self.assertEqual(r3.encoding, 'iso-8859-1')
self.assertEqual(r4.headers_encoding(), None)
self.assertEqual(r5.headers_encoding(), None)
self.assertEqual(r5.encoding, "utf-8")
assert r4.body_encoding() is not None and r4.body_encoding() != 'ascii'
self.assertEqual(r1._headers_encoding(), "utf-8")
self.assertEqual(r2._headers_encoding(), None)
self.assertEqual(r2._declared_encoding(), 'utf-8')
self._assert_response_encoding(r2, 'utf-8')
self.assertEqual(r3._headers_encoding(), "iso-8859-1")
self.assertEqual(r3._declared_encoding(), "iso-8859-1")
self.assertEqual(r4._headers_encoding(), None)
self.assertEqual(r5._headers_encoding(), None)
self._assert_response_encoding(r5, "utf-8")
assert r4._body_inferred_encoding() is not None and r4._body_inferred_encoding() != 'ascii'
self._assert_response_values(r1, 'utf-8', u"\xa3")
self._assert_response_values(r2, 'utf-8', u"\xa3")
self._assert_response_values(r3, 'iso-8859-1', u"\xa3")

25
scrapy/tests/test_utils_encoding.py Normal file
View File

@ -0,0 +1,25 @@
import unittest
from scrapy.utils.encoding import encoding_exists, resolve_encoding
class UtilsEncodingTestCase(unittest.TestCase):
_ENCODING_ALIASES = {
'foo': 'cp1252',
'bar': 'none',
}
def test_resolve_encoding(self):
self.assertEqual(resolve_encoding('latin1', self._ENCODING_ALIASES),
'latin1')
self.assertEqual(resolve_encoding('foo', self._ENCODING_ALIASES),
'cp1252')
def test_encoding_exists(self):
assert encoding_exists('latin1', self._ENCODING_ALIASES)
assert encoding_exists('foo', self._ENCODING_ALIASES)
assert not encoding_exists('bar', self._ENCODING_ALIASES)
assert not encoding_exists('none', self._ENCODING_ALIASES)
if __name__ == "__main__":
unittest.main()

23
scrapy/tests/test_utils_response.py
View File

@ -29,7 +29,7 @@ class ResponseUtilsTest(unittest.TestCase):
self.assertTrue(isinstance(body_or_str(u'text', unicode=True), unicode))
def test_get_base_url(self):
response = Response(url='http://example.org', body="""\
response = HtmlResponse(url='http://example.org', body="""\
<html>\
<head><title>Dummy</title><base href='http://example.org/something' /></head>\
<body>blahablsdfsal&amp;</body>\
@ -42,17 +42,17 @@ class ResponseUtilsTest(unittest.TestCase):
<head><title>Dummy</title><meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>
<body>blahablsdfsal&amp;</body>
</html>"""
response = Response(url='http://example.org', body=body)
response = TextResponse(url='http://example.org', body=body)
self.assertEqual(get_meta_refresh(response), (5, 'http://example.org/newpage'))
# refresh without url should return (None, None)
body = """<meta http-equiv="refresh" content="5" />"""
response = Response(url='http://example.org', body=body)
response = TextResponse(url='http://example.org', body=body)
self.assertEqual(get_meta_refresh(response), (None, None))
body = """<meta http-equiv="refresh" content="5;
url=http://example.org/newpage" /></head>"""
response = Response(url='http://example.org', body=body)
response = TextResponse(url='http://example.org', body=body)
self.assertEqual(get_meta_refresh(response), (5, 'http://example.org/newpage'))
# meta refresh in multiple lines
@ -60,17 +60,17 @@ class ResponseUtilsTest(unittest.TestCase):
<META
HTTP-EQUIV="Refresh"
CONTENT="1; URL=http://example.org/newpage">"""
response = Response(url='http://example.org', body=body)
response = TextResponse(url='http://example.org', body=body)
self.assertEqual(get_meta_refresh(response), (1, 'http://example.org/newpage'))
# entities in the redirect url
body = """<meta http-equiv="refresh" content="3; url=&#39;http://www.example.com/other&#39;">"""
response = Response(url='http://example.com', body=body)
response = TextResponse(url='http://example.com', body=body)
self.assertEqual(get_meta_refresh(response), (3, 'http://www.example.com/other'))
# relative redirects
body = """<meta http-equiv="refresh" content="3; url=other.html">"""
response = Response(url='http://example.com/page/this.html', body=body)
response = TextResponse(url='http://example.com/page/this.html', body=body)
self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/page/other.html'))
# non-standard encodings (utf-16)
@ -81,7 +81,7 @@ class ResponseUtilsTest(unittest.TestCase):
# non-ascii chars in the url (default encoding - utf8)
body = """<meta http-equiv="refresh" content="3; url=http://example.com/to\xc2\xa3">"""
response = Response(url='http://example.com', body=body)
response = TextResponse(url='http://example.com', body=body)
self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/to%C2%A3'))
# non-ascii chars in the url (custom encoding - latin1)
@ -89,13 +89,8 @@ class ResponseUtilsTest(unittest.TestCase):
response = TextResponse(url='http://example.com', body=body, encoding='latin1')
self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/to%C2%A3'))
# wrong encodings (possibly caused by truncated chunks)
body = """<meta http-equiv="refresh" content="3; url=http://example.com/this\xc2_THAT">"""
response = Response(url='http://example.com', body=body)
self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/thisTHAT'))
# responses without refresh tag should return None None
response = Response(url='http://example.org')
response = TextResponse(url='http://example.org')
self.assertEqual(get_meta_refresh(response), (None, None))
response = TextResponse(url='http://example.org')
self.assertEqual(get_meta_refresh(response), (None, None))

23
scrapy/utils/encoding.py
View File

@ -1,11 +1,20 @@
import codecs
def add_encoding_alias(encoding, alias, overwrite=False):
from scrapy.conf import settings
_ENCODING_ALIASES = dict(settings['ENCODING_ALIASES_BASE'])
_ENCODING_ALIASES.update(settings['ENCODING_ALIASES'])
def encoding_exists(encoding, _aliases=_ENCODING_ALIASES):
"""Returns ``True`` if encoding is valid, otherwise returns ``False``"""
try:
codecs.lookup(alias)
alias_exists = True
codecs.lookup(resolve_encoding(encoding, _aliases))
except LookupError:
alias_exists = False
if overwrite or not alias_exists:
codec = codecs.lookup(encoding)
codecs.register(lambda x: codec if x == alias else None)
return False
return True
def resolve_encoding(alias, _aliases=_ENCODING_ALIASES):
"""Return the encoding the given alias maps to, or the alias as passed if
no mapping is found.
"""
return _aliases.get(alias.lower(), alias)

14
scrapy/utils/response.py
View File

@ -18,7 +18,8 @@ from scrapy.xlib.BeautifulSoup import BeautifulSoup
from scrapy.http import Response, HtmlResponse
def body_or_str(obj, unicode=True):
assert isinstance(obj, (Response, basestring)), "obj must be Response or basestring, not %s" % type(obj).__name__
assert isinstance(obj, (Response, basestring)), \
"obj must be Response or basestring, not %s" % type(obj).__name__
if isinstance(obj, Response):
return obj.body_as_unicode() if unicode else obj.body
elif isinstance(obj, str):
@ -26,16 +27,17 @@ def body_or_str(obj, unicode=True):
else:
return obj if unicode else obj.encode('utf-8')
BASEURL_RE = re.compile(r'<base\s+href\s*=\s*[\"\']\s*([^\"\'\s]+)\s*[\"\']', re.I)
BASEURL_RE = re.compile(ur'<base\s+href\s*=\s*[\"\']\s*([^\"\'\s]+)\s*[\"\']', re.I)
_baseurl_cache = weakref.WeakKeyDictionary()
def get_base_url(response):
""" Return the base url of the given response used to resolve relative links. """
if response not in _baseurl_cache:
match = BASEURL_RE.search(response.body[0:4096])
match = BASEURL_RE.search(response.body_as_unicode()[0:4096])
_baseurl_cache[response] = match.group(1) if match else response.url
return _baseurl_cache[response]
META_REFRESH_RE = re.compile(ur'<meta[^>]*http-equiv[^>]*refresh[^>]*content\s*=\s*(?P<quote>["\'])(?P<int>\d+)\s*;\s*url=(?P<url>.*?)(?P=quote)', re.DOTALL | re.IGNORECASE)
META_REFRESH_RE = re.compile(ur'<meta[^>]*http-equiv[^>]*refresh[^>]*content\s*=\s*(?P<quote>["\'])(?P<int>\d+)\s*;\s*url=(?P<url>.*?)(?P=quote)', \
re.DOTALL | re.IGNORECASE)
_metaref_cache = weakref.WeakKeyDictionary()
def get_meta_refresh(response):
"""Parse the http-equiv parameter of the HTML meta element from the given
@ -46,9 +48,7 @@ def get_meta_refresh(response):
If no meta redirect is found, (None, None) is returned.
"""
if response not in _metaref_cache:
encoding = getattr(response, 'encoding', None) or 'utf-8'
body_chunk = remove_entities(unicode(response.body[0:4096], encoding, \
errors='ignore'))
body_chunk = remove_entities(response.body_as_unicode()[0:4096])
match = META_REFRESH_RE.search(body_chunk)
if match:
interval = int(match.group('int'))