RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

added support for xml-declared encodings 

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40276
This commit is contained in:
samus_ 2008-09-24 18:19:19 +00:00
parent a4b709e03f
commit 79485ceb5c
2 changed files with 13 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
scrapy/trunk/scrapy/http/response.py
View File

@ -106,7 +106,16 @@ class ResponseBody(object):
This handles conversion to unicode and various character encodings.
"""
CHARSET_RE = re.compile(r'<meta\s+http-equiv\s*=\s*[\"\']?\s*Content-Type\s*[\"\']?\s+content\s*=(?P<mime>[^;]+);\s*charset=(?P<charset>[\w-]+)', re.I)
_template = r'%s\s*=\s*[\"\']?\s*%s\s*[\"\']?'
_httpequiv_re = _template % ('http-equiv', 'Content-Type')
_content_re = _template % ('content', r'(?P<mime>[^;]+);\s*charset=(?P<charset>[\w-]+)')
_encoding_re = _template % ('encoding', r'(?P<charset>[\w-]+)')
XMLDECL_RE = re.compile(r'<\?xml\s.*?%s' % _encoding_re, re.I)
METATAG_RE = re.compile(r'<meta\s+%s\s+%s' % (_httpequiv_re, _content_re), re.I)
METATAG_RE2 = re.compile(r'<meta\s+%s\s+%s' % (_content_re, _httpequiv_re), re.I)
def __init__(self, content, declared_encoding=None):
self._content = content
@ -163,7 +172,8 @@ class ResponseBody(object):
return self._expected_encoding
proposed = self.declared_encoding
if not proposed:
match = self.CHARSET_RE.search(self._content[:5000])
chunk = self._content[:5000]
match = self.XMLDECL_RE.search(chunk) or self.METATAG_RE.search(chunk) or self.METATAG_RE2.search(chunk)
if match:
proposed = match.group("charset")
self._expected_encoding = proposed

2
scrapy/trunk/scrapy/utils/misc.py
View File

@ -79,7 +79,7 @@ def extract_regex(regex, text, encoding):
* if the regex contains a named group called "extract" that will be returned
* if the regex contains multiple numbered groups, all those will be returned (flattened)
* if the refex doesn't contain any group the entire regex matching is returned
* if the regex doesn't contain any group the entire regex matching is returned
"""
if isinstance(regex, basestring):