RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improved encoding support by explicitly passing encoding to all str_to_unicode() and unicode_to_str() calls

This commit is contained in:
Pablo Hoffman 2010-03-24 12:14:07 -03:00
parent 4fa833c849
commit 45411926b5
7 changed files with 37 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
scrapy/__init__.py
View File

@ -18,6 +18,7 @@ from scrapy.xlib import twisted_250_monkeypatches
from scrapy.utils.encoding import add_encoding_alias
add_encoding_alias('gb2312', 'zh-cn')
add_encoding_alias('cp1251', 'win-1251')
add_encoding_alias('cp1252', 'iso8859-1', overwrite=True)
# optional_features is a set containing Scrapy optional features
optional_features = set()

1
scrapy/conf/default_settings.py
View File

@ -101,6 +101,7 @@ ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager'
ITEM_PIPELINES = []
LOG_ENABLED = True
LOG_ENCODING = 'utf-8'
LOG_FORMATTER_CRAWLED = 'scrapy.contrib.logformatter.crawled_logline'
LOG_STDOUT = False
LOG_LEVEL = 'DEBUG'

12
scrapy/contrib/linkextractors/image.py
View File

@ -3,7 +3,6 @@ This module implements the HtmlImageLinkExtractor for extracting
image links only.
"""
import urlparse
from scrapy.link import Link
from scrapy.utils.url import canonicalize_url, urljoin_rfc
@ -25,13 +24,13 @@ class HTMLImageLinkExtractor(object):
self.unique = unique
self.canonicalize = canonicalize
def extract_from_selector(self, selector, parent=None):
def extract_from_selector(self, selector, encoding, parent=None):
ret = []
def _add_link(url_sel, alt_sel=None):
url = flatten([url_sel.extract()])
alt = flatten([alt_sel.extract()]) if alt_sel else (u'', )
if url:
ret.append(Link(unicode_to_str(url[0]), alt[0]))
ret.append(Link(unicode_to_str(url[0], encoding), alt[0]))
if selector.xmlNode.type == 'element':
if selector.xmlNode.name == 'img':
@ -41,7 +40,7 @@ class HTMLImageLinkExtractor(object):
children = selector.select('child::*')
if len(children):
for child in children:
ret.extend(self.extract_from_selector(child, parent=selector))
ret.extend(self.extract_from_selector(child, encoding, parent=selector))
elif selector.xmlNode.name == 'a' and not parent:
_add_link(selector.select('@href'), selector.select('@title'))
else:
@ -52,7 +51,8 @@ class HTMLImageLinkExtractor(object):
def extract_links(self, response):
xs = HtmlXPathSelector(response)
base_url = xs.select('//base/@href').extract()
base_url = unicode_to_str(base_url[0]) if base_url else unicode_to_str(response.url)
base_url = unicode_to_str(base_url[0], response.encoding) if base_url \
else unicode_to_str(response.url, response.encoding)
links = []
for location in self.locations:
@ -64,7 +64,7 @@ class HTMLImageLinkExtractor(object):
continue
for selector in selectors:
links.extend(self.extract_from_selector(selector))
links.extend(self.extract_from_selector(selector, response.encoding))
seen, ret = set(), []
for link in links:

4
scrapy/selector/__init__.py
View File

@ -29,8 +29,8 @@ class XPathSelector(object_ref):
self.doc = Libxml2Document(response, factory=self._get_libxml2_doc)
self.xmlNode = self.doc.xmlDoc
elif text:
response = TextResponse(url='about:blank', body=unicode_to_str(text), \
encoding='utf-8')
response = TextResponse(url='about:blank', \
body=unicode_to_str(text, 'utf-8'), encoding='utf-8')
self.doc = Libxml2Document(response, factory=self._get_libxml2_doc)
self.xmlNode = self.doc.xmlDoc
self.expr = expr

35
scrapy/utils/markup.py
View File

@ -60,10 +60,10 @@ def remove_entities(text, keep=(), remove_illegal=True, encoding='utf-8'):
return _ent_re.sub(convert_entity, str_to_unicode(text, encoding))
def has_entities(text):
return bool(_ent_re.search(str_to_unicode(text)))
def has_entities(text, encoding=None):
return bool(_ent_re.search(str_to_unicode(text, encoding)))
def replace_tags(text, token=''):
def replace_tags(text, token='', encoding=None):
"""Replace all markup tags found in the given text by the given token. By
default token is a null string so it just remove all tags.
@ -71,43 +71,44 @@ def replace_tags(text, token=''):
Always returns a unicode string.
"""
return _tag_re.sub(token, str_to_unicode(text))
return _tag_re.sub(token, str_to_unicode(text, encoding))
def remove_comments(text):
def remove_comments(text, encoding=None):
""" Remove HTML Comments. """
return re.sub('<!--.*?-->', u'', str_to_unicode(text), re.DOTALL)
return re.sub('<!--.*?-->', u'', str_to_unicode(text, encoding), re.DOTALL)
def remove_tags(text, which_ones=()):
def remove_tags(text, which_ones=(), encoding=None):
""" Remove HTML Tags only.
which_ones -- is a tuple of which tags we want to remove.
if is empty remove all tags.
"""
if which_ones:
tags = ['<%s>|<%s .*?>|</%s>' % (tag,tag,tag) for tag in which_ones]
tags = ['<%s>|<%s .*?>|</%s>' % (tag, tag, tag) for tag in which_ones]
regex = '|'.join(tags)
else:
regex = '<.*?>'
retags = re.compile(regex, re.DOTALL | re.IGNORECASE)
return retags.sub(u'', str_to_unicode(text))
return retags.sub(u'', str_to_unicode(text, encoding))
def remove_tags_with_content(text, which_ones=()):
def remove_tags_with_content(text, which_ones=(), encoding=None):
""" Remove tags and its content.
which_ones -- is a tuple of which tags with its content we want to remove.
if is empty do nothing.
"""
text = str_to_unicode(text)
text = str_to_unicode(text, encoding)
if which_ones:
tags = '|'.join(['<%s.*?</%s>' % (tag,tag) for tag in which_ones])
tags = '|'.join(['<%s.*?</%s>' % (tag, tag) for tag in which_ones])
retags = re.compile(tags, re.DOTALL | re.IGNORECASE)
text = retags.sub(u'', text)
return text
def replace_escape_chars(text, which_ones=('\n','\t','\r'), replace_by=u''):
def replace_escape_chars(text, which_ones=('\n', '\t', '\r'), replace_by=u'', \
encoding=None):
""" Remove escape chars. Default : \\n, \\t, \\r
which_ones -- is a tuple of which escape chars we want to remove.
@ -117,10 +118,10 @@ def replace_escape_chars(text, which_ones=('\n','\t','\r'), replace_by=u''):
It defaults to '', so the escape chars are removed.
"""
for ec in which_ones:
text = text.replace(ec, str_to_unicode(replace_by))
return str_to_unicode(text)
text = text.replace(ec, str_to_unicode(replace_by, encoding))
return str_to_unicode(text, encoding)
def unquote_markup(text, keep=(), remove_illegal=True):
def unquote_markup(text, keep=(), remove_illegal=True, encoding=None):
"""
This function receives markup as a text (always a unicode string or a utf-8 encoded string) and does the following:
- removes entities (except the ones in 'keep') from any part of it that it's not inside a CDATA
@ -138,7 +139,7 @@ def unquote_markup(text, keep=(), remove_illegal=True):
offset = match_e
yield txt[offset:]
text = str_to_unicode(text)
text = str_to_unicode(text, encoding)
ret_text = u''
for fragment in _get_fragments(text, _cdata_re):
if isinstance(fragment, basestring):

8
scrapy/utils/python.py
View File

@ -64,13 +64,15 @@ def unique(list_, key=lambda x: x):
return result
def str_to_unicode(text, encoding='utf-8'):
def str_to_unicode(text, encoding=None):
"""Return the unicode representation of text in the given encoding. Unlike
.encode(encoding) this function can be applied directly to a unicode
object without the risk of double-decoding problems (which can happen if
you don't use the default 'ascii' encoding)
"""
if encoding is None:
encoding = 'utf-8'
if isinstance(text, str):
return text.decode(encoding)
elif isinstance(text, unicode):
@ -78,13 +80,15 @@ def str_to_unicode(text, encoding='utf-8'):
else:
raise TypeError('str_to_unicode must receive a str or unicode object, got %s' % type(text).__name__)
def unicode_to_str(text, encoding='utf-8'):
def unicode_to_str(text, encoding=None):
"""Return the str representation of text in the given encoding. Unlike
.encode(encoding) this function can be applied directly to a str
object without the risk of double-decoding problems (which can happen if
you don't use the default 'ascii' encoding)
"""
if encoding is None:
encoding = 'utf-8'
if isinstance(text, unicode):
return text.encode(encoding)
elif isinstance(text, str):

5
scrapy/utils/url.py
View File

@ -130,7 +130,8 @@ def add_or_replace_parameter(url, name, new_value, sep='&', url_is_quoted=False)
name+'='+new_value)
return next_url
def canonicalize_url(url, keep_blank_values=True, keep_fragments=False):
def canonicalize_url(url, keep_blank_values=True, keep_fragments=False, \
encoding=None):
"""Canonicalize the given url by applying the following procedures:
- sort query arguments, first by key, then by value
@ -147,7 +148,7 @@ def canonicalize_url(url, keep_blank_values=True, keep_fragments=False):
For examples see the tests in scrapy.tests.test_utils_url
"""
url = unicode_to_str(url)
url = unicode_to_str(url, encoding)
scheme, netloc, path, params, query, fragment = urlparse.urlparse(url)
keyvals = cgi.parse_qsl(query, keep_blank_values)
keyvals.sort()