From 25a38e53d5d026da56e4791fa579cb1df623d771 Mon Sep 17 00:00:00 2001 From: elpolilla Date: Mon, 29 Dec 2008 15:54:16 +0000 Subject: [PATCH] Fixed minor encoding issues in adaptors --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40570 --- scrapy/trunk/scrapy/contrib/adaptors/misc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/trunk/scrapy/contrib/adaptors/misc.py b/scrapy/trunk/scrapy/contrib/adaptors/misc.py index 2d064eda7..22e392385 100644 --- a/scrapy/trunk/scrapy/contrib/adaptors/misc.py +++ b/scrapy/trunk/scrapy/contrib/adaptors/misc.py @@ -4,7 +4,7 @@ import re from scrapy.xpath.selector import XPathSelector, XPathSelectorList from scrapy.utils.url import canonicalize_url from scrapy.utils.misc import extract_regex -from scrapy.utils.python import flatten, str_to_unicode +from scrapy.utils.python import flatten, str_to_unicode, unicode_to_str from scrapy.item.adaptors import adaptize def to_unicode(value): @@ -35,7 +35,7 @@ def clean_spaces(value): Output: list of unicodes """ _clean_spaces_re = re.compile("\s+", re.U) - return [ _clean_spaces_re.sub(' ', v.decode('utf-8')) for v in value ] + return [ _clean_spaces_re.sub(' ', str_to_unicode(v)) for v in value ] def strip(value): """ @@ -78,9 +78,9 @@ def canonicalize_urls(value): Output: list of unicodes(urls) """ if hasattr(value, '__iter__'): - return [canonicalize_url(str(url)) for url in value] + return [canonicalize_url(unicode_to_str(url)) for url in value] elif isinstance(value, basestring): - return canonicalize_url(str(value)) + return canonicalize_url(unicode_to_str(value)) return '' class Delist(object):