diff --git a/scrapy/trunk/scrapy/tests/test_utils_python.py b/scrapy/trunk/scrapy/tests/test_utils_python.py index 92e76781c..5533e1cc6 100644 --- a/scrapy/trunk/scrapy/tests/test_utils_python.py +++ b/scrapy/trunk/scrapy/tests/test_utils_python.py @@ -7,6 +7,9 @@ class UtilsPythonTestCase(unittest.TestCase): # converting an utf-8 encoded string to unicode self.assertEqual(str_to_unicode('lel\xc3\xb1e'), u'lel\xf1e') + # converting a latin-1 encoded string to unicode + self.assertEqual(str_to_unicode('lel\xf1e', 'latin-1'), u'lel\xf1e') + # converting a unicode to unicode should return the same object self.assertEqual(str_to_unicode(u'\xf1e\xf1e\xf1e'), u'\xf1e\xf1e\xf1e') @@ -17,6 +20,9 @@ class UtilsPythonTestCase(unittest.TestCase): # converting a unicode object to an utf-8 encoded string self.assertEqual(unicode_to_str(u'\xa3 49'), '\xc2\xa3 49') + # converting a unicode object to a latin-1 encoded string + self.assertEqual(unicode_to_str(u'\xa3 49', 'latin-1'), '\xa3 49') + # converting a regular string to string should return the same object self.assertEqual(unicode_to_str('lel\xf1e'), 'lel\xf1e') diff --git a/scrapy/trunk/scrapy/utils/python.py b/scrapy/trunk/scrapy/utils/python.py index 387f2991e..8272bacce 100644 --- a/scrapy/trunk/scrapy/utils/python.py +++ b/scrapy/trunk/scrapy/utils/python.py @@ -53,17 +53,17 @@ def unique(list_): return result -def str_to_unicode(text): +def str_to_unicode(text, encoding='utf-8'): if isinstance(text, str): - return text.decode('utf-8') + return text.decode(encoding) elif isinstance(text, unicode): return text else: raise TypeError('str_to_unicode can only receive a string object') -def unicode_to_str(text): +def unicode_to_str(text, encoding='utf-8'): if isinstance(text, unicode): - return text.encode('utf-8') + return text.encode(encoding) elif isinstance(text, str): return text else: diff --git a/scrapy/trunk/scrapy/utils/url.py b/scrapy/trunk/scrapy/utils/url.py index 6e9b732a9..47256c71c 100644 --- a/scrapy/trunk/scrapy/utils/url.py +++ b/scrapy/trunk/scrapy/utils/url.py @@ -54,7 +54,7 @@ def safe_url_string(url, use_encoding='utf8'): values in the escaping. For urls on html pages, you should use the original encoding of that page. """ - s = url.encode(use_encoding) if isinstance(url, unicode) else url + s = unicode_to_str(url, use_encoding) return urllib.quote(s, _safe_chars)