From 47881c6e64f80ad8cce72c5eb0732eb5d87bee7b Mon Sep 17 00:00:00 2001 From: elpolilla Date: Fri, 2 Jan 2009 18:59:34 +0000 Subject: [PATCH] . Modified unicode conversion functions to accept the conversion encoding as a parameter . Modified safe_url_string to make use of this change --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40610 --- scrapy/trunk/scrapy/tests/test_utils_python.py | 6 ++++++ scrapy/trunk/scrapy/utils/python.py | 8 ++++---- scrapy/trunk/scrapy/utils/url.py | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/scrapy/trunk/scrapy/tests/test_utils_python.py b/scrapy/trunk/scrapy/tests/test_utils_python.py index 92e76781c..5533e1cc6 100644 --- a/scrapy/trunk/scrapy/tests/test_utils_python.py +++ b/scrapy/trunk/scrapy/tests/test_utils_python.py @@ -7,6 +7,9 @@ class UtilsPythonTestCase(unittest.TestCase): # converting an utf-8 encoded string to unicode self.assertEqual(str_to_unicode('lel\xc3\xb1e'), u'lel\xf1e') + # converting a latin-1 encoded string to unicode + self.assertEqual(str_to_unicode('lel\xf1e', 'latin-1'), u'lel\xf1e') + # converting a unicode to unicode should return the same object self.assertEqual(str_to_unicode(u'\xf1e\xf1e\xf1e'), u'\xf1e\xf1e\xf1e') @@ -17,6 +20,9 @@ class UtilsPythonTestCase(unittest.TestCase): # converting a unicode object to an utf-8 encoded string self.assertEqual(unicode_to_str(u'\xa3 49'), '\xc2\xa3 49') + # converting a unicode object to a latin-1 encoded string + self.assertEqual(unicode_to_str(u'\xa3 49', 'latin-1'), '\xa3 49') + # converting a regular string to string should return the same object self.assertEqual(unicode_to_str('lel\xf1e'), 'lel\xf1e') diff --git a/scrapy/trunk/scrapy/utils/python.py b/scrapy/trunk/scrapy/utils/python.py index 387f2991e..8272bacce 100644 --- a/scrapy/trunk/scrapy/utils/python.py +++ b/scrapy/trunk/scrapy/utils/python.py @@ -53,17 +53,17 @@ def unique(list_): return result -def str_to_unicode(text): +def str_to_unicode(text, encoding='utf-8'): if isinstance(text, str): - return text.decode('utf-8') + return text.decode(encoding) elif isinstance(text, unicode): return text else: raise TypeError('str_to_unicode can only receive a string object') -def unicode_to_str(text): +def unicode_to_str(text, encoding='utf-8'): if isinstance(text, unicode): - return text.encode('utf-8') + return text.encode(encoding) elif isinstance(text, str): return text else: diff --git a/scrapy/trunk/scrapy/utils/url.py b/scrapy/trunk/scrapy/utils/url.py index 6e9b732a9..47256c71c 100644 --- a/scrapy/trunk/scrapy/utils/url.py +++ b/scrapy/trunk/scrapy/utils/url.py @@ -54,7 +54,7 @@ def safe_url_string(url, use_encoding='utf8'): values in the escaping. For urls on html pages, you should use the original encoding of that page. """ - s = url.encode(use_encoding) if isinstance(url, unicode) else url + s = unicode_to_str(url, use_encoding) return urllib.quote(s, _safe_chars)