. Modified unicode conversion functions to accept the conversion encoding as a parameter

. Modified safe_url_string to make use of this change

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40610

scrapy/trunk/scrapy/tests/test_utils_python.py

@@ -7,6 +7,9 @@ class UtilsPythonTestCase(unittest.TestCase):
         # converting an utf-8 encoded string to unicode
         self.assertEqual(str_to_unicode('lel\xc3\xb1e'), u'lel\xf1e')
 
+        # converting a latin-1 encoded string to unicode
+        self.assertEqual(str_to_unicode('lel\xf1e', 'latin-1'), u'lel\xf1e')
+
         # converting a unicode to unicode should return the same object
         self.assertEqual(str_to_unicode(u'\xf1e\xf1e\xf1e'), u'\xf1e\xf1e\xf1e')
 
@@ -17,6 +20,9 @@ class UtilsPythonTestCase(unittest.TestCase):
         # converting a unicode object to an utf-8 encoded string
         self.assertEqual(unicode_to_str(u'\xa3 49'), '\xc2\xa3 49')
 
+        # converting a unicode object to a latin-1 encoded string
+        self.assertEqual(unicode_to_str(u'\xa3 49', 'latin-1'), '\xa3 49')
+
         # converting a regular string to string should return the same object
         self.assertEqual(unicode_to_str('lel\xf1e'), 'lel\xf1e')
 

scrapy/trunk/scrapy/utils/python.py

@@ -53,17 +53,17 @@ def unique(list_):
     return result
 
 
-def str_to_unicode(text):
+def str_to_unicode(text, encoding='utf-8'):
     if isinstance(text, str):
-        return text.decode('utf-8')
+        return text.decode(encoding)
     elif isinstance(text, unicode):
         return text
     else:
         raise TypeError('str_to_unicode can only receive a string object')
 
-def unicode_to_str(text):
+def unicode_to_str(text, encoding='utf-8'):
     if isinstance(text, unicode):
-        return text.encode('utf-8')
+        return text.encode(encoding)
     elif isinstance(text, str):
         return text
     else:

scrapy/trunk/scrapy/utils/url.py

@@ -54,7 +54,7 @@ def safe_url_string(url, use_encoding='utf8'):
     values in the escaping. For urls on html pages, you should use the original
     encoding of that page.
     """
-    s = url.encode(use_encoding) if isinstance(url, unicode) else url
+    s = unicode_to_str(url, use_encoding)
     return urllib.quote(s,  _safe_chars)