Added str_to_unicode and unicode_to_str functions, and used them in utils/markup

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40546

scrapy/trunk/scrapy/utils/markup.py

@@ -5,6 +5,8 @@ Functions for dealing with markup text
 import re
 import htmlentitydefs
 
+from scrapy.utils.python import str_to_unicode
+
 _ent_re = re.compile(r'&(#?)([^&;]+);')
 _tag_re = re.compile(r'<[a-zA-Z\/!].*?>', re.DOTALL)
 
@@ -45,16 +47,10 @@ def remove_entities(text, keep=(), remove_illegal=True):
             else:
                 return u'&%s;' % m.group(2)
 
-    if not isinstance(text, unicode):
-        text = text.decode('utf-8')
-
-    return _ent_re.sub(convert_entity, text)
+    return _ent_re.sub(convert_entity, str_to_unicode(text))
 
 def has_entities(text):
-    if not isinstance(text, unicode):
-        text = text.decode('utf-8')
-
-    return bool(_ent_re.search(text))
+    return bool(_ent_re.search(str_to_unicode(text)))
 
 def replace_tags(text, token=''):
     """Replace all markup tags found in the given text by the given token. By
@@ -64,18 +60,12 @@ def replace_tags(text, token=''):
 
     Always returns a unicode string.
     """
-    if not isinstance(text, unicode):
-        text = text.decode('utf-8')
-
-    return _tag_re.sub(token, text)
+    return _tag_re.sub(token, str_to_unicode(text))
 
 
 def remove_comments(text):
     """ Remove HTML Comments. """
-    if not isinstance(text, unicode):
-        text = text.decode('utf-8')
-
-    return re.sub('<!--.*?-->', u'', text, re.DOTALL)
+    return re.sub('<!--.*?-->', u'', str_to_unicode(text), re.DOTALL)
       
 def remove_tags(text, which_ones=()):
     """ Remove HTML Tags only. 
@@ -83,16 +73,13 @@ def remove_tags(text, which_ones=()):
         which_ones -- is a tuple of which tags we want to remove.
                       if is empty remove all tags.
     """
-    if not isinstance(text, unicode):
-        text = text.decode('utf-8')
-
     if len(which_ones) > 0:
         tags = [ '<%s>|<%s .*?>|</%s>' % (tag,tag,tag) for tag in which_ones ]
         reg_exp_remove_tags = '|'.join(tags)
     else:
         reg_exp_remove_tags = '<.*?>'
     re_tags = re.compile(reg_exp_remove_tags, re.DOTALL)
-    return re_tags.sub(u'', text)
+    return re_tags.sub(u'', str_to_unicode(text))
 
 def remove_tags_with_content(text, which_ones=()):
     """ Remove tags and its content.
@@ -100,12 +87,9 @@ def remove_tags_with_content(text, which_ones=()):
         which_ones -- is a tuple of which tags with its content we want to remove.
                       if is empty do nothing.
     """
-    if not isinstance(text, unicode):
-        text = text.decode('utf-8')
-
     tags = [ '<%s.*?</%s>' % (tag,tag) for tag in which_ones ]
     re_tags_remove = re.compile('|'.join(tags), re.DOTALL)
-    return re_tags_remove.sub(u'', text)
+    return re_tags_remove.sub(u'', str_to_unicode(text))
 
 def remove_escape_chars(text, which_ones=('\n','\t','\r')):
     """ Remove escape chars. Default : \\n, \\t, \\r
@@ -113,11 +97,8 @@ def remove_escape_chars(text, which_ones=('\n','\t','\r')):
         which_ones -- is a tuple of which escape chars we want to remove.
                       By default removes \n, \t, \r.
     """
-    if not isinstance(text, unicode):
-        text = text.decode('utf-8')
-
     re_escape_chars = re.compile('[%s]' % ''.join(which_ones))
-    return re_escape_chars.sub(u'', text)
+    return re_escape_chars.sub(u'', str_to_unicode(text))
 
 def unquote_markup(text, keep=(), remove_illegal=True):
     """
@@ -139,9 +120,7 @@ def unquote_markup(text, keep=(), remove_illegal=True):
         fragments.append(txt[offset:])
         return fragments
 
-    if not isinstance(text, unicode):
-        text = text.decode('utf-8')
-
+    text = str_to_unicode(text)
     ret_text = u''
     for fragment in _get_fragments(text, _cdata_re):
         if isinstance(fragment, basestring):

scrapy/trunk/scrapy/utils/python.py

@@ -53,6 +53,22 @@ def unique(list_):
     return result
 
 
+def str_to_unicode(text):
+    if isinstance(text, str):
+        return text.decode('utf-8')
+    elif isinstance(text, unicode):
+        return text
+    else:
+        raise TypeError('str_to_unicode can only receive a string object')
+
+def unicode_to_str(text):
+    if isinstance(text, unicode):
+        return text.encode('utf-8')
+    elif isinstance(text, str):
+        return text
+    else:
+        raise TypeError('unicode_to_str can only receive a unicode object')
+
 def re_rsearch(pattern, text, chunk_size=1024):
     """
     This function does a reverse search in a text using a regular expression