diff --git a/scrapy/trunk/scrapy/utils/markup.py b/scrapy/trunk/scrapy/utils/markup.py index f115ed13f..f3e585426 100644 --- a/scrapy/trunk/scrapy/utils/markup.py +++ b/scrapy/trunk/scrapy/utils/markup.py @@ -5,6 +5,8 @@ Functions for dealing with markup text import re import htmlentitydefs +from scrapy.utils.python import str_to_unicode + _ent_re = re.compile(r'&(#?)([^&;]+);') _tag_re = re.compile(r'<[a-zA-Z\/!].*?>', re.DOTALL) @@ -45,16 +47,10 @@ def remove_entities(text, keep=(), remove_illegal=True): else: return u'&%s;' % m.group(2) - if not isinstance(text, unicode): - text = text.decode('utf-8') - - return _ent_re.sub(convert_entity, text) + return _ent_re.sub(convert_entity, str_to_unicode(text)) def has_entities(text): - if not isinstance(text, unicode): - text = text.decode('utf-8') - - return bool(_ent_re.search(text)) + return bool(_ent_re.search(str_to_unicode(text))) def replace_tags(text, token=''): """Replace all markup tags found in the given text by the given token. By @@ -64,18 +60,12 @@ def replace_tags(text, token=''): Always returns a unicode string. """ - if not isinstance(text, unicode): - text = text.decode('utf-8') - - return _tag_re.sub(token, text) + return _tag_re.sub(token, str_to_unicode(text)) def remove_comments(text): """ Remove HTML Comments. """ - if not isinstance(text, unicode): - text = text.decode('utf-8') - - return re.sub('', u'', text, re.DOTALL) + return re.sub('', u'', str_to_unicode(text), re.DOTALL) def remove_tags(text, which_ones=()): """ Remove HTML Tags only. @@ -83,16 +73,13 @@ def remove_tags(text, which_ones=()): which_ones -- is a tuple of which tags we want to remove. if is empty remove all tags. """ - if not isinstance(text, unicode): - text = text.decode('utf-8') - if len(which_ones) > 0: tags = [ '<%s>|<%s .*?>|' % (tag,tag,tag) for tag in which_ones ] reg_exp_remove_tags = '|'.join(tags) else: reg_exp_remove_tags = '<.*?>' re_tags = re.compile(reg_exp_remove_tags, re.DOTALL) - return re_tags.sub(u'', text) + return re_tags.sub(u'', str_to_unicode(text)) def remove_tags_with_content(text, which_ones=()): """ Remove tags and its content. @@ -100,12 +87,9 @@ def remove_tags_with_content(text, which_ones=()): which_ones -- is a tuple of which tags with its content we want to remove. if is empty do nothing. """ - if not isinstance(text, unicode): - text = text.decode('utf-8') - tags = [ '<%s.*?' % (tag,tag) for tag in which_ones ] re_tags_remove = re.compile('|'.join(tags), re.DOTALL) - return re_tags_remove.sub(u'', text) + return re_tags_remove.sub(u'', str_to_unicode(text)) def remove_escape_chars(text, which_ones=('\n','\t','\r')): """ Remove escape chars. Default : \\n, \\t, \\r @@ -113,11 +97,8 @@ def remove_escape_chars(text, which_ones=('\n','\t','\r')): which_ones -- is a tuple of which escape chars we want to remove. By default removes \n, \t, \r. """ - if not isinstance(text, unicode): - text = text.decode('utf-8') - re_escape_chars = re.compile('[%s]' % ''.join(which_ones)) - return re_escape_chars.sub(u'', text) + return re_escape_chars.sub(u'', str_to_unicode(text)) def unquote_markup(text, keep=(), remove_illegal=True): """ @@ -139,9 +120,7 @@ def unquote_markup(text, keep=(), remove_illegal=True): fragments.append(txt[offset:]) return fragments - if not isinstance(text, unicode): - text = text.decode('utf-8') - + text = str_to_unicode(text) ret_text = u'' for fragment in _get_fragments(text, _cdata_re): if isinstance(fragment, basestring): diff --git a/scrapy/trunk/scrapy/utils/python.py b/scrapy/trunk/scrapy/utils/python.py index 0420ff9c2..387f2991e 100644 --- a/scrapy/trunk/scrapy/utils/python.py +++ b/scrapy/trunk/scrapy/utils/python.py @@ -53,6 +53,22 @@ def unique(list_): return result +def str_to_unicode(text): + if isinstance(text, str): + return text.decode('utf-8') + elif isinstance(text, unicode): + return text + else: + raise TypeError('str_to_unicode can only receive a string object') + +def unicode_to_str(text): + if isinstance(text, unicode): + return text.encode('utf-8') + elif isinstance(text, str): + return text + else: + raise TypeError('unicode_to_str can only receive a unicode object') + def re_rsearch(pattern, text, chunk_size=1024): """ This function does a reverse search in a text using a regular expression