From 05f4a26ccad042b7842d4a70cf613dee0f46efa4 Mon Sep 17 00:00:00 2001 From: Andres Moreira Date: Mon, 6 Oct 2008 11:58:11 +0000 Subject: [PATCH] Fixed bug for unicode support.The empty string ('') in some platforms is decoding as ascii, independently of the default encoding of python, changed to u''. --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40309 --- scrapy/trunk/scrapy/utils/markup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/trunk/scrapy/utils/markup.py b/scrapy/trunk/scrapy/utils/markup.py index 756a4132f..5ec5b1719 100644 --- a/scrapy/trunk/scrapy/utils/markup.py +++ b/scrapy/trunk/scrapy/utils/markup.py @@ -63,7 +63,7 @@ def replace_tags(text, token=''): def remove_comments(text): """ Remove HTML Comments. """ - return re.sub('', '', text.decode('utf-8'), re.DOTALL) + return re.sub('', u'', text.decode('utf-8'), re.DOTALL) def remove_tags(text, which_ones=()): """ Remove HTML Tags only. @@ -77,7 +77,7 @@ def remove_tags(text, which_ones=()): else: reg_exp_remove_tags = '<.*?>' re_tags = re.compile(reg_exp_remove_tags, re.DOTALL) - return re_tags.sub('', text.decode('utf-8')) + return re_tags.sub(u'', text.decode('utf-8')) def remove_tags_with_content(text, which_ones=()): """ Remove tags and its content. @@ -87,7 +87,7 @@ def remove_tags_with_content(text, which_ones=()): """ tags = [ '<%s.*?' % (tag,tag) for tag in which_ones ] re_tags_remove = re.compile('|'.join(tags), re.DOTALL) - return re_tags_remove.sub('', text.decode('utf-8')) + return re_tags_remove.sub(u'', text.decode('utf-8')) def remove_escape_chars(text, which_ones=('\n','\t','\r')): """ Remove escape chars. Default : \\n, \\t, \\r @@ -96,5 +96,5 @@ def remove_escape_chars(text, which_ones=('\n','\t','\r')): By default removes \n, \t, \r. """ re_escape_chars = re.compile('[%s]' % ''.join(which_ones)) - return re_escape_chars.sub('', text.decode('utf-8')) + return re_escape_chars.sub(u'', text.decode('utf-8'))