diff --git a/scrapy/trunk/scrapy/tests/test_utils_markup.py b/scrapy/trunk/scrapy/tests/test_utils_markup.py index 51c4f466e..0624560d8 100644 --- a/scrapy/trunk/scrapy/tests/test_utils_markup.py +++ b/scrapy/trunk/scrapy/tests/test_utils_markup.py @@ -1,6 +1,7 @@ import unittest -from scrapy.utils.markup import remove_entities, replace_tags +from scrapy.utils.markup import remove_entities, replace_tags, remove_comments +from scrapy.utils.markup import remove_tags_with_content, remove_escape_chars, remove_tags class UtilsMarkupTest(unittest.TestCase): @@ -28,7 +29,7 @@ class UtilsMarkupTest(unittest.TestCase): - def test_remove_tags(self): + def test_replace_tags(self): # make sure it always return uncode assert isinstance(replace_tags('no entities'), unicode) @@ -41,3 +42,72 @@ class UtilsMarkupTest(unittest.TestCase): # multiline tags self.assertEqual(replace_tags('Click here'), u'Click here') + + def test_remove_comments(self): + # make sure it always return unicode + assert isinstance(remove_comments('without comments'), unicode) + assert isinstance(remove_comments(''), unicode) + + # text without comments + self.assertEqual(remove_comments(u'text without comments'), u'text without comments') + + # text with comments + self.assertEqual(remove_comments(u''), u'') + self.assertEqual(remove_comments(u'Hello'),u'Hello') + + def test_remove_tags(self): + # make sure it always return unicode + assert isinstance(remove_tags('no tags'), unicode) + assert isinstance(remove_tags('no tags', which_ones=('p',)), unicode) + assert isinstance(remove_tags('
one tag
'), unicode) + assert isinstance(remove_tags('one tag
', which_ones=('p')), unicode) + assert isinstance(remove_tags('link', which_ones=('b',)), unicode) + + # text without tags + self.assertEqual(remove_tags(u'no tags'), u'no tags') + self.assertEqual(remove_tags(u'no tags', which_ones=('p','b',)), u'no tags') + + # text with tags + self.assertEqual(remove_tags(u'one p tag
'), u'one p tag') + self.assertEqual(remove_tags(u'one p tag
', which_ones=('b',)), u'one p tag
') + + self.assertEqual(remove_tags(u'not will removedi will removed', which_ones=('i',)), + u'not will removedi will removed') + + # text with tags and attributes + self.assertEqual(remove_tags(u'texty
'), u'texty') + self.assertEqual(remove_tags(u'texty
', which_ones=('b',)), + u'texty
') + + def test_remove_tags_with_content(self): + # make sure it always return unicode + assert isinstance(remove_tags_with_content('no tags'), unicode) + assert isinstance(remove_tags_with_content('no tags', which_ones=('p',)), unicode) + assert isinstance(remove_tags_with_content('one tag
', which_ones=('p',)), unicode) + assert isinstance(remove_tags_with_content('link', which_ones=('b',)), unicode) + + # text without tags + self.assertEqual(remove_tags_with_content(u'no tags'), u'no tags') + self.assertEqual(remove_tags_with_content(u'no tags', which_ones=('p','b',)), u'no tags') + + # text with tags + self.assertEqual(remove_tags_with_content(u'one p tag
'), u'one p tag
') + self.assertEqual(remove_tags_with_content(u'one p tag
', which_ones=('p',)), u'') + + self.assertEqual(remove_tags_with_content(u'not will removedi will removed', which_ones=('i',)), + u'not will removed') + + def test_remove_escape_chars(self): + # make sure it always return unicode + assert isinstance(remove_escape_chars('no ec'), unicode) + assert isinstance(remove_escape_chars('no ec', which_ones=('\n','\t',)), unicode) + + # text without escape chars + self.assertEqual(remove_escape_chars(u'no ec'), u'no ec') + self.assertEqual(remove_escape_chars(u'no ec', which_ones=('\n',)), u'no ec') + + # text with escape chars + self.assertEqual(remove_escape_chars(u'escape\n\n'), u'escape') + self.assertEqual(remove_escape_chars(u'escape\n', which_ones=('\t',)), u'escape\n') + self.assertEqual(remove_escape_chars(u'escape\tchars\n', which_ones=('\t')), 'escapechars\n') + diff --git a/scrapy/trunk/scrapy/utils/markup.py b/scrapy/trunk/scrapy/utils/markup.py index af208dff9..756a4132f 100644 --- a/scrapy/trunk/scrapy/utils/markup.py +++ b/scrapy/trunk/scrapy/utils/markup.py @@ -63,7 +63,7 @@ def replace_tags(text, token=''): def remove_comments(text): """ Remove HTML Comments. """ - return re.sub('', '', text, re.DOTALL) + return re.sub('', '', text.decode('utf-8'), re.DOTALL) def remove_tags(text, which_ones=()): """ Remove HTML Tags only. @@ -77,7 +77,7 @@ def remove_tags(text, which_ones=()): else: reg_exp_remove_tags = '<.*?>' re_tags = re.compile(reg_exp_remove_tags, re.DOTALL) - return re_tags.sub('', text) + return re_tags.sub('', text.decode('utf-8')) def remove_tags_with_content(text, which_ones=()): """ Remove tags and its content. @@ -87,14 +87,14 @@ def remove_tags_with_content(text, which_ones=()): """ tags = [ '<%s.*?%s>' % (tag,tag) for tag in which_ones ] re_tags_remove = re.compile('|'.join(tags), re.DOTALL) - return re_tags_remove.sub('', text) + return re_tags_remove.sub('', text.decode('utf-8')) def remove_escape_chars(text, which_ones=('\n','\t','\r')): """ Remove escape chars. Default : \\n, \\t, \\r which_ones -- is a tuple of which escape chars we want to remove. - if is empty do nothing. + By default removes \n, \t, \r. """ re_escape_chars = re.compile('[%s]' % ''.join(which_ones)) - return re_escape_chars.sub('', text) + return re_escape_chars.sub('', text.decode('utf-8'))