diff --git a/scrapy/tests/test_utils_markup.py b/scrapy/tests/test_utils_markup.py index b363106fd..5f8a40422 100644 --- a/scrapy/tests/test_utils_markup.py +++ b/scrapy/tests/test_utils_markup.py @@ -91,6 +91,11 @@ class UtilsMarkupTest(unittest.TestCase): self.assertEqual(remove_tags(u'a
b
c'), u'abc') self.assertEqual(remove_tags(u'a
b
c', which_ones=('br',)), u'abc') + # test keep arg + self.assertEqual(remove_tags(u'

a
b
c

', keep=('br',)), u'a
b
c') + self.assertEqual(remove_tags(u'

a
b
c

', keep=('p',)), u'

abc

') + self.assertEqual(remove_tags(u'

a
b
c

', keep=('p','br','div')), u'

a
b
c

') + def test_remove_tags_with_content(self): # make sure it always return unicode assert isinstance(remove_tags_with_content('no tags'), unicode) diff --git a/scrapy/utils/markup.py b/scrapy/utils/markup.py index 7904267da..1e381074f 100644 --- a/scrapy/utils/markup.py +++ b/scrapy/utils/markup.py @@ -78,20 +78,34 @@ def remove_comments(text, encoding=None): """ Remove HTML Comments. """ return re.sub('', u'', str_to_unicode(text, encoding), re.DOTALL) -def remove_tags(text, which_ones=(), encoding=None): +def remove_tags(text, which_ones=(), keep=(), encoding=None): """ Remove HTML Tags only. - which_ones -- is a tuple of which tags we want to remove. - if is empty remove all tags. + which_ones and keep are both tuples, there are four cases: + + which_ones, keep (1 - not empty, 0 - empty) + 1, 0 - remove all tags in which_ones + 0, 1 - remove all tags except the ones in keep + 0, 0 - remove all tags + 1, 1 - not allowd """ - if which_ones: - tags = ['<%s/?>|<%s .*?>|' % (tag, tag, tag) for tag in which_ones] - regex = '|'.join(tags) - else: - regex = '<.*?>' + + assert not (which_ones and keep), 'which_ones and keep can not be given at the same time' + + def will_remove(tag): + if which_ones: + return tag in which_ones + else: + return tag not in keep + + def remove_tag(m): + tag = m.group(1) + return u'' if will_remove(tag) else m.group(0) + + regex = '/]+).*?>' retags = re.compile(regex, re.DOTALL | re.IGNORECASE) - return retags.sub(u'', str_to_unicode(text, encoding)) + return retags.sub(remove_tag, str_to_unicode(text, encoding)) def remove_tags_with_content(text, which_ones=(), encoding=None): """ Remove tags and its content.