Fix remove_tags like functions can't remove empty tag such as <br/>

Signed-off-by: Ping Yin <pkufranky@gmail.com>

scrapy/tests/test_utils_markup.py

@@ -87,6 +87,10 @@ class UtilsMarkupTest(unittest.TestCase):
         self.assertEqual(remove_tags(u'<p align="center" class="one">texty</p>', which_ones=('b',)),
                          u'<p align="center" class="one">texty</p>')
 
+        # text with empty tags
+        self.assertEqual(remove_tags(u'a<br />b<br/>c'), u'abc')
+        self.assertEqual(remove_tags(u'a<br />b<br/>c', which_ones=('br',)), u'abc')
+
     def test_remove_tags_with_content(self):
         # make sure it always return unicode
         assert isinstance(remove_tags_with_content('no tags'), unicode)
@@ -105,6 +109,9 @@ class UtilsMarkupTest(unittest.TestCase):
         self.assertEqual(remove_tags_with_content(u'<b>not will removed</b><i>i will removed</i>', which_ones=('i',)),
                          u'<b>not will removed</b>')
 
+        # text with empty tags
+        self.assertEqual(remove_tags_with_content(u'<br/>a<br />', which_ones=('br',)), u'a')
+
     def test_replace_escape_chars(self):
         # make sure it always return unicode
         assert isinstance(replace_escape_chars('no ec'), unicode)

scrapy/utils/markup.py

@@ -85,7 +85,7 @@ def remove_tags(text, which_ones=(), encoding=None):
                       if is empty remove all tags.
     """
     if which_ones:
-        tags = ['<%s>|<%s .*?>|</%s>' % (tag, tag, tag) for tag in which_ones]
+        tags = ['<%s/?>|<%s .*?>|</%s>' % (tag, tag, tag) for tag in which_ones]
         regex = '|'.join(tags)
     else:
         regex = '<.*?>'
@@ -101,7 +101,7 @@ def remove_tags_with_content(text, which_ones=(), encoding=None):
     """
     text = str_to_unicode(text, encoding)
     if which_ones:
-        tags = '|'.join(['<%s.*?</%s>' % (tag, tag) for tag in which_ones])
+        tags = '|'.join([r'<%s.*?</%s>|<%s\s*/>' % (tag, tag, tag) for tag in which_ones])
         retags = re.compile(tags, re.DOTALL | re.IGNORECASE)
         text = retags.sub(u'', text)
     return text