added scrapy.utils.markup module

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40129

scrapy/trunk/scrapy/tests/test_utils_markup.py

@@ -0,0 +1,40 @@
+import unittest
+
+from scrapy.utils.markup import remove_entities, replace_tags
+
+class UtilsMarkupTest(unittest.TestCase):
+
+    def test_remove_entities(self):
+        # make sure it always return uncode
+        assert isinstance(remove_entities('no entities'), unicode)
+        assert isinstance(remove_entities('Price: £100!'),  unicode)
+
+        # regular conversions
+        self.assertEqual(remove_entities(u'As low as £100!'),
+                         u'As low as \xa3100!')
+        self.assertEqual(remove_entities('As low as £100!'),
+                         u'As low as \xa3100!')
+
+        # keep some entities
+        self.assertEqual(remove_entities('<b>Low &lt; High &amp; Medium &pound; six</b>', keep=['lt', 'amp']),
+                         u'<b>Low &lt; High &amp; Medium \xa3 six</b>')
+
+        # illegal entities
+        self.assertEqual(remove_entities('a &lt; b &illegal; c &#12345678; six', remove_illegal=False),
+                         u'a < b &illegal; c &#12345678; six')
+        self.assertEqual(remove_entities('a &lt; b &illegal; c &#12345678; six', remove_illegal=True),
+                         u'a < b  c  six')
+
+    def test_remove_tags(self):
+        # make sure it always return uncode
+        assert isinstance(replace_tags('no entities'), unicode)
+
+        self.assertEqual(replace_tags(u'This text contains <a>some tag</a>'),
+                         u'This text contains some tag')
+
+        self.assertEqual(replace_tags('This text is very im<b>port</b>ant', ' '),
+                         u'This text is very im port ant')
+
+        # multiline tags
+        self.assertEqual(replace_tags('Click <a class="one"\r\n href="url">here</a>'),
+                         u'Click here')

scrapy/trunk/scrapy/utils/markup.py

@@ -0,0 +1,59 @@
+"""
+Functions for dealing with markup text
+"""
+
+import re
+import htmlentitydefs
+
+_ent_re = re.compile(r'&(#?)(.+?);')
+_tag_re = re.compile(r'<[a-zA-Z\/!].*?>', re.DOTALL)
+
+def remove_entities(text, keep=(), remove_illegal=True):
+    """Remove entities from the given text.
+
+    'text' can be a unicode string or a regular string encoded as 'utf-8'
+
+    If 'keep' is passed (with a list of entity names) those entities will
+    be kept (they won't be removed).
+    
+    It supports both numeric (&#nnnn;) and named (&nbsp; &gt;) entities.
+    
+    If remove_illegal is True, entities that can't be converted are removed.
+    If remove_illegal is False, entities that can't be converted are kept "as
+    is". For more information see the tests.
+
+    Always returns a unicode string (with the entities removed).
+    """
+
+    def convert_entity(m):
+        if m.group(1)=='#':
+            try:
+                return unichr(int(m.group(2)))
+            except ValueError:
+                if remove_illegal:
+                    return u''
+                else:
+                    return u'&#%s;' % m.group(2)
+        try:
+            if m.group(2) in keep:
+                return '&%s;' % m.group(2)
+            else:
+                return unichr(htmlentitydefs.name2codepoint[m.group(2)])
+        except KeyError:
+            if remove_illegal:
+                return u''
+            else:
+                return u'&%s;' % m.group(2)
+
+    return _ent_re.sub(convert_entity, text.decode('utf-8'))
+
+
+def replace_tags(text, token=''):
+    """Replace all markup tags found in the given text by the given token. By
+    default token is a null string so it just remove all tags.
+
+    'text' can be a unicode string or a regular string encoded as 'utf-8'
+
+    Always returns a unicode string.
+    """
+    return _tag_re.sub(token, text.decode('utf-8'))