Added "unquote_markup" function to scrapy.utils.markup

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40532

scrapy/trunk/scrapy/tests/test_utils_markup.py

@@ -2,6 +2,7 @@ import unittest
 
 from scrapy.utils.markup import remove_entities, replace_tags, remove_comments
 from scrapy.utils.markup import remove_tags_with_content, remove_escape_chars, remove_tags
+from scrapy.utils.markup import unquote_markup
 
 class UtilsMarkupTest(unittest.TestCase):
 
@@ -111,3 +112,22 @@ class UtilsMarkupTest(unittest.TestCase):
         self.assertEqual(remove_escape_chars(u'escape\n', which_ones=('\t',)), u'escape\n')
         self.assertEqual(remove_escape_chars(u'escape\tchars\n', which_ones=('\t')), 'escapechars\n')
 
+    def test_unquote_markup(self):
+        sample_txt1 = u"""<node1>hi, this is sample text with entities: &amp; &copy;
+<![CDATA[although this is inside a cdata! &amp; &quot;]]></node1>"""
+        sample_txt2 = u'<node2>blah&amp;blah<![CDATA[blahblahblah!&pound;]]>moreblah&lt;&gt;</node2>'
+        sample_txt3 = u'something&pound;&amp;more<node3><![CDATA[things, stuff, and such]]>what&quot;ever</node3><node4'
+
+        # make sure it always return unicode
+        assert isinstance(unquote_markup(sample_txt1.encode('latin-1')), unicode)
+        assert isinstance(unquote_markup(sample_txt2), unicode)
+
+        self.assertEqual(unquote_markup(sample_txt1), u"""<node1>hi, this is sample text with entities: & \xa9
+although this is inside a cdata! &amp; &quot;</node1>""")
+
+        self.assertEqual(unquote_markup(sample_txt2), u'<node2>blah&blahblahblahblah!&pound;moreblah<></node2>')
+
+        self.assertEqual(unquote_markup(sample_txt1 + sample_txt2), u"""<node1>hi, this is sample text with entities: & \xa9
+although this is inside a cdata! &amp; &quot;</node1><node2>blah&blahblahblahblah!&pound;moreblah<></node2>""")
+
+        self.assertEqual(unquote_markup(sample_txt3), u'something\xa3&more<node3>things, stuff, and suchwhat"ever</node3><node4')

scrapy/trunk/scrapy/utils/markup.py

@@ -98,3 +98,32 @@ def remove_escape_chars(text, which_ones=('\n','\t','\r')):
     re_escape_chars = re.compile('[%s]' % ''.join(which_ones))
     return re_escape_chars.sub(u'', text.decode('utf-8'))
 
+def unquote_markup(text):
+    """
+    This function receives markup as a text and does the following:
+     - removes entities from any part of it that it's not inside a CDATA
+     - searches for CDATAs and extracts their text (if any) without modifying it.
+     - removes the found CDATAs
+    """
+    _cdata_re = re.compile(r'((?P<cdata_s><!\[CDATA\[)(?P<cdata_d>.*?)(?P<cdata_e>\]\]>))', re.DOTALL)
+
+    def _get_fragments(txt, pattern):
+        fragments = []
+        offset = 0
+        for match in pattern.finditer(txt):
+            match_s, match_e = match.span(1)
+            fragments.append(txt[offset:match_s])
+            fragments.append(match)
+            offset = match_e
+        fragments.append(txt[offset:])
+        return fragments
+
+    ret_text = ''
+    for fragment in _get_fragments(text.decode('utf-8'), _cdata_re):
+        if isinstance(fragment, basestring):
+            # it's not a CDATA (so we try to remove its entities)
+            ret_text += remove_entities(fragment)
+        else:
+            # it's a CDATA (so we just extract its content)
+            ret_text += fragment.group('cdata_d')
+    return unicode(ret_text)