diff --git a/scrapy/trunk/scrapy/tests/test_utils_markup.py b/scrapy/trunk/scrapy/tests/test_utils_markup.py
index 0624560d8..b466dce88 100644
--- a/scrapy/trunk/scrapy/tests/test_utils_markup.py
+++ b/scrapy/trunk/scrapy/tests/test_utils_markup.py
@@ -2,6 +2,7 @@ import unittest
from scrapy.utils.markup import remove_entities, replace_tags, remove_comments
from scrapy.utils.markup import remove_tags_with_content, remove_escape_chars, remove_tags
+from scrapy.utils.markup import unquote_markup
class UtilsMarkupTest(unittest.TestCase):
@@ -111,3 +112,22 @@ class UtilsMarkupTest(unittest.TestCase):
self.assertEqual(remove_escape_chars(u'escape\n', which_ones=('\t',)), u'escape\n')
self.assertEqual(remove_escape_chars(u'escape\tchars\n', which_ones=('\t')), 'escapechars\n')
+ def test_unquote_markup(self):
+ sample_txt1 = u"""hi, this is sample text with entities: & ©
+"""
+ sample_txt2 = u'blah&blahmoreblah<>'
+ sample_txt3 = u'something£&morewhat"everhi, this is sample text with entities: & \xa9
+although this is inside a cdata! & """")
+
+ self.assertEqual(unquote_markup(sample_txt2), u'blah&blahblahblahblah!£moreblah<>')
+
+ self.assertEqual(unquote_markup(sample_txt1 + sample_txt2), u"""hi, this is sample text with entities: & \xa9
+although this is inside a cdata! & "blah&blahblahblahblah!£moreblah<>""")
+
+ self.assertEqual(unquote_markup(sample_txt3), u'something\xa3&morethings, stuff, and suchwhat"ever.*?)(?P\]\]>))', re.DOTALL)
+
+ def _get_fragments(txt, pattern):
+ fragments = []
+ offset = 0
+ for match in pattern.finditer(txt):
+ match_s, match_e = match.span(1)
+ fragments.append(txt[offset:match_s])
+ fragments.append(match)
+ offset = match_e
+ fragments.append(txt[offset:])
+ return fragments
+
+ ret_text = ''
+ for fragment in _get_fragments(text.decode('utf-8'), _cdata_re):
+ if isinstance(fragment, basestring):
+ # it's not a CDATA (so we try to remove its entities)
+ ret_text += remove_entities(fragment)
+ else:
+ # it's a CDATA (so we just extract its content)
+ ret_text += fragment.group('cdata_d')
+ return unicode(ret_text)