diff --git a/scrapy/trunk/scrapy/utils/markup.py b/scrapy/trunk/scrapy/utils/markup.py index bf5802e0f..ff3300100 100644 --- a/scrapy/trunk/scrapy/utils/markup.py +++ b/scrapy/trunk/scrapy/utils/markup.py @@ -98,10 +98,10 @@ def remove_escape_chars(text, which_ones=('\n','\t','\r')): re_escape_chars = re.compile('[%s]' % ''.join(which_ones)) return re_escape_chars.sub(u'', text.decode('utf-8')) -def unquote_markup(text): +def unquote_markup(text, keep=(), remove_illegal=True): """ This function receives markup as a text and does the following: - - removes entities from any part of it that it's not inside a CDATA + - removes entities (except the ones in 'keep') from any part of it that it's not inside a CDATA - searches for CDATAs and extracts their text (if any) without modifying it. - removes the found CDATAs """ @@ -122,7 +122,7 @@ def unquote_markup(text): for fragment in _get_fragments(text.decode('utf-8'), _cdata_re): if isinstance(fragment, basestring): # it's not a CDATA (so we try to remove its entities) - ret_text += remove_entities(fragment) + ret_text += remove_entities(fragment, keep=keep, remove_illegal=remove_illegal) else: # it's a CDATA (so we just extract its content) ret_text += fragment.group('cdata_d')