From bda22a539c9e371c05bf776264d86da9323c8ee4 Mon Sep 17 00:00:00 2001 From: elpolilla Date: Thu, 18 Dec 2008 16:12:49 +0000 Subject: [PATCH] Added more options to unquote_markup that are passed to remove_entities --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40533 --- scrapy/trunk/scrapy/utils/markup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/trunk/scrapy/utils/markup.py b/scrapy/trunk/scrapy/utils/markup.py index bf5802e0f..ff3300100 100644 --- a/scrapy/trunk/scrapy/utils/markup.py +++ b/scrapy/trunk/scrapy/utils/markup.py @@ -98,10 +98,10 @@ def remove_escape_chars(text, which_ones=('\n','\t','\r')): re_escape_chars = re.compile('[%s]' % ''.join(which_ones)) return re_escape_chars.sub(u'', text.decode('utf-8')) -def unquote_markup(text): +def unquote_markup(text, keep=(), remove_illegal=True): """ This function receives markup as a text and does the following: - - removes entities from any part of it that it's not inside a CDATA + - removes entities (except the ones in 'keep') from any part of it that it's not inside a CDATA - searches for CDATAs and extracts their text (if any) without modifying it. - removes the found CDATAs """ @@ -122,7 +122,7 @@ def unquote_markup(text): for fragment in _get_fragments(text.decode('utf-8'), _cdata_re): if isinstance(fragment, basestring): # it's not a CDATA (so we try to remove its entities) - ret_text += remove_entities(fragment) + ret_text += remove_entities(fragment, keep=keep, remove_illegal=remove_illegal) else: # it's a CDATA (so we just extract its content) ret_text += fragment.group('cdata_d')