Added more options to unquote_markup that are passed to remove_entities

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40533

scrapy/trunk/scrapy/utils/markup.py

@@ -98,10 +98,10 @@ def remove_escape_chars(text, which_ones=('\n','\t','\r')):
     re_escape_chars = re.compile('[%s]' % ''.join(which_ones))
     return re_escape_chars.sub(u'', text.decode('utf-8'))
 
-def unquote_markup(text):
+def unquote_markup(text, keep=(), remove_illegal=True):
     """
     This function receives markup as a text and does the following:
-     - removes entities from any part of it that it's not inside a CDATA
+     - removes entities (except the ones in 'keep') from any part of it that it's not inside a CDATA
      - searches for CDATAs and extracts their text (if any) without modifying it.
      - removes the found CDATAs
     """
@@ -122,7 +122,7 @@ def unquote_markup(text):
     for fragment in _get_fragments(text.decode('utf-8'), _cdata_re):
         if isinstance(fragment, basestring):
             # it's not a CDATA (so we try to remove its entities)
-            ret_text += remove_entities(fragment)
+            ret_text += remove_entities(fragment, keep=keep, remove_illegal=remove_illegal)
         else:
             # it's a CDATA (so we just extract its content)
             ret_text += fragment.group('cdata_d')