From 4842fe0679b40e3d973430eb1efddc5f39ffda00 Mon Sep 17 00:00:00 2001 From: german Date: Mon, 22 Sep 2008 18:04:30 +0000 Subject: [PATCH] removed unquote_html --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40268 --- scrapy/trunk/scrapy/utils/misc.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/scrapy/trunk/scrapy/utils/misc.py b/scrapy/trunk/scrapy/utils/misc.py index f7ae7953d..0cd87568f 100644 --- a/scrapy/trunk/scrapy/utils/misc.py +++ b/scrapy/trunk/scrapy/utils/misc.py @@ -9,6 +9,7 @@ from twisted.python import failure from scrapy.core.exceptions import UsageError from scrapy.utils.python import flatten +from scrapy.utils.markup import remove_entities def dict_updatedefault(D, E, **F): """ @@ -172,14 +173,6 @@ def convert_entity(m, keep_reserved=False): except KeyError: return '&%s;' % m.group(2) - -def unquote_html(s, keep_reserved=False): - """Convert a HTML quoted string into normal string (ISO-8859-1). - - Works with &#XX; and with   > etc. - """ - return re.sub(re.compile(r'&(#?)(.+?);', re.U), lambda m: convert_entity(m, keep_reserved), s) - def extract_regex(regex, text, encoding): """Extract a list of unicode strings from the given text/encoding using the following policies: @@ -198,9 +191,9 @@ def extract_regex(regex, text, encoding): strings = flatten(strings) if isinstance(text, unicode): - return [unquote_html(s, keep_reserved=True) for s in strings] + return [remove_entities(s, keep=['lt', 'amp']) for s in strings] else: - return [unquote_html(unicode(s, encoding), keep_reserved=True) for s in strings] + return [remove_entities(unicode(s, encoding), keep=['lt', 'amp']) for s in strings] _regex_type = type(re.compile("", 0))