Added function convert_entity from decobot.utils.text_extraction

(to complete and fix revision 124)

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40126

scrapy/trunk/scrapy/utils/misc.py

@@ -2,6 +2,7 @@
 Auxiliary functions which doesn't fit anywhere else
 """
 import re
+import htmlentitydefs
 
 from twisted.internet import defer, reactor
 from twisted.python import failure
@@ -154,6 +155,24 @@ def load_class(class_path):
 
     return cls
 
+def convert_entity(m, keep_reserved=False):
+    """
+    Convert a HTML entity into unicode string
+    """
+    if m.group(1)=='#':
+        try:
+            return unichr(int(m.group(2)))
+        except ValueError:
+            return '&#%s;' % m.group(2)
+    try:
+        if not (keep_reserved and m.group(2) in ['lt', 'amp']):
+            return unichr(htmlentitydefs.name2codepoint[m.group(2)])
+        else:
+            return '&%s;' % m.group(2)
+    except KeyError:
+        return '&%s;' % m.group(2)
+
+
 def unquote_html(s, keep_reserved=False):
     """Convert a HTML quoted string into normal string (ISO-8859-1).