From b59f62dc91869d499a859929e44c05df9ddc2017 Mon Sep 17 00:00:00 2001
From: olveyra <none@none>
Date: Sun, 27 Jul 2008 14:54:11 +0000
Subject: [PATCH] Added function convert_entity from
 decobot.utils.text_extraction (to complete and fix revision 124)

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40126
---
 scrapy/trunk/scrapy/utils/misc.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/scrapy/trunk/scrapy/utils/misc.py b/scrapy/trunk/scrapy/utils/misc.py
index cd374e66f..18e2a78dc 100644
--- a/scrapy/trunk/scrapy/utils/misc.py
+++ b/scrapy/trunk/scrapy/utils/misc.py
@@ -2,6 +2,7 @@
 Auxiliary functions which doesn't fit anywhere else
 """
 import re
+import htmlentitydefs
 
 from twisted.internet import defer, reactor
 from twisted.python import failure
@@ -154,6 +155,24 @@ def load_class(class_path):
 
     return cls
 
+def convert_entity(m, keep_reserved=False):
+    """
+    Convert a HTML entity into unicode string
+    """
+    if m.group(1)=='#':
+        try:
+            return unichr(int(m.group(2)))
+        except ValueError:
+            return '&#%s;' % m.group(2)
+    try:
+        if not (keep_reserved and m.group(2) in ['lt', 'amp']):
+            return unichr(htmlentitydefs.name2codepoint[m.group(2)])
+        else:
+            return '&%s;' % m.group(2)
+    except KeyError:
+        return '&%s;' % m.group(2)
+
+
 def unquote_html(s, keep_reserved=False):
     """Convert a HTML quoted string into normal string (ISO-8859-1).