RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

remove_entities: added support for common browser hack for numeric character references in the 80-9F range

This commit is contained in:
Pablo Hoffman 2009-08-06 14:31:59 -03:00
parent c3f7bfea1f
commit f2c2609e42
2 changed files with 8 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/tests/test_utils_markup.py
View File

@ -30,6 +30,8 @@ class UtilsMarkupTest(unittest.TestCase):
u'a < b c six')
self.assertEqual(remove_entities('x&#x2264;y'), u'x\u2264y')
# check browser hack for numeric character references in the 80-9F range
self.assertEqual(remove_entities('x&#153;y', encoding='cp1252'), u'x\u2122y')
def test_replace_tags(self):
# make sure it always return uncode

6
scrapy/utils/markup.py
View File

@ -37,6 +37,12 @@ def remove_entities(text, keep=(), remove_illegal=True, encoding='utf-8'):
number = int(entity_body, 16)
else:
number = int(entity_body, 10)
# Numeric character references in the 80-9F range are typically
# interpreted by browsers as representing the characters mapped
# to bytes 80-9F in the Windows-1252 encoding. For more info
# see: http://en.wikipedia.org/wiki/Character_encodings_in_HTML
if 0x80 <= number <= 0x9f:
return chr(number).decode('cp1252')
except ValueError:
number = None
else: