diff --git a/scrapy/tests/test_utils_markup.py b/scrapy/tests/test_utils_markup.py index 95fb2a50b..5c2c7df32 100644 --- a/scrapy/tests/test_utils_markup.py +++ b/scrapy/tests/test_utils_markup.py @@ -30,6 +30,8 @@ class UtilsMarkupTest(unittest.TestCase): u'a < b c six') self.assertEqual(remove_entities('x≤y'), u'x\u2264y') + # check browser hack for numeric character references in the 80-9F range + self.assertEqual(remove_entities('x™y', encoding='cp1252'), u'x\u2122y') def test_replace_tags(self): # make sure it always return uncode diff --git a/scrapy/utils/markup.py b/scrapy/utils/markup.py index eb7ec5ca0..c978628ed 100644 --- a/scrapy/utils/markup.py +++ b/scrapy/utils/markup.py @@ -37,6 +37,12 @@ def remove_entities(text, keep=(), remove_illegal=True, encoding='utf-8'): number = int(entity_body, 16) else: number = int(entity_body, 10) + # Numeric character references in the 80-9F range are typically + # interpreted by browsers as representing the characters mapped + # to bytes 80-9F in the Windows-1252 encoding. For more info + # see: http://en.wikipedia.org/wiki/Character_encodings_in_HTML + if 0x80 <= number <= 0x9f: + return chr(number).decode('cp1252') except ValueError: number = None else: