diff --git a/scrapy/trunk/scrapy/utils/markup.py b/scrapy/trunk/scrapy/utils/markup.py index 6ae05a8b7..ee878c09e 100644 --- a/scrapy/trunk/scrapy/utils/markup.py +++ b/scrapy/trunk/scrapy/utils/markup.py @@ -5,7 +5,7 @@ Functions for dealing with markup text import re import htmlentitydefs -_ent_re = re.compile(r'&(#?)(.+?);') +_ent_re = re.compile(r'&(#?)([^&;]+);') _tag_re = re.compile(r'<[a-zA-Z\/!].*?>', re.DOTALL) def remove_entities(text, keep=(), remove_illegal=True): @@ -15,9 +15,9 @@ def remove_entities(text, keep=(), remove_illegal=True): If 'keep' is passed (with a list of entity names) those entities will be kept (they won't be removed). - + It supports both numeric (&#nnnn;) and named (  >) entities. - + If remove_illegal is True, entities that can't be converted are removed. If remove_illegal is False, entities that can't be converted are kept "as is". For more information see the tests.