From 4e7e539b037ead9852b8965649f08a5abaa9dac8 Mon Sep 17 00:00:00 2001 From: Damian Canabal Date: Wed, 24 Sep 2008 12:25:10 +0000 Subject: [PATCH] html entities regexp improvement --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40274 --- scrapy/trunk/scrapy/utils/markup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/trunk/scrapy/utils/markup.py b/scrapy/trunk/scrapy/utils/markup.py index 6ae05a8b7..ee878c09e 100644 --- a/scrapy/trunk/scrapy/utils/markup.py +++ b/scrapy/trunk/scrapy/utils/markup.py @@ -5,7 +5,7 @@ Functions for dealing with markup text import re import htmlentitydefs -_ent_re = re.compile(r'&(#?)(.+?);') +_ent_re = re.compile(r'&(#?)([^&;]+);') _tag_re = re.compile(r'<[a-zA-Z\/!].*?>', re.DOTALL) def remove_entities(text, keep=(), remove_illegal=True): @@ -15,9 +15,9 @@ def remove_entities(text, keep=(), remove_illegal=True): If 'keep' is passed (with a list of entity names) those entities will be kept (they won't be removed). - + It supports both numeric (&#nnnn;) and named (  >) entities. - + If remove_illegal is True, entities that can't be converted are removed. If remove_illegal is False, entities that can't be converted are kept "as is". For more information see the tests.