From 453714b25256f9e3ca3a6a9728bce57cc3d55cc3 Mon Sep 17 00:00:00 2001 From: Andres Moreira Date: Fri, 3 Oct 2008 11:57:51 +0000 Subject: [PATCH] Added new functions to parse html. --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40295 --- scrapy/trunk/scrapy/utils/markup.py | 39 +++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/scrapy/trunk/scrapy/utils/markup.py b/scrapy/trunk/scrapy/utils/markup.py index 2faf5e2b9..af208dff9 100644 --- a/scrapy/trunk/scrapy/utils/markup.py +++ b/scrapy/trunk/scrapy/utils/markup.py @@ -59,3 +59,42 @@ def replace_tags(text, token=''): Always returns a unicode string. """ return _tag_re.sub(token, text.decode('utf-8')) + + +def remove_comments(text): + """ Remove HTML Comments. """ + return re.sub('', '', text, re.DOTALL) + +def remove_tags(text, which_ones=()): + """ Remove HTML Tags only. + + which_ones -- is a tuple of which tags we want to remove. + if is empty remove all tags. + """ + if len(which_ones) > 0: + tags = [ '<%s>|<%s .*?>|' % (tag,tag,tag) for tag in which_ones ] + reg_exp_remove_tags = '|'.join(tags) + else: + reg_exp_remove_tags = '<.*?>' + re_tags = re.compile(reg_exp_remove_tags, re.DOTALL) + return re_tags.sub('', text) + +def remove_tags_with_content(text, which_ones=()): + """ Remove tags and its content. + + which_ones -- is a tuple of which tags with its content we want to remove. + if is empty do nothing. + """ + tags = [ '<%s.*?' % (tag,tag) for tag in which_ones ] + re_tags_remove = re.compile('|'.join(tags), re.DOTALL) + return re_tags_remove.sub('', text) + +def remove_escape_chars(text, which_ones=('\n','\t','\r')): + """ Remove escape chars. Default : \\n, \\t, \\r + + which_ones -- is a tuple of which escape chars we want to remove. + if is empty do nothing. + """ + re_escape_chars = re.compile('[%s]' % ''.join(which_ones)) + return re_escape_chars.sub('', text) +