From a2bd70ba21af273b97ecb4f35ad6521940bf3cd6 Mon Sep 17 00:00:00 2001 From: olveyra Date: Thu, 21 Aug 2008 15:07:55 +0000 Subject: [PATCH] moved clean_markup to scrapy.utils.markup --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40176 --- scrapy/trunk/scrapy/contrib/adaptors.py | 60 ++++--------------------- scrapy/trunk/scrapy/utils/markup.py | 55 +++++++++++++++++++++++ 2 files changed, 63 insertions(+), 52 deletions(-) diff --git a/scrapy/trunk/scrapy/contrib/adaptors.py b/scrapy/trunk/scrapy/contrib/adaptors.py index 7878b56d7..c49266b5f 100644 --- a/scrapy/trunk/scrapy/contrib/adaptors.py +++ b/scrapy/trunk/scrapy/contrib/adaptors.py @@ -10,6 +10,7 @@ from scrapy.utils.python import flatten, unique from scrapy.xpath import XPathSelector from scrapy.utils.misc import unquote_html, location_str from scrapy.conf import settings +from scrapy.utils.markup import clean_markup class ExtendedAdaptor(BaseAdaptor): @@ -30,7 +31,7 @@ class ExtendedAdaptor(BaseAdaptor): class ExtractAdaptor(ExtendedAdaptor): - def function(self, attrname, location, **pipeargs): + def function(self, location, **pipeargs): return self.do(self.extract, location, **pipeargs) def extract(self, location, **kwargs): @@ -65,62 +66,17 @@ class ExtractAdaptor(ExtendedAdaptor): raise TypeError, "unsupported location type: %s" % type(location) return strings - - -_clean_spaces_re = re.compile("\s+", re.U) -_remove_root_re = re.compile(r'^\s*<.*?>(.*)\s*$', re.DOTALL) -_xml_remove_tags_re = re.compile(r'<[a-zA-Z\/!][^>]*?>') -_xml_remove_cdata_re = re.compile(')', re.S) class HtmlCleanAdaptor(ExtendedAdaptor): - def function(self, attrname, string, **pipeargs): + def function(self, string, **pipeargs): return self.do(self.clean, string, **pipeargs) - def clean(self, string, **kwargs): - """Clean (list of) strings removing newlines, spaces, etc""" - if isinstance(string, list): - return [self.clean(s, **kwargs) for s in string] - - xml = self._remove_tags(string, **kwargs) - - if kwargs.get('remove_root', True) and not kwargs.get('remove_tags', True): - m = _remove_root_re.search(xml) - if m: - xml = m.group(1) - - if kwargs.get('remove_spaces', True): - xml = _clean_spaces_re.sub(' ', xml) - if kwargs.get('strip', True): - xml = xml.strip() - - return xml - - def _remove_tags(self, xml, **kwargs): - if kwargs.get('remove_tags', True): - xml = _xml_remove_tags_re.sub(' ', xml) - return xml + def clean(self, string, **pipeargs): + return clean_markup(string, **pipeargs) class XmlCleanAdaptor(HtmlCleanAdaptor): - def _remove_tags(self, xml, **kwargs): - #process in pieces the text that contains CDATA. The first check is to avoid unnecesary regex check - if _xml_remove_cdata_re.search(xml): - pieces = [] - - for piece in _xml_cdata_split_re.split(xml): - - m = _xml_remove_cdata_re.search(piece) - if m: - if kwargs.get('remove_cdata', True):#remove cdata special tag - pieces.append(HtmlCleanAdaptor._remove_tags(self, m.groups()[0], **kwargs)) - else: - pieces.append(piece)#conserve intact the cdata - else: - pieces.append(HtmlCleanAdaptor._remove_tags(self, piece, **kwargs)) - - xml = "".join(pieces) - else: - xml = HtmlCleanAdaptor._remove_tags(self, xml, **kwargs) - return xml + def clean(self, string, **pipeargs): + pipeargs["xml_doc"] = True + return clean_markup(string, **pipeargs) diff --git a/scrapy/trunk/scrapy/utils/markup.py b/scrapy/trunk/scrapy/utils/markup.py index 6ae05a8b7..accbdae00 100644 --- a/scrapy/trunk/scrapy/utils/markup.py +++ b/scrapy/trunk/scrapy/utils/markup.py @@ -57,3 +57,58 @@ def replace_tags(text, token=''): Always returns a unicode string. """ return _tag_re.sub(token, text.decode('utf-8')) + +_clean_spaces_re = re.compile("\s+", re.U) +_remove_root_re = re.compile(r'^\s*<.*?>(.*)\s*$', re.DOTALL) +_xml_remove_tags_re = re.compile(r'<[a-zA-Z\/!][^>]*?>') +_xml_remove_cdata_re = re.compile(')', re.S) + +def remove_tags(xml, **kwargs): + if kwargs.get('remove_tags', True): + xml = _xml_remove_tags_re.sub(' ', xml) + return xml + +def xml_remove_tags(xml, **kwargs): + #process in pieces the text that contains CDATA. The first check is to avoid unnecesary regex check + if _xml_remove_cdata_re.search(xml): + pieces = [] + + for piece in _xml_cdata_split_re.split(xml): + + m = _xml_remove_cdata_re.search(piece) + if m: + if kwargs.get('remove_cdata', True):#remove cdata special tag + pieces.append(remove_tags(m.groups()[0], **kwargs)) + else: + pieces.append(piece)#conserve intact the cdata + else: + pieces.append(remove_tags(piece, **kwargs)) + + xml = "".join(pieces) + else: + xml = remove_tags(xml, **kwargs) + return xml + + +def clean_markup(string, **kwargs): + """Clean (list of) strings removing newlines, spaces, etc""" + + _remove_tags = xml_remove_tags if kwargs.get("xml_doc") else remove_tags + + if isinstance(string, list): + return [clean_markup(s, **kwargs) for s in string] + + string = _remove_tags(string, **kwargs) + + if kwargs.get('remove_root', True) and not kwargs.get('remove_tags', True): + m = _remove_root_re.search(string) + if m: + string = m.group(1) + + if kwargs.get('remove_spaces', True): + string = _clean_spaces_re.sub(' ', string) + if kwargs.get('strip', True): + string = string.strip() + + return string