diff --git a/scrapy/trunk/scrapy/contrib/adaptors.py b/scrapy/trunk/scrapy/contrib/adaptors.py index c49266b5f..304d06bb5 100644 --- a/scrapy/trunk/scrapy/contrib/adaptors.py +++ b/scrapy/trunk/scrapy/contrib/adaptors.py @@ -10,7 +10,6 @@ from scrapy.utils.python import flatten, unique from scrapy.xpath import XPathSelector from scrapy.utils.misc import unquote_html, location_str from scrapy.conf import settings -from scrapy.utils.markup import clean_markup class ExtendedAdaptor(BaseAdaptor): @@ -65,18 +64,4 @@ class ExtractAdaptor(ExtendedAdaptor): else: raise TypeError, "unsupported location type: %s" % type(location) - return strings - -class HtmlCleanAdaptor(ExtendedAdaptor): - - def function(self, string, **pipeargs): - return self.do(self.clean, string, **pipeargs) - - def clean(self, string, **pipeargs): - return clean_markup(string, **pipeargs) - -class XmlCleanAdaptor(HtmlCleanAdaptor): - - def clean(self, string, **pipeargs): - pipeargs["xml_doc"] = True - return clean_markup(string, **pipeargs) + return strings \ No newline at end of file diff --git a/scrapy/trunk/scrapy/utils/markup.py b/scrapy/trunk/scrapy/utils/markup.py index accbdae00..6ae05a8b7 100644 --- a/scrapy/trunk/scrapy/utils/markup.py +++ b/scrapy/trunk/scrapy/utils/markup.py @@ -57,58 +57,3 @@ def replace_tags(text, token=''): Always returns a unicode string. """ return _tag_re.sub(token, text.decode('utf-8')) - -_clean_spaces_re = re.compile("\s+", re.U) -_remove_root_re = re.compile(r'^\s*<.*?>(.*)\s*$', re.DOTALL) -_xml_remove_tags_re = re.compile(r'<[a-zA-Z\/!][^>]*?>') -_xml_remove_cdata_re = re.compile(')', re.S) - -def remove_tags(xml, **kwargs): - if kwargs.get('remove_tags', True): - xml = _xml_remove_tags_re.sub(' ', xml) - return xml - -def xml_remove_tags(xml, **kwargs): - #process in pieces the text that contains CDATA. The first check is to avoid unnecesary regex check - if _xml_remove_cdata_re.search(xml): - pieces = [] - - for piece in _xml_cdata_split_re.split(xml): - - m = _xml_remove_cdata_re.search(piece) - if m: - if kwargs.get('remove_cdata', True):#remove cdata special tag - pieces.append(remove_tags(m.groups()[0], **kwargs)) - else: - pieces.append(piece)#conserve intact the cdata - else: - pieces.append(remove_tags(piece, **kwargs)) - - xml = "".join(pieces) - else: - xml = remove_tags(xml, **kwargs) - return xml - - -def clean_markup(string, **kwargs): - """Clean (list of) strings removing newlines, spaces, etc""" - - _remove_tags = xml_remove_tags if kwargs.get("xml_doc") else remove_tags - - if isinstance(string, list): - return [clean_markup(s, **kwargs) for s in string] - - string = _remove_tags(string, **kwargs) - - if kwargs.get('remove_root', True) and not kwargs.get('remove_tags', True): - m = _remove_root_re.search(string) - if m: - string = m.group(1) - - if kwargs.get('remove_spaces', True): - string = _clean_spaces_re.sub(' ', string) - if kwargs.get('strip', True): - string = string.strip() - - return string