reverted clean_markup code movement

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40177

scrapy/trunk/scrapy/contrib/adaptors.py

@@ -10,7 +10,6 @@ from scrapy.utils.python import flatten, unique
 from scrapy.xpath import XPathSelector
 from scrapy.utils.misc import unquote_html, location_str
 from scrapy.conf import settings
-from scrapy.utils.markup import clean_markup
 
 class ExtendedAdaptor(BaseAdaptor):
 
@@ -65,18 +64,4 @@ class ExtractAdaptor(ExtendedAdaptor):
         else:
             raise TypeError, "unsupported location type: %s" % type(location)
 
-        return strings
-
-class HtmlCleanAdaptor(ExtendedAdaptor):
-
-    def function(self, string, **pipeargs):
-        return self.do(self.clean, string, **pipeargs)
-    
-    def clean(self, string, **pipeargs):
-        return clean_markup(string, **pipeargs)
-        
-class XmlCleanAdaptor(HtmlCleanAdaptor):
-    
-    def clean(self, string, **pipeargs):
-        pipeargs["xml_doc"] = True
-        return clean_markup(string, **pipeargs)
+        return strings

scrapy/trunk/scrapy/utils/markup.py

@@ -57,58 +57,3 @@ def replace_tags(text, token=''):
     Always returns a unicode string.
     """
     return _tag_re.sub(token, text.decode('utf-8'))
-
-_clean_spaces_re = re.compile("\s+", re.U)
-_remove_root_re = re.compile(r'^\s*<.*?>(.*)</.*>\s*$', re.DOTALL)
-_xml_remove_tags_re = re.compile(r'<[a-zA-Z\/!][^>]*?>')
-_xml_remove_cdata_re = re.compile('<!\[CDATA\[(.*)\]\]', re.S)
-_xml_cdata_split_re = re.compile('(<!\[CDATA\[.*?\]\]>)', re.S)
-
-def remove_tags(xml, **kwargs):
-    if kwargs.get('remove_tags', True):
-        xml = _xml_remove_tags_re.sub(' ', xml)
-    return xml
-
-def xml_remove_tags(xml, **kwargs):
-    #process in pieces the text that contains CDATA. The first check is to avoid unnecesary regex check
-    if _xml_remove_cdata_re.search(xml):
-        pieces = []
-        
-        for piece in _xml_cdata_split_re.split(xml):
-            
-            m = _xml_remove_cdata_re.search(piece)
-            if m:
-                if kwargs.get('remove_cdata', True):#remove cdata special tag
-                    pieces.append(remove_tags(m.groups()[0], **kwargs))
-                else:
-                    pieces.append(piece)#conserve intact the cdata
-            else:
-                pieces.append(remove_tags(piece, **kwargs))
-
-        xml = "".join(pieces)
-    else:
-        xml = remove_tags(xml, **kwargs)
-    return xml
-
-
-def clean_markup(string, **kwargs):
-    """Clean (list of) strings removing newlines, spaces, etc"""
-    
-    _remove_tags = xml_remove_tags if kwargs.get("xml_doc") else remove_tags
-
-    if isinstance(string, list):
-        return [clean_markup(s, **kwargs) for s in string]
-    
-    string = _remove_tags(string, **kwargs)
-
-    if kwargs.get('remove_root', True) and not kwargs.get('remove_tags', True):
-        m = _remove_root_re.search(string)
-        if m:
-            string = m.group(1)
-
-    if kwargs.get('remove_spaces', True):
-        string = _clean_spaces_re.sub(' ', string)
-    if kwargs.get('strip', True):
-        string = string.strip()
-
-    return string