- Added generic clean adaptors

- removed attribute name from adaptor function method (adaptors should
not nor need to know attribute names)

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40171

scrapy/trunk/scrapy/contrib/adaptors.py

@@ -3,6 +3,8 @@ Generic Adaptors for item adaptation pipe. Performs tasks
 usually needed by most cases
 """
 
+import re
+
 from scrapy.item.models import BaseAdaptor
 from scrapy.utils.python import flatten, unique
 from scrapy.xpath import XPathSelector
@@ -12,18 +14,10 @@ from scrapy.conf import settings
 class ExtendedAdaptor(BaseAdaptor):
 
     def do(self, function, a, *args, **kwargs):
-        """Execute a function of the pipeline examining kwargs for possible 
-        pre/post functions. Also prints a lot of useful debugging info."""
+        """Executes an adaptor printing a lot of useful debugging info."""
     
         debug = kwargs.get('debug')
         fname = function.__name__
-        f = kwargs.get('pre_%s' % fname)
-        if f and a:
-            if debug:
-                print "  pre_%s | input  >" % fname, a
-            a = f(a)
-            if debug:
-                print "  pre_%s | output >" % fname, a
     
         if debug:
             print repr(kwargs)
@@ -32,21 +26,14 @@ class ExtendedAdaptor(BaseAdaptor):
         if debug:
             print "  %07s | output>" % fname, a
     
-        f = kwargs.get('post_%s' % fname)
-        if f and a:
-            if debug:
-                print "  post_%s | input  >" % fname, a
-            a = f(a)
-            if debug:
-                print "  post_%s | output >" % fname, a
         return a
         
 class ExtractAdaptor(ExtendedAdaptor):
 
-    def function(self, item, attrname, location, **pipeargs):
-        return self.do(self._extract, location, **pipeargs)
+    def function(self, item, location, **pipeargs):
+        return self.do(self.extract, location, **pipeargs)
 
-    def _extract(self, location, **kwargs):
+    def extract(self, location, **kwargs):
         """Extract a list of strings from the location passed.
         Receives a list of XPathSelectors or an XPathSelector,
         or a list of strings or a string.
@@ -58,7 +45,7 @@ class ExtractAdaptor(ExtendedAdaptor):
         if not location:
             return []
         if isinstance(location, (list, tuple)):
-            strings = flatten([self._extract(o, **kwargs) for o in location])
+            strings = flatten([self.extract(o, **kwargs) for o in location])
             if kwargs.get('remove_dupes', False):
                 strings = unique(strings)
             if kwargs.get('first', False):
@@ -79,3 +66,59 @@ class ExtractAdaptor(ExtendedAdaptor):
 
         return strings
         
+   
+_clean_spaces_re = re.compile("\s+", re.U)
+_remove_root_re = re.compile(r'^\s*<.*?>(.*)</.*>\s*$', re.DOTALL)
+_xml_remove_tags_re = re.compile(r'<[a-zA-Z\/!][^>]*?>')
+_xml_remove_cdata_re = re.compile('<!\[CDATA\[(.*)\]\]', re.S)
+_xml_cdata_split_re = re.compile('(<!\[CDATA\[.*?\]\]>)', re.S)
+
+class HtmlCleanAdaptor(ExtendedAdaptor):
+
+    def function(self, item, location, **pipeargs):
+        return self.do(self.clean, string, **pipeargs)
+    
+    def clean(self, string, **kwargs):
+        """Clean (list of) strings removing newlines, spaces, etc"""
+        if isinstance(string, list):
+            return [self.clean(s, **kwargs) for s in string]
+        
+        xml = self._remove_tags(string, **kwargs)
+
+        if kwargs.get('remove_root', True) and not kwargs.get('remove_tags', True):
+            m = _remove_root_re.search(xml)
+            if m:
+                xml = m.group(1)
+
+        if kwargs.get('remove_spaces', True):
+            xml = _clean_spaces_re.sub(' ', xml)
+        if kwargs.get('strip', True):
+            xml = xml.strip()
+    
+        return xml
+    
+    def _remove_tags(self, xml, **kwargs):
+        if kwargs.get('remove_tags', True):
+            xml = _xml_remove_tags_re.sub(' ', xml)
+        return xml
+        
+class XmlCleanAdaptor(HtmlCleanAdaptor):
+    
+    def _remove_tags(self, xml, **kwargs):
+        #process in pieces the text that contains CDATA. The first check is to avoid unnecesary regex check
+        if _xml_remove_cdata_re.search(xml):
+            pieces = []
+            
+            for piece in _xml_cdata_split_re.split(xml):
+                
+                m = _xml_remove_cdata_re.search(piece)
+                if m:
+                    if kwargs.get('remove_cdata', True):#remove cdata special tag
+                        pieces.append(HtmlCleanAdaptor._remove_tags(self, m.groups()[0], **kwargs))
+                    else:
+                        pieces.append(piece)#conserve intact the cdata
+                else:
+                    pieces.append(HtmlCleanAdaptor._remove_tags(self, piece, **kwargs))
+
+            xml = "".join(pieces)
+        return xml

scrapy/trunk/scrapy/item/models.py

@@ -1,19 +1,14 @@
 class BaseAdaptor(object):
-    def function(self, item, attrname, value, **pipeargs):
+    def function(self, item, value, **pipeargs):
         raise NotImplemented
 
 #default adaptors
 class ExtractAdaptor(BaseAdaptor):
-    def function(self, item, attrname, value, **pipeargs):
+    def function(self, item, value, **pipeargs):
         if hasattr(value, 'extract'):
             value = value.extract()
         return value
 
-class AssignAdaptor(BaseAdaptor):
-    def function(self, item, attrname, value, **pipeargs):
-        if not hasattr(item, attrname):
-            setattr(item, attrname, value)
-
 class ScrapedItem(object):
     """
     This is the base class for all scraped items.
@@ -22,7 +17,7 @@ class ScrapedItem(object):
     * guid (unique global indentifier)
     * url (URL where that item was scraped from)
     """
-    adaptors_pipe = [ExtractAdaptor(), AssignAdaptor()]
+    adaptors_pipe = [ExtractAdaptor()]
     
     def set_adaptors_pipe(adaptors_pipes):
         ScrapedItem.adaptors_pipes = adaptors_pipes
@@ -30,4 +25,6 @@ class ScrapedItem(object):
     def attribute(self, name, value, **pipeargs):
 
         for adaptor in ScrapedItem.adaptors_pipe:
-            value = adaptor.function(self, name, value, **pipeargs)
+            value = adaptor.function(self, value, **pipeargs)
+        if not hasattr(item, name):
+            setattr(item, name, value)