- Introduction of class BaseAdaptor

- Contrib Adaptors
- location_str moved from decobot to scrapy
- Added setting DEFAULT_DATA_ENCODING

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40161

scrapy/trunk/scrapy/contrib/adaptors.py

@@ -0,0 +1,79 @@
+"""
+Adaptors for item adaptation pipe.
+"""
+
+from scrapy.item.models import BaseAdaptor
+from scrapy.utils.python import flatten, unique
+from scrapy.xpath import XPathSelector
+from scrapy.utils.misc import unquote_html, location_str
+
+class ExtendedAdaptor(BaseAdaptor):
+
+    def do(self, function, a, *args, **kwargs):
+        """Execute a function of the pipeline examining kwargs for possible 
+        pre/post functions. Also prints a lot of useful debugging info."""
+    
+        debug = kwargs.get('debug')
+        fname = function.__name__
+        f = kwargs.get('pre_%s' % fname)
+        if f and a:
+            if debug:
+                print "  pre_%s | input  >" % fname, a
+            a = f(a)
+            if debug:
+                print "  pre_%s | output >" % fname, a
+    
+        if debug:
+            print repr(kwargs)
+            print "  %07s | input >" % fname, location_str(a)
+        a = function(a, *args, **kwargs)
+        if debug:
+            print "  %07s | output>" % fname, a
+    
+        f = kwargs.get('post_%s' % fname)
+        if f and a:
+            if debug:
+                print "  post_%s | input  >" % fname, a
+            a = f(a)
+            if debug:
+                print "  post_%s | output >" % fname, a
+        return a
+        
+class ExtractAdaptor(ExtendedAdaptor):
+
+    def function(self, item, attrname, location, **pipeargs):
+        return self.do(self._extract, location, **pipeargs)
+
+    def _extract(self, location, **kwargs):
+        """Extract a list of strings from the location passed.
+        Receives a list of XPathSelectors or an XPathSelector,
+        or a list of strings or a string.
+        
+        Return a list of strings extracted.
+
+        This function *always* returns a list.
+        """
+        if not location:
+            return []
+        if isinstance(location, (list, tuple)):
+            strings = flatten([self._extract(o, **kwargs) for o in location])
+            if kwargs.get('remove_dupes', False):
+                strings = unique(strings)
+            if kwargs.get('first', False):
+                return strings[:1]
+            else:
+                return strings
+        # XPathSelector
+        elif isinstance(location, XPathSelector):
+            strings = [location.extract()]
+        # Strings
+        elif isinstance(location, unicode):
+            strings = [unquote_html(location, keep_reserved=True)]
+        elif isinstance(location, str):
+            encoding = kwargs.get("encoding", settings.get('DEFAULT_DATA_ENCODING'))
+            strings = [unquote_html(unicode(location, encoding), keep_reserved=True)]
+        else:
+            raise TypeError, "unsupported location type: %s" % type(location)
+
+        return strings
+        

scrapy/trunk/scrapy/item/models.py

@@ -1,3 +1,19 @@
+class BaseAdaptor:
+    def function(self, item, attrname, value, **pipeargs):
+        raise NotImplemented
+
+#default adaptors
+class ExtractAdaptor(BaseAdaptor):
+    def function(self, item, attrname, value, **pipeargs):
+        if hasattr(value, 'extract'):
+            value = value.extract()
+        return value
+
+class AssignAdaptor(BaseAdaptor):
+    def function(self, item, attrname, value, **pipeargs):
+        if not hasattr(item, attrname):
+            setattr(item, attrname, value)
+
 class ScrapedItem(object):
     """
     This is the base class for all scraped items.
@@ -6,11 +22,12 @@ class ScrapedItem(object):
     * guid (unique global indentifier)
     * url (URL where that item was scraped from)
     """
+    adaptors_pipe = [ExtractAdaptor(), AssignAdaptor()]
     
-    def assign(self, name, value):
+    def set_adaptors_pipe(adaptors_pipes):
-        """Assign an attribute. Can receive a Selector in value which will
+        ScrapedItem.adaptors_pipes = adaptors_pipes
-        extract its data """
 
-        if hasattr(value, 'extract'):
+    def attribute(self, name, value, **pipeargs):
-            value = value.extract()
+
-        setattr(self, name, value)
+        for adaptor in ScrapedItem.adaptors_pipe:
+            value = adaptor.function(self, name, value, **pipeargs)

scrapy/trunk/scrapy/templates/settings.tmpl

@@ -7,6 +7,8 @@ import __project_name__
 BOT_NAME = 'scrapy-bot'
 BOT_VERSION = '1.0'
 
+DEFAULT_DATA_ENCODING = 'ISO-8859-1'
+
 COMMANDS_MODULE = '__project_name__.commands'
 COMMANDS_SETTINGS_MODULE = '__project_name__.conf.commands'
 SPIDER_MODULES = ['__project_name__.spiders']

scrapy/trunk/scrapy/utils/misc.py

@@ -202,4 +202,12 @@ def extract_regex(regex, text, encoding):
     else:
         return [unquote_html(unicode(s, encoding), keep_reserved=True) for s in strings]
 
+_regex_type = type(re.compile("", 0))
 
+def location_str(location):
+    """Return a human friendly representation of a parser location"""
+
+    if isinstance(location, _regex_type):
+        return "(regex) " + location.pattern
+    else:
+        return location