Improved adaptors functionality:

- Added many basic adaptors (like extract, extract_links, regex, etc.)
- Added some basic pipelines (for single data, lists, urls, etc.)
- Now XPathSelectors store the response with which they were created (if any)

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40315

scrapy/trunk/scrapy/item/adaptors.py

@@ -1,13 +1,29 @@
+import re
+
 from traceback import format_exc
 
-class AdaptorPipe:
+from scrapy.xpath.selector import XPathSelector, XPathSelectorList
+from scrapy.utils.python import unique, flatten
+from scrapy.utils.markup import replace_tags, remove_entities
+from scrapy.utils.misc import extract_regex
 
-    def __init__(self, adaptors_dict=None):
+class AdaptorPipe:
+    def __init__(self, adaptors_pipe=None):
         """
         Receives a dictionary that maps attribute_name to a list of adaptor functions
         """
-        self.pipes = adaptors_dict or {}
+        self.pipes = adaptors_pipe or {}
 
+    def append_adaptor(self, attrname, adaptor):
+        """
+        Add an adaptor at the end of the provided attribute's pipeline
+        """
+        if callable(adaptor):
+            if self.pipes.get(attrname):
+                self.pipes[attrname].append(adaptor)
+            else:
+                self.pipes[attrname] = [adaptor]
+                
     def execute(self, attrname, value, debug=False):
         """
         Execute pipeline for attribute name "attrname" and value "value".
@@ -26,3 +42,192 @@ class AdaptorPipe:
                 return
 
         return value
+
+
+############
+# Adaptors #
+############
+def extract(location):
+    """
+    This adaptor extracts a list of strings
+    from 'location', which can be either a list (or tuple),
+    or an XPathSelector.
+    
+    This function *always* returns a list.
+    """
+    if not location:
+        return []
+    elif isinstance(location, (list, tuple)):
+        return flatten([extract(o) for o in location])
+    elif isinstance(location, XPathSelector):
+        return location.extract()
+    elif isinstance(location, str):
+        return [location]
+
+def _absolutize_links(rel_links, current_url, base_url):
+    abs_links = []
+    for link in rel_links:
+        if link.startswith('/'):
+            abs_links.append('%s%s' % (base_url, link))
+        elif link.startswith('http://'):
+            abs_links.append(link)
+        else:
+            abs_links.append('%s/%s' % (current_url, link))
+    return abs_links
+    
+def extract_links(locations):
+    """
+    This adaptor receives either an XPathSelector containing
+    the desired locations for finding urls, or a tuple like (xpath, regexp)
+    containing the xpath locations to look in, and a regular expression
+    to parse those locations.
+    
+    In any case, this adaptor returns a list of absolute urls extracted.
+    """
+    ret = []
+    if locations:
+        regexp = None
+        if isinstance(locations, XPathSelector):
+            locations = XPathSelectorList([locations])
+        elif isinstance(locations, tuple):
+            locations, regexp = locations
+    
+        if isinstance(locations, XPathSelectorList):
+            if regexp:
+                ret = locations.re(regexp)
+            else:
+                for selector in locations:
+                    if selector.xmlNode.type == 'element':
+                        if selector.xmlNode.name == 'a':
+                            children = selector.x('child::*')
+                            if len(children) > 1:
+                                ret.extend(selector.x('.//@href'))
+                                ret.extend(selector.x('.//@src'))
+                            else:
+                                ret.extend(selector.x('@href'))
+                        elif selector.xmlNode.name == 'img':
+                            ret.extend(selector.x('@src'))
+                        else:
+                            ret.extend(selector.x('.//@href'))
+                            ret.extend(selector.x('.//@src'))
+                    elif selector.xmlNode.type == 'attribute' and selector.xmlNode.name in ['href', 'src']:
+                        ret.append(selector)
+                ret = [selector.extract() for selector in ret]
+            current_url, base_url = re.search(r'((http://(?:www\.)?[\w\d\.-]+?)(?:/|$).*)/', locations[0].response.url).groups()
+            ret = _absolutize_links(ret, current_url, base_url)
+    return ret
+
+def to_unicode(value):
+    """
+    Receives a list of strings, converts
+    it to unicode, and returns a new list.
+    """
+    if isinstance(value, (list, tuple)):
+        return [ unicode(v) for v in value ]
+    else:
+        raise TypeError('to_unicode adaptor must receive a list or a tuple.')
+    
+def regex(expr):
+    """
+    This factory function returns a ready-to-use
+    adaptor for the specified regular expression.
+    This adaptor will accept either an XPathSelectorList
+    or a list of strings, and will apply the provided regular
+    expression to each of its members.
+    
+    This adaptor always returns a list of strings.
+    """
+    def _regex(value):
+        if isinstance(value, (XPathSelector, XPathSelectorList)):
+            return value.re(expr)
+        elif isinstance(value, list) and value:
+            return extract_regex(expr, value, 'utf-8')
+        return value
+    return _regex
+
+def unquote_all(value):
+    """
+    Receives a list of strings, removes all of the
+    entities the strings may have, and returns
+    a new list
+    """
+    return [ remove_entities(v) for v in value ]
+
+def unquote(value):
+    """
+    Receives a list of strings, removes all of the entities
+    the strings may have (except for < and &), and
+    returns a new list
+    """
+    return [ remove_entities(v, keep=['lt', 'amp']) for v in value ]
+
+def remove_tags(value):
+    return [ replace_tags(v) for v in value ]
+    
+_remove_root_re = re.compile(r'^\s*<.*?>(.*)</.*>\s*$', re.DOTALL)
+def _remove_root(value):
+    m = _remove_root_re.search(value)
+    if m:
+        value = m.group(1)
+    return value
+
+def remove_root(value):
+    return [ _remove_root(v) for v in value ]
+    
+_clean_spaces_re = re.compile("\s+", re.U)
+def remove_multispaces(value):
+    return [ _clean_spaces_re.sub(' ', v) for v in value ]
+
+def strip(value):
+    return [ v.strip() for v in value ]
+
+def drop_empty_elements(value):
+    return [ v for v in value if v ]
+
+def delist(value):
+    return ' '.join(value)
+    
+
+
+#############
+# Pipelines #
+#############
+"""
+The following methods automatically generate adaptor pipelines
+for some basic datatypes, according to the parameters you pass them.
+"""
+def single_pipeline(do_unquote=True):
+    pipe = [ extract,
+             unique,
+             to_unicode,
+             drop_empty_elements,
+             remove_tags,
+             remove_root,
+             remove_multispaces,
+             strip,
+           ]
+    if do_unquote:
+        pipe.append(unquote)
+    return pipe + [delist]
+           
+def url_pipeline():
+    return [ extract_links,
+             unique,
+             to_unicode,
+             drop_empty_elements,
+           ]
+               
+def list_pipeline():
+    return [ extract,
+             unique,
+             to_unicode,
+             drop_empty_elements,
+             unquote,
+             remove_tags,
+             remove_root,
+             strip,
+           ]
+                
+def list_join_pipeline(delimiter='\t'):
+    return list_pipeline() + [delimiter.join]
+    

scrapy/trunk/scrapy/item/models.py

@@ -7,15 +7,18 @@ class ScrapedItem(object):
     that identifies uniquely the given scraped item.
     """
 
-    def setadaptors(self, adaptors_dict):
-        """Set adaptors to use for this item. Receives a dict of adaptors and
-        returns the item itself"""
-        self.adaptors_dict = AdaptorPipe(adaptors_dict)
+    def setadaptors(self, adaptors_pipe):
+        """
+        Set adaptors to use for this item. Receives a dict of adaptors and
+        returns the item itself.
+        """
+        object.__setattr__(self, '_adaptors_pipe', AdaptorPipe(adaptors_pipe))
         return self
-
+    
+    def append_adaptor(self, attrname, adaptor):
+        self._adaptors_pipe.append_adaptor(attrname, adaptor)
+            
     def attribute(self, attrname, value):
-        value = self.adaptors_pipe.execute(attrname, value)
-        if not hasattr(self, attrname):
+        value = self._adaptors_pipe.execute(attrname, value)
+        if not hasattr(self, attrname) or not getattr(self, attrname):
             setattr(self, attrname, value)
-
-                

scrapy/trunk/scrapy/xpath/selector.py

@@ -32,6 +32,7 @@ class XPathSelector(object):
             self.doc = Libxml2Document(response, constructor=constructor)
             self.xmlNode = self.doc.xmlDoc
         self.expr = expr
+        self.response = response
 
     def x(self, xpath):
         """Perform the given XPath query on the current XPathSelector and
@@ -41,9 +42,10 @@ class XPathSelector(object):
             xpath_result = self.doc.xpathContext.xpathEval(xpath)
             cls = type(self)
             if hasattr(xpath_result, '__iter__'):
-                return XPathSelectorList([cls(node=node, parent=self, expr=xpath) for node in xpath_result])
+                return XPathSelectorList([cls(node=node, parent=self, expr=xpath, response=self.response)
+                                          for node in xpath_result])
             else:
-                return XPathSelectorList([cls(node=xpath_result, parent=self, expr=xpath)])
+                return XPathSelectorList([cls(node=xpath_result, parent=self, expr=xpath, response=self.response)])
         else:
             return XPathSelectorList([])