From da7b87962a8f08482a96e2e033e39326ce9412a3 Mon Sep 17 00:00:00 2001
From: olveyra <none@none>
Date: Wed, 10 Sep 2008 14:06:28 +0000
Subject: [PATCH] - AdaptorPipe compilation feature to resolve the problem of
 efficience and mantain the simplicity of the actual pipeline definition
 approach (for people that prefer to bore with lots of lines to define and
 edit a pipeline, they can edit directly the compiled version)

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40220
---
 scrapy/trunk/scrapy/item/adaptors.py | 54 ++++++++++++++++++----------
 scrapy/trunk/scrapy/item/models.py   | 10 ++----
 2 files changed, 39 insertions(+), 25 deletions(-)

diff --git a/scrapy/trunk/scrapy/item/adaptors.py b/scrapy/trunk/scrapy/item/adaptors.py
index 3f17738f9..1db6c550e 100644
--- a/scrapy/trunk/scrapy/item/adaptors.py
+++ b/scrapy/trunk/scrapy/item/adaptors.py
@@ -15,26 +15,31 @@ class Adaptor(object):
         self.match_function = match_function
     def function(self, value):
         return self.basefunction(value)
+    def __repr__(self):
+        return "'%s' Adaptor" %self.name
         
 class AdaptorPipe:
 
-    def __init__(self, adaptors=None, adaptorclass=None):
+    def __init__(self, attribute_names, adaptors=None, adaptorclass=None):
         """
         If "adaptors" is given, constructs pipeline from this.
         "adaptors" is an ordered tuple of 3-elements tuples, each of which
         has the same parameters you give to the insertadaptor method, except 
         'after' and 'before', because you define the adaptors order in the tuple.
-        . Example:
+        Example:
         (
           (my_function, "my_function", lambda x: x in my_list)
           ...
         )
         """
+        self.__attribute_names = [ n for n in attribute_names ]
         self.__adaptorspipe = []
         self.__adaptorclass = adaptorclass or Adaptor
+        self.pipes = {}
         if adaptors:
             for entry in adaptors:
-                self.insertadaptor(*entry)
+                self.insertadaptor(compile_pipe=False, *entry)
+            self._compile_pipe()
 
     @property
     def adaptors_names(self):
@@ -43,7 +48,7 @@ class AdaptorPipe:
             _adaptors.append(a.name)
         return _adaptors
     
-    def insertadaptor(self, function, name, match_function=lambda x: True, after=None, before=None):
+    def insertadaptor(self, function, name, match_function=lambda x: True, after=None, before=None, compile_pipe=True):
         """
         Inserts a "function" as an adaptor that will apply when match_function returns True (by
         default always apply)
@@ -62,25 +67,38 @@ class AdaptorPipe:
             elif before:
                 pos = self.adaptors_names.index(before)
             self.__adaptorspipe.insert(pos, adaptor)
+            if compile_pipe:
+                self._compile_pipe()
             return pos
 
-    def execute(self, match_condition, value, debug=False):
+    def removeadaptor(self, adaptorname):
+        pos = self.adaptors_names.index(adaptorname)
+        self.__adaptorspipe.pop(pos)
+        self._compile_pipe()
+
+    def _compile_pipe(self):
+        for attrname in self.__attribute_names:
+            adaptors_pipe = []
+            for adaptor in self.__adaptorspipe:
+                if adaptor.match_function(attrname):
+                    adaptors_pipe.append(adaptor)
+            self.pipes[attrname] = adaptors_pipe
+            
+    def execute(self, attrname, value, debug=False):
         """
         Execute pipeline for attribute name "attrname" and value "value".
-        Pass the given pipeargs to each adaptor function in the pipe.
         """
-        for adaptor in self.__adaptorspipe:
-            if adaptor.match_function(match_condition):
-                try:
-                    if debug:
-                        print "  %07s | input >" % adaptor.name, repr(value)
-                    value = adaptor.function(value)
-                    if debug:
-                        print "  %07s | output>" % adaptor.name, repr(value)
+        for adaptor in self.pipes.get(attrname, []):
+            try:
+                if debug:
+                    print "  %07s | input >" % adaptor.name, repr(value)
+                value = adaptor.function(value)
+                if debug:
+                    print "  %07s | output>" % adaptor.name, repr(value)
 
-                except Exception, e:
-                    print "Error in '%s' adaptor. Traceback text:" % adaptor.name
-                    print format_exc()
-                    return
+            except Exception, e:
+                print "Error in '%s' adaptor. Traceback text:" % adaptor.name
+                print format_exc()
+                return
 
         return value
diff --git a/scrapy/trunk/scrapy/item/models.py b/scrapy/trunk/scrapy/item/models.py
index b556d8f3d..3c40182fb 100644
--- a/scrapy/trunk/scrapy/item/models.py
+++ b/scrapy/trunk/scrapy/item/models.py
@@ -3,14 +3,10 @@ from scrapy.item.adaptors import AdaptorPipe
 class ScrapedItem(object):
     """
     This is the base class for all scraped items.
-
-    The only required attributes are:
-    * guid (unique global indentifier)
-    * url (URL where that item was scraped from)
     """
-    adaptors_pipe = AdaptorPipe()
+    adaptors_pipe = AdaptorPipe([])
 
-    def attribute(self, attrname, value, match_condition=None):
-        value = self.adaptors_pipe.execute(match_condition or attrname, value)
+    def attribute(self, attrname, value):
+        value = self.adaptors_pipe.execute(attrname, value)
         if not hasattr(self, attrname):
             setattr(self, attrname, value)