- AdaptorPipe compilation feature to resolve the problem of efficience

and mantain the simplicity of the actual pipeline definition approach (for
people that prefer to bore with lots of lines to define and edit a
pipeline, they can edit directly the compiled version)

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40220

scrapy/trunk/scrapy/item/adaptors.py

@@ -15,26 +15,31 @@ class Adaptor(object):
         self.match_function = match_function
     def function(self, value):
         return self.basefunction(value)
+    def __repr__(self):
+        return "'%s' Adaptor" %self.name
         
 class AdaptorPipe:
 
-    def __init__(self, adaptors=None, adaptorclass=None):
+    def __init__(self, attribute_names, adaptors=None, adaptorclass=None):
         """
         If "adaptors" is given, constructs pipeline from this.
         "adaptors" is an ordered tuple of 3-elements tuples, each of which
         has the same parameters you give to the insertadaptor method, except 
         'after' and 'before', because you define the adaptors order in the tuple.
-        . Example:
+        Example:
         (
           (my_function, "my_function", lambda x: x in my_list)
           ...
         )
         """
+        self.__attribute_names = [ n for n in attribute_names ]
         self.__adaptorspipe = []
         self.__adaptorclass = adaptorclass or Adaptor
+        self.pipes = {}
         if adaptors:
             for entry in adaptors:
-                self.insertadaptor(*entry)
+                self.insertadaptor(compile_pipe=False, *entry)
+            self._compile_pipe()
 
     @property
     def adaptors_names(self):
@@ -43,7 +48,7 @@ class AdaptorPipe:
             _adaptors.append(a.name)
         return _adaptors
     
-    def insertadaptor(self, function, name, match_function=lambda x: True, after=None, before=None):
+    def insertadaptor(self, function, name, match_function=lambda x: True, after=None, before=None, compile_pipe=True):
         """
         Inserts a "function" as an adaptor that will apply when match_function returns True (by
         default always apply)
@@ -62,25 +67,38 @@ class AdaptorPipe:
             elif before:
                 pos = self.adaptors_names.index(before)
             self.__adaptorspipe.insert(pos, adaptor)
+            if compile_pipe:
+                self._compile_pipe()
             return pos
 
-    def execute(self, match_condition, value, debug=False):
+    def removeadaptor(self, adaptorname):
+        pos = self.adaptors_names.index(adaptorname)
+        self.__adaptorspipe.pop(pos)
+        self._compile_pipe()
+
+    def _compile_pipe(self):
+        for attrname in self.__attribute_names:
+            adaptors_pipe = []
+            for adaptor in self.__adaptorspipe:
+                if adaptor.match_function(attrname):
+                    adaptors_pipe.append(adaptor)
+            self.pipes[attrname] = adaptors_pipe
+            
+    def execute(self, attrname, value, debug=False):
         """
         Execute pipeline for attribute name "attrname" and value "value".
-        Pass the given pipeargs to each adaptor function in the pipe.
         """
-        for adaptor in self.__adaptorspipe:
-            if adaptor.match_function(match_condition):
-                try:
-                    if debug:
-                        print "  %07s | input >" % adaptor.name, repr(value)
-                    value = adaptor.function(value)
-                    if debug:
-                        print "  %07s | output>" % adaptor.name, repr(value)
+        for adaptor in self.pipes.get(attrname, []):
+            try:
+                if debug:
+                    print "  %07s | input >" % adaptor.name, repr(value)
+                value = adaptor.function(value)
+                if debug:
+                    print "  %07s | output>" % adaptor.name, repr(value)
 
-                except Exception, e:
-                    print "Error in '%s' adaptor. Traceback text:" % adaptor.name
-                    print format_exc()
-                    return
+            except Exception, e:
+                print "Error in '%s' adaptor. Traceback text:" % adaptor.name
+                print format_exc()
+                return
 
         return value

scrapy/trunk/scrapy/item/models.py

@@ -3,14 +3,10 @@ from scrapy.item.adaptors import AdaptorPipe
 class ScrapedItem(object):
     """
     This is the base class for all scraped items.
-
-    The only required attributes are:
-    * guid (unique global indentifier)
-    * url (URL where that item was scraped from)
     """
-    adaptors_pipe = AdaptorPipe()
+    adaptors_pipe = AdaptorPipe([])
 
-    def attribute(self, attrname, value, match_condition=None):
-        value = self.adaptors_pipe.execute(match_condition or attrname, value)
+    def attribute(self, attrname, value):
+        value = self.adaptors_pipe.execute(attrname, value)
         if not hasattr(self, attrname):
             setattr(self, attrname, value)