diff --git a/scrapy/trunk/scrapy/item/adaptors.py b/scrapy/trunk/scrapy/item/adaptors.py index 3f17738f9..1db6c550e 100644 --- a/scrapy/trunk/scrapy/item/adaptors.py +++ b/scrapy/trunk/scrapy/item/adaptors.py @@ -15,26 +15,31 @@ class Adaptor(object): self.match_function = match_function def function(self, value): return self.basefunction(value) + def __repr__(self): + return "'%s' Adaptor" %self.name class AdaptorPipe: - def __init__(self, adaptors=None, adaptorclass=None): + def __init__(self, attribute_names, adaptors=None, adaptorclass=None): """ If "adaptors" is given, constructs pipeline from this. "adaptors" is an ordered tuple of 3-elements tuples, each of which has the same parameters you give to the insertadaptor method, except 'after' and 'before', because you define the adaptors order in the tuple. - . Example: + Example: ( (my_function, "my_function", lambda x: x in my_list) ... ) """ + self.__attribute_names = [ n for n in attribute_names ] self.__adaptorspipe = [] self.__adaptorclass = adaptorclass or Adaptor + self.pipes = {} if adaptors: for entry in adaptors: - self.insertadaptor(*entry) + self.insertadaptor(compile_pipe=False, *entry) + self._compile_pipe() @property def adaptors_names(self): @@ -43,7 +48,7 @@ class AdaptorPipe: _adaptors.append(a.name) return _adaptors - def insertadaptor(self, function, name, match_function=lambda x: True, after=None, before=None): + def insertadaptor(self, function, name, match_function=lambda x: True, after=None, before=None, compile_pipe=True): """ Inserts a "function" as an adaptor that will apply when match_function returns True (by default always apply) @@ -62,25 +67,38 @@ class AdaptorPipe: elif before: pos = self.adaptors_names.index(before) self.__adaptorspipe.insert(pos, adaptor) + if compile_pipe: + self._compile_pipe() return pos - def execute(self, match_condition, value, debug=False): + def removeadaptor(self, adaptorname): + pos = self.adaptors_names.index(adaptorname) + self.__adaptorspipe.pop(pos) + self._compile_pipe() + + def _compile_pipe(self): + for attrname in self.__attribute_names: + adaptors_pipe = [] + for adaptor in self.__adaptorspipe: + if adaptor.match_function(attrname): + adaptors_pipe.append(adaptor) + self.pipes[attrname] = adaptors_pipe + + def execute(self, attrname, value, debug=False): """ Execute pipeline for attribute name "attrname" and value "value". - Pass the given pipeargs to each adaptor function in the pipe. """ - for adaptor in self.__adaptorspipe: - if adaptor.match_function(match_condition): - try: - if debug: - print " %07s | input >" % adaptor.name, repr(value) - value = adaptor.function(value) - if debug: - print " %07s | output>" % adaptor.name, repr(value) + for adaptor in self.pipes.get(attrname, []): + try: + if debug: + print " %07s | input >" % adaptor.name, repr(value) + value = adaptor.function(value) + if debug: + print " %07s | output>" % adaptor.name, repr(value) - except Exception, e: - print "Error in '%s' adaptor. Traceback text:" % adaptor.name - print format_exc() - return + except Exception, e: + print "Error in '%s' adaptor. Traceback text:" % adaptor.name + print format_exc() + return return value diff --git a/scrapy/trunk/scrapy/item/models.py b/scrapy/trunk/scrapy/item/models.py index b556d8f3d..3c40182fb 100644 --- a/scrapy/trunk/scrapy/item/models.py +++ b/scrapy/trunk/scrapy/item/models.py @@ -3,14 +3,10 @@ from scrapy.item.adaptors import AdaptorPipe class ScrapedItem(object): """ This is the base class for all scraped items. - - The only required attributes are: - * guid (unique global indentifier) - * url (URL where that item was scraped from) """ - adaptors_pipe = AdaptorPipe() + adaptors_pipe = AdaptorPipe([]) - def attribute(self, attrname, value, match_condition=None): - value = self.adaptors_pipe.execute(match_condition or attrname, value) + def attribute(self, attrname, value): + value = self.adaptors_pipe.execute(attrname, value) if not hasattr(self, attrname): setattr(self, attrname, value)