From ddfaf6049fa036b7cec4731d8f772c5afdb57ccb Mon Sep 17 00:00:00 2001 From: Ping Yin Date: Fri, 23 Apr 2010 01:42:55 +0800 Subject: [PATCH] {add,replace}_value: add processors args and allow field_name to be None * value is first proccessed by processors before passing to input processor * if field_name is None, values for multiple fields may be added/replaced. The keys of the processed value are as the field names * add get_value function for the processor logic Signed-off-by: Ping Yin --- scrapy/contrib/loader/__init__.py | 37 +++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/scrapy/contrib/loader/__init__.py b/scrapy/contrib/loader/__init__.py index 88493b5d3..92659f619 100644 --- a/scrapy/contrib/loader/__init__.py +++ b/scrapy/contrib/loader/__init__.py @@ -25,14 +25,43 @@ class ItemLoader(object): self.context = context self._values = defaultdict(list) - def add_value(self, field_name, value): + def add_value(self, field_name, value, *processors): + value = self.get_value(value, *processors) + if not value: + return + if not field_name: + for k,v in value.iteritems(): + self._add_value(k, v) + else: + self._add_value(field_name, value) + + def replace_value(self, field_name, value, *processors): + value = self.get_value(value, *processors) + if not value: + return + if not field_name: + for k,v in value.iteritems(): + self._replace_value(k, v) + else: + self._replace_value(field_name, value) + + def _add_value(self, field_name, value): value = arg_to_iter(value) processed_value = self._process_input_value(field_name, value) - self._values[field_name] += arg_to_iter(processed_value) + if processed_value: + self._values[field_name] += arg_to_iter(processed_value) - def replace_value(self, field_name, value): + def _replace_value(self, field_name, value): self._values.pop(field_name, None) - self.add_value(field_name, value) + self._add_value(field_name, value) + + def get_value(self, value, *processors): + for proc in processors: + if value is None: + break + proc = wrap_loader_context(proc, self.context) + value = proc(value) + return value def load_item(self): item = self.item