diff --git a/scrapy/contrib_exp/newitem/adaptors.py b/scrapy/contrib_exp/newitem/adaptors.py index 42873e6ad..4121180d3 100644 --- a/scrapy/contrib_exp/newitem/adaptors.py +++ b/scrapy/contrib_exp/newitem/adaptors.py @@ -1,3 +1,5 @@ + +from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import get_func_args @@ -31,14 +33,12 @@ class ItemAdaptorMeta(type): adaptor = adaptize(attrs[item_field]) cls._field_adaptors[item_field] = adaptor setattr(cls, item_field, staticmethod(adaptor)) - return cls def __getattr__(cls, name): if name in cls.item_class.fields: return cls.default_adaptor - - raise AttributeError + raise AttributeError(name) class ItemAdaptor(object): @@ -55,13 +55,10 @@ class ItemAdaptor(object): def __setattr__(self, name, value): if (name.startswith('_') or name == 'item_instance' \ - or name == 'default_adaptor'): + or name == 'default_adaptor'): return object.__setattr__(self, name, value) - try: - fa = self._field_adaptors[name] - except KeyError: - fa = self.default_adaptor + fa = self._field_adaptors.get(name, self.default_adaptor) adaptor_args = {'response': self._response, 'item': self.item_instance} ovalue = fa(value, adaptor_args=adaptor_args) @@ -69,13 +66,13 @@ class ItemAdaptor(object): def __getattribute__(self, name): if (name.startswith('_') or name.startswith('item_') \ - or name == 'default_adaptor'): + or name == 'default_adaptor'): return object.__getattribute__(self, name) return getattr(self.item_instance, name) -def adaptor(*funcs, **adaptor_args): +def adaptor(*funcs, **default_adaptor_args): """A pipe adaptor implementing the tree adaption logic It takes multiples unnamed arguments used as functions of the pipe, and @@ -87,32 +84,29 @@ def adaptor(*funcs, **adaptor_args): Always returns a list of values """ - pipe_adaptor_args = adaptor_args _funcs = [] for func in funcs: - takes_args = 'adaptor_args' in get_func_args(func) - _funcs.append((func, takes_args)) + accepts_args = 'adaptor_args' in get_func_args(func) + _funcs.append((func, accepts_args)) def _adaptor(value, adaptor_args=None): - values = value if isinstance(value, (list, tuple)) else [value] - aargs = dict(t for d in [pipe_adaptor_args, adaptor_args or {}] for t in d.items()) + values = arg_to_iter(value) + aargs = default_adaptor_args + if adaptor_args: + aargs = aargs.copy() + aargs.update(adaptor_args) pipe_kwargs = {'adaptor_args': aargs} - - for func, takes_args in _funcs: + for func, accepts_args in _funcs: next = [] - kwargs = pipe_kwargs if takes_args else {} - + kwargs = pipe_kwargs if accepts_args else {} for val in values: val = func(val, **kwargs) - - if isinstance(val, (list, tuple)): + if hasattr(val, '__iter__'): next.extend(val) elif val is not None: next.append(val) - values = next return list(values) return _adaptor -