diff --git a/scrapy/trunk/scrapy/contrib_exp/newitem/extractors.py b/scrapy/trunk/scrapy/contrib_exp/newitem/extractors.py index 09b36a9ca..5a9ff3bc9 100644 --- a/scrapy/trunk/scrapy/contrib_exp/newitem/extractors.py +++ b/scrapy/trunk/scrapy/contrib_exp/newitem/extractors.py @@ -70,7 +70,18 @@ class ExtractorField(object): return list(values) -def treeadapt(*funcs, **adaptor_args): +def adaptor(*funcs, **adaptor_args): + """A pipe adaptor implementing the tree adaption logic + + It takes multiples unnamed arguments used as functions of the pipe, and a + keywords used as adaptor_args to be passed to functions that supports it + + If an adaptor function returns a list of values, each value is used as + input for next adaptor function + + Always returns a list of values + """ + pipe_adaptor_args = adaptor_args _funcs = [] for func in funcs: @@ -79,8 +90,8 @@ def treeadapt(*funcs, **adaptor_args): def _adaptor(value, adaptor_args=None): values = value if isinstance(value, (list, tuple)) else [value] - pipe_adaptor_args.update(adaptor_args or {}) - pipe_kwargs = {'adaptor_args': pipe_adaptor_args} + aargs = dict(t for d in [pipe_adaptor_args, adaptor_args or {}] for t in d.iteritems()) + pipe_kwargs = { 'adaptor_args': aargs } for func, takes_args in _funcs: next = [] diff --git a/scrapy/trunk/scrapy/tests/test_itemextractor.py b/scrapy/trunk/scrapy/tests/test_itemextractor.py new file mode 100644 index 000000000..47b0e420f --- /dev/null +++ b/scrapy/trunk/scrapy/tests/test_itemextractor.py @@ -0,0 +1,43 @@ + +import unittest +from scrapy.contrib_exp.newitem.extractors import adaptor, ItemExtractor + + +class TreeadaptTest(unittest.TestCase): + + def test_1_passtrough(self): + ad = adaptor() + self.assertEqual(ad('string'), ['string']) + + def test_2_composing(self): + addone = lambda v: v+1 + ad = adaptor(addone) + self.assertEqual(ad(0), [1]) + + addtwo = lambda v: v+2 + ad2 = adaptor(addone, addtwo) + self.assertEqual(ad2(0), [3]) + + ad3 = adaptor(ad, ad2) + self.assertEqual(ad3(0), [4]) + + def test_3_adaptor_args(self): + addn = lambda v, adaptor_args: v + adaptor_args.get('add_value', 0) + + ad = adaptor(addn) + self.assertEqual(ad(0), [0]) + self.assertEqual(ad(0, {'add_value': 3}), [3]) + self.assertEqual(ad(0), [0]) + + ad = adaptor(addn, add_value=5) + self.assertEqual(ad(0), [5]) + self.assertEqual(ad(0, {'add_value': 3}), [3]) + self.assertEqual(ad(0), [5]) + + + def test_4_treelogic(self): + split1 = lambda v: v.split('&') + split2 = lambda v: v.split('=') + ad = adaptor(split1, split2) + self.assertEqual(ad('name=joe&job=joker'), ['name', 'joe', 'job', 'joker']) + diff --git a/scrapy/trunk/scrapy/utils/python.py b/scrapy/trunk/scrapy/utils/python.py index 06ba3ccd4..db45bb843 100644 --- a/scrapy/trunk/scrapy/utils/python.py +++ b/scrapy/trunk/scrapy/utils/python.py @@ -245,4 +245,6 @@ def get_func_args(func): func_args, _, _, _ = inspect.getargspec(func.__call__) except Exception: func_args = [] + else: + raise TypeError('%s is not callable' % type(func)) return func_args