From 9081e84e27ef49b4b10cfafddceca8e3aade4fdd Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Tue, 4 Aug 2009 20:02:49 -0300 Subject: [PATCH] ItemLoader: sorted out module locations, and added more tests --HG-- rename : scrapy/newitem/reducers.py => scrapy/newitem/loader/reducers.py --- scrapy/newitem/loader/__init__.py | 85 +++------------- scrapy/newitem/loader/expanders.py | 62 ++++++++++++ scrapy/newitem/{ => loader}/reducers.py | 4 +- scrapy/tests/test_itemloader.py | 123 ++++++++++++++---------- 4 files changed, 152 insertions(+), 122 deletions(-) create mode 100644 scrapy/newitem/loader/expanders.py rename scrapy/newitem/{ => loader}/reducers.py (87%) diff --git a/scrapy/newitem/loader/__init__.py b/scrapy/newitem/loader/__init__.py index 9403140a9..2ff6ecfd9 100644 --- a/scrapy/newitem/loader/__init__.py +++ b/scrapy/newitem/loader/__init__.py @@ -1,64 +1,7 @@ from collections import defaultdict -from types import UnboundMethodType -from scrapy.utils.misc import arg_to_iter -from scrapy.utils.python import get_func_args -from scrapy.utils.datatypes import MergeDict from scrapy.newitem import Item -from scrapy.newitem.reducers import take_first - -def tree_expander(*functions, **default_loader_args): - """Create an ItemLoader expander from a list of functions using the tree - expansion algorithm described below. - - The functions can optionally accept a ``loader_args`` argument which (if - present) will be used to pass loader arguments when the function is called. - - The tree expansion algorithm consists in an ordered list of functions, each - of which receives one value and can return zero, one or more values (as a - list or iterable). If a function returns more than one value, the next - function in the pipeline will be called with each of those values, - potentially returning more values. Hence the name "tree expansion - algorithm". - """ - def wrap_y(f): - return lambda x, y, z: f(y) - - def wrap_xy(f): - return lambda x, y, z: f(x, y) - - def wrap_yz(f): - return lambda x, y, z: f(y, z) - - wrapped_funcs = [] - for func in functions: - if isinstance(func, UnboundMethodType): - func = func.im_func - if 'loader_args' in get_func_args(func): - wfunc = func - else: - wfunc = wrap_xy(func) - else: - if 'loader_args' in get_func_args(func): - wfunc = wrap_yz(func) - else: - wfunc = wrap_y(func) - wrapped_funcs.append(wfunc) - - def _expander(loader, value, loader_args): - values = arg_to_iter(value) - largs = default_loader_args - if loader_args: - largs = MergeDict(loader_args, default_loader_args) - for func in wrapped_funcs: - next_values = [] - for v in values: - next_values += arg_to_iter(func(loader, v, largs)) - values = next_values - return list(values) - - return _expander - +from scrapy.newitem.loader.reducers import take_first class ItemLoader(object): @@ -71,23 +14,28 @@ class ItemLoader(object): self._values = defaultdict(list) def add_value(self, field_name, value, **new_loader_args): - evalue = self._expand_value(field_name, value, new_loader_args) - self._values[field_name].extend(evalue) + self._values[field_name] += self._expand_value(field_name, value, \ + new_loader_args) def replace_value(self, field_name, value, **new_loader_args): - evalue = self._expand_value(field_name, value, new_loader_args) - self._values[field_name] = evalue + self._values[field_name] = self._expand_value(field_name, value, \ + new_loader_args) def get_item(self): item = self._item for field_name in self._values: - item[field_name] = self.get_value(field_name) + item[field_name] = self.get_reduced_value(field_name) return item - def get_value(self, field_name): - values = self._values[field_name] + def get_expanded_value(self, field_name): + return self._values[field_name] + + def get_reduced_value(self, field_name): reducer = self.get_reducer(field_name) - return reducer(values) + return reducer(self._values[field_name]) + + def get_expander(self, field_name): + return getattr(self, 'expand_%s' % field_name, self.expand) def get_reducer(self, field_name): try: @@ -95,9 +43,6 @@ class ItemLoader(object): except AttributeError: return self._item.fields[field_name].get('reducer', self.reduce) - def get_expander(self, field_name): - return getattr(self, 'expand_%s' % field_name, self.expand) - def _expand_value(self, field_name, value, new_loader_args): if new_loader_args: loader_args = self._loader_args.copy() @@ -110,5 +55,5 @@ class ItemLoader(object): def expand(self, value, loader_args): # default expander return value - def reduce(self, values): + def reduce(self, values): # default reducer return take_first(values) diff --git a/scrapy/newitem/loader/expanders.py b/scrapy/newitem/loader/expanders.py new file mode 100644 index 000000000..fbab377d4 --- /dev/null +++ b/scrapy/newitem/loader/expanders.py @@ -0,0 +1,62 @@ +""" +ItemLoader expanders +""" + +from types import UnboundMethodType + +from scrapy.utils.misc import arg_to_iter +from scrapy.utils.python import get_func_args +from scrapy.utils.datatypes import MergeDict + +def tree_expander(*functions, **default_loader_args): + """Create an ItemLoader expander from a list of functions using the tree + expansion algorithm described below. + + The functions can optionally accept a ``loader_args`` argument which (if + present) will be used to pass loader arguments when the function is called. + + The tree expansion algorithm consists in an ordered list of functions, each + of which receives one value and can return zero, one or more values (as a + list or iterable). If a function returns more than one value, the next + function in the pipeline will be called with each of those values, + potentially returning more values. Hence the name "tree expansion + algorithm". + """ + def wrap_y(f): + return lambda x, y, z: f(y) + + def wrap_xy(f): + return lambda x, y, z: f(x, y) + + def wrap_yz(f): + return lambda x, y, z: f(y, z) + + wrapped_funcs = [] + for func in functions: + if isinstance(func, UnboundMethodType): + func = func.im_func + if 'loader_args' in get_func_args(func): + wfunc = func + else: + wfunc = wrap_xy(func) + else: + if 'loader_args' in get_func_args(func): + wfunc = wrap_yz(func) + else: + wfunc = wrap_y(func) + wrapped_funcs.append(wfunc) + + def _expander(loader, value, loader_args): + values = arg_to_iter(value) + largs = default_loader_args + if loader_args: + largs = MergeDict(loader_args, default_loader_args) + for func in wrapped_funcs: + next_values = [] + for v in values: + next_values += arg_to_iter(func(loader, v, largs)) + values = next_values + return list(values) + + return _expander + diff --git a/scrapy/newitem/reducers.py b/scrapy/newitem/loader/reducers.py similarity index 87% rename from scrapy/newitem/reducers.py rename to scrapy/newitem/loader/reducers.py index e1394488f..f0b78cb21 100644 --- a/scrapy/newitem/reducers.py +++ b/scrapy/newitem/loader/reducers.py @@ -1,4 +1,6 @@ -"""Some common reducers""" +""" +ItemLoader reducers +""" def take_first(values): for value in values: diff --git a/scrapy/tests/test_itemloader.py b/scrapy/tests/test_itemloader.py index 56faaf1de..fa5d777c9 100644 --- a/scrapy/tests/test_itemloader.py +++ b/scrapy/tests/test_itemloader.py @@ -1,6 +1,7 @@ import unittest -from scrapy.newitem.loader import ItemLoader, tree_expander +from scrapy.newitem.loader import ItemLoader +from scrapy.newitem.loader.expanders import tree_expander from scrapy.newitem import Item, Field @@ -31,104 +32,124 @@ class InheritDefaultedItemLoader(DefaultedItemLoader): class ItemLoaderTest(unittest.TestCase): - def test_basic(self): - ib = TestItemLoader() + def test_get_item(self): + il = TestItemLoader() - ib.add_value('name', u'marta') - self.assertEqual(ib.get_value('name'), u'Marta') - - item = ib.get_item() + il.add_value('name', u'marta') + item = il.get_item() self.assertEqual(item['name'], u'Marta') + def test_add_value(self): + il = TestItemLoader() + + il.add_value('name', u'marta') + self.assertEqual(il.get_expanded_value('name'), [u'Marta']) + self.assertEqual(il.get_reduced_value('name'), u'Marta') + + il.add_value('name', u'pepe') + self.assertEqual(il.get_expanded_value('name'), [u'Marta', u'Pepe']) + self.assertEqual(il.get_reduced_value('name'), u'Marta') + + def test_replace_value(self): + il = TestItemLoader() + + il.replace_value('name', u'marta') + self.assertEqual(il.get_expanded_value('name'), [u'Marta']) + self.assertEqual(il.get_reduced_value('name'), u'Marta') + + il.replace_value('name', u'pepe') + self.assertEqual(il.get_expanded_value('name'), [u'Pepe']) + self.assertEqual(il.get_reduced_value('name'), u'Pepe') + def test_multiple_functions(self): class TestItemLoader(BaseItemLoader): expand_name = tree_expander(lambda v: v.title(), lambda v: v[:-1]) - ib = TestItemLoader() + il = TestItemLoader() - ib.add_value('name', u'marta') - self.assertEqual(ib.get_value('name'), u'Mart') + il.add_value('name', u'marta') + self.assertEqual(il.get_reduced_value('name'), u'Mart') - item = ib.get_item() + item = il.get_item() self.assertEqual(item['name'], u'Mart') def test_defaulted(self): - dib = DefaultedItemLoader() + dil = DefaultedItemLoader() - dib.add_value('name', u'marta') - self.assertEqual(dib.get_value('name'), u'mart') + dil.add_value('name', u'marta') + self.assertEqual(dil.get_reduced_value('name'), u'mart') def test_inherited_default(self): - dib = InheritDefaultedItemLoader() + dil = InheritDefaultedItemLoader() - dib.add_value('name', u'marta') - self.assertEqual(dib.get_value('name'), u'mart') + dil.add_value('name', u'marta') + self.assertEqual(dil.get_reduced_value('name'), u'mart') def test_inheritance(self): class ChildItemLoader(TestItemLoader): expand_url = tree_expander(lambda v: v.lower()) - ib = ChildItemLoader() + il = ChildItemLoader() - ib.add_value('url', u'HTTP://scrapy.ORG') - self.assertEqual(ib.get_value('url'), u'http://scrapy.org') + il.add_value('url', u'HTTP://scrapy.ORG') + self.assertEqual(il.get_reduced_value('url'), u'http://scrapy.org') - ib.add_value('name', u'marta') - self.assertEqual(ib.get_value('name'), u'Marta') + il.add_value('name', u'marta') + self.assertEqual(il.get_reduced_value('name'), u'Marta') class ChildChildItemLoader(ChildItemLoader): expand_url = tree_expander(lambda v: v.upper()) expand_summary = tree_expander(lambda v: v) - ib = ChildChildItemLoader() + il = ChildChildItemLoader() - ib.add_value('url', u'http://scrapy.org') - self.assertEqual(ib.get_value('url'), u'HTTP://SCRAPY.ORG') + il.add_value('url', u'http://scrapy.org') + self.assertEqual(il.get_reduced_value('url'), u'HTTP://SCRAPY.ORG') - ib.add_value('name', u'marta') - self.assertEqual(ib.get_value('name'), u'Marta') + il.add_value('name', u'marta') + self.assertEqual(il.get_reduced_value('name'), u'Marta') def test_identity(self): class IdentityDefaultedItemLoader(DefaultedItemLoader): expand_name = tree_expander() - ib = IdentityDefaultedItemLoader() + il = IdentityDefaultedItemLoader() - ib.add_value('name', u'marta') - self.assertEqual(ib.get_value('name'), u'marta') + il.add_value('name', u'marta') + self.assertEqual(il.get_reduced_value('name'), u'marta') def test_staticmethods(self): class ChildItemLoader(TestItemLoader): expand_name = tree_expander(TestItemLoader.expand_name, unicode.swapcase) - ib = ChildItemLoader() + il = ChildItemLoader() - ib.add_value('name', u'marta') - self.assertEqual(ib.get_value('name'), u'mARTA') + il.add_value('name', u'marta') + self.assertEqual(il.get_reduced_value('name'), u'mARTA') def test_staticdefaults(self): class ChildDefaultedItemLoader(DefaultedItemLoader): expand_name = tree_expander(DefaultedItemLoader.expand, unicode.swapcase) - ib = ChildDefaultedItemLoader() + il = ChildDefaultedItemLoader() - ib.add_value('name', u'marta') - self.assertEqual(ib.get_value('name'), u'MART') + il.add_value('name', u'marta') + self.assertEqual(il.get_reduced_value('name'), u'MART') def test_reducer(self): - ib = TestItemLoader() + il = TestItemLoader() - ib.add_value('name', [u'mar', u'ta']) - self.assertEqual(ib.get_value('name'), u'Mar') + il.add_value('name', [u'mar', u'ta']) + self.assertEqual(il.get_reduced_value('name'), u'Mar') class TakeFirstItemLoader(TestItemLoader): reduce_name = staticmethod(u" ".join) - ib = TakeFirstItemLoader() + il = TakeFirstItemLoader() - ib.add_value('name', [u'mar', u'ta']) - self.assertEqual(ib.get_value('name'), u'Mar Ta') + il.add_value('name', [u'mar', u'ta']) + self.assertEqual(il.get_reduced_value('name'), u'Mar Ta') def test_loader_args(self): def expander_func_with_args(value, loader_args): @@ -139,16 +160,16 @@ class ItemLoaderTest(unittest.TestCase): class ChildItemLoader(TestItemLoader): expand_url = tree_expander(expander_func_with_args) - ib = ChildItemLoader(val=u'val') - ib.add_value('url', u'text') - self.assertEqual(ib.get_value('url'), 'val') + il = ChildItemLoader(val=u'val') + il.add_value('url', u'text') + self.assertEqual(il.get_reduced_value('url'), 'val') - ib = ChildItemLoader() - ib.add_value('url', u'text', val=u'val') - self.assertEqual(ib.get_value('url'), 'val') + il = ChildItemLoader() + il.add_value('url', u'text', val=u'val') + self.assertEqual(il.get_reduced_value('url'), 'val') def test_add_value_unknown_field(self): - ib = TestItemLoader() - ib.add_value('wrong_field', [u'lala', u'lolo']) + il = TestItemLoader() + il.add_value('wrong_field', [u'lala', u'lolo']) - self.assertRaises(KeyError, ib.get_item) + self.assertRaises(KeyError, il.get_item)