mirror of https://github.com/scrapy/scrapy.git
ItemLoader: sorted out module locations, and added more tests
--HG-- rename : scrapy/newitem/reducers.py => scrapy/newitem/loader/reducers.py
This commit is contained in:
Pablo Hoffman
parent
ac9f4c9cc2
commit
9081e84e27
|
|
@ -1,64 +1,7 @@
|
|||
from collections import defaultdict
|
||||
from types import UnboundMethodType
|
||||
|
||||
from scrapy.utils.misc import arg_to_iter
|
||||
from scrapy.utils.python import get_func_args
|
||||
from scrapy.utils.datatypes import MergeDict
|
||||
from scrapy.newitem import Item
|
||||
from scrapy.newitem.reducers import take_first
|
||||
|
||||
def tree_expander(*functions, **default_loader_args):
|
||||
"""Create an ItemLoader expander from a list of functions using the tree
|
||||
expansion algorithm described below.
|
||||
|
||||
The functions can optionally accept a ``loader_args`` argument which (if
|
||||
present) will be used to pass loader arguments when the function is called.
|
||||
|
||||
The tree expansion algorithm consists in an ordered list of functions, each
|
||||
of which receives one value and can return zero, one or more values (as a
|
||||
list or iterable). If a function returns more than one value, the next
|
||||
function in the pipeline will be called with each of those values,
|
||||
potentially returning more values. Hence the name "tree expansion
|
||||
algorithm".
|
||||
"""
|
||||
def wrap_y(f):
|
||||
return lambda x, y, z: f(y)
|
||||
|
||||
def wrap_xy(f):
|
||||
return lambda x, y, z: f(x, y)
|
||||
|
||||
def wrap_yz(f):
|
||||
return lambda x, y, z: f(y, z)
|
||||
|
||||
wrapped_funcs = []
|
||||
for func in functions:
|
||||
if isinstance(func, UnboundMethodType):
|
||||
func = func.im_func
|
||||
if 'loader_args' in get_func_args(func):
|
||||
wfunc = func
|
||||
else:
|
||||
wfunc = wrap_xy(func)
|
||||
else:
|
||||
if 'loader_args' in get_func_args(func):
|
||||
wfunc = wrap_yz(func)
|
||||
else:
|
||||
wfunc = wrap_y(func)
|
||||
wrapped_funcs.append(wfunc)
|
||||
|
||||
def _expander(loader, value, loader_args):
|
||||
values = arg_to_iter(value)
|
||||
largs = default_loader_args
|
||||
if loader_args:
|
||||
largs = MergeDict(loader_args, default_loader_args)
|
||||
for func in wrapped_funcs:
|
||||
next_values = []
|
||||
for v in values:
|
||||
next_values += arg_to_iter(func(loader, v, largs))
|
||||
values = next_values
|
||||
return list(values)
|
||||
|
||||
return _expander
|
||||
|
||||
from scrapy.newitem.loader.reducers import take_first
|
||||
|
||||
class ItemLoader(object):
|
||||
|
||||
|
|
@ -71,23 +14,28 @@ class ItemLoader(object):
|
|||
self._values = defaultdict(list)
|
||||
|
||||
def add_value(self, field_name, value, **new_loader_args):
|
||||
evalue = self._expand_value(field_name, value, new_loader_args)
|
||||
self._values[field_name].extend(evalue)
|
||||
self._values[field_name] += self._expand_value(field_name, value, \
|
||||
new_loader_args)
|
||||
|
||||
def replace_value(self, field_name, value, **new_loader_args):
|
||||
evalue = self._expand_value(field_name, value, new_loader_args)
|
||||
self._values[field_name] = evalue
|
||||
self._values[field_name] = self._expand_value(field_name, value, \
|
||||
new_loader_args)
|
||||
|
||||
def get_item(self):
|
||||
item = self._item
|
||||
for field_name in self._values:
|
||||
item[field_name] = self.get_value(field_name)
|
||||
item[field_name] = self.get_reduced_value(field_name)
|
||||
return item
|
||||
|
||||
def get_value(self, field_name):
|
||||
values = self._values[field_name]
|
||||
def get_expanded_value(self, field_name):
|
||||
return self._values[field_name]
|
||||
|
||||
def get_reduced_value(self, field_name):
|
||||
reducer = self.get_reducer(field_name)
|
||||
return reducer(values)
|
||||
return reducer(self._values[field_name])
|
||||
|
||||
def get_expander(self, field_name):
|
||||
return getattr(self, 'expand_%s' % field_name, self.expand)
|
||||
|
||||
def get_reducer(self, field_name):
|
||||
try:
|
||||
|
|
@ -95,9 +43,6 @@ class ItemLoader(object):
|
|||
except AttributeError:
|
||||
return self._item.fields[field_name].get('reducer', self.reduce)
|
||||
|
||||
def get_expander(self, field_name):
|
||||
return getattr(self, 'expand_%s' % field_name, self.expand)
|
||||
|
||||
def _expand_value(self, field_name, value, new_loader_args):
|
||||
if new_loader_args:
|
||||
loader_args = self._loader_args.copy()
|
||||
|
|
@ -110,5 +55,5 @@ class ItemLoader(object):
|
|||
def expand(self, value, loader_args): # default expander
|
||||
return value
|
||||
|
||||
def reduce(self, values):
|
||||
def reduce(self, values): # default reducer
|
||||
return take_first(values)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,62 @@
|
|||
"""
|
||||
ItemLoader expanders
|
||||
"""
|
||||
|
||||
from types import UnboundMethodType
|
||||
|
||||
from scrapy.utils.misc import arg_to_iter
|
||||
from scrapy.utils.python import get_func_args
|
||||
from scrapy.utils.datatypes import MergeDict
|
||||
|
||||
def tree_expander(*functions, **default_loader_args):
|
||||
"""Create an ItemLoader expander from a list of functions using the tree
|
||||
expansion algorithm described below.
|
||||
|
||||
The functions can optionally accept a ``loader_args`` argument which (if
|
||||
present) will be used to pass loader arguments when the function is called.
|
||||
|
||||
The tree expansion algorithm consists in an ordered list of functions, each
|
||||
of which receives one value and can return zero, one or more values (as a
|
||||
list or iterable). If a function returns more than one value, the next
|
||||
function in the pipeline will be called with each of those values,
|
||||
potentially returning more values. Hence the name "tree expansion
|
||||
algorithm".
|
||||
"""
|
||||
def wrap_y(f):
|
||||
return lambda x, y, z: f(y)
|
||||
|
||||
def wrap_xy(f):
|
||||
return lambda x, y, z: f(x, y)
|
||||
|
||||
def wrap_yz(f):
|
||||
return lambda x, y, z: f(y, z)
|
||||
|
||||
wrapped_funcs = []
|
||||
for func in functions:
|
||||
if isinstance(func, UnboundMethodType):
|
||||
func = func.im_func
|
||||
if 'loader_args' in get_func_args(func):
|
||||
wfunc = func
|
||||
else:
|
||||
wfunc = wrap_xy(func)
|
||||
else:
|
||||
if 'loader_args' in get_func_args(func):
|
||||
wfunc = wrap_yz(func)
|
||||
else:
|
||||
wfunc = wrap_y(func)
|
||||
wrapped_funcs.append(wfunc)
|
||||
|
||||
def _expander(loader, value, loader_args):
|
||||
values = arg_to_iter(value)
|
||||
largs = default_loader_args
|
||||
if loader_args:
|
||||
largs = MergeDict(loader_args, default_loader_args)
|
||||
for func in wrapped_funcs:
|
||||
next_values = []
|
||||
for v in values:
|
||||
next_values += arg_to_iter(func(loader, v, largs))
|
||||
values = next_values
|
||||
return list(values)
|
||||
|
||||
return _expander
|
||||
|
||||
|
|
@ -1,4 +1,6 @@
|
|||
"""Some common reducers"""
|
||||
"""
|
||||
ItemLoader reducers
|
||||
"""
|
||||
|
||||
def take_first(values):
|
||||
for value in values:
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
import unittest
|
||||
|
||||
from scrapy.newitem.loader import ItemLoader, tree_expander
|
||||
from scrapy.newitem.loader import ItemLoader
|
||||
from scrapy.newitem.loader.expanders import tree_expander
|
||||
from scrapy.newitem import Item, Field
|
||||
|
||||
|
||||
|
|
@ -31,104 +32,124 @@ class InheritDefaultedItemLoader(DefaultedItemLoader):
|
|||
|
||||
class ItemLoaderTest(unittest.TestCase):
|
||||
|
||||
def test_basic(self):
|
||||
ib = TestItemLoader()
|
||||
def test_get_item(self):
|
||||
il = TestItemLoader()
|
||||
|
||||
ib.add_value('name', u'marta')
|
||||
self.assertEqual(ib.get_value('name'), u'Marta')
|
||||
|
||||
item = ib.get_item()
|
||||
il.add_value('name', u'marta')
|
||||
item = il.get_item()
|
||||
self.assertEqual(item['name'], u'Marta')
|
||||
|
||||
def test_add_value(self):
|
||||
il = TestItemLoader()
|
||||
|
||||
il.add_value('name', u'marta')
|
||||
self.assertEqual(il.get_expanded_value('name'), [u'Marta'])
|
||||
self.assertEqual(il.get_reduced_value('name'), u'Marta')
|
||||
|
||||
il.add_value('name', u'pepe')
|
||||
self.assertEqual(il.get_expanded_value('name'), [u'Marta', u'Pepe'])
|
||||
self.assertEqual(il.get_reduced_value('name'), u'Marta')
|
||||
|
||||
def test_replace_value(self):
|
||||
il = TestItemLoader()
|
||||
|
||||
il.replace_value('name', u'marta')
|
||||
self.assertEqual(il.get_expanded_value('name'), [u'Marta'])
|
||||
self.assertEqual(il.get_reduced_value('name'), u'Marta')
|
||||
|
||||
il.replace_value('name', u'pepe')
|
||||
self.assertEqual(il.get_expanded_value('name'), [u'Pepe'])
|
||||
self.assertEqual(il.get_reduced_value('name'), u'Pepe')
|
||||
|
||||
def test_multiple_functions(self):
|
||||
class TestItemLoader(BaseItemLoader):
|
||||
expand_name = tree_expander(lambda v: v.title(), lambda v: v[:-1])
|
||||
|
||||
ib = TestItemLoader()
|
||||
il = TestItemLoader()
|
||||
|
||||
ib.add_value('name', u'marta')
|
||||
self.assertEqual(ib.get_value('name'), u'Mart')
|
||||
il.add_value('name', u'marta')
|
||||
self.assertEqual(il.get_reduced_value('name'), u'Mart')
|
||||
|
||||
item = ib.get_item()
|
||||
item = il.get_item()
|
||||
self.assertEqual(item['name'], u'Mart')
|
||||
|
||||
def test_defaulted(self):
|
||||
dib = DefaultedItemLoader()
|
||||
dil = DefaultedItemLoader()
|
||||
|
||||
dib.add_value('name', u'marta')
|
||||
self.assertEqual(dib.get_value('name'), u'mart')
|
||||
dil.add_value('name', u'marta')
|
||||
self.assertEqual(dil.get_reduced_value('name'), u'mart')
|
||||
|
||||
def test_inherited_default(self):
|
||||
dib = InheritDefaultedItemLoader()
|
||||
dil = InheritDefaultedItemLoader()
|
||||
|
||||
dib.add_value('name', u'marta')
|
||||
self.assertEqual(dib.get_value('name'), u'mart')
|
||||
dil.add_value('name', u'marta')
|
||||
self.assertEqual(dil.get_reduced_value('name'), u'mart')
|
||||
|
||||
def test_inheritance(self):
|
||||
class ChildItemLoader(TestItemLoader):
|
||||
expand_url = tree_expander(lambda v: v.lower())
|
||||
|
||||
ib = ChildItemLoader()
|
||||
il = ChildItemLoader()
|
||||
|
||||
ib.add_value('url', u'HTTP://scrapy.ORG')
|
||||
self.assertEqual(ib.get_value('url'), u'http://scrapy.org')
|
||||
il.add_value('url', u'HTTP://scrapy.ORG')
|
||||
self.assertEqual(il.get_reduced_value('url'), u'http://scrapy.org')
|
||||
|
||||
ib.add_value('name', u'marta')
|
||||
self.assertEqual(ib.get_value('name'), u'Marta')
|
||||
il.add_value('name', u'marta')
|
||||
self.assertEqual(il.get_reduced_value('name'), u'Marta')
|
||||
|
||||
class ChildChildItemLoader(ChildItemLoader):
|
||||
expand_url = tree_expander(lambda v: v.upper())
|
||||
expand_summary = tree_expander(lambda v: v)
|
||||
|
||||
ib = ChildChildItemLoader()
|
||||
il = ChildChildItemLoader()
|
||||
|
||||
ib.add_value('url', u'http://scrapy.org')
|
||||
self.assertEqual(ib.get_value('url'), u'HTTP://SCRAPY.ORG')
|
||||
il.add_value('url', u'http://scrapy.org')
|
||||
self.assertEqual(il.get_reduced_value('url'), u'HTTP://SCRAPY.ORG')
|
||||
|
||||
ib.add_value('name', u'marta')
|
||||
self.assertEqual(ib.get_value('name'), u'Marta')
|
||||
il.add_value('name', u'marta')
|
||||
self.assertEqual(il.get_reduced_value('name'), u'Marta')
|
||||
|
||||
def test_identity(self):
|
||||
class IdentityDefaultedItemLoader(DefaultedItemLoader):
|
||||
expand_name = tree_expander()
|
||||
|
||||
ib = IdentityDefaultedItemLoader()
|
||||
il = IdentityDefaultedItemLoader()
|
||||
|
||||
ib.add_value('name', u'marta')
|
||||
self.assertEqual(ib.get_value('name'), u'marta')
|
||||
il.add_value('name', u'marta')
|
||||
self.assertEqual(il.get_reduced_value('name'), u'marta')
|
||||
|
||||
def test_staticmethods(self):
|
||||
class ChildItemLoader(TestItemLoader):
|
||||
expand_name = tree_expander(TestItemLoader.expand_name, unicode.swapcase)
|
||||
|
||||
ib = ChildItemLoader()
|
||||
il = ChildItemLoader()
|
||||
|
||||
ib.add_value('name', u'marta')
|
||||
self.assertEqual(ib.get_value('name'), u'mARTA')
|
||||
il.add_value('name', u'marta')
|
||||
self.assertEqual(il.get_reduced_value('name'), u'mARTA')
|
||||
|
||||
|
||||
def test_staticdefaults(self):
|
||||
class ChildDefaultedItemLoader(DefaultedItemLoader):
|
||||
expand_name = tree_expander(DefaultedItemLoader.expand, unicode.swapcase)
|
||||
|
||||
ib = ChildDefaultedItemLoader()
|
||||
il = ChildDefaultedItemLoader()
|
||||
|
||||
ib.add_value('name', u'marta')
|
||||
self.assertEqual(ib.get_value('name'), u'MART')
|
||||
il.add_value('name', u'marta')
|
||||
self.assertEqual(il.get_reduced_value('name'), u'MART')
|
||||
|
||||
def test_reducer(self):
|
||||
ib = TestItemLoader()
|
||||
il = TestItemLoader()
|
||||
|
||||
ib.add_value('name', [u'mar', u'ta'])
|
||||
self.assertEqual(ib.get_value('name'), u'Mar')
|
||||
il.add_value('name', [u'mar', u'ta'])
|
||||
self.assertEqual(il.get_reduced_value('name'), u'Mar')
|
||||
|
||||
class TakeFirstItemLoader(TestItemLoader):
|
||||
reduce_name = staticmethod(u" ".join)
|
||||
|
||||
ib = TakeFirstItemLoader()
|
||||
il = TakeFirstItemLoader()
|
||||
|
||||
ib.add_value('name', [u'mar', u'ta'])
|
||||
self.assertEqual(ib.get_value('name'), u'Mar Ta')
|
||||
il.add_value('name', [u'mar', u'ta'])
|
||||
self.assertEqual(il.get_reduced_value('name'), u'Mar Ta')
|
||||
|
||||
def test_loader_args(self):
|
||||
def expander_func_with_args(value, loader_args):
|
||||
|
|
@ -139,16 +160,16 @@ class ItemLoaderTest(unittest.TestCase):
|
|||
class ChildItemLoader(TestItemLoader):
|
||||
expand_url = tree_expander(expander_func_with_args)
|
||||
|
||||
ib = ChildItemLoader(val=u'val')
|
||||
ib.add_value('url', u'text')
|
||||
self.assertEqual(ib.get_value('url'), 'val')
|
||||
il = ChildItemLoader(val=u'val')
|
||||
il.add_value('url', u'text')
|
||||
self.assertEqual(il.get_reduced_value('url'), 'val')
|
||||
|
||||
ib = ChildItemLoader()
|
||||
ib.add_value('url', u'text', val=u'val')
|
||||
self.assertEqual(ib.get_value('url'), 'val')
|
||||
il = ChildItemLoader()
|
||||
il.add_value('url', u'text', val=u'val')
|
||||
self.assertEqual(il.get_reduced_value('url'), 'val')
|
||||
|
||||
def test_add_value_unknown_field(self):
|
||||
ib = TestItemLoader()
|
||||
ib.add_value('wrong_field', [u'lala', u'lolo'])
|
||||
il = TestItemLoader()
|
||||
il.add_value('wrong_field', [u'lala', u'lolo'])
|
||||
|
||||
self.assertRaises(KeyError, ib.get_item)
|
||||
self.assertRaises(KeyError, il.get_item)
|
||||
|
|
|
|||
Loading…
Reference in New Issue