mirror of https://github.com/scrapy/scrapy.git
ItemLoader: some more code cleanups, and added many more tests
This commit is contained in:
Pablo Hoffman
parent
9081e84e27
commit
7bc7af0162
|
|
@ -3,7 +3,7 @@ from UserDict import DictMixin
|
|||
from scrapy.item.models import BaseItem
|
||||
|
||||
class Field(dict):
|
||||
pass
|
||||
"""Container of field metadata"""
|
||||
|
||||
class _ItemMeta(type):
|
||||
|
||||
|
|
@ -30,8 +30,7 @@ class Item(DictMixin, BaseItem):
|
|||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self._values = {}
|
||||
|
||||
if args or kwargs: # don't instantiate dict for simple (most common) case
|
||||
if args or kwargs: # avoid instantiating dict for most common case
|
||||
for k, v in dict(*args, **kwargs).iteritems():
|
||||
self[k] = v
|
||||
|
||||
|
|
@ -70,8 +69,3 @@ class Item(DictMixin, BaseItem):
|
|||
"""
|
||||
values = ', '.join('%s=%r' % field for field in self.iteritems())
|
||||
return "%s(%s)" % (self.__class__.__name__, values)
|
||||
|
||||
def get_id(self):
|
||||
"""Returns the unique id for this item."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
|
|
|||
|
|
@ -1,15 +1,19 @@
|
|||
from collections import defaultdict
|
||||
|
||||
from scrapy.utils.datatypes import MergeDict
|
||||
from scrapy.newitem import Item
|
||||
from scrapy.newitem.loader.reducers import take_first
|
||||
from scrapy.newitem.loader.reducers import TakeFirst
|
||||
from scrapy.newitem.loader.expanders import IdentityExpander
|
||||
|
||||
class ItemLoader(object):
|
||||
|
||||
item_class = Item
|
||||
default_item_class = Item
|
||||
default_expander = IdentityExpander()
|
||||
default_reducer = TakeFirst()
|
||||
|
||||
def __init__(self, **loader_args):
|
||||
self._response = loader_args.get('response')
|
||||
self._item = loader_args.get('item') or self.item_class()
|
||||
self._item = loader_args.setdefault('item', self.default_item_class())
|
||||
self._loader_args = loader_args
|
||||
self._values = defaultdict(list)
|
||||
|
||||
|
|
@ -35,25 +39,22 @@ class ItemLoader(object):
|
|||
return reducer(self._values[field_name])
|
||||
|
||||
def get_expander(self, field_name):
|
||||
return getattr(self, 'expand_%s' % field_name, self.expand)
|
||||
expander = getattr(self, '%s_exp' % field_name, None)
|
||||
if not expander:
|
||||
expander = self._item.fields[field_name].get('expander', \
|
||||
self.default_expander)
|
||||
return expander
|
||||
|
||||
def get_reducer(self, field_name):
|
||||
try:
|
||||
return getattr(self, 'reduce_%s' % field_name)
|
||||
except AttributeError:
|
||||
return self._item.fields[field_name].get('reducer', self.reduce)
|
||||
reducer = getattr(self, '%s_red' % field_name, None)
|
||||
if not reducer:
|
||||
reducer = self._item.fields[field_name].get('reducer', \
|
||||
self.default_reducer)
|
||||
return reducer
|
||||
|
||||
def _expand_value(self, field_name, value, new_loader_args):
|
||||
loader_args = self._loader_args
|
||||
if new_loader_args:
|
||||
loader_args = self._loader_args.copy()
|
||||
loader_args.update(new_loader_args)
|
||||
else: # shortcut for most common case
|
||||
loader_args = self._loader_args
|
||||
loader_args = MergeDict(new_loader_args, self._loader_args)
|
||||
expander = self.get_expander(field_name)
|
||||
return expander(value, loader_args=loader_args)
|
||||
|
||||
def expand(self, value, loader_args): # default expander
|
||||
return value
|
||||
|
||||
def reduce(self, values): # default reducer
|
||||
return take_first(values)
|
||||
|
|
|
|||
|
|
@ -1,62 +1,59 @@
|
|||
"""
|
||||
ItemLoader expanders
|
||||
This module provides some commonly used Expanders
|
||||
"""
|
||||
|
||||
from types import UnboundMethodType
|
||||
|
||||
from scrapy.utils.misc import arg_to_iter
|
||||
from scrapy.utils.python import get_func_args
|
||||
from scrapy.utils.datatypes import MergeDict
|
||||
|
||||
def tree_expander(*functions, **default_loader_args):
|
||||
"""Create an ItemLoader expander from a list of functions using the tree
|
||||
expansion algorithm described below.
|
||||
class TreeExpander(object):
|
||||
"""An expander which applies the given list of functions consecutively to
|
||||
each value returned by the previous function.
|
||||
|
||||
The functions can optionally accept a ``loader_args`` argument which (if
|
||||
present) will be used to pass loader arguments when the function is called.
|
||||
|
||||
The tree expansion algorithm consists in an ordered list of functions, each
|
||||
of which receives one value and can return zero, one or more values (as a
|
||||
list or iterable). If a function returns more than one value, the next
|
||||
function in the pipeline will be called with each of those values,
|
||||
potentially returning more values. Hence the name "tree expansion
|
||||
algorithm".
|
||||
The algorithm consists in an ordered list of functions, each of which
|
||||
receives one value and can return zero, one or more values (as a list or
|
||||
iterable). If a function returns more than one value, the next function in
|
||||
the list will be called with each of those values, potentially returning
|
||||
more values and thus expanding the execution into different branches, which
|
||||
is why this expander is called Tree Expander.
|
||||
|
||||
The expander functions can optionally receive a ``loader_args`` argument,
|
||||
which will contain the current active loader arguments.
|
||||
"""
|
||||
def wrap_y(f):
|
||||
return lambda x, y, z: f(y)
|
||||
|
||||
def wrap_xy(f):
|
||||
return lambda x, y, z: f(x, y)
|
||||
|
||||
def wrap_yz(f):
|
||||
return lambda x, y, z: f(y, z)
|
||||
|
||||
wrapped_funcs = []
|
||||
for func in functions:
|
||||
if isinstance(func, UnboundMethodType):
|
||||
func = func.im_func
|
||||
def __init__(self, *functions, **default_loader_args):
|
||||
self.default_loader_args = default_loader_args
|
||||
self.wrapped_funcs = []
|
||||
for func in functions:
|
||||
if 'loader_args' in get_func_args(func):
|
||||
wfunc = func
|
||||
wfunc = self.wrap_with_args(func)
|
||||
else:
|
||||
wfunc = wrap_xy(func)
|
||||
else:
|
||||
if 'loader_args' in get_func_args(func):
|
||||
wfunc = wrap_yz(func)
|
||||
else:
|
||||
wfunc = wrap_y(func)
|
||||
wrapped_funcs.append(wfunc)
|
||||
wfunc = self.wrap_no_args(func)
|
||||
self.wrapped_funcs.append(wfunc)
|
||||
|
||||
def wrap_no_args(self, f):
|
||||
return lambda x, y: f(x)
|
||||
|
||||
def _expander(loader, value, loader_args):
|
||||
def wrap_with_args(self, f):
|
||||
return lambda x, y: f(x, loader_args=y)
|
||||
|
||||
def __call__(self, value, loader_args):
|
||||
values = arg_to_iter(value)
|
||||
largs = default_loader_args
|
||||
largs = self.default_loader_args
|
||||
if loader_args:
|
||||
largs = MergeDict(loader_args, default_loader_args)
|
||||
for func in wrapped_funcs:
|
||||
largs = MergeDict(loader_args, self.default_loader_args)
|
||||
for func in self.wrapped_funcs:
|
||||
next_values = []
|
||||
for v in values:
|
||||
next_values += arg_to_iter(func(loader, v, largs))
|
||||
next_values += arg_to_iter(func(v, largs))
|
||||
values = next_values
|
||||
return list(values)
|
||||
|
||||
return _expander
|
||||
|
||||
class IdentityExpander(object):
|
||||
"""An expander which returns the original values unchanged. It doesn't
|
||||
support any constructor arguments.
|
||||
"""
|
||||
|
||||
def __call__(self, values, loader_args):
|
||||
return arg_to_iter(values)
|
||||
|
|
|
|||
|
|
@ -1,14 +1,33 @@
|
|||
"""
|
||||
ItemLoader reducers
|
||||
This module provides some commonly used Reducers
|
||||
"""
|
||||
|
||||
def take_first(values):
|
||||
for value in values:
|
||||
if value:
|
||||
return value
|
||||
class TakeFirst(object):
|
||||
"""Return the first non-null value from the list to reduce"""
|
||||
|
||||
def identity(values):
|
||||
return values
|
||||
def __call__(self, values):
|
||||
for value in values:
|
||||
if value:
|
||||
return value
|
||||
|
||||
def join_strings(values):
|
||||
return u' '.join(values)
|
||||
|
||||
class Identity(object):
|
||||
"""Return the list to reduce untouched"""
|
||||
|
||||
def __call__(self, values):
|
||||
return values
|
||||
|
||||
|
||||
class JoinStrings(object):
|
||||
"""Return a string with the contents of the list to reduce joined with the
|
||||
separator given in the constructor, which defaults to u' '.
|
||||
|
||||
When using the default separator, this reducer is equivalent to the
|
||||
function: u' '.join
|
||||
"""
|
||||
|
||||
def __init__(self, separator=u' '):
|
||||
self.separator = separator
|
||||
|
||||
def __call__(self, values):
|
||||
return self.separator.join(values)
|
||||
|
|
|
|||
|
|
@ -1,175 +1,218 @@
|
|||
import unittest
|
||||
|
||||
from scrapy.newitem.loader import ItemLoader
|
||||
from scrapy.newitem.loader.expanders import tree_expander
|
||||
from scrapy.newitem.loader.expanders import TreeExpander, IdentityExpander
|
||||
from scrapy.newitem.loader.reducers import JoinStrings, Identity
|
||||
from scrapy.newitem import Item, Field
|
||||
|
||||
# test items
|
||||
|
||||
class BaseItem(Item):
|
||||
class NameItem(Item):
|
||||
name = Field()
|
||||
|
||||
|
||||
class TestItem(BaseItem):
|
||||
class TestItem(NameItem):
|
||||
url = Field()
|
||||
summary = Field()
|
||||
|
||||
# test loaders
|
||||
|
||||
class BaseItemLoader(ItemLoader):
|
||||
item_class = TestItem
|
||||
class NameItemLoader(ItemLoader):
|
||||
default_item_class = TestItem
|
||||
|
||||
class TestItemLoader(NameItemLoader):
|
||||
name_exp = TreeExpander(lambda v: v.title())
|
||||
|
||||
class TestItemLoader(BaseItemLoader):
|
||||
expand_name = tree_expander(lambda v: v.title())
|
||||
class DefaultedItemLoader(NameItemLoader):
|
||||
default_expander = TreeExpander(lambda v: v[:-1])
|
||||
|
||||
# test expanders
|
||||
|
||||
class DefaultedItemLoader(BaseItemLoader):
|
||||
expand = tree_expander(lambda v: v[:-1])
|
||||
|
||||
|
||||
class InheritDefaultedItemLoader(DefaultedItemLoader):
|
||||
pass
|
||||
|
||||
def expander_func_with_args(value, other=None, loader_args=None):
|
||||
if 'key' in loader_args:
|
||||
return loader_args['key']
|
||||
return value
|
||||
|
||||
class ItemLoaderTest(unittest.TestCase):
|
||||
|
||||
def test_get_item(self):
|
||||
il = TestItemLoader()
|
||||
def test_get_item_using_default_loader(self):
|
||||
i = TestItem()
|
||||
i['summary'] = u'lala'
|
||||
il = ItemLoader(item=i)
|
||||
il.add_value('name', u'marta')
|
||||
item = il.get_item()
|
||||
assert item is i
|
||||
self.assertEqual(item['summary'], u'lala')
|
||||
self.assertEqual(item['name'], u'marta')
|
||||
|
||||
def test_get_item_using_custom_loader(self):
|
||||
il = TestItemLoader()
|
||||
il.add_value('name', u'marta')
|
||||
item = il.get_item()
|
||||
self.assertEqual(item['name'], u'Marta')
|
||||
|
||||
def test_add_value(self):
|
||||
il = TestItemLoader()
|
||||
|
||||
il.add_value('name', u'marta')
|
||||
self.assertEqual(il.get_expanded_value('name'), [u'Marta'])
|
||||
self.assertEqual(il.get_reduced_value('name'), u'Marta')
|
||||
|
||||
il.add_value('name', u'pepe')
|
||||
self.assertEqual(il.get_expanded_value('name'), [u'Marta', u'Pepe'])
|
||||
self.assertEqual(il.get_reduced_value('name'), u'Marta')
|
||||
|
||||
def test_replace_value(self):
|
||||
il = TestItemLoader()
|
||||
|
||||
il.replace_value('name', u'marta')
|
||||
self.assertEqual(il.get_expanded_value('name'), [u'Marta'])
|
||||
self.assertEqual(il.get_reduced_value('name'), u'Marta')
|
||||
|
||||
il.replace_value('name', u'pepe')
|
||||
self.assertEqual(il.get_expanded_value('name'), [u'Pepe'])
|
||||
self.assertEqual(il.get_reduced_value('name'), u'Pepe')
|
||||
|
||||
def test_multiple_functions(self):
|
||||
class TestItemLoader(BaseItemLoader):
|
||||
expand_name = tree_expander(lambda v: v.title(), lambda v: v[:-1])
|
||||
def test_tree_expander_multiple_functions(self):
|
||||
class TestItemLoader(NameItemLoader):
|
||||
name_exp = TreeExpander(lambda v: v.title(), lambda v: v[:-1])
|
||||
|
||||
il = TestItemLoader()
|
||||
|
||||
il.add_value('name', u'marta')
|
||||
self.assertEqual(il.get_reduced_value('name'), u'Mart')
|
||||
|
||||
item = il.get_item()
|
||||
self.assertEqual(item['name'], u'Mart')
|
||||
|
||||
def test_defaulted(self):
|
||||
dil = DefaultedItemLoader()
|
||||
def test_default_expander(self):
|
||||
il = DefaultedItemLoader()
|
||||
il.add_value('name', u'marta')
|
||||
self.assertEqual(il.get_reduced_value('name'), u'mart')
|
||||
|
||||
dil.add_value('name', u'marta')
|
||||
self.assertEqual(dil.get_reduced_value('name'), u'mart')
|
||||
def test_inherited_default_expander(self):
|
||||
class InheritDefaultedItemLoader(DefaultedItemLoader):
|
||||
pass
|
||||
|
||||
def test_inherited_default(self):
|
||||
dil = InheritDefaultedItemLoader()
|
||||
il = InheritDefaultedItemLoader()
|
||||
il.add_value('name', u'marta')
|
||||
self.assertEqual(il.get_reduced_value('name'), u'mart')
|
||||
|
||||
dil.add_value('name', u'marta')
|
||||
self.assertEqual(dil.get_reduced_value('name'), u'mart')
|
||||
|
||||
def test_inheritance(self):
|
||||
def test_expander_inheritance(self):
|
||||
class ChildItemLoader(TestItemLoader):
|
||||
expand_url = tree_expander(lambda v: v.lower())
|
||||
url_exp = TreeExpander(lambda v: v.lower())
|
||||
|
||||
il = ChildItemLoader()
|
||||
|
||||
il.add_value('url', u'HTTP://scrapy.ORG')
|
||||
self.assertEqual(il.get_reduced_value('url'), u'http://scrapy.org')
|
||||
|
||||
il.add_value('name', u'marta')
|
||||
self.assertEqual(il.get_reduced_value('name'), u'Marta')
|
||||
|
||||
class ChildChildItemLoader(ChildItemLoader):
|
||||
expand_url = tree_expander(lambda v: v.upper())
|
||||
expand_summary = tree_expander(lambda v: v)
|
||||
url_exp = TreeExpander(lambda v: v.upper())
|
||||
summary_exp = TreeExpander(lambda v: v)
|
||||
|
||||
il = ChildChildItemLoader()
|
||||
|
||||
il.add_value('url', u'http://scrapy.org')
|
||||
self.assertEqual(il.get_reduced_value('url'), u'HTTP://SCRAPY.ORG')
|
||||
|
||||
il.add_value('name', u'marta')
|
||||
self.assertEqual(il.get_reduced_value('name'), u'Marta')
|
||||
|
||||
def test_identity(self):
|
||||
def test_empty_tree_expander(self):
|
||||
class IdentityDefaultedItemLoader(DefaultedItemLoader):
|
||||
expand_name = tree_expander()
|
||||
name_exp = TreeExpander()
|
||||
|
||||
il = IdentityDefaultedItemLoader()
|
||||
|
||||
il.add_value('name', u'marta')
|
||||
self.assertEqual(il.get_reduced_value('name'), u'marta')
|
||||
|
||||
def test_staticmethods(self):
|
||||
def test_identity_expander(self):
|
||||
class IdentityDefaultedItemLoader(DefaultedItemLoader):
|
||||
name_exp = IdentityExpander()
|
||||
|
||||
il = IdentityDefaultedItemLoader()
|
||||
il.add_value('name', u'marta')
|
||||
self.assertEqual(il.get_reduced_value('name'), u'marta')
|
||||
|
||||
def test_extend_expanders(self):
|
||||
class ChildItemLoader(TestItemLoader):
|
||||
expand_name = tree_expander(TestItemLoader.expand_name, unicode.swapcase)
|
||||
name_exp = TreeExpander(TestItemLoader.name_exp, unicode.swapcase)
|
||||
|
||||
il = ChildItemLoader()
|
||||
|
||||
il.add_value('name', u'marta')
|
||||
self.assertEqual(il.get_reduced_value('name'), u'mARTA')
|
||||
|
||||
|
||||
def test_staticdefaults(self):
|
||||
class ChildDefaultedItemLoader(DefaultedItemLoader):
|
||||
expand_name = tree_expander(DefaultedItemLoader.expand, unicode.swapcase)
|
||||
name_exp = TreeExpander(DefaultedItemLoader.default_expander, unicode.swapcase)
|
||||
|
||||
il = ChildDefaultedItemLoader()
|
||||
|
||||
il.add_value('name', u'marta')
|
||||
self.assertEqual(il.get_reduced_value('name'), u'MART')
|
||||
|
||||
def test_reducer(self):
|
||||
def test_reducer_using_function(self):
|
||||
il = TestItemLoader()
|
||||
|
||||
il.add_value('name', [u'mar', u'ta'])
|
||||
self.assertEqual(il.get_reduced_value('name'), u'Mar')
|
||||
|
||||
class TakeFirstItemLoader(TestItemLoader):
|
||||
reduce_name = staticmethod(u" ".join)
|
||||
name_red = u" ".join
|
||||
|
||||
il = TakeFirstItemLoader()
|
||||
|
||||
il.add_value('name', [u'mar', u'ta'])
|
||||
self.assertEqual(il.get_reduced_value('name'), u'Mar Ta')
|
||||
|
||||
def test_loader_args(self):
|
||||
def expander_func_with_args(value, loader_args):
|
||||
if 'val' in loader_args:
|
||||
return loader_args['val']
|
||||
return value
|
||||
def test_reducer_using_classes(self):
|
||||
il = TestItemLoader()
|
||||
il.add_value('name', [u'mar', u'ta'])
|
||||
self.assertEqual(il.get_reduced_value('name'), u'Mar')
|
||||
|
||||
class TakeFirstItemLoader(TestItemLoader):
|
||||
name_red = JoinStrings()
|
||||
|
||||
il = TakeFirstItemLoader()
|
||||
il.add_value('name', [u'mar', u'ta'])
|
||||
self.assertEqual(il.get_reduced_value('name'), u'Mar Ta')
|
||||
|
||||
class TakeFirstItemLoader(TestItemLoader):
|
||||
name_red = JoinStrings("<br>")
|
||||
|
||||
il = TakeFirstItemLoader()
|
||||
il.add_value('name', [u'mar', u'ta'])
|
||||
self.assertEqual(il.get_reduced_value('name'), u'Mar<br>Ta')
|
||||
|
||||
def test_default_reducer(self):
|
||||
il = TestItemLoader()
|
||||
il.add_value('name', [u'mar', u'ta'])
|
||||
self.assertEqual(il.get_reduced_value('name'), u'Mar')
|
||||
|
||||
class LalaItemLoader(TestItemLoader):
|
||||
default_reducer = Identity()
|
||||
|
||||
il = LalaItemLoader()
|
||||
il.add_value('name', [u'mar', u'ta'])
|
||||
self.assertEqual(il.get_reduced_value('name'), [u'Mar', u'Ta'])
|
||||
|
||||
def test_expander_args_on_declaration(self):
|
||||
class ChildItemLoader(TestItemLoader):
|
||||
expand_url = tree_expander(expander_func_with_args)
|
||||
url_exp = TreeExpander(expander_func_with_args, key=u'val')
|
||||
|
||||
il = ChildItemLoader(val=u'val')
|
||||
il = ChildItemLoader()
|
||||
il.add_value('url', u'text', key=u'val')
|
||||
self.assertEqual(il.get_reduced_value('url'), 'val')
|
||||
|
||||
def test_expander_args_on_instantiation(self):
|
||||
class ChildItemLoader(TestItemLoader):
|
||||
url_exp = TreeExpander(expander_func_with_args)
|
||||
|
||||
il = ChildItemLoader(key=u'val')
|
||||
il.add_value('url', u'text')
|
||||
self.assertEqual(il.get_reduced_value('url'), 'val')
|
||||
|
||||
def test_expander_args_on_assign(self):
|
||||
class ChildItemLoader(TestItemLoader):
|
||||
url_exp = TreeExpander(expander_func_with_args)
|
||||
|
||||
il = ChildItemLoader()
|
||||
il.add_value('url', u'text', val=u'val')
|
||||
il.add_value('url', u'text', key=u'val')
|
||||
self.assertEqual(il.get_reduced_value('url'), 'val')
|
||||
|
||||
def test_add_value_unknown_field(self):
|
||||
def test_add_value_on_unknown_field(self):
|
||||
il = TestItemLoader()
|
||||
il.add_value('wrong_field', [u'lala', u'lolo'])
|
||||
self.assertRaises(KeyError, il.add_value, 'wrong_field', [u'lala', u'lolo'])
|
||||
|
||||
self.assertRaises(KeyError, il.get_item)
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Reference in New Issue