mirror of https://github.com/scrapy/scrapy.git
added first implementation of ItemBuilder
This commit is contained in:
Ismael Carnales
parent
32894643a0
commit
f05695d75e
|
|
@ -0,0 +1,130 @@
|
|||
from collections import defaultdict
|
||||
|
||||
from scrapy.utils.misc import arg_to_iter
|
||||
from scrapy.utils.python import get_func_args
|
||||
|
||||
|
||||
class BuilderField(object):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.expander = self.tree_expander(*args)
|
||||
self.reducer = kwargs.get('reducer')
|
||||
|
||||
def tree_expander(self, *funcs, **default_expander_args):
|
||||
"""A pipe expander implementing tree logic
|
||||
|
||||
It takes multiples unnamed arguments used as functions of the pipe, and
|
||||
keywords used as expander_args to be passed to functions that supports it
|
||||
|
||||
If an expander function returns a list of values, each value is used as
|
||||
input for next expander function
|
||||
|
||||
Always returns a list of values
|
||||
"""
|
||||
|
||||
_funcs = []
|
||||
for func in funcs:
|
||||
accepts_args = 'expander_args' in get_func_args(func)
|
||||
_funcs.append((func, accepts_args))
|
||||
|
||||
def _expander(value, expander_args=None):
|
||||
values = arg_to_iter(value)
|
||||
aargs = default_expander_args
|
||||
if expander_args:
|
||||
aargs = aargs.copy()
|
||||
aargs.update(expander_args)
|
||||
pipe_kwargs = {'expander_args': aargs}
|
||||
for func, accepts_args in _funcs:
|
||||
next = []
|
||||
kwargs = pipe_kwargs if accepts_args else {}
|
||||
for val in values:
|
||||
val = func(val, **kwargs)
|
||||
if hasattr(val, '__iter__'):
|
||||
next.extend(val)
|
||||
elif val is not None:
|
||||
next.append(val)
|
||||
values = next
|
||||
return list(values)
|
||||
|
||||
return _expander
|
||||
|
||||
|
||||
class ItemBuilderMeta(type):
|
||||
def __new__(mcs, class_name, bases, attrs):
|
||||
cls = type.__new__(mcs, class_name, bases, attrs)
|
||||
cls._builder_fields = cls._builder_fields.copy()
|
||||
|
||||
if cls.item_class:
|
||||
for name, field in cls.item_class.fields.iteritems():
|
||||
bfield = None
|
||||
if name in attrs:
|
||||
bfield = attrs[name]
|
||||
else:
|
||||
if name not in cls._builder_fields and cls.default_builder:
|
||||
bfield = cls.default_builder
|
||||
#actually add the field to the class
|
||||
setattr(cls, name, bfield)
|
||||
if bfield:
|
||||
cls._builder_fields.add(name)
|
||||
return cls
|
||||
|
||||
|
||||
class ItemBuilder(object):
|
||||
__metaclass__ = ItemBuilderMeta
|
||||
|
||||
item_class = None
|
||||
default_builder = None
|
||||
|
||||
_builder_fields = set()
|
||||
|
||||
def __init__(self, response=None, item=None, **expander_args):
|
||||
self._response = response
|
||||
self._item = item if item else self.item_class()
|
||||
|
||||
self._expander_args = {'response': self._response}
|
||||
if expander_args:
|
||||
self._expander_args.update(expander_args)
|
||||
|
||||
self._values = defaultdict(list)
|
||||
|
||||
def add_value(self, field_name, value, **new_expander_args):
|
||||
field = self._get_builder_field(field_name)
|
||||
evalue = self._expand_value(field, value, **new_expander_args)
|
||||
self._values[field_name].extend(evalue)
|
||||
|
||||
def get_item(self):
|
||||
item = self._item
|
||||
for field in self._values:
|
||||
item[field] = self.get_value(field)
|
||||
return item
|
||||
|
||||
def get_value(self, field_name):
|
||||
field = self._get_builder_field(field_name)
|
||||
reducer = field.reducer or self._item.fields[field_name].from_unicode_list
|
||||
return reducer(self._values[field_name])
|
||||
|
||||
def replace_value(self, field_name, value, **new_expander_args):
|
||||
field = self._get_builder_field(field_name)
|
||||
evalue = self._expand_value(field, value, **new_expander_args)
|
||||
self._values[field_name] = evalue
|
||||
|
||||
def _get_builder_field(self, name):
|
||||
if name not in self._builder_fields:
|
||||
raise KeyError
|
||||
|
||||
return getattr(self, name)
|
||||
|
||||
def _expand_value(self, field, value, **new_expander_args):
|
||||
if new_expander_args:
|
||||
expander_args = self._expander_args.copy()
|
||||
expander_args.update(new_expander_args)
|
||||
else:
|
||||
expander_args = self._expander_args
|
||||
|
||||
evalue = field.expander(value, expander_args=expander_args)
|
||||
|
||||
if not isinstance(evalue, list):
|
||||
evalue = [evalue]
|
||||
|
||||
return evalue
|
||||
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
def take_first(values):
|
||||
for value in values:
|
||||
if value is not u'':
|
||||
return value
|
||||
|
||||
|
||||
def return_values(values):
|
||||
return values
|
||||
|
||||
|
||||
def join_strings(values):
|
||||
return u' '.join(values)
|
||||
|
||||
|
|
@ -0,0 +1,174 @@
|
|||
import unittest
|
||||
import string
|
||||
from scrapy.newitem.builder import ItemBuilder, BuilderField
|
||||
from scrapy.newitem.builder import reducers
|
||||
from scrapy.newitem import Item, fields
|
||||
|
||||
|
||||
class BaseItem(Item):
|
||||
name = fields.TextField()
|
||||
|
||||
|
||||
class TestItem(BaseItem):
|
||||
url = fields.TextField()
|
||||
summary = fields.TextField()
|
||||
|
||||
|
||||
class BaseItemBuilder(ItemBuilder):
|
||||
item_class = TestItem
|
||||
|
||||
|
||||
class TestItemBuilder(BaseItemBuilder):
|
||||
name = BuilderField(lambda v: v.title())
|
||||
|
||||
|
||||
class DefaultedItemBuilder(BaseItemBuilder):
|
||||
default_builder = BuilderField(lambda v: v[:-1])
|
||||
|
||||
|
||||
class InheritDefaultedItemBuilder(DefaultedItemBuilder):
|
||||
pass
|
||||
|
||||
|
||||
class ListFieldTestItem(Item):
|
||||
names = fields.ListField(fields.TextField())
|
||||
|
||||
|
||||
class ListFieldItemBuilder(ItemBuilder):
|
||||
item_class = ListFieldTestItem
|
||||
|
||||
names = BuilderField(lambda v: v.title())
|
||||
|
||||
|
||||
class ItemBuilderTest(unittest.TestCase):
|
||||
|
||||
def test_basic(self):
|
||||
ib = TestItemBuilder()
|
||||
|
||||
ib.add_value('name', u'marta')
|
||||
self.assertEqual(ib.get_value('name'), u'Marta')
|
||||
|
||||
item = ib.get_item()
|
||||
self.assertEqual(item['name'], u'Marta')
|
||||
|
||||
def test_multiple_functions(self):
|
||||
class TestItemBuilder(BaseItemBuilder):
|
||||
name = BuilderField(lambda v: v.title(), lambda v: v[:-1])
|
||||
|
||||
ib = TestItemBuilder()
|
||||
|
||||
ib.add_value('name', u'marta')
|
||||
self.assertEqual(ib.get_value('name'), u'Mart')
|
||||
|
||||
item = ib.get_item()
|
||||
self.assertEqual(item['name'], u'Mart')
|
||||
|
||||
def test_defaulted(self):
|
||||
dib = DefaultedItemBuilder()
|
||||
assert dib.default_builder
|
||||
|
||||
dib.add_value('name', u'marta')
|
||||
self.assertEqual(dib.get_value('name'), u'mart')
|
||||
|
||||
def test_inherited_default(self):
|
||||
dib = InheritDefaultedItemBuilder()
|
||||
assert dib.default_builder
|
||||
|
||||
dib.add_value('name', u'marta')
|
||||
self.assertEqual(dib.get_value('name'), u'mart')
|
||||
|
||||
def test_inheritance(self):
|
||||
class ChildItemBuilder(TestItemBuilder):
|
||||
url = BuilderField(lambda v: v.lower())
|
||||
|
||||
ib = ChildItemBuilder()
|
||||
assert 'url' in ib._builder_fields
|
||||
assert 'name' in ib._builder_fields
|
||||
|
||||
ib.add_value('url', u'HTTP://scrapy.ORG')
|
||||
self.assertEqual(ib.get_value('url'), u'http://scrapy.org')
|
||||
|
||||
ib.add_value('name', u'marta')
|
||||
self.assertEqual(ib.get_value('name'), u'Marta')
|
||||
|
||||
class ChildChildItemBuilder(ChildItemBuilder):
|
||||
url = BuilderField(lambda v: v.upper())
|
||||
summary = BuilderField(lambda v: v)
|
||||
|
||||
ib = ChildChildItemBuilder()
|
||||
assert 'url' in ib._builder_fields
|
||||
assert 'name' in ib._builder_fields
|
||||
assert 'summary' in ib._builder_fields
|
||||
|
||||
ib.add_value('url', u'http://scrapy.org')
|
||||
self.assertEqual(ib.get_value('url'), u'HTTP://SCRAPY.ORG')
|
||||
|
||||
ib.add_value('name', u'marta')
|
||||
self.assertEqual(ib.get_value('name'), u'Marta')
|
||||
|
||||
def test_multiplevaluedadaptor(self):
|
||||
ib = ListFieldItemBuilder()
|
||||
|
||||
ib.add_value('names', [u'name1', u'name2'])
|
||||
self.assertEqual(ib.get_value('names'), [u'Name1', u'Name2'])
|
||||
|
||||
def test_identity(self):
|
||||
class IdentityDefaultedItemBuilder(DefaultedItemBuilder):
|
||||
name = BuilderField()
|
||||
|
||||
ib = IdentityDefaultedItemBuilder()
|
||||
|
||||
ib.add_value('name', u'marta')
|
||||
self.assertEqual(ib.get_value('name'), u'marta')
|
||||
|
||||
def test_staticmethods(self):
|
||||
class ChildItemBuilder(TestItemBuilder):
|
||||
name = BuilderField(TestItemBuilder.name.expander, string.swapcase)
|
||||
|
||||
ib = ChildItemBuilder()
|
||||
|
||||
ib.add_value('name', u'marta')
|
||||
self.assertEqual(ib.get_value('name'), u'mARTA')
|
||||
|
||||
|
||||
def test_staticdefaults(self):
|
||||
class ChildDefaultedItemBuilder(DefaultedItemBuilder):
|
||||
name = BuilderField(DefaultedItemBuilder.name.expander, string.swapcase)
|
||||
|
||||
ib = ChildDefaultedItemBuilder()
|
||||
|
||||
ib.add_value('name', u'marta')
|
||||
self.assertEqual(ib.get_value('name'), u'MART')
|
||||
|
||||
def test_reducer(self):
|
||||
ib = TestItemBuilder()
|
||||
|
||||
ib.add_value('name', [u'mar', u'ta'])
|
||||
self.assertEqual(ib.get_value('name'), u'Mar Ta')
|
||||
|
||||
class TakeFirstItemBuilder(TestItemBuilder):
|
||||
name = BuilderField(TestItemBuilder.name.expander,
|
||||
reducer=reducers.take_first)
|
||||
|
||||
ib = TakeFirstItemBuilder()
|
||||
|
||||
ib.add_value('name', [u'mar', u'ta'])
|
||||
self.assertEqual(ib.get_value('name'), u'Mar')
|
||||
|
||||
def test_expander_args(self):
|
||||
def expander_with_args(value, expander_args=None):
|
||||
if 'val' in expander_args:
|
||||
return expander_args['val']
|
||||
return value
|
||||
|
||||
class ChildItemBuilder(TestItemBuilder):
|
||||
url = BuilderField(expander_with_args)
|
||||
|
||||
ib = ChildItemBuilder(val=u'val')
|
||||
ib.add_value('url', u'text')
|
||||
self.assertEqual(ib.get_value('url'), 'val')
|
||||
|
||||
ib = ChildItemBuilder()
|
||||
ib.add_value('url', u'text', val=u'val')
|
||||
self.assertEqual(ib.get_value('url'), 'val')
|
||||
|
||||
Loading…
Reference in New Issue