diff --git a/scrapy/newitem/builder/__init__.py b/scrapy/newitem/builder/__init__.py new file mode 100644 index 000000000..85ef494bb --- /dev/null +++ b/scrapy/newitem/builder/__init__.py @@ -0,0 +1,130 @@ +from collections import defaultdict + +from scrapy.utils.misc import arg_to_iter +from scrapy.utils.python import get_func_args + + +class BuilderField(object): + + def __init__(self, *args, **kwargs): + self.expander = self.tree_expander(*args) + self.reducer = kwargs.get('reducer') + + def tree_expander(self, *funcs, **default_expander_args): + """A pipe expander implementing tree logic + + It takes multiples unnamed arguments used as functions of the pipe, and + keywords used as expander_args to be passed to functions that supports it + + If an expander function returns a list of values, each value is used as + input for next expander function + + Always returns a list of values + """ + + _funcs = [] + for func in funcs: + accepts_args = 'expander_args' in get_func_args(func) + _funcs.append((func, accepts_args)) + + def _expander(value, expander_args=None): + values = arg_to_iter(value) + aargs = default_expander_args + if expander_args: + aargs = aargs.copy() + aargs.update(expander_args) + pipe_kwargs = {'expander_args': aargs} + for func, accepts_args in _funcs: + next = [] + kwargs = pipe_kwargs if accepts_args else {} + for val in values: + val = func(val, **kwargs) + if hasattr(val, '__iter__'): + next.extend(val) + elif val is not None: + next.append(val) + values = next + return list(values) + + return _expander + + +class ItemBuilderMeta(type): + def __new__(mcs, class_name, bases, attrs): + cls = type.__new__(mcs, class_name, bases, attrs) + cls._builder_fields = cls._builder_fields.copy() + + if cls.item_class: + for name, field in cls.item_class.fields.iteritems(): + bfield = None + if name in attrs: + bfield = attrs[name] + else: + if name not in cls._builder_fields and cls.default_builder: + bfield = cls.default_builder + #actually add the field to the class + setattr(cls, name, bfield) + if bfield: + cls._builder_fields.add(name) + return cls + + +class ItemBuilder(object): + __metaclass__ = ItemBuilderMeta + + item_class = None + default_builder = None + + _builder_fields = set() + + def __init__(self, response=None, item=None, **expander_args): + self._response = response + self._item = item if item else self.item_class() + + self._expander_args = {'response': self._response} + if expander_args: + self._expander_args.update(expander_args) + + self._values = defaultdict(list) + + def add_value(self, field_name, value, **new_expander_args): + field = self._get_builder_field(field_name) + evalue = self._expand_value(field, value, **new_expander_args) + self._values[field_name].extend(evalue) + + def get_item(self): + item = self._item + for field in self._values: + item[field] = self.get_value(field) + return item + + def get_value(self, field_name): + field = self._get_builder_field(field_name) + reducer = field.reducer or self._item.fields[field_name].from_unicode_list + return reducer(self._values[field_name]) + + def replace_value(self, field_name, value, **new_expander_args): + field = self._get_builder_field(field_name) + evalue = self._expand_value(field, value, **new_expander_args) + self._values[field_name] = evalue + + def _get_builder_field(self, name): + if name not in self._builder_fields: + raise KeyError + + return getattr(self, name) + + def _expand_value(self, field, value, **new_expander_args): + if new_expander_args: + expander_args = self._expander_args.copy() + expander_args.update(new_expander_args) + else: + expander_args = self._expander_args + + evalue = field.expander(value, expander_args=expander_args) + + if not isinstance(evalue, list): + evalue = [evalue] + + return evalue + diff --git a/scrapy/newitem/builder/reducers.py b/scrapy/newitem/builder/reducers.py new file mode 100644 index 000000000..de4670030 --- /dev/null +++ b/scrapy/newitem/builder/reducers.py @@ -0,0 +1,13 @@ +def take_first(values): + for value in values: + if value is not u'': + return value + + +def return_values(values): + return values + + +def join_strings(values): + return u' '.join(values) + diff --git a/scrapy/tests/test_itembuilder.py b/scrapy/tests/test_itembuilder.py new file mode 100644 index 000000000..8a041c7bc --- /dev/null +++ b/scrapy/tests/test_itembuilder.py @@ -0,0 +1,174 @@ +import unittest +import string +from scrapy.newitem.builder import ItemBuilder, BuilderField +from scrapy.newitem.builder import reducers +from scrapy.newitem import Item, fields + + +class BaseItem(Item): + name = fields.TextField() + + +class TestItem(BaseItem): + url = fields.TextField() + summary = fields.TextField() + + +class BaseItemBuilder(ItemBuilder): + item_class = TestItem + + +class TestItemBuilder(BaseItemBuilder): + name = BuilderField(lambda v: v.title()) + + +class DefaultedItemBuilder(BaseItemBuilder): + default_builder = BuilderField(lambda v: v[:-1]) + + +class InheritDefaultedItemBuilder(DefaultedItemBuilder): + pass + + +class ListFieldTestItem(Item): + names = fields.ListField(fields.TextField()) + + +class ListFieldItemBuilder(ItemBuilder): + item_class = ListFieldTestItem + + names = BuilderField(lambda v: v.title()) + + +class ItemBuilderTest(unittest.TestCase): + + def test_basic(self): + ib = TestItemBuilder() + + ib.add_value('name', u'marta') + self.assertEqual(ib.get_value('name'), u'Marta') + + item = ib.get_item() + self.assertEqual(item['name'], u'Marta') + + def test_multiple_functions(self): + class TestItemBuilder(BaseItemBuilder): + name = BuilderField(lambda v: v.title(), lambda v: v[:-1]) + + ib = TestItemBuilder() + + ib.add_value('name', u'marta') + self.assertEqual(ib.get_value('name'), u'Mart') + + item = ib.get_item() + self.assertEqual(item['name'], u'Mart') + + def test_defaulted(self): + dib = DefaultedItemBuilder() + assert dib.default_builder + + dib.add_value('name', u'marta') + self.assertEqual(dib.get_value('name'), u'mart') + + def test_inherited_default(self): + dib = InheritDefaultedItemBuilder() + assert dib.default_builder + + dib.add_value('name', u'marta') + self.assertEqual(dib.get_value('name'), u'mart') + + def test_inheritance(self): + class ChildItemBuilder(TestItemBuilder): + url = BuilderField(lambda v: v.lower()) + + ib = ChildItemBuilder() + assert 'url' in ib._builder_fields + assert 'name' in ib._builder_fields + + ib.add_value('url', u'HTTP://scrapy.ORG') + self.assertEqual(ib.get_value('url'), u'http://scrapy.org') + + ib.add_value('name', u'marta') + self.assertEqual(ib.get_value('name'), u'Marta') + + class ChildChildItemBuilder(ChildItemBuilder): + url = BuilderField(lambda v: v.upper()) + summary = BuilderField(lambda v: v) + + ib = ChildChildItemBuilder() + assert 'url' in ib._builder_fields + assert 'name' in ib._builder_fields + assert 'summary' in ib._builder_fields + + ib.add_value('url', u'http://scrapy.org') + self.assertEqual(ib.get_value('url'), u'HTTP://SCRAPY.ORG') + + ib.add_value('name', u'marta') + self.assertEqual(ib.get_value('name'), u'Marta') + + def test_multiplevaluedadaptor(self): + ib = ListFieldItemBuilder() + + ib.add_value('names', [u'name1', u'name2']) + self.assertEqual(ib.get_value('names'), [u'Name1', u'Name2']) + + def test_identity(self): + class IdentityDefaultedItemBuilder(DefaultedItemBuilder): + name = BuilderField() + + ib = IdentityDefaultedItemBuilder() + + ib.add_value('name', u'marta') + self.assertEqual(ib.get_value('name'), u'marta') + + def test_staticmethods(self): + class ChildItemBuilder(TestItemBuilder): + name = BuilderField(TestItemBuilder.name.expander, string.swapcase) + + ib = ChildItemBuilder() + + ib.add_value('name', u'marta') + self.assertEqual(ib.get_value('name'), u'mARTA') + + + def test_staticdefaults(self): + class ChildDefaultedItemBuilder(DefaultedItemBuilder): + name = BuilderField(DefaultedItemBuilder.name.expander, string.swapcase) + + ib = ChildDefaultedItemBuilder() + + ib.add_value('name', u'marta') + self.assertEqual(ib.get_value('name'), u'MART') + + def test_reducer(self): + ib = TestItemBuilder() + + ib.add_value('name', [u'mar', u'ta']) + self.assertEqual(ib.get_value('name'), u'Mar Ta') + + class TakeFirstItemBuilder(TestItemBuilder): + name = BuilderField(TestItemBuilder.name.expander, + reducer=reducers.take_first) + + ib = TakeFirstItemBuilder() + + ib.add_value('name', [u'mar', u'ta']) + self.assertEqual(ib.get_value('name'), u'Mar') + + def test_expander_args(self): + def expander_with_args(value, expander_args=None): + if 'val' in expander_args: + return expander_args['val'] + return value + + class ChildItemBuilder(TestItemBuilder): + url = BuilderField(expander_with_args) + + ib = ChildItemBuilder(val=u'val') + ib.add_value('url', u'text') + self.assertEqual(ib.get_value('url'), 'val') + + ib = ChildItemBuilder() + ib.add_value('url', u'text', val=u'val') + self.assertEqual(ib.get_value('url'), 'val') +