From ac9f4c9cc2802108c9868ae5999a83711b421db6 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Tue, 4 Aug 2009 19:26:31 -0300 Subject: [PATCH] Refatored scrapy.newitem package: - left only one type of Field - just a dict wrapper to contain field metadata - removed Item Builder and tests - adapted Item Loader to work with new Field class --- scrapy/newitem/__init__.py | 78 ++++++- scrapy/newitem/builder/__init__.py | 130 ----------- scrapy/newitem/fields.py | 168 -------------- scrapy/newitem/loader/__init__.py | 19 +- scrapy/newitem/models.py | 71 ------ scrapy/newitem/{builder => }/reducers.py | 9 +- scrapy/tests/test_itemadaptor.py | 150 ------------ scrapy/tests/test_itembuilder.py | 174 -------------- scrapy/tests/test_itemloader.py | 35 +-- scrapy/tests/test_newitem.py | 281 ++--------------------- 10 files changed, 117 insertions(+), 998 deletions(-) delete mode 100644 scrapy/newitem/builder/__init__.py delete mode 100644 scrapy/newitem/fields.py delete mode 100644 scrapy/newitem/models.py rename scrapy/newitem/{builder => }/reducers.py (69%) delete mode 100644 scrapy/tests/test_itemadaptor.py delete mode 100644 scrapy/tests/test_itembuilder.py diff --git a/scrapy/newitem/__init__.py b/scrapy/newitem/__init__.py index 70fc71565..e32dd9660 100644 --- a/scrapy/newitem/__init__.py +++ b/scrapy/newitem/__init__.py @@ -1 +1,77 @@ -from scrapy.newitem.models import Item +from UserDict import DictMixin + +from scrapy.item.models import BaseItem + +class Field(dict): + pass + +class _ItemMeta(type): + + def __new__(mcs, class_name, bases, attrs): + fields = {} + new_attrs = {} + for n, v in attrs.iteritems(): + if isinstance(v, Field): + fields[n] = v + else: + new_attrs[n] = v + + cls = type.__new__(mcs, class_name, bases, new_attrs) + cls.fields = cls.fields.copy() + cls.fields.update(fields) + return cls + + +class Item(DictMixin, BaseItem): + + __metaclass__ = _ItemMeta + + fields = {} + + def __init__(self, *args, **kwargs): + self._values = {} + + if args or kwargs: # don't instantiate dict for simple (most common) case + for k, v in dict(*args, **kwargs).iteritems(): + self[k] = v + + def __getitem__(self, key): + try: + return self._values[key] + except KeyError: + field = self.fields[key] + default_factory = field.get('default_factory') + if default_factory: + return default_factory() + else: + raise KeyError(key) + + def __setitem__(self, key, value): + if key in self.fields: + self._values[key] = value + else: + raise KeyError("%s does not support field: %s" % \ + (self.__class__.__name__, key)) + + def __delitem__(self, key): + del self._values[key] + + def __getattr__(self, name): + if name in self.fields: + raise AttributeError("Use [%r] to access item field value" % name) + raise AttributeError(name) + + def keys(self): + return self._values.keys() + + def __repr__(self): + """Generate a representation of this item that can be used to + reconstruct the item by evaluating it + """ + values = ', '.join('%s=%r' % field for field in self.iteritems()) + return "%s(%s)" % (self.__class__.__name__, values) + + def get_id(self): + """Returns the unique id for this item.""" + raise NotImplementedError + diff --git a/scrapy/newitem/builder/__init__.py b/scrapy/newitem/builder/__init__.py deleted file mode 100644 index 85ef494bb..000000000 --- a/scrapy/newitem/builder/__init__.py +++ /dev/null @@ -1,130 +0,0 @@ -from collections import defaultdict - -from scrapy.utils.misc import arg_to_iter -from scrapy.utils.python import get_func_args - - -class BuilderField(object): - - def __init__(self, *args, **kwargs): - self.expander = self.tree_expander(*args) - self.reducer = kwargs.get('reducer') - - def tree_expander(self, *funcs, **default_expander_args): - """A pipe expander implementing tree logic - - It takes multiples unnamed arguments used as functions of the pipe, and - keywords used as expander_args to be passed to functions that supports it - - If an expander function returns a list of values, each value is used as - input for next expander function - - Always returns a list of values - """ - - _funcs = [] - for func in funcs: - accepts_args = 'expander_args' in get_func_args(func) - _funcs.append((func, accepts_args)) - - def _expander(value, expander_args=None): - values = arg_to_iter(value) - aargs = default_expander_args - if expander_args: - aargs = aargs.copy() - aargs.update(expander_args) - pipe_kwargs = {'expander_args': aargs} - for func, accepts_args in _funcs: - next = [] - kwargs = pipe_kwargs if accepts_args else {} - for val in values: - val = func(val, **kwargs) - if hasattr(val, '__iter__'): - next.extend(val) - elif val is not None: - next.append(val) - values = next - return list(values) - - return _expander - - -class ItemBuilderMeta(type): - def __new__(mcs, class_name, bases, attrs): - cls = type.__new__(mcs, class_name, bases, attrs) - cls._builder_fields = cls._builder_fields.copy() - - if cls.item_class: - for name, field in cls.item_class.fields.iteritems(): - bfield = None - if name in attrs: - bfield = attrs[name] - else: - if name not in cls._builder_fields and cls.default_builder: - bfield = cls.default_builder - #actually add the field to the class - setattr(cls, name, bfield) - if bfield: - cls._builder_fields.add(name) - return cls - - -class ItemBuilder(object): - __metaclass__ = ItemBuilderMeta - - item_class = None - default_builder = None - - _builder_fields = set() - - def __init__(self, response=None, item=None, **expander_args): - self._response = response - self._item = item if item else self.item_class() - - self._expander_args = {'response': self._response} - if expander_args: - self._expander_args.update(expander_args) - - self._values = defaultdict(list) - - def add_value(self, field_name, value, **new_expander_args): - field = self._get_builder_field(field_name) - evalue = self._expand_value(field, value, **new_expander_args) - self._values[field_name].extend(evalue) - - def get_item(self): - item = self._item - for field in self._values: - item[field] = self.get_value(field) - return item - - def get_value(self, field_name): - field = self._get_builder_field(field_name) - reducer = field.reducer or self._item.fields[field_name].from_unicode_list - return reducer(self._values[field_name]) - - def replace_value(self, field_name, value, **new_expander_args): - field = self._get_builder_field(field_name) - evalue = self._expand_value(field, value, **new_expander_args) - self._values[field_name] = evalue - - def _get_builder_field(self, name): - if name not in self._builder_fields: - raise KeyError - - return getattr(self, name) - - def _expand_value(self, field, value, **new_expander_args): - if new_expander_args: - expander_args = self._expander_args.copy() - expander_args.update(new_expander_args) - else: - expander_args = self._expander_args - - evalue = field.expander(value, expander_args=expander_args) - - if not isinstance(evalue, list): - evalue = [evalue] - - return evalue - diff --git a/scrapy/newitem/fields.py b/scrapy/newitem/fields.py deleted file mode 100644 index a5e34a31b..000000000 --- a/scrapy/newitem/fields.py +++ /dev/null @@ -1,168 +0,0 @@ -import datetime -import decimal -import re -import time - - -class BaseField(object): - def __init__(self, default=None): - self._default = self.to_python(default) if default is not None else None - - def to_python(self, value): - raise NotImplementedError() - - def from_unicode_list(self, unicode_list): - return self.to_python(unicode_list[0]) if unicode_list else None - - def get_default(self): - return self._default - - -class ListField(BaseField): - def __init__(self, field, default=None): - self.field = field - super(ListField, self).__init__(default) - - def to_python(self, value): - if hasattr(value, '__iter__'): - return [self.field.to_python(v) for v in value] - else: - raise TypeError("Cannot instatiante %s with %s" \ - % (self.__class__.__name__, type(value).__name__)) - - def from_unicode_list(self, unicode_list): - return self.to_python(unicode_list) - - -class BooleanField(BaseField): - def to_python(self, value): - return bool(value) - - def from_unicode_list(self, unicode_list): - return self.to_python(unicode_list) - - -class DateField(BaseField): - ansi_date_re = re.compile(r'^\d{4}-\d{1,2}-\d{1,2}$') - - def to_python(self, value): - if isinstance(value, datetime.datetime): - return value.date() - elif isinstance(value, datetime.date): - return value - elif isinstance(value, basestring): - if not self.ansi_date_re.search(value): - raise ValueError("Enter a valid date in YYYY-MM-DD format.") - - year, month, day = map(int, value.split('-')) - try: - return datetime.date(year, month, day) - except ValueError, e: - raise ValueError("Invalid date: %s" % str(e)) - else: - raise TypeError("Cannot instatiante %s with %s" \ - % (self.__class__.__name__, type(value).__name__)) - - -class DateTimeField(BaseField): - def to_python(self, value): - if isinstance(value, datetime.datetime): - return value - elif isinstance(value, datetime.date): - return datetime.datetime(value.year, value.month, value.day) - elif isinstance(value, basestring): - # Attempt to parse a datetime: - value = str(value) - # split usecs, because they are not recognized by strptime. - if '.' in value: - try: - value, usecs = value.split('.') - usecs = int(usecs) - except ValueError: - raise ValueError('Enter a valid date/time in YYYY-MM-DD HH:MM[:ss[.uuuuuu]] format.') - else: - usecs = 0 - kwargs = {'microsecond': usecs} - try: # Seconds are optional, so try converting seconds first. - return datetime.datetime(*time.strptime(value, '%Y-%m-%d %H:%M:%S')[:6], - **kwargs) - - except ValueError: - try: # Try without seconds. - return datetime.datetime(*time.strptime(value, '%Y-%m-%d %H:%M')[:5], - **kwargs) - except ValueError: # Try without hour/minutes/seconds. - try: - return datetime.datetime(*time.strptime(value, '%Y-%m-%d')[:3], - **kwargs) - except ValueError: - raise ValueError('Enter a valid date/time in YYYY-MM-DD HH:MM[:ss[.uuuuuu]] format.') - else: - raise TypeError("Cannot instatiante %s with %s" \ - % (self.__class__.__name__, type(value).__name__)) - - - -class DecimalField(BaseField): - def to_python(self, value): - return decimal.Decimal(value) - - -class FloatField(BaseField): - def to_python(self, value): - return float(value) - - -class IntegerField(BaseField): - def to_python(self, value): - return int(value) - - -class TextField(BaseField): - def to_python(self, value): - if isinstance(value, unicode): - return value - elif isinstance(value, (long, float)): - return unicode(value) - # Note: True and False are instances of int! - elif isinstance(value, int) and not isinstance(value, bool): - return unicode(value) - else: - raise TypeError("%s values cannot be created from '%s' objects" % \ - (self.__class__.__name__, value.__class__.__name__)) - - def from_unicode_list(self, unicode_list): - return u' '.join((self.to_python(x) for x in unicode_list)) - -class TimeField(BaseField): - def to_python(self, value): - if isinstance(value, datetime.time): - return value - if isinstance(value, datetime.datetime): - return value.time - elif isinstance(value, basestring): - # Attempt to parse a datetime: - value = str(value) - # split usecs, because they are not recognized by strptime. - if '.' in value: - try: - value, usecs = value.split('.') - usecs = int(usecs) - except ValueError: - raise ValueError('Enter a valid time in HH:MM[:ss[.uuuuuu]] format.') - else: - usecs = 0 - kwargs = {'microsecond': usecs} - - try: # Seconds are optional, so try converting seconds first. - return datetime.time(*time.strptime(value, '%H:%M:%S')[3:6], - **kwargs) - except ValueError: - try: # Try without seconds. - return datetime.time(*time.strptime(value, '%H:%M')[3:5], - **kwargs) - except ValueError: - raise ValueError('Enter a valid time in HH:MM[:ss[.uuuuuu]] format.') - else: - raise TypeError("Cannot instatiante %s with %s" \ - % (self.__class__.__name__, type(value).__name__)) diff --git a/scrapy/newitem/loader/__init__.py b/scrapy/newitem/loader/__init__.py index 3b501a989..9403140a9 100644 --- a/scrapy/newitem/loader/__init__.py +++ b/scrapy/newitem/loader/__init__.py @@ -4,8 +4,8 @@ from types import UnboundMethodType from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import get_func_args from scrapy.utils.datatypes import MergeDict - -from scrapy.newitem.models import Item +from scrapy.newitem import Item +from scrapy.newitem.reducers import take_first def tree_expander(*functions, **default_loader_args): """Create an ItemLoader expander from a list of functions using the tree @@ -86,16 +86,14 @@ class ItemLoader(object): def get_value(self, field_name): values = self._values[field_name] - field = self._item.fields[field_name] reducer = self.get_reducer(field_name) - # XXX: calling different methods based on reducer is ugly - if reducer: - return field.to_python(reducer(values)) - else: - return field.from_unicode_list(values) + return reducer(values) def get_reducer(self, field_name): - return getattr(self, 'reduce_%s' % field_name, None) + try: + return getattr(self, 'reduce_%s' % field_name) + except AttributeError: + return self._item.fields[field_name].get('reducer', self.reduce) def get_expander(self, field_name): return getattr(self, 'expand_%s' % field_name, self.expand) @@ -111,3 +109,6 @@ class ItemLoader(object): def expand(self, value, loader_args): # default expander return value + + def reduce(self, values): + return take_first(values) diff --git a/scrapy/newitem/models.py b/scrapy/newitem/models.py deleted file mode 100644 index ef07462ac..000000000 --- a/scrapy/newitem/models.py +++ /dev/null @@ -1,71 +0,0 @@ -from UserDict import DictMixin - -from scrapy.item.models import BaseItem -from scrapy.newitem.fields import BaseField - - -class _ItemMeta(type): - - def __new__(mcs, class_name, bases, attrs): - fields = {} - new_attrs = {} - for n, v in attrs.iteritems(): - if isinstance(v, BaseField): - fields[n] = v - else: - new_attrs[n] = v - - cls = type.__new__(mcs, class_name, bases, new_attrs) - cls.fields = cls.fields.copy() - cls.fields.update(fields) - return cls - - -class Item(DictMixin, BaseItem): - - __metaclass__ = _ItemMeta - - fields = {} - - def __init__(self, *args, **kwargs): - self._values = {} - - if args or kwargs: # don't instantiate dict for simple (most common) case - for k, v in dict(*args, **kwargs).iteritems(): - self[k] = v - - def __getitem__(self, key): - try: - return self._values[key] - except KeyError: - default = self.fields[key].get_default() - if default is not None: - return default - else: - raise KeyError(key) - - def __setitem__(self, key, value): - self._values[key] = self.fields[key].to_python(value) - - def __delitem__(self, key): - del self._values[key] - - def __getattr__(self, name): - if name in self.fields: - raise AttributeError("Use [%r] to access item field value" % name) - raise AttributeError(name) - - def keys(self): - return self._values.keys() - - def __repr__(self): - """Generate a representation of this item that can be used to - reconstruct the item by evaluating it - """ - values = ', '.join('%s=%r' % field for field in self.iteritems()) - return "%s(%s)" % (self.__class__.__name__, values) - - def get_id(self): - """Returns the unique id for this item.""" - raise NotImplementedError - diff --git a/scrapy/newitem/builder/reducers.py b/scrapy/newitem/reducers.py similarity index 69% rename from scrapy/newitem/builder/reducers.py rename to scrapy/newitem/reducers.py index de4670030..e1394488f 100644 --- a/scrapy/newitem/builder/reducers.py +++ b/scrapy/newitem/reducers.py @@ -1,13 +1,12 @@ +"""Some common reducers""" + def take_first(values): for value in values: - if value is not u'': + if value: return value - -def return_values(values): +def identity(values): return values - def join_strings(values): return u' '.join(values) - diff --git a/scrapy/tests/test_itemadaptor.py b/scrapy/tests/test_itemadaptor.py deleted file mode 100644 index c64e3f2d9..000000000 --- a/scrapy/tests/test_itemadaptor.py +++ /dev/null @@ -1,150 +0,0 @@ -import unittest -import string -from scrapy.contrib_exp.newitem.adaptors import adaptor, ItemAdaptor -from scrapy.newitem import Item, fields - - -class BaseItem(Item): - name = fields.TextField() - - -class TestItem(BaseItem): - url = fields.TextField() - summary = fields.TextField() - - -class BaseAdaptor(ItemAdaptor): - item_class = TestItem - - -class TestAdaptor(BaseAdaptor): - name = lambda v: v.title() - - -class DefaultedAdaptor(BaseAdaptor): - default_adaptor = lambda v: v[:-1] - - -class InheritDefaultAdaptor(DefaultedAdaptor): - pass - - -class ListFieldTestItem(Item): - names = fields.ListField(fields.TextField()) - - -class ListFieldItemAdaptor(ItemAdaptor): - item_class = ListFieldTestItem - - names = adaptor(lambda v: v.title()) - - -class ItemAdaptorTest(unittest.TestCase): - - def test_basic(self): - ia = TestAdaptor() - ia.name = u'marta' - self.assertEqual(ia.item_instance['name'], u'Marta') - self.assertEqual(ia.name, u'Marta') - - def test_defaultadaptor(self): - dta = DefaultedAdaptor() - assert dta.default_adaptor - dta.name = u'marta' - self.assertEqual(dta.name, u'mart') - - def test_inheritdefaultadaptor(self): - ida = InheritDefaultAdaptor() - ida.name = u'marta' - assert ida.name == u'mart' - - def test_inheritance(self): - class ChildTestAdaptor(TestAdaptor): - url = lambda v: v.lower() - - ia = ChildTestAdaptor() - assert 'url' in ia._field_adaptors - assert 'name' in ia._field_adaptors - - ia.url = u'HTTP://scrapy.ORG' - self.assertEqual(ia.url, u'http://scrapy.org') - - ia.name = u'marta' - self.assertEqual(ia.name, u'Marta') - - class ChildChildTestAdaptor(ChildTestAdaptor): - url = lambda v: v.upper() - summary = lambda v: v - - ia = ChildChildTestAdaptor() - assert 'url' in ia._field_adaptors - assert 'name' in ia._field_adaptors - assert 'summary' in ia._field_adaptors - - ia.url = u'HTTP://scrapy.ORG' - self.assertEqual(ia.url, u'HTTP://SCRAPY.ORG') - - ia.name = u'marta' - self.assertEqual(ia.name, u'Marta') - -# FIXME: deprecated tests - will be replaced by ItemBuilder tests -# -# def test_staticmethods(self): -# class ChildAdaptor(TestAdaptor): -# name = adaptor(TestAdaptor.name, string.swapcase) -# -# ia = ChildAdaptor() -# ia.name = u'Marta' -# self.assertEqual(ia.name, u'mARTA') -# -# def test_staticdefaults(self): -# class ChildAdaptorDefaulted(DefaultedAdaptor): -# name = adaptor(DefaultedAdaptor.name, string.swapcase) -# -# dia = ChildAdaptorDefaulted() -# dia.name = u'marta' -# self.assertEqual(dia.name, u'MART') - - def test_multiplevaluedadaptor(self): - ma = ListFieldItemAdaptor() - ma.names = [u'name1', u'name2'] - assert ma.names == [u'Name1', u'Name2'] - - -class TreeadaptTest(unittest.TestCase): - - def test_1_passtrough(self): - ad = adaptor() - self.assertEqual(ad('string'), ['string']) - - def test_2_composing(self): - addone = lambda v: v+1 - ad = adaptor(addone) - self.assertEqual(ad(0), [1]) - - addtwo = lambda v: v+2 - ad2 = adaptor(addone, addtwo) - self.assertEqual(ad2(0), [3]) - - ad3 = adaptor(ad, ad2) - self.assertEqual(ad3(0), [4]) - - def test_3_adaptor_args(self): - addn = lambda v, adaptor_args: v + adaptor_args.get('add_value', 0) - - ad = adaptor(addn) - self.assertEqual(ad(0), [0]) - self.assertEqual(ad(0, {'add_value': 3}), [3]) - self.assertEqual(ad(0), [0]) - - ad = adaptor(addn, add_value=5) - self.assertEqual(ad(0), [5]) - self.assertEqual(ad(0, {'add_value': 3}), [3]) - self.assertEqual(ad(0), [5]) - - def test_4_treelogic(self): - split1 = lambda v: v.split('&') - split2 = lambda v: v.split('=') - ad = adaptor(split1, split2) - self.assertEqual(ad('name=joe&job=joker'), ['name', 'joe', 'job', 'joker']) - diff --git a/scrapy/tests/test_itembuilder.py b/scrapy/tests/test_itembuilder.py deleted file mode 100644 index 8a041c7bc..000000000 --- a/scrapy/tests/test_itembuilder.py +++ /dev/null @@ -1,174 +0,0 @@ -import unittest -import string -from scrapy.newitem.builder import ItemBuilder, BuilderField -from scrapy.newitem.builder import reducers -from scrapy.newitem import Item, fields - - -class BaseItem(Item): - name = fields.TextField() - - -class TestItem(BaseItem): - url = fields.TextField() - summary = fields.TextField() - - -class BaseItemBuilder(ItemBuilder): - item_class = TestItem - - -class TestItemBuilder(BaseItemBuilder): - name = BuilderField(lambda v: v.title()) - - -class DefaultedItemBuilder(BaseItemBuilder): - default_builder = BuilderField(lambda v: v[:-1]) - - -class InheritDefaultedItemBuilder(DefaultedItemBuilder): - pass - - -class ListFieldTestItem(Item): - names = fields.ListField(fields.TextField()) - - -class ListFieldItemBuilder(ItemBuilder): - item_class = ListFieldTestItem - - names = BuilderField(lambda v: v.title()) - - -class ItemBuilderTest(unittest.TestCase): - - def test_basic(self): - ib = TestItemBuilder() - - ib.add_value('name', u'marta') - self.assertEqual(ib.get_value('name'), u'Marta') - - item = ib.get_item() - self.assertEqual(item['name'], u'Marta') - - def test_multiple_functions(self): - class TestItemBuilder(BaseItemBuilder): - name = BuilderField(lambda v: v.title(), lambda v: v[:-1]) - - ib = TestItemBuilder() - - ib.add_value('name', u'marta') - self.assertEqual(ib.get_value('name'), u'Mart') - - item = ib.get_item() - self.assertEqual(item['name'], u'Mart') - - def test_defaulted(self): - dib = DefaultedItemBuilder() - assert dib.default_builder - - dib.add_value('name', u'marta') - self.assertEqual(dib.get_value('name'), u'mart') - - def test_inherited_default(self): - dib = InheritDefaultedItemBuilder() - assert dib.default_builder - - dib.add_value('name', u'marta') - self.assertEqual(dib.get_value('name'), u'mart') - - def test_inheritance(self): - class ChildItemBuilder(TestItemBuilder): - url = BuilderField(lambda v: v.lower()) - - ib = ChildItemBuilder() - assert 'url' in ib._builder_fields - assert 'name' in ib._builder_fields - - ib.add_value('url', u'HTTP://scrapy.ORG') - self.assertEqual(ib.get_value('url'), u'http://scrapy.org') - - ib.add_value('name', u'marta') - self.assertEqual(ib.get_value('name'), u'Marta') - - class ChildChildItemBuilder(ChildItemBuilder): - url = BuilderField(lambda v: v.upper()) - summary = BuilderField(lambda v: v) - - ib = ChildChildItemBuilder() - assert 'url' in ib._builder_fields - assert 'name' in ib._builder_fields - assert 'summary' in ib._builder_fields - - ib.add_value('url', u'http://scrapy.org') - self.assertEqual(ib.get_value('url'), u'HTTP://SCRAPY.ORG') - - ib.add_value('name', u'marta') - self.assertEqual(ib.get_value('name'), u'Marta') - - def test_multiplevaluedadaptor(self): - ib = ListFieldItemBuilder() - - ib.add_value('names', [u'name1', u'name2']) - self.assertEqual(ib.get_value('names'), [u'Name1', u'Name2']) - - def test_identity(self): - class IdentityDefaultedItemBuilder(DefaultedItemBuilder): - name = BuilderField() - - ib = IdentityDefaultedItemBuilder() - - ib.add_value('name', u'marta') - self.assertEqual(ib.get_value('name'), u'marta') - - def test_staticmethods(self): - class ChildItemBuilder(TestItemBuilder): - name = BuilderField(TestItemBuilder.name.expander, string.swapcase) - - ib = ChildItemBuilder() - - ib.add_value('name', u'marta') - self.assertEqual(ib.get_value('name'), u'mARTA') - - - def test_staticdefaults(self): - class ChildDefaultedItemBuilder(DefaultedItemBuilder): - name = BuilderField(DefaultedItemBuilder.name.expander, string.swapcase) - - ib = ChildDefaultedItemBuilder() - - ib.add_value('name', u'marta') - self.assertEqual(ib.get_value('name'), u'MART') - - def test_reducer(self): - ib = TestItemBuilder() - - ib.add_value('name', [u'mar', u'ta']) - self.assertEqual(ib.get_value('name'), u'Mar Ta') - - class TakeFirstItemBuilder(TestItemBuilder): - name = BuilderField(TestItemBuilder.name.expander, - reducer=reducers.take_first) - - ib = TakeFirstItemBuilder() - - ib.add_value('name', [u'mar', u'ta']) - self.assertEqual(ib.get_value('name'), u'Mar') - - def test_expander_args(self): - def expander_with_args(value, expander_args=None): - if 'val' in expander_args: - return expander_args['val'] - return value - - class ChildItemBuilder(TestItemBuilder): - url = BuilderField(expander_with_args) - - ib = ChildItemBuilder(val=u'val') - ib.add_value('url', u'text') - self.assertEqual(ib.get_value('url'), 'val') - - ib = ChildItemBuilder() - ib.add_value('url', u'text', val=u'val') - self.assertEqual(ib.get_value('url'), 'val') - diff --git a/scrapy/tests/test_itemloader.py b/scrapy/tests/test_itemloader.py index 2f07aded0..56faaf1de 100644 --- a/scrapy/tests/test_itemloader.py +++ b/scrapy/tests/test_itemloader.py @@ -1,18 +1,16 @@ import unittest -import string from scrapy.newitem.loader import ItemLoader, tree_expander -from scrapy.newitem.builder import reducers -from scrapy.newitem import Item, fields +from scrapy.newitem import Item, Field class BaseItem(Item): - name = fields.TextField() + name = Field() class TestItem(BaseItem): - url = fields.TextField() - summary = fields.TextField() + url = Field() + summary = Field() class BaseItemLoader(ItemLoader): @@ -30,15 +28,6 @@ class DefaultedItemLoader(BaseItemLoader): class InheritDefaultedItemLoader(DefaultedItemLoader): pass -class ListFieldTestItem(Item): - names = fields.ListField(fields.TextField()) - - -class ListFieldItemLoader(ItemLoader): - item_class = ListFieldTestItem - - expand_names = tree_expander(lambda v: v.title()) - class ItemLoaderTest(unittest.TestCase): @@ -99,12 +88,6 @@ class ItemLoaderTest(unittest.TestCase): ib.add_value('name', u'marta') self.assertEqual(ib.get_value('name'), u'Marta') - def test_multiplevaluedadaptor(self): - ib = ListFieldItemLoader() - - ib.add_value('names', [u'name1', u'name2']) - self.assertEqual(ib.get_value('names'), [u'Name1', u'Name2']) - def test_identity(self): class IdentityDefaultedItemLoader(DefaultedItemLoader): expand_name = tree_expander() @@ -116,7 +99,7 @@ class ItemLoaderTest(unittest.TestCase): def test_staticmethods(self): class ChildItemLoader(TestItemLoader): - expand_name = tree_expander(TestItemLoader.expand_name, string.swapcase) + expand_name = tree_expander(TestItemLoader.expand_name, unicode.swapcase) ib = ChildItemLoader() @@ -126,7 +109,7 @@ class ItemLoaderTest(unittest.TestCase): def test_staticdefaults(self): class ChildDefaultedItemLoader(DefaultedItemLoader): - expand_name = tree_expander(DefaultedItemLoader.expand, string.swapcase) + expand_name = tree_expander(DefaultedItemLoader.expand, unicode.swapcase) ib = ChildDefaultedItemLoader() @@ -137,15 +120,15 @@ class ItemLoaderTest(unittest.TestCase): ib = TestItemLoader() ib.add_value('name', [u'mar', u'ta']) - self.assertEqual(ib.get_value('name'), u'Mar Ta') + self.assertEqual(ib.get_value('name'), u'Mar') class TakeFirstItemLoader(TestItemLoader): - reduce_name = staticmethod(reducers.take_first) + reduce_name = staticmethod(u" ".join) ib = TakeFirstItemLoader() ib.add_value('name', [u'mar', u'ta']) - self.assertEqual(ib.get_value('name'), u'Mar') + self.assertEqual(ib.get_value('name'), u'Mar Ta') def test_loader_args(self): def expander_func_with_args(value, loader_args): diff --git a/scrapy/tests/test_newitem.py b/scrapy/tests/test_newitem.py index fb6fc5926..bd801df78 100644 --- a/scrapy/tests/test_newitem.py +++ b/scrapy/tests/test_newitem.py @@ -2,15 +2,14 @@ import datetime import decimal import unittest -from scrapy.newitem import Item, fields -from scrapy.newitem.fields import BaseField +from scrapy.newitem import Item, Field class NewItemTest(unittest.TestCase): def test_simple(self): class TestItem(Item): - name = fields.TextField() + name = Field() i = TestItem() i['name'] = u'name' @@ -18,7 +17,7 @@ class NewItemTest(unittest.TestCase): def test_init(self): class TestItem(Item): - name = fields.TextField() + name = Field() i = TestItem() self.assertRaises(KeyError, i.__getitem__, 'name') @@ -35,18 +34,6 @@ class NewItemTest(unittest.TestCase): self.assertRaises(KeyError, TestItem, {'name': u'john doe', 'other': u'foo'}) - self.assertRaises(TypeError, TestItem, name=set()) - - def test_list(self): - class TestListItem(Item): - name = fields.TextField() - names = fields.ListField(fields.TextField()) - - i = TestListItem() - i['name'] = u'name' - i['names'] = [u'name1', u'name2'] - self.assertEqual(i['names'], [u'name1', u'name2']) - def test_invalid_field(self): class TestItem(Item): pass @@ -57,22 +44,19 @@ class NewItemTest(unittest.TestCase): def test_default_value(self): class TestItem(Item): - name = fields.TextField(default=u'John') + name = Field(default_factory=lambda: u'John') i = TestItem() self.assertEqual(i['name'], u'John') - def test_wrong_default(self): - self.assertRaises(TypeError, fields.TextField, default=set()) - def test_repr(self): class TestItem(Item): - name = fields.TextField() - number = fields.IntegerField() + name = Field() + number = Field() i = TestItem() i['name'] = u'John Doe' - i['number'] = '123' + i['number'] = 123 itemrepr = repr(i) self.assertEqual(itemrepr, "TestItem(name=u'John Doe', number=123)") @@ -83,7 +67,7 @@ class NewItemTest(unittest.TestCase): def test_private_attr(self): class TestItem(Item): - name = fields.TextField() + name = Field() i = TestItem() i._private = 'test' @@ -91,7 +75,7 @@ class NewItemTest(unittest.TestCase): def test_custom_methods(self): class TestItem(Item): - name = fields.TextField() + name = Field() def get_name(self): return self['name'] @@ -108,9 +92,9 @@ class NewItemTest(unittest.TestCase): def test_metaclass(self): class TestItem(Item): - name = fields.TextField() - keys = fields.TextField() - values = fields.TextField() + name = Field() + keys = Field() + values = Field() i = TestItem() i['name'] = u'John' @@ -124,12 +108,12 @@ class NewItemTest(unittest.TestCase): def test_metaclass_inheritance(self): class BaseItem(Item): - name = fields.TextField() - keys = fields.TextField() - values = fields.TextField() + name = Field() + keys = Field() + values = Field() class TestItem(BaseItem): - keys = fields.IntegerField() + keys = Field() i = TestItem() i['keys'] = 3 @@ -138,240 +122,9 @@ class NewItemTest(unittest.TestCase): def test_to_dict(self): class TestItem(Item): - name = fields.TextField() + name = Field() i = TestItem() i['name'] = u'John' self.assertEqual(dict(i), {'name': u'John'}) - def test_id(self): - class TestItem(Item): - name = fields.TextField() - - i = TestItem() - self.assertRaises(NotImplementedError, i.get_id) - - class IdItem(Item): - id = fields.IntegerField() - - def get_id(self): - return self['id'] - - i = IdItem() - i['id'] = 11 - self.assertEqual(i.get_id(), 11) - - -class NewItemFieldsTest(unittest.TestCase): - - def test_base_field(self): - f = fields.BaseField() - - self.assert_(f.get_default() is None) - self.assertRaises(NotImplementedError, f.to_python, 1) - - def test_boolean_field(self): - class TestItem(Item): - field = fields.BooleanField() - - i = TestItem() - - i['field'] = True - self.assert_(i['field'] is True) - - i['field'] = 1 - self.assert_(i['field'] is True) - - i['field'] = False - self.assert_(i['field'] is False) - - i['field'] = 0 - self.assert_(i['field'] is False) - - i['field'] = None - self.assert_(i['field'] is False) - - def test_date_field(self): - class TestItem(Item): - field = fields.DateField() - - i = TestItem() - - d_today = datetime.date.today() - i['field'] = d_today - self.assertEqual(i['field'], d_today) - - dt_today = datetime.datetime.today() - i['field'] = dt_today - self.assertEqual(i['field'], dt_today.date()) - - i['field'] = '2009-05-21' - self.assertEqual(i['field'], datetime.date(2009, 5, 21)) - - self.assertRaises(ValueError, i.__setitem__, 'field', '21-05-2009') - - self.assertRaises(ValueError, i.__setitem__, 'field', '2009-05-51') - - self.assertRaises(TypeError, i.__setitem__, 'field', None) - - def test_datetime_field(self): - class TestItem(Item): - field = fields.DateTimeField() - - i = TestItem() - - dt_today = datetime.datetime.today() - i['field'] = dt_today - self.assertEqual(i['field'], dt_today) - - d_today = datetime.date.today() - i['field'] = d_today - self.assertEqual(i['field'], datetime.datetime(d_today.year, - d_today.month, d_today.day)) - - i['field'] = '2009-05-21 11:08:10.100' - self.assertEqual(i['field'], datetime.datetime(2009, 5, 21, 11, 8, 10, - 100)) - - i['field'] = '2009-05-21 11:08:10' - self.assertEqual(i['field'], datetime.datetime(2009, 5, 21, 11, 8, 10)) - - i['field'] = '2009-05-21 11:08' - self.assertEqual(i['field'], datetime.datetime(2009, 5, 21, 11, 8)) - - i['field'] = '2009-05-21' - self.assertEqual(i['field'], datetime.datetime(2009, 5, 21)) - - self.assertRaises(ValueError, i.__setitem__, 'field', '2009-05-21 11:08:10.usecs') - - self.assertRaises(ValueError, i.__setitem__, 'field', '21-05-2009') - - self.assertRaises(ValueError, i.__setitem__, 'field', '2009-05-51') - - self.assertRaises(TypeError, i.__setitem__, 'field', None) - - def test_decimal_field(self): - class TestItem(Item): - field = fields.DecimalField() - - i = TestItem() - - i['field'] = decimal.Decimal('3.14') - self.assertEqual(i['field'], decimal.Decimal('3.14')) - - i['field'] = '3.14' - self.assertEqual(i['field'], decimal.Decimal('3.14')) - - self.assertRaises(decimal.InvalidOperation, i.__setitem__, 'field', 'text') - - self.assertRaises(TypeError, i.__setitem__, 'field', None) - - def test_float_field(self): - class TestItem(Item): - field = fields.FloatField() - - i = TestItem() - - i['field'] = 3.14 - self.assertEqual(i['field'], 3.14) - - i['field'] = '3.14' - self.assertEqual(i['field'], 3.14) - - self.assertRaises(ValueError, i.__setitem__, 'field', 'text') - - self.assertRaises(TypeError, i.__setitem__, 'field', None) - - def test_integer_field(self): - class TestItem(Item): - field = fields.IntegerField() - - i = TestItem() - - i['field'] = 3 - self.assertEqual(i['field'], 3) - - i['field'] = '3' - self.assertEqual(i['field'], 3) - - self.assertRaises(ValueError, i.__setitem__, 'field', 'text') - - self.assertRaises(TypeError, i.__setitem__, 'field', None) - - def test_text_field(self): - class TestItem(Item): - field = fields.TextField() - - i = TestItem() - - # valid castings - i['field'] = u'hello' - self.assertEqual(i['field'], u'hello') - self.assert_(isinstance(i['field'], unicode)) - - i['field'] = 3 - self.assertEqual(i['field'], u'3') - self.assert_(isinstance(i['field'], unicode)) - - i['field'] = 3.2 - self.assertEqual(i['field'], u'3.2') - self.assert_(isinstance(i['field'], unicode)) - - i['field'] = 100L - self.assertEqual(i['field'], u'100') - self.assert_(isinstance(i['field'], unicode)) - - # invalid castings - self.assertRaises(TypeError, i.__setitem__, 'field', [u'hello', u'world']) - self.assertRaises(TypeError, i.__setitem__, 'field', 'string') # must be unicode! - self.assertRaises(TypeError, i.__setitem__, 'field', set()) - self.assertRaises(TypeError, i.__setitem__, 'field', True) - self.assertRaises(TypeError, i.__setitem__, 'field', None) - - - def test_from_unicode_list(self): - field = fields.BaseField() - self.assertEqual(field.from_unicode_list([]), None) - - field = fields.TextField() - self.assertEqual(field.from_unicode_list([]), u'') - self.assertEqual(field.from_unicode_list([u'hello', u'world']), u'hello world') - - field = fields.ListField(fields.TextField()) - self.assertEqual(field.from_unicode_list([]), []) - self.assertEqual(field.from_unicode_list([u'hello', u'world']), [u'hello', u'world']) - - field = fields.IntegerField() - self.assertEqual(field.from_unicode_list([u'123']), 123) - - def test_time_field(self): - class TestItem(Item): - field = fields.TimeField() - - i = TestItem() - - dt_t = datetime.time(11, 8, 10, 100) - i['field'] = dt_t - self.assertEqual(i['field'], dt_t) - - self.assertRaises(TypeError, i.__setitem__, 'field', None) - - dt_dt = datetime.datetime.today() - i['field'] = dt_dt - self.assertEqual(i['field'], dt_dt.time) - - i['field'] = '11:08:10.100' - self.assertEqual(i['field'], datetime.time(11, 8, 10, 100)) - - i['field'] = '11:08:10' - self.assertEqual(i['field'], datetime.time(11, 8, 10)) - - i['field'] = '11:08' - self.assertEqual(i['field'], datetime.time(11, 8)) - - self.assertRaises(ValueError, i.__setitem__, 'field', '11:08:10.usecs') - - self.assertRaises(ValueError, i.__setitem__, 'field', '25:08:10') - - self.assertRaises(ValueError, i.__setitem__, 'field', 'string') -