changed the newitem API to a dict-like interface

This commit is contained in:
Ismael Carnales 2009-07-21 12:20:49 -03:00
parent 6fcbd03bff
commit 894a5d81d3
5 changed files with 225 additions and 160 deletions

View File

@ -53,9 +53,9 @@ And then you instantiate the item and assign values to its fields, which will be
converted to the expected Python types depending of their class::
>>> item = NewsItem()
>>> item.headline = u'Headline'
>>> item.content = u'Content'
>>> item.published = '2009-07-08'
>>> item['headline'] = u'Headline'
>>> item['content'] = u'Content'
>>> item['published'] = '2009-07-08'
>>> item
NewsItem({'headline': u'Headline', 'content': u'Content', 'published': datetime.date(2009, 7, 8)})
@ -72,8 +72,8 @@ Each field accepts a ``default`` argument, that sets the default value of the
field.
Fields which contain a default value will always return that value when not
set, while fields which don't contain a default value will always return
``None`` when not set:
set, while fields which don't contain a default value will raise ``KeyError``,
you can use ``get`` method to avoid this.
.. code-block:: python
@ -89,12 +89,14 @@ set, while fields which don't contain a default value will always return
.. code-block:: python
>>> it = NewsItem()
>>> it.content is None
True
>>> it.author
>>> it['author']
u'Myself'
>>> it.published is None
True
>>> it.views
>>> it['views']
0
>>> it['headline']
Traceback (most recent call last):
...
KeyError: 'content
>>> it.get('content') is None
True

View File

@ -27,7 +27,7 @@ class ItemAdaptorMeta(type):
cls._field_adaptors = cls._field_adaptors.copy()
if cls.item_class:
for item_field in cls.item_class.get_fields():
for item_field in cls.item_class.fields:
if item_field in attrs:
adaptor = adaptize(attrs[item_field])
cls._field_adaptors[item_field] = adaptor
@ -35,7 +35,7 @@ class ItemAdaptorMeta(type):
return cls
def __getattr__(cls, name):
if name in cls.item_class.get_fields():
if name in cls.item_class.fields:
return cls.default_adaptor
raise AttributeError(name)
@ -61,14 +61,14 @@ class ItemAdaptor(object):
adaptor_args = {'response': self._response, 'item': self.item_instance}
ovalue = fa(value, adaptor_args=adaptor_args)
setattr(self.item_instance, name, ovalue)
self.item_instance[name] = ovalue
def __getattribute__(self, name):
if (name.startswith('_') or name.startswith('item_') \
or name == 'default_adaptor'):
return object.__getattribute__(self, name)
return getattr(self.item_instance, name)
return self.item_instance[name]
def adaptor(*funcs, **default_adaptor_args):

View File

@ -1,3 +1,5 @@
from UserDict import DictMixin
from scrapy.item import ScrapedItem
from scrapy.contrib_exp.newitem.fields import BaseField
@ -5,58 +7,60 @@ from scrapy.contrib_exp.newitem.fields import BaseField
class _ItemMeta(type):
def __new__(meta, class_name, bases, attrs):
cls = type.__new__(meta, class_name, bases, attrs)
cls._fields = cls._fields.copy()
fields = {}
new_attrs = {}
for n, v in attrs.iteritems():
if isinstance(v, BaseField):
cls._fields[n] = v
delattr(cls, n)
fields[n] = v
else:
new_attrs[n] = v
cls = type.__new__(meta, class_name, bases, new_attrs)
cls.fields = cls.fields.copy()
cls.fields.update(fields)
return cls
class Item(ScrapedItem):
""" This is the base class for all scraped items. """
class Item(DictMixin, ScrapedItem):
__metaclass__ = _ItemMeta
_fields = {}
fields = {}
def __init__(self, values=None):
def __init__(self, *args, **kwargs):
self._values = {}
if isinstance(values, dict):
for k, v in values.iteritems():
setattr(self, k, v)
elif values is not None:
raise TypeError("Items must be instantiated with dicts, got %s" % \
type(values).__name__)
# load default values
for name, field in self.fields.iteritems():
default = field.get_default()
if default:
self._values[name] = default
# load init values
if args or kwargs: # don't instantiate dict for simple (most common) case
for k, v in dict(*args, **kwargs).iteritems():
self[k] = v
def __setattr__(self, name, value):
if name.startswith('_'):
return ScrapedItem.__setattr__(self, name, value)
def __getitem__(self, key):
return self._values[key]
if name in self._fields:
self._values[name] = self._fields[name].to_python(value)
else:
raise AttributeError(name)
def __setitem__(self, key, value):
self._values[key] = self.fields[key].to_python(value)
def __getattr__(self, name):
try:
return self._values[name]
except KeyError:
try:
return self._fields[name].get_default()
except KeyError:
raise AttributeError(name)
def __delitem__(self, key):
del self._values[key]
def keys(self):
return self._values.keys()
def __repr__(self):
"""Generate a representation of this item that can be used to
reconstruct the item by evaluating it
"""
values = dict((field, getattr(self, field)) for field in self._fields)
return "%s(%s)" % (self.__class__.__name__, repr(values))
values = ', '.join('%s=%r' % field for field in self.iteritems())
return "%s(%s)" % (self.__class__.__name__, values)
@classmethod
def get_fields(cls):
"""Returns the item fields"""
return cls._fields
def get_id(self):
"""Returns the unique id for this item."""
raise NotImplementedError

View File

@ -44,7 +44,7 @@ class ItemAdaptorTest(unittest.TestCase):
def test_basic(self):
ia = TestAdaptor()
ia.name = u'marta'
self.assertEqual(ia.item_instance.name, u'Marta')
self.assertEqual(ia.item_instance['name'], u'Marta')
self.assertEqual(ia.name, u'Marta')
def test_defaultadaptor(self):

View File

@ -13,23 +13,29 @@ class NewItemTest(unittest.TestCase):
name = fields.TextField()
i = TestItem()
i.name = u'name'
self.assertEqual(i.name, u'name')
i['name'] = u'name'
self.assertEqual(i['name'], u'name')
def test_init(self):
class TestItem(Item):
name = fields.TextField()
i = TestItem()
self.assert_(i.name is None)
self.assertRaises(KeyError, i.__getitem__, 'name')
i2 = TestItem({'name': u'john doe'})
self.assertEqual(i2.name, u'john doe')
i2 = TestItem(name=u'john doe')
self.assertEqual(i2['name'], u'john doe')
self.assertRaises(TypeError, TestItem, name=u'john doe')
i3 = TestItem({'name': u'john doe'})
self.assertEqual(i3['name'], u'john doe')
self.assertRaises(AttributeError, TestItem, {'name': u'john doe',
'other': u'foo'})
i4 = TestItem(i3)
self.assertEqual(i4['name'], u'john doe')
self.assertRaises(KeyError, TestItem, {'name': u'john doe',
'other': u'foo'})
self.assertRaises(TypeError, TestItem, name=3)
def test_multi(self):
class TestMultiItem(Item):
@ -37,26 +43,24 @@ class NewItemTest(unittest.TestCase):
names = fields.MultiValuedField(fields.TextField)
i = TestMultiItem()
i.name = u'name'
i.names = [u'name1', u'name2']
self.assertEqual(i.names, [u'name1', u'name2'])
i['name'] = u'name'
i['names'] = [u'name1', u'name2']
self.assertEqual(i['names'], [u'name1', u'name2'])
def test_invalid_field(self):
class TestItem(Item):
pass
i = TestItem()
self.assertRaises(AttributeError, setattr, i, 'field', 'text')
self.assertRaises(AttributeError, getattr, i, 'field')
self.assertRaises(KeyError, i.__setitem__, 'field', 'text')
self.assertRaises(KeyError, i.__getitem__, 'field')
def test_default_value(self):
class TestItem(Item):
name = fields.TextField(default=u'John')
i = TestItem()
self.assertEqual(i.name, u'John')
self.assertEqual(i['name'], u'John')
def test_wrong_default(self):
def set_wrong_default():
@ -70,8 +74,8 @@ class NewItemTest(unittest.TestCase):
name = fields.TextField()
i = TestItem()
i.name = (u'John', u'Doe')
self.assertEqual(i.name, u'John Doe')
i['name'] = (u'John', u'Doe')
self.assertEqual(i['name'], u'John Doe')
def test_repr(self):
class TestItem(Item):
@ -79,15 +83,15 @@ class NewItemTest(unittest.TestCase):
number = fields.IntegerField()
i = TestItem()
i.name = u'John Doe'
i.number = '123'
i['name'] = u'John Doe'
i['number'] = '123'
itemrepr = repr(i)
self.assertEqual(itemrepr,
"TestItem({'name': u'John Doe', 'number': 123})")
"TestItem(name=u'John Doe', number=123)")
i2 = eval(itemrepr)
self.assertEqual(i2.name, 'John Doe')
self.assertEqual(i2.number, 123)
self.assertEqual(i2['name'], 'John Doe')
self.assertEqual(i2['number'], 123)
def test_private_attr(self):
class TestItem(Item):
@ -102,18 +106,73 @@ class NewItemTest(unittest.TestCase):
name = fields.TextField()
def get_name(self):
return self.name
return self['name']
def change_name(self, name):
self.name = name
self['name'] = name
i = TestItem()
self.assertEqual(i.get_name(), None)
i.name = u'lala'
self.assertRaises(KeyError, i.get_name)
i['name'] = u'lala'
self.assertEqual(i.get_name(), u'lala')
i.change_name(u'other')
self.assertEqual(i.get_name(), 'other')
def test_metaclass(self):
class TestItem(Item):
name = fields.TextField()
keys = fields.TextField()
values = fields.TextField()
i = TestItem()
i['name'] = u'John'
self.assertEqual(i.keys(), ['name'])
self.assertEqual(i.values(), ['John'])
i['keys'] = u'Keys'
i['values'] = u'Values'
self.assertEqual(i.keys(), ['keys', 'values', 'name'])
self.assertEqual(i.values(), [u'Keys', u'Values', u'John'])
def test_metaclass_inheritance(self):
class BaseItem(Item):
name = fields.TextField()
keys = fields.TextField()
values = fields.TextField()
class TestItem(BaseItem):
keys = fields.IntegerField()
i = TestItem()
i['keys'] = 3
self.assertEqual(i.keys(), ['keys'])
self.assertEqual(i.values(), [3])
def test_to_dict(self):
class TestItem(Item):
name = fields.TextField()
i = TestItem()
i['name'] = u'John'
self.assertEqual(dict(i), {'name': u'John'})
def test_id(self):
class TestItem(Item):
name = fields.TextField()
i = TestItem()
self.assertRaises(NotImplementedError, i.get_id)
class IdItem(Item):
id = fields.IntegerField()
def get_id(self):
return self['id']
i = IdItem()
i['id'] = 11
self.assertEqual(i.get_id(), 11)
class NewItemFieldsTest(unittest.TestCase):
@ -129,20 +188,20 @@ class NewItemFieldsTest(unittest.TestCase):
i = TestItem()
i.field = True
self.assert_(i.field is True)
i['field'] = True
self.assert_(i['field'] is True)
i.field = 1
self.assert_(i.field is True)
i['field'] = 1
self.assert_(i['field'] is True)
i.field = False
self.assert_(i.field is False)
i['field'] = False
self.assert_(i['field'] is False)
i.field = 0
self.assert_(i.field is False)
i['field'] = 0
self.assert_(i['field'] is False)
i.field = None
self.assert_(i.field is False)
i['field'] = None
self.assert_(i['field'] is False)
def test_date_field(self):
class TestItem(Item):
@ -151,21 +210,21 @@ class NewItemFieldsTest(unittest.TestCase):
i = TestItem()
d_today = datetime.date.today()
i.field = d_today
self.assertEqual(i.field, d_today)
i['field'] = d_today
self.assertEqual(i['field'], d_today)
dt_today = datetime.datetime.today()
i.field = dt_today
self.assertEqual(i.field, dt_today.date())
i['field'] = dt_today
self.assertEqual(i['field'], dt_today.date())
i.field = '2009-05-21'
self.assertEqual(i.field, datetime.date(2009, 5, 21))
i['field'] = '2009-05-21'
self.assertEqual(i['field'], datetime.date(2009, 5, 21))
self.assertRaises(ValueError, setattr, i, 'field', '21-05-2009')
self.assertRaises(ValueError, i.__setitem__, 'field', '21-05-2009')
self.assertRaises(ValueError, setattr, i, 'field', '2009-05-51')
self.assertRaises(ValueError, i.__setitem__, 'field', '2009-05-51')
self.assertRaises(TypeError, setattr, i, 'field', None)
self.assertRaises(TypeError, i.__setitem__, 'field', None)
def test_datetime_field(self):
class TestItem(Item):
@ -174,34 +233,34 @@ class NewItemFieldsTest(unittest.TestCase):
i = TestItem()
dt_today = datetime.datetime.today()
i.field = dt_today
self.assertEqual(i.field, dt_today)
i['field'] = dt_today
self.assertEqual(i['field'], dt_today)
d_today = datetime.date.today()
i.field = d_today
self.assertEqual(i.field, datetime.datetime(d_today.year,
i['field'] = d_today
self.assertEqual(i['field'], datetime.datetime(d_today.year,
d_today.month, d_today.day))
i.field = '2009-05-21 11:08:10.100'
self.assertEqual(i.field, datetime.datetime(2009, 5, 21, 11, 8, 10,
i['field'] = '2009-05-21 11:08:10.100'
self.assertEqual(i['field'], datetime.datetime(2009, 5, 21, 11, 8, 10,
100))
i.field = '2009-05-21 11:08:10'
self.assertEqual(i.field, datetime.datetime(2009, 5, 21, 11, 8, 10))
i['field'] = '2009-05-21 11:08:10'
self.assertEqual(i['field'], datetime.datetime(2009, 5, 21, 11, 8, 10))
i.field = '2009-05-21 11:08'
self.assertEqual(i.field, datetime.datetime(2009, 5, 21, 11, 8))
i['field'] = '2009-05-21 11:08'
self.assertEqual(i['field'], datetime.datetime(2009, 5, 21, 11, 8))
i.field = '2009-05-21'
self.assertEqual(i.field, datetime.datetime(2009, 5, 21))
i['field'] = '2009-05-21'
self.assertEqual(i['field'], datetime.datetime(2009, 5, 21))
self.assertRaises(ValueError, setattr, i, 'field', '2009-05-21 11:08:10.usecs')
self.assertRaises(ValueError, i.__setitem__, 'field', '2009-05-21 11:08:10.usecs')
self.assertRaises(ValueError, setattr, i, 'field', '21-05-2009')
self.assertRaises(ValueError, i.__setitem__, 'field', '21-05-2009')
self.assertRaises(ValueError, setattr, i, 'field', '2009-05-51')
self.assertRaises(ValueError, i.__setitem__, 'field', '2009-05-51')
self.assertRaises(TypeError, setattr, i, 'field', None)
self.assertRaises(TypeError, i.__setitem__, 'field', None)
def test_decimal_field(self):
class TestItem(Item):
@ -209,15 +268,15 @@ class NewItemFieldsTest(unittest.TestCase):
i = TestItem()
i.field = decimal.Decimal('3.14')
self.assertEqual(i.field, decimal.Decimal('3.14'))
i['field'] = decimal.Decimal('3.14')
self.assertEqual(i['field'], decimal.Decimal('3.14'))
i.field = '3.14'
self.assertEqual(i.field, decimal.Decimal('3.14'))
i['field'] = '3.14'
self.assertEqual(i['field'], decimal.Decimal('3.14'))
self.assertRaises(decimal.InvalidOperation, setattr, i, 'field', 'text')
self.assertRaises(decimal.InvalidOperation, i.__setitem__, 'field', 'text')
self.assertRaises(TypeError, setattr, i, 'field', None)
self.assertRaises(TypeError, i.__setitem__, 'field', None)
def test_float_field(self):
class TestItem(Item):
@ -225,15 +284,15 @@ class NewItemFieldsTest(unittest.TestCase):
i = TestItem()
i.field = 3.14
self.assertEqual(i.field, 3.14)
i['field'] = 3.14
self.assertEqual(i['field'], 3.14)
i.field = '3.14'
self.assertEqual(i.field, 3.14)
i['field'] = '3.14'
self.assertEqual(i['field'], 3.14)
self.assertRaises(ValueError, setattr, i, 'field', 'text')
self.assertRaises(ValueError, i.__setitem__, 'field', 'text')
self.assertRaises(TypeError, setattr, i, 'field', None)
self.assertRaises(TypeError, i.__setitem__, 'field', None)
def test_integer_field(self):
class TestItem(Item):
@ -241,15 +300,15 @@ class NewItemFieldsTest(unittest.TestCase):
i = TestItem()
i.field = 3
self.assertEqual(i.field, 3)
i['field'] = 3
self.assertEqual(i['field'], 3)
i.field = '3'
self.assertEqual(i.field, 3)
i['field'] = '3'
self.assertEqual(i['field'], 3)
self.assertRaises(ValueError, setattr, i, 'field', 'text')
self.assertRaises(ValueError, i.__setitem__, 'field', 'text')
self.assertRaises(TypeError, setattr, i, 'field', None)
self.assertRaises(TypeError, i.__setitem__, 'field', None)
def test_text_field(self):
class TestItem(Item):
@ -257,32 +316,32 @@ class NewItemFieldsTest(unittest.TestCase):
i = TestItem()
i.field = u'hello'
self.assertEqual(i.field, u'hello')
self.assert_(isinstance(i.field, unicode))
i['field'] = u'hello'
self.assertEqual(i['field'], u'hello')
self.assert_(isinstance(i['field'], unicode))
# must be unicode!
self.assertRaises(TypeError, setattr, i, 'field', 'string')
self.assertRaises(TypeError, i.__setitem__, 'field', 'string')
self.assertRaises(TypeError, setattr, i, 'field', None)
self.assertRaises(TypeError, i.__setitem__, 'field', None)
def set_invalid_value():
i.field = 3
i['field'] = 3
self.assertRaises(TypeError, set_invalid_value)
i = TestItem()
i.field = [u'hello', u'world']
self.assertEqual(i.field, u'hello world')
self.assert_(isinstance(i.field, unicode))
i['field'] = [u'hello', u'world']
self.assertEqual(i['field'], u'hello world')
self.assert_(isinstance(i['field'], unicode))
self.assertRaises(TypeError, setattr, i, 'field', [u'hello', 3, u'world'])
self.assertRaises(TypeError, setattr, i, 'field', [u'hello', 'world'])
self.assertRaises(TypeError, i.__setitem__, 'field', [u'hello', 3, u'world'])
self.assertRaises(TypeError, i.__setitem__, 'field', [u'hello', 'world'])
i = TestItem()
i.field = []
self.assert_(isinstance(i.field, unicode))
self.assertEqual(i.field, '')
i['field'] = []
self.assert_(isinstance(i['field'], unicode))
self.assertEqual(i['field'], '')
def test_time_field(self):
class TestItem(Item):
@ -291,27 +350,27 @@ class NewItemFieldsTest(unittest.TestCase):
i = TestItem()
dt_t = datetime.time(11, 8, 10, 100)
i.field = dt_t
self.assertEqual(i.field, dt_t)
i['field'] = dt_t
self.assertEqual(i['field'], dt_t)
self.assertRaises(TypeError, setattr, i, 'field', None)
self.assertRaises(TypeError, i.__setitem__, 'field', None)
dt_dt = datetime.datetime.today()
i.field = dt_dt
self.assertEqual(i.field, dt_dt.time)
i['field'] = dt_dt
self.assertEqual(i['field'], dt_dt.time)
i.field = '11:08:10.100'
self.assertEqual(i.field, datetime.time(11, 8, 10, 100))
i['field'] = '11:08:10.100'
self.assertEqual(i['field'], datetime.time(11, 8, 10, 100))
i.field = '11:08:10'
self.assertEqual(i.field, datetime.time(11, 8, 10))
i['field'] = '11:08:10'
self.assertEqual(i['field'], datetime.time(11, 8, 10))
i.field = '11:08'
self.assertEqual(i.field, datetime.time(11, 8))
i['field'] = '11:08'
self.assertEqual(i['field'], datetime.time(11, 8))
self.assertRaises(ValueError, setattr, i, 'field', '11:08:10.usecs')
self.assertRaises(ValueError, i.__setitem__, 'field', '11:08:10.usecs')
self.assertRaises(ValueError, setattr, i, 'field', '25:08:10')
self.assertRaises(ValueError, i.__setitem__, 'field', '25:08:10')
self.assertRaises(ValueError, setattr, i, 'field', 'string')
self.assertRaises(ValueError, i.__setitem__, 'field', 'string')