RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

added from_unicode_list() method to Field objects

This commit is contained in:
Pablo Hoffman 2009-07-30 16:58:24 -03:00
parent c0dcd76424
commit 73172b244d
4 changed files with 96 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
docs/experimental/ref/newitem/fields.rst
View File

@ -29,6 +29,16 @@ TextField
A unicode text.
This class overrides the following methods from :class:`BaseField`:
.. method:: from_unicode_list(unicode_list)
Return a unicode string composed by joining the elements of
``unicode_list`` with spaces.
For more info about this method see :class:`BaseField.from_unicode_list`.
IntegerField
------------
@ -136,6 +146,22 @@ BaseField class
This method must always return object of the expected field type.
.. method:: from_unicode_list(unicode_list)
Take the input list of unicode strings and convert it to a proper value
with the type expected by this field. If no proper value if found,
``None`` is returned instead.
The default behaviour is to return the value of the first item of the
list, passed through the :meth:`to_python` method, or ``None`` if the
list is empty::
return self.to_python(unicode_list[0]) if unicode_list else None
This default behaviour is provided because it's the more common one, but
it's typical for :class:`BaseField` subclasses to override this method,
such as the :meth:`TextField.from_unicode_list` method.
.. method:: get_default()
Return the default value for this field, or ``None`` if the field

35
scrapy/newitem/fields.py
View File

@ -11,6 +11,9 @@ class BaseField(object):
def to_python(self, value):
raise NotImplementedError()
def from_unicode_list(self, unicode_list):
return self.to_python(unicode_list[0]) if unicode_list else None
def get_default(self):
return self._default
@ -21,10 +24,14 @@ class MultiValuedField(BaseField):
super(MultiValuedField, self).__init__(default)
def to_python(self, value):
if value is None:
return []
else:
if hasattr(value, '__iter__'):
return [self._field.to_python(v) for v in value]
else:
raise TypeError("Cannot instatiante %s with %s" \
% (self.__class__.__name__, type(value).__name__))
def from_unicode_list(self, unicode_list):
return self.to_python(unicode_list)
# FIXME: temporary alias required for ItemExporters (to be removed on ListField merge)
ListField = MultiValuedField
@ -33,6 +40,9 @@ class BooleanField(BaseField):
def to_python(self, value):
return bool(value)
def from_unicode_list(self, unicode_list):
return self.to_python(unicode_list)
class DateField(BaseField):
ansi_date_re = re.compile(r'^\d{4}-\d{1,2}-\d{1,2}$')
@ -112,18 +122,19 @@ class IntegerField(BaseField):
class TextField(BaseField):
def to_python(self, value):
if hasattr(value, '__iter__'):
return self.to_python(self.to_single(value))
elif isinstance(value, unicode):
if isinstance(value, unicode):
return value
elif isinstance(value, (long, float)):
return unicode(value)
# Note: True and False are instances of int!
elif isinstance(value, int) and not isinstance(value, bool):
return unicode(value)
else:
raise TypeError("%s requires a unicode (or iterable of unicodes), got %s" \
% (self.__class__.__name__, type(value).__name__))
def to_single(self, value):
"""Converts the input iterable into a single value"""
return u' '.join((self.to_python(x) for x in value))
raise TypeError("%s values cannot be created from '%s' objects" % \
(self.__class__.__name__, value.__class__.__name__))
def from_unicode_list(self, unicode_list):
return u' '.join((self.to_python(x) for x in unicode_list))
class TimeField(BaseField):
def to_python(self, value):

32
scrapy/tests/test_itemadaptor.py
View File

@ -87,21 +87,23 @@ class ItemAdaptorTest(unittest.TestCase):
ia.name = u'marta'
self.assertEqual(ia.name, u'Marta')
def test_staticmethods(self):
class ChildAdaptor(TestAdaptor):
name = adaptor(TestAdaptor.name, string.swapcase)
ia = ChildAdaptor()
ia.name = u'Marta'
self.assertEqual(ia.name, u'mARTA')
def test_staticdefaults(self):
class ChildAdaptorDefaulted(DefaultedAdaptor):
name = adaptor(DefaultedAdaptor.name, string.swapcase)
dia = ChildAdaptorDefaulted()
dia.name = u'marta'
self.assertEqual(dia.name, u'MART')
# FIXME: deprecated tests - will be replaced by ItemBuilder tests
#
# def test_staticmethods(self):
# class ChildAdaptor(TestAdaptor):
# name = adaptor(TestAdaptor.name, string.swapcase)
#
# ia = ChildAdaptor()
# ia.name = u'Marta'
# self.assertEqual(ia.name, u'mARTA')
#
# def test_staticdefaults(self):
# class ChildAdaptorDefaulted(DefaultedAdaptor):
# name = adaptor(DefaultedAdaptor.name, string.swapcase)
#
# dia = ChildAdaptorDefaulted()
# dia.name = u'marta'
# self.assertEqual(dia.name, u'MART')
def test_multiplevaluedadaptor(self):
ma = MultiValuedItemAdaptor()

59
scrapy/tests/test_newitem.py
View File

@ -35,7 +35,7 @@ class NewItemTest(unittest.TestCase):
self.assertRaises(KeyError, TestItem, {'name': u'john doe',
'other': u'foo'})
self.assertRaises(TypeError, TestItem, name=3)
self.assertRaises(TypeError, TestItem, name=set())
def test_multi(self):
class TestMultiItem(Item):
@ -63,19 +63,7 @@ class NewItemTest(unittest.TestCase):
self.assertEqual(i['name'], u'John')
def test_wrong_default(self):
def set_wrong_default():
class TestItem(Item):
name = fields.TextField(default=3)
self.assertRaises(TypeError, set_wrong_default)
def test_to_python_iter(self):
class TestItem(Item):
name = fields.TextField()
i = TestItem()
i['name'] = (u'John', u'Doe')
self.assertEqual(i['name'], u'John Doe')
self.assertRaises(TypeError, fields.TextField, default=set())
def test_repr(self):
class TestItem(Item):
@ -316,32 +304,45 @@ class NewItemFieldsTest(unittest.TestCase):
i = TestItem()
# valid castings
i['field'] = u'hello'
self.assertEqual(i['field'], u'hello')
self.assert_(isinstance(i['field'], unicode))
# must be unicode!
self.assertRaises(TypeError, i.__setitem__, 'field', 'string')
i['field'] = 3
self.assertEqual(i['field'], u'3')
self.assert_(isinstance(i['field'], unicode))
i['field'] = 3.2
self.assertEqual(i['field'], u'3.2')
self.assert_(isinstance(i['field'], unicode))
i['field'] = 100L
self.assertEqual(i['field'], u'100')
self.assert_(isinstance(i['field'], unicode))
# invalid castings
self.assertRaises(TypeError, i.__setitem__, 'field', [u'hello', u'world'])
self.assertRaises(TypeError, i.__setitem__, 'field', 'string') # must be unicode!
self.assertRaises(TypeError, i.__setitem__, 'field', set())
self.assertRaises(TypeError, i.__setitem__, 'field', True)
self.assertRaises(TypeError, i.__setitem__, 'field', None)
def set_invalid_value():
i['field'] = 3
self.assertRaises(TypeError, set_invalid_value)
def test_from_unicode_list(self):
field = fields.BaseField()
self.assertEqual(field.from_unicode_list([]), None)
i = TestItem()
i['field'] = [u'hello', u'world']
self.assertEqual(i['field'], u'hello world')
self.assert_(isinstance(i['field'], unicode))
field = fields.TextField()
self.assertEqual(field.from_unicode_list([]), u'')
self.assertEqual(field.from_unicode_list([u'hello', u'world']), u'hello world')
self.assertRaises(TypeError, i.__setitem__, 'field', [u'hello', 3, u'world'])
self.assertRaises(TypeError, i.__setitem__, 'field', [u'hello', 'world'])
field = fields.MultiValuedField(fields.TextField)
self.assertEqual(field.from_unicode_list([]), [])
self.assertEqual(field.from_unicode_list([u'hello', u'world']), [u'hello', u'world'])
i = TestItem()
i['field'] = []
self.assert_(isinstance(i['field'], unicode))
self.assertEqual(i['field'], '')
field = fields.IntegerField()
self.assertEqual(field.from_unicode_list([u'123']), 123)
def test_time_field(self):
class TestItem(Item):