From 73172b244dc55d444ebd98f9a91bd3967e8bc096 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Thu, 30 Jul 2009 16:58:24 -0300 Subject: [PATCH] added from_unicode_list() method to Field objects --- docs/experimental/ref/newitem/fields.rst | 26 +++++++++++ scrapy/newitem/fields.py | 35 +++++++++----- scrapy/tests/test_itemadaptor.py | 32 +++++++------ scrapy/tests/test_newitem.py | 59 ++++++++++++------------ 4 files changed, 96 insertions(+), 56 deletions(-) diff --git a/docs/experimental/ref/newitem/fields.rst b/docs/experimental/ref/newitem/fields.rst index c7aba31df..cd54c2ccb 100644 --- a/docs/experimental/ref/newitem/fields.rst +++ b/docs/experimental/ref/newitem/fields.rst @@ -29,6 +29,16 @@ TextField A unicode text. + This class overrides the following methods from :class:`BaseField`: + + .. method:: from_unicode_list(unicode_list) + + Return a unicode string composed by joining the elements of + ``unicode_list`` with spaces. + + For more info about this method see :class:`BaseField.from_unicode_list`. + + IntegerField ------------ @@ -136,6 +146,22 @@ BaseField class This method must always return object of the expected field type. + .. method:: from_unicode_list(unicode_list) + + Take the input list of unicode strings and convert it to a proper value + with the type expected by this field. If no proper value if found, + ``None`` is returned instead. + + The default behaviour is to return the value of the first item of the + list, passed through the :meth:`to_python` method, or ``None`` if the + list is empty:: + + return self.to_python(unicode_list[0]) if unicode_list else None + + This default behaviour is provided because it's the more common one, but + it's typical for :class:`BaseField` subclasses to override this method, + such as the :meth:`TextField.from_unicode_list` method. + .. method:: get_default() Return the default value for this field, or ``None`` if the field diff --git a/scrapy/newitem/fields.py b/scrapy/newitem/fields.py index 222c4e0a2..c7870817f 100644 --- a/scrapy/newitem/fields.py +++ b/scrapy/newitem/fields.py @@ -11,6 +11,9 @@ class BaseField(object): def to_python(self, value): raise NotImplementedError() + def from_unicode_list(self, unicode_list): + return self.to_python(unicode_list[0]) if unicode_list else None + def get_default(self): return self._default @@ -21,10 +24,14 @@ class MultiValuedField(BaseField): super(MultiValuedField, self).__init__(default) def to_python(self, value): - if value is None: - return [] - else: + if hasattr(value, '__iter__'): return [self._field.to_python(v) for v in value] + else: + raise TypeError("Cannot instatiante %s with %s" \ + % (self.__class__.__name__, type(value).__name__)) + + def from_unicode_list(self, unicode_list): + return self.to_python(unicode_list) # FIXME: temporary alias required for ItemExporters (to be removed on ListField merge) ListField = MultiValuedField @@ -33,6 +40,9 @@ class BooleanField(BaseField): def to_python(self, value): return bool(value) + def from_unicode_list(self, unicode_list): + return self.to_python(unicode_list) + class DateField(BaseField): ansi_date_re = re.compile(r'^\d{4}-\d{1,2}-\d{1,2}$') @@ -112,18 +122,19 @@ class IntegerField(BaseField): class TextField(BaseField): def to_python(self, value): - if hasattr(value, '__iter__'): - return self.to_python(self.to_single(value)) - elif isinstance(value, unicode): + if isinstance(value, unicode): return value + elif isinstance(value, (long, float)): + return unicode(value) + # Note: True and False are instances of int! + elif isinstance(value, int) and not isinstance(value, bool): + return unicode(value) else: - raise TypeError("%s requires a unicode (or iterable of unicodes), got %s" \ - % (self.__class__.__name__, type(value).__name__)) - - def to_single(self, value): - """Converts the input iterable into a single value""" - return u' '.join((self.to_python(x) for x in value)) + raise TypeError("%s values cannot be created from '%s' objects" % \ + (self.__class__.__name__, value.__class__.__name__)) + def from_unicode_list(self, unicode_list): + return u' '.join((self.to_python(x) for x in unicode_list)) class TimeField(BaseField): def to_python(self, value): diff --git a/scrapy/tests/test_itemadaptor.py b/scrapy/tests/test_itemadaptor.py index 8e37aaf0b..48d3ebb92 100644 --- a/scrapy/tests/test_itemadaptor.py +++ b/scrapy/tests/test_itemadaptor.py @@ -87,21 +87,23 @@ class ItemAdaptorTest(unittest.TestCase): ia.name = u'marta' self.assertEqual(ia.name, u'Marta') - def test_staticmethods(self): - class ChildAdaptor(TestAdaptor): - name = adaptor(TestAdaptor.name, string.swapcase) - - ia = ChildAdaptor() - ia.name = u'Marta' - self.assertEqual(ia.name, u'mARTA') - - def test_staticdefaults(self): - class ChildAdaptorDefaulted(DefaultedAdaptor): - name = adaptor(DefaultedAdaptor.name, string.swapcase) - - dia = ChildAdaptorDefaulted() - dia.name = u'marta' - self.assertEqual(dia.name, u'MART') +# FIXME: deprecated tests - will be replaced by ItemBuilder tests +# +# def test_staticmethods(self): +# class ChildAdaptor(TestAdaptor): +# name = adaptor(TestAdaptor.name, string.swapcase) +# +# ia = ChildAdaptor() +# ia.name = u'Marta' +# self.assertEqual(ia.name, u'mARTA') +# +# def test_staticdefaults(self): +# class ChildAdaptorDefaulted(DefaultedAdaptor): +# name = adaptor(DefaultedAdaptor.name, string.swapcase) +# +# dia = ChildAdaptorDefaulted() +# dia.name = u'marta' +# self.assertEqual(dia.name, u'MART') def test_multiplevaluedadaptor(self): ma = MultiValuedItemAdaptor() diff --git a/scrapy/tests/test_newitem.py b/scrapy/tests/test_newitem.py index 61bd7d7c6..79c3326e7 100644 --- a/scrapy/tests/test_newitem.py +++ b/scrapy/tests/test_newitem.py @@ -35,7 +35,7 @@ class NewItemTest(unittest.TestCase): self.assertRaises(KeyError, TestItem, {'name': u'john doe', 'other': u'foo'}) - self.assertRaises(TypeError, TestItem, name=3) + self.assertRaises(TypeError, TestItem, name=set()) def test_multi(self): class TestMultiItem(Item): @@ -63,19 +63,7 @@ class NewItemTest(unittest.TestCase): self.assertEqual(i['name'], u'John') def test_wrong_default(self): - def set_wrong_default(): - class TestItem(Item): - name = fields.TextField(default=3) - - self.assertRaises(TypeError, set_wrong_default) - - def test_to_python_iter(self): - class TestItem(Item): - name = fields.TextField() - - i = TestItem() - i['name'] = (u'John', u'Doe') - self.assertEqual(i['name'], u'John Doe') + self.assertRaises(TypeError, fields.TextField, default=set()) def test_repr(self): class TestItem(Item): @@ -316,32 +304,45 @@ class NewItemFieldsTest(unittest.TestCase): i = TestItem() + # valid castings i['field'] = u'hello' self.assertEqual(i['field'], u'hello') self.assert_(isinstance(i['field'], unicode)) - # must be unicode! - self.assertRaises(TypeError, i.__setitem__, 'field', 'string') + i['field'] = 3 + self.assertEqual(i['field'], u'3') + self.assert_(isinstance(i['field'], unicode)) + i['field'] = 3.2 + self.assertEqual(i['field'], u'3.2') + self.assert_(isinstance(i['field'], unicode)) + + i['field'] = 100L + self.assertEqual(i['field'], u'100') + self.assert_(isinstance(i['field'], unicode)) + + # invalid castings + self.assertRaises(TypeError, i.__setitem__, 'field', [u'hello', u'world']) + self.assertRaises(TypeError, i.__setitem__, 'field', 'string') # must be unicode! + self.assertRaises(TypeError, i.__setitem__, 'field', set()) + self.assertRaises(TypeError, i.__setitem__, 'field', True) self.assertRaises(TypeError, i.__setitem__, 'field', None) - def set_invalid_value(): - i['field'] = 3 - self.assertRaises(TypeError, set_invalid_value) + def test_from_unicode_list(self): + field = fields.BaseField() + self.assertEqual(field.from_unicode_list([]), None) - i = TestItem() - i['field'] = [u'hello', u'world'] - self.assertEqual(i['field'], u'hello world') - self.assert_(isinstance(i['field'], unicode)) + field = fields.TextField() + self.assertEqual(field.from_unicode_list([]), u'') + self.assertEqual(field.from_unicode_list([u'hello', u'world']), u'hello world') - self.assertRaises(TypeError, i.__setitem__, 'field', [u'hello', 3, u'world']) - self.assertRaises(TypeError, i.__setitem__, 'field', [u'hello', 'world']) + field = fields.MultiValuedField(fields.TextField) + self.assertEqual(field.from_unicode_list([]), []) + self.assertEqual(field.from_unicode_list([u'hello', u'world']), [u'hello', u'world']) - i = TestItem() - i['field'] = [] - self.assert_(isinstance(i['field'], unicode)) - self.assertEqual(i['field'], '') + field = fields.IntegerField() + self.assertEqual(field.from_unicode_list([u'123']), 123) def test_time_field(self): class TestItem(Item):