diff --git a/scrapy/trunk/scrapy/contrib_exp/newitem/adaptors.py b/scrapy/trunk/scrapy/contrib_exp/newitem/adaptors.py index 874a8fd28..b7db1624b 100644 --- a/scrapy/trunk/scrapy/contrib_exp/newitem/adaptors.py +++ b/scrapy/trunk/scrapy/contrib_exp/newitem/adaptors.py @@ -7,53 +7,51 @@ def is_adaptor(func): class ItemAdaptor(Declarative): + item_class = None + default_adaptor = None + field_adaptors = {} + def __classinit__(cls, attrs): - # defines adaptors as staticmethods - for n, v in attrs.items(): - if is_adaptor(v): - setattr(cls, n, staticmethod(v)) + cls.field_adaptors = cls.field_adaptors.copy() + if cls.item_class: + # set new adaptors + for n, v in attrs.items(): + if n in cls.item_class.fields.keys() and is_adaptor(v): + cls.set_adaptor(n, v) + + # if default_adaptor is set, use it for the unadapted fields + if cls.default_adaptor: + for field in cls.item_class.fields.keys(): + if field not in cls.field_adaptors.keys(): + cls.set_adaptor(field, cls.default_adaptor.im_func) + + @classmethod + def set_adaptor(cls, name, func): + cls.field_adaptors[name] = func + # define adaptor as a staticmethod + setattr(cls, name, staticmethod(func)) def __init__(self, response=None, item=None): self.item_instance = item if item else self.item_class() self._response = response - self._field_adaptors = self._get_field_adaptors() - self._default_adaptor = self._get_default_adaptor() - - def _get_field_adaptors(self): - fa = {} - for field in self.item_instance.fields.keys(): - try: - fa[field] = object.__getattribute__(self, field) - except: - pass - - return fa - - def _get_default_adaptor(self): - try: - return object.__getattribute__(self, 'default_adaptor') - except: - return None def __setattr__(self, name, value): - if name.startswith('_') or name == 'item_instance' \ - or name == 'default_adaptor': + if (name.startswith('_') or name == 'item_instance' \ + or name == 'default_adaptor' or name == 'field_adaptors'): return object.__setattr__(self, name, value) try: - fa = self._field_adaptors[name] + fa = self.field_adaptors[name] except KeyError: - if self._default_adaptor: - fa = self._default_adaptor - else: - return setattr(self.item_instance, name, value) + return setattr(self.item_instance, name, value) adaptor_args = {'response': self._response, 'item': self.item_instance} ovalue = fa(value, adaptor_args=adaptor_args) setattr(self.item_instance, name, ovalue) def __getattribute__(self, name): - if name.startswith('_') or name.startswith('item_'): + if (name.startswith('_') or name.startswith('item_') \ + or name == 'default_adaptor' or name == 'field_adaptors'): return object.__getattribute__(self, name) return getattr(self.item_instance, name) diff --git a/scrapy/trunk/scrapy/tests/test_itemadaptor.py b/scrapy/trunk/scrapy/tests/test_itemadaptor.py index 8bd4699cb..4600e425e 100644 --- a/scrapy/trunk/scrapy/tests/test_itemadaptor.py +++ b/scrapy/trunk/scrapy/tests/test_itemadaptor.py @@ -1,4 +1,3 @@ - import unittest import string from scrapy.contrib_exp.newitem.adaptors import adaptor, ItemAdaptor @@ -10,6 +9,7 @@ class TestItem(Item): url = StringField() summary = StringField() + class TestAdaptor(ItemAdaptor): item_class = TestItem name = lambda v, adaptor_args: v.title() @@ -29,7 +29,7 @@ class ItemAdaptorTest(unittest.TestCase): default_adaptor = lambda v, adaptor_args: v.title() dta = DefaultedTestAdaptor() - assert dta._default_adaptor + assert dta.default_adaptor dta.name = 'marta' assert dta.name == 'Marta' @@ -38,8 +38,8 @@ class ItemAdaptorTest(unittest.TestCase): url = lambda v, adaptor_args: v.lower() ia = ChildTestAdaptor() - assert 'url' in ia._field_adaptors - assert 'name' in ia._field_adaptors + assert 'url' in ia.field_adaptors + assert 'name' in ia.field_adaptors ia.url = 'HTTP://scrapy.ORG' self.assertEqual(ia.url, 'http://scrapy.org') @@ -47,16 +47,14 @@ class ItemAdaptorTest(unittest.TestCase): ia.name = 'marta' self.assertEqual(ia.name, 'Marta') - class ChildChildTestAdaptor(ChildTestAdaptor): url = lambda v, adaptor_args: v.upper() summary = lambda v, adaptor_args: v ia = ChildChildTestAdaptor() - assert 'url' in ia._field_adaptors - assert 'name' in ia._field_adaptors - assert 'summary' in ia._field_adaptors - + assert 'url' in ia.field_adaptors + assert 'name' in ia.field_adaptors + assert 'summary' in ia.field_adaptors ia.url = 'HTTP://scrapy.ORG' self.assertEqual(ia.url, 'HTTP://SCRAPY.ORG') @@ -75,7 +73,17 @@ class ItemAdaptorTest(unittest.TestCase): ia.name = 'Marta' self.assertEqual(ia.name, 'mARTA') + def test_staticdefaults(self): + class ParentAdaptorDefaulted(ItemAdaptor): + item_class = TestItem + default_adaptor = lambda v, adaptor_args: v.title() + class ChildAdaptorDefaulted(ParentAdaptorDefaulted): + name = adaptor(ParentAdaptorDefaulted.name, string.swapcase) + + dia = ChildAdaptorDefaulted() + dia.name = 'marta' + self.assertEqual(dia.name, 'mARTA') class TreeadaptTest(unittest.TestCase): @@ -109,7 +117,6 @@ class TreeadaptTest(unittest.TestCase): self.assertEqual(ad(0, {'add_value': 3}), [3]) self.assertEqual(ad(0), [5]) - def test_4_treelogic(self): split1 = lambda v: v.split('&') split2 = lambda v: v.split('=')