. Moved init and repr methods from RobustScrapedItem to ScrapedItem

. Refactored ScrapedItem's attribute method, added docstring and tests

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40648

scrapy/trunk/scrapy/contrib/item/models.py

@@ -42,16 +42,7 @@ class RobustScrapedItem(ScrapedItem):
     }
     
     def __init__(self, data=None):
-        """
-        A scraped item can be initialised with a dictionary that will be
-        squirted directly into the object.
-        """
-        if isinstance(data, dict):
-            for attr, value in data.iteritems():
-                setattr(self, attr, value)
-        elif data is not None:
-            raise UsageError("Initialize with dict, not %s" % data.__class__.__name__)
-
+        super(RobustScrapedItem, self).__init__(data)
         self.__dict__['_version'] = None
 
     def __getattr__(self, attr):
@@ -113,15 +104,6 @@ class RobustScrapedItem(ScrapedItem):
     def __sub__(self, other):
         return RobustItemDelta(other, self)
 
-    def __repr__(self):
-        # Generate this format so that it can be deserialized easily:
-        # ClassName({...})
-        reprdict = {}
-        for k, v in self.__dict__.iteritems():
-            if not k.startswith('_'):
-                reprdict[k] = v
-        return "%s(%s)" % (self.__class__.__name__, repr(reprdict))
-
     def __str__(self) :
         return "%s: GUID=%s, url=%s" % ( self.__class__.__name__ , self.guid, self.url )
             

scrapy/trunk/scrapy/item/models.py

@@ -7,7 +7,30 @@ class ScrapedItem(object):
     'guid' attribute is required, and is an attribute
     that identifies uniquely the given scraped item.
     """
-    _adaptors_dict = {}
+
+    def __init__(self, data=None):
+        """
+        A ScrapedItem can be initialised with a dictionary that will be
+        squirted directly into the object.
+        """
+        self._adaptors_dict = {}
+
+        if isinstance(data, dict):
+            for attr, value in data.iteritems():
+                setattr(self, attr, value)
+        elif data is not None:
+            raise UsageError("Initialize with dict, not %s" % data.__class__.__name__)
+
+    def __repr__(self):
+        """
+        Generate the following format so that items can be deserialized
+        easily: ClassName({'attrib': value, ...})
+        """
+        reprdict = dict(items for items in self.__dict__.iteritems() if not items[0].startswith('_'))
+        return "%s(%s)" % (self.__class__.__name__, repr(reprdict))
+
+    def __sub__(self, other):
+        raise NotImplementedError
 
     def set_adaptors(self, adaptors_dict):
         """
@@ -35,29 +58,43 @@ class ScrapedItem(object):
                 pipe.insert(position, adaptor)
             self.set_attrib_adaptors(attrib, pipe)
 
-    def attribute(self, attrname, value, **kwargs):
-        override = kwargs.pop('override', False)
-        add = kwargs.pop('add', False)
+    def attribute(self, attrname, value, override=False, add=False, **kwargs):
+        """
+        Set the given value to the provided attribute (`attrname`) by filtering it
+        through its adaptor pipeline first (if it has any).
+
+        If the attribute has been already set, it won't be overwritten unless
+        `override` is True.
+        If there was an old value and `add` is True (but `override` isn't), `value`
+        will be appended/extended (depending on its type), to the old value, as long as
+        the old value is a list.
+        If both of the values are strings they will be joined by using the `add` delimiter
+        (which may be a string, or True, in which case '' will be used as the delimiter).
+
+        The kwargs parameter is passed to the adaptors pipeline, which manages to transmit
+        it to the adaptors themselves.
+        """
         pipe = self._adaptors_dict.get(attrname)
+        old_value = getattr(self, attrname, None)
+
         if pipe:
-            val = pipe(value, **kwargs)
-            if val or val is False:
-                curr_val = getattr(self, attrname, None)
-                if not curr_val:
-                    setattr(self, attrname, val)
-                else:
-                    if override:
-                        setattr(self, attrname, val)
-                    elif add:
-                        if all(isinstance(var, basestring) for var in (curr_val, val)):
-                            setattr(self, attrname, '%s\t%s' % (curr_val, val))
-                        elif all(hasattr(var, '__iter__') for var in (curr_val, val)):
-                            setattr(self, attrname, curr_val + val)
-        elif value:
+            value = pipe(value, **kwargs)
+
+        if old_value:
+            if override:
+                setattr(self, attrname, value)
+            elif add:
+                if hasattr(old_value, '__iter__'):
+                    if hasattr(value, '__iter__'):
+                        self.__dict__[attrname].extend(list(value))
+                    else:
+                        self.__dict__[attrname].append(value)
+                elif isinstance(old_value, basestring) and isinstance(value, basestring):
+                    delimiter = add if isinstance(add, basestring) else ''
+                    setattr(self, attrname, '%s%s%s' % (old_value, delimiter, value))
+        else:
             setattr(self, attrname, value)
 
-    def __sub__(self, other):
-        raise NotImplementedError
 
 class ItemDelta(object):
     pass

scrapy/trunk/scrapy/tests/test_scrapeditem.py

@@ -0,0 +1,70 @@
+# -*- coding: utf8 -*-
+import unittest
+
+from scrapy.item.models import ScrapedItem
+from scrapy.item.adaptors import AdaptorPipe
+from scrapy.contrib import adaptors
+
+class ScrapedItemTestCase(unittest.TestCase):
+    def setUp(self):
+        self.item = ScrapedItem()
+
+    def test_attribute_basic(self):
+        self.item.attribute('name', 'John')
+        self.assertEqual(self.item.name, 'John')
+
+    def test_attribute_override(self):
+        self.item.attribute('name', 'John')
+        self.item.attribute('name', 'Charlie')
+        self.assertEqual(self.item.name, 'John')
+
+        self.item.attribute('name', 'Charlie', override=True)
+        self.assertEqual(self.item.name, 'Charlie')
+
+    def test_attribute_add(self):
+        self.item.attribute('name', 'John')
+      
+        self.item.attribute('name', 'Doe', add=True)
+        self.assertEqual(self.item.name, 'JohnDoe')
+
+        self.item.attribute('name', 'Smith', add=' ')
+        self.assertEqual(self.item.name, 'JohnDoe Smith')
+
+        self.item.attribute('children', ['Ken', 'Tom'])
+        self.item.attribute('children', 'Bobby')
+        self.item.attribute('children', 'Jimmy', add=True)
+        self.assertEqual(self.item.children, ['Ken', 'Tom', 'Jimmy'])
+
+        self.item.attribute('children', ['Johnny', 'Rodrigo'], add=True)
+        self.assertEqual(self.item.children, ['Ken', 'Tom', 'Jimmy', 'Johnny', 'Rodrigo'])
+
+    def test_set_adaptors(self):
+        self.assertEqual(self.item._adaptors_dict, {})
+
+        delist = adaptors.Delist()
+        self.item.set_adaptors({'name': [adaptors.extract, delist]})
+        self.assertEqual(self.item._adaptors_dict, {'name': [adaptors.extract, delist]})
+
+        self.item.set_adaptors({'description': [adaptors.extract]})
+        self.assertEqual(self.item._adaptors_dict, {'description': [adaptors.extract]})
+
+    def test_set_attrib_adaptors(self):
+        self.assertEqual(self.item._adaptors_dict, {})
+
+        self.item.set_attrib_adaptors('name', [adaptors.extract, adaptors.strip])
+        self.assertEqual(self.item._adaptors_dict['name'],
+            AdaptorPipe([adaptors.extract, adaptors.strip]))
+
+        unquote = adaptors.Unquote()
+        self.item.set_attrib_adaptors('name', [adaptors.extract, unquote])
+        self.assertEqual(self.item._adaptors_dict['name'],
+            AdaptorPipe([adaptors.extract, unquote]))
+
+    def test_add_adaptor(self):
+        self.assertEqual(self.item._adaptors_dict, {})
+
+        self.item.add_adaptor('name', adaptors.strip)
+        self.assertEqual(self.item._adaptors_dict['name'], [adaptors.strip])
+        self.item.add_adaptor('name', adaptors.extract, position=0)
+        self.assertEqual(self.item._adaptors_dict['name'], [adaptors.extract, adaptors.strip])
+