From f1962c1c0af52e1ae6700d680685f65182b5097c Mon Sep 17 00:00:00 2001 From: elpolilla Date: Mon, 5 Jan 2009 16:54:54 +0000 Subject: [PATCH] . Moved init and repr methods from RobustScrapedItem to ScrapedItem . Refactored ScrapedItem's attribute method, added docstring and tests --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40648 --- scrapy/trunk/scrapy/contrib/item/models.py | 20 +---- scrapy/trunk/scrapy/item/models.py | 77 ++++++++++++++----- scrapy/trunk/scrapy/tests/test_scrapeditem.py | 70 +++++++++++++++++ 3 files changed, 128 insertions(+), 39 deletions(-) create mode 100644 scrapy/trunk/scrapy/tests/test_scrapeditem.py diff --git a/scrapy/trunk/scrapy/contrib/item/models.py b/scrapy/trunk/scrapy/contrib/item/models.py index d7fe8e695..d58c705b5 100644 --- a/scrapy/trunk/scrapy/contrib/item/models.py +++ b/scrapy/trunk/scrapy/contrib/item/models.py @@ -42,16 +42,7 @@ class RobustScrapedItem(ScrapedItem): } def __init__(self, data=None): - """ - A scraped item can be initialised with a dictionary that will be - squirted directly into the object. - """ - if isinstance(data, dict): - for attr, value in data.iteritems(): - setattr(self, attr, value) - elif data is not None: - raise UsageError("Initialize with dict, not %s" % data.__class__.__name__) - + super(RobustScrapedItem, self).__init__(data) self.__dict__['_version'] = None def __getattr__(self, attr): @@ -113,15 +104,6 @@ class RobustScrapedItem(ScrapedItem): def __sub__(self, other): return RobustItemDelta(other, self) - def __repr__(self): - # Generate this format so that it can be deserialized easily: - # ClassName({...}) - reprdict = {} - for k, v in self.__dict__.iteritems(): - if not k.startswith('_'): - reprdict[k] = v - return "%s(%s)" % (self.__class__.__name__, repr(reprdict)) - def __str__(self) : return "%s: GUID=%s, url=%s" % ( self.__class__.__name__ , self.guid, self.url ) diff --git a/scrapy/trunk/scrapy/item/models.py b/scrapy/trunk/scrapy/item/models.py index 09554c993..4fb13ebb0 100644 --- a/scrapy/trunk/scrapy/item/models.py +++ b/scrapy/trunk/scrapy/item/models.py @@ -7,7 +7,30 @@ class ScrapedItem(object): 'guid' attribute is required, and is an attribute that identifies uniquely the given scraped item. """ - _adaptors_dict = {} + + def __init__(self, data=None): + """ + A ScrapedItem can be initialised with a dictionary that will be + squirted directly into the object. + """ + self._adaptors_dict = {} + + if isinstance(data, dict): + for attr, value in data.iteritems(): + setattr(self, attr, value) + elif data is not None: + raise UsageError("Initialize with dict, not %s" % data.__class__.__name__) + + def __repr__(self): + """ + Generate the following format so that items can be deserialized + easily: ClassName({'attrib': value, ...}) + """ + reprdict = dict(items for items in self.__dict__.iteritems() if not items[0].startswith('_')) + return "%s(%s)" % (self.__class__.__name__, repr(reprdict)) + + def __sub__(self, other): + raise NotImplementedError def set_adaptors(self, adaptors_dict): """ @@ -35,29 +58,43 @@ class ScrapedItem(object): pipe.insert(position, adaptor) self.set_attrib_adaptors(attrib, pipe) - def attribute(self, attrname, value, **kwargs): - override = kwargs.pop('override', False) - add = kwargs.pop('add', False) + def attribute(self, attrname, value, override=False, add=False, **kwargs): + """ + Set the given value to the provided attribute (`attrname`) by filtering it + through its adaptor pipeline first (if it has any). + + If the attribute has been already set, it won't be overwritten unless + `override` is True. + If there was an old value and `add` is True (but `override` isn't), `value` + will be appended/extended (depending on its type), to the old value, as long as + the old value is a list. + If both of the values are strings they will be joined by using the `add` delimiter + (which may be a string, or True, in which case '' will be used as the delimiter). + + The kwargs parameter is passed to the adaptors pipeline, which manages to transmit + it to the adaptors themselves. + """ pipe = self._adaptors_dict.get(attrname) + old_value = getattr(self, attrname, None) + if pipe: - val = pipe(value, **kwargs) - if val or val is False: - curr_val = getattr(self, attrname, None) - if not curr_val: - setattr(self, attrname, val) - else: - if override: - setattr(self, attrname, val) - elif add: - if all(isinstance(var, basestring) for var in (curr_val, val)): - setattr(self, attrname, '%s\t%s' % (curr_val, val)) - elif all(hasattr(var, '__iter__') for var in (curr_val, val)): - setattr(self, attrname, curr_val + val) - elif value: + value = pipe(value, **kwargs) + + if old_value: + if override: + setattr(self, attrname, value) + elif add: + if hasattr(old_value, '__iter__'): + if hasattr(value, '__iter__'): + self.__dict__[attrname].extend(list(value)) + else: + self.__dict__[attrname].append(value) + elif isinstance(old_value, basestring) and isinstance(value, basestring): + delimiter = add if isinstance(add, basestring) else '' + setattr(self, attrname, '%s%s%s' % (old_value, delimiter, value)) + else: setattr(self, attrname, value) - def __sub__(self, other): - raise NotImplementedError class ItemDelta(object): pass diff --git a/scrapy/trunk/scrapy/tests/test_scrapeditem.py b/scrapy/trunk/scrapy/tests/test_scrapeditem.py new file mode 100644 index 000000000..a644d10da --- /dev/null +++ b/scrapy/trunk/scrapy/tests/test_scrapeditem.py @@ -0,0 +1,70 @@ +# -*- coding: utf8 -*- +import unittest + +from scrapy.item.models import ScrapedItem +from scrapy.item.adaptors import AdaptorPipe +from scrapy.contrib import adaptors + +class ScrapedItemTestCase(unittest.TestCase): + def setUp(self): + self.item = ScrapedItem() + + def test_attribute_basic(self): + self.item.attribute('name', 'John') + self.assertEqual(self.item.name, 'John') + + def test_attribute_override(self): + self.item.attribute('name', 'John') + self.item.attribute('name', 'Charlie') + self.assertEqual(self.item.name, 'John') + + self.item.attribute('name', 'Charlie', override=True) + self.assertEqual(self.item.name, 'Charlie') + + def test_attribute_add(self): + self.item.attribute('name', 'John') + + self.item.attribute('name', 'Doe', add=True) + self.assertEqual(self.item.name, 'JohnDoe') + + self.item.attribute('name', 'Smith', add=' ') + self.assertEqual(self.item.name, 'JohnDoe Smith') + + self.item.attribute('children', ['Ken', 'Tom']) + self.item.attribute('children', 'Bobby') + self.item.attribute('children', 'Jimmy', add=True) + self.assertEqual(self.item.children, ['Ken', 'Tom', 'Jimmy']) + + self.item.attribute('children', ['Johnny', 'Rodrigo'], add=True) + self.assertEqual(self.item.children, ['Ken', 'Tom', 'Jimmy', 'Johnny', 'Rodrigo']) + + def test_set_adaptors(self): + self.assertEqual(self.item._adaptors_dict, {}) + + delist = adaptors.Delist() + self.item.set_adaptors({'name': [adaptors.extract, delist]}) + self.assertEqual(self.item._adaptors_dict, {'name': [adaptors.extract, delist]}) + + self.item.set_adaptors({'description': [adaptors.extract]}) + self.assertEqual(self.item._adaptors_dict, {'description': [adaptors.extract]}) + + def test_set_attrib_adaptors(self): + self.assertEqual(self.item._adaptors_dict, {}) + + self.item.set_attrib_adaptors('name', [adaptors.extract, adaptors.strip]) + self.assertEqual(self.item._adaptors_dict['name'], + AdaptorPipe([adaptors.extract, adaptors.strip])) + + unquote = adaptors.Unquote() + self.item.set_attrib_adaptors('name', [adaptors.extract, unquote]) + self.assertEqual(self.item._adaptors_dict['name'], + AdaptorPipe([adaptors.extract, unquote])) + + def test_add_adaptor(self): + self.assertEqual(self.item._adaptors_dict, {}) + + self.item.add_adaptor('name', adaptors.strip) + self.assertEqual(self.item._adaptors_dict['name'], [adaptors.strip]) + self.item.add_adaptor('name', adaptors.extract, position=0) + self.assertEqual(self.item._adaptors_dict['name'], [adaptors.extract, adaptors.strip]) +