From 4ef7c46303f33c8bf5f45a9b81a2637be6ef9cf7 Mon Sep 17 00:00:00 2001 From: elpolilla Date: Thu, 13 Nov 2008 10:35:24 +0000 Subject: [PATCH] Added ItemAttribute objects --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40373 --- scrapy/trunk/scrapy/contrib/item/__init__.py | 2 +- scrapy/trunk/scrapy/contrib/item/models.py | 37 +------- scrapy/trunk/scrapy/item/__init__.py | 2 +- scrapy/trunk/scrapy/item/models.py | 99 ++++++++++++++++---- 4 files changed, 87 insertions(+), 53 deletions(-) diff --git a/scrapy/trunk/scrapy/contrib/item/__init__.py b/scrapy/trunk/scrapy/contrib/item/__init__.py index 8f029a687..a6226fec9 100644 --- a/scrapy/trunk/scrapy/contrib/item/__init__.py +++ b/scrapy/trunk/scrapy/contrib/item/__init__.py @@ -1 +1 @@ -from scrapy.contrib.item.models import RobustScrapedItem, RobustItemDelta, ValidationError, ValidationPipeline, SetGUIDPipeline +from scrapy.contrib.item.models import RobustScrapedItem, RobustItemDelta, ValidationError, ValidationPipeline diff --git a/scrapy/trunk/scrapy/contrib/item/models.py b/scrapy/trunk/scrapy/contrib/item/models.py index 65ff653de..2cd5bb159 100644 --- a/scrapy/trunk/scrapy/contrib/item/models.py +++ b/scrapy/trunk/scrapy/contrib/item/models.py @@ -8,7 +8,7 @@ import hashlib from pydispatch import dispatcher from pprint import PrettyPrinter -from scrapy.item import ScrapedItem, ItemDelta +from scrapy.item import ScrapedItem, ItemDelta, ItemAttribute from scrapy.spider import spiders from scrapy.core import signals from scrapy.core.exceptions import UsageError, DropItem @@ -30,11 +30,6 @@ class ValidationPipeline(object): item.validate() return item -class SetGUIDPipeline(object): - def process_item(self, domain, response, item): - spiders.fromdomain(domain).set_guid(item) - return item - class RobustScrapedItem(ScrapedItem): """ A more robust scraped item class with a built-in validation mechanism and @@ -42,8 +37,8 @@ class RobustScrapedItem(ScrapedItem): """ ATTRIBUTES = { - 'guid': basestring, # a global unique identifier - 'url': basestring, # the main URL where this item was scraped from + 'guid': ItemAttribute(attrib_name='guid', attrib_type=basestring), # a global unique identifier + 'url': ItemAttribute(attrib_name='url', attrib_type=basestring), # the main URL where this item was scraped from } def __init__(self, data=None): @@ -64,7 +59,7 @@ class RobustScrapedItem(ScrapedItem): # Note that this method is called only when the attribute is not found in # self.__dict__ or the class/instance methods. if attr in self.ATTRIBUTES: - return None + return () if hasattr(self.ATTRIBUTES[attr].attrib_type, '__iter__') else None else: raise AttributeError(attr) @@ -72,30 +67,8 @@ class RobustScrapedItem(ScrapedItem): """ Set an attribute checking it matches the attribute type declared in self.ATTRIBUTES """ - if not attr.startswith('_') and attr not in self.ATTRIBUTES: - raise AttributeError('Attribute "%s" is not a valid attribute name. You must add it to %s.ATTRIBUTES' % (attr, self.__class__.__name__)) - - if value is None: - self.__dict__.pop(attr, None) - return - - if attr == '_adaptors_dict': - return object.__setattr__(self, '_adaptors_dict', value) - - type1 = self.ATTRIBUTES[attr] - if hasattr(type1, '__iter__'): - if not hasattr(value, '__iter__'): - raise TypeError('Attribute "%s" must be a sequence' % attr) - type2 = type1[0] - for i in value: - if not isinstance(i, type2): - raise TypeError('Attribute "%s" cannot contain %s, only %s' % (attr, i.__class__.__name__, type2.__name__)) - else: - if not isinstance(value, type1): - raise TypeError('Attribute "%s" must be %s, not %s' % (attr, type1.__name__, value.__class__.__name__)) - - self.__dict__[attr] = value self.__dict__['_version'] = None + super(RobustScrapedItem, self).__setattr__(attr, value) def __delattr__(self, attr): """ diff --git a/scrapy/trunk/scrapy/item/__init__.py b/scrapy/trunk/scrapy/item/__init__.py index 423ebfb01..5449a80f1 100644 --- a/scrapy/trunk/scrapy/item/__init__.py +++ b/scrapy/trunk/scrapy/item/__init__.py @@ -1 +1 @@ -from scrapy.item.models import ScrapedItem, ItemDelta +from scrapy.item.models import ScrapedItem, ItemDelta, ItemAttribute diff --git a/scrapy/trunk/scrapy/item/models.py b/scrapy/trunk/scrapy/item/models.py index 00638c977..c85831070 100644 --- a/scrapy/trunk/scrapy/item/models.py +++ b/scrapy/trunk/scrapy/item/models.py @@ -1,5 +1,59 @@ -from scrapy.item.adaptors import AdaptorDict +import types + +from traceback import format_exc +from scrapy import log from scrapy.conf import settings +from scrapy.core.exceptions import NotConfigured + +class ItemAttribute(object): + def __init__(self, attrib_name, attrib_type, adaptor=None): + self.attrib_name = attrib_name + self.attrib_type = attrib_type + self.adaptor = adaptor + + def adapt(self, value, **kwargs): + debug = kwargs.get('debug') or all([settings.getbool('LOG_ENABLED'), settings.get('LOGLEVEL') == 'TRACE']) + if not self.adaptor: + if debug: + log.msg('No adaptor defined for attribute %s.' % self.attrib_name, log.WARNING) + return value + + try: + if debug: + print " %07s | input >" % self.attrib_name, repr(value) + value = self.adaptor(value, **kwargs) + if debug: + print " %07s | output >" % self.attrib_name, repr(value) + + except Exception: + print "Error in '%s' adaptor. Traceback text:" % name + print format_exc() + self.value = None + return False + + return value + + def check(self, value): + if not self.attrib_name: + raise NotConfigured('You must define "attrib_name" attribute in order to use an ItemAttribute') + + if not self.attrib_type: + raise NotConfigured('You must define "attrib_type" attribute in order to use an ItemAttribute') + else: + if hasattr(self.attrib_type, '__iter__'): + if not hasattr(value, '__iter__'): + raise TypeError('Attribute "%s" must be a sequence' % self.attrib_name) + iter_type = self.attrib_type[0] + for i in value: + if not isinstance(i, iter_type): + raise TypeError('Attribute "%s" cannot contain %s, only %s' % (self.name, i.__class__.__name__, iter_type.__name__)) + else: + if not isinstance(value, self.attrib_type): + raise TypeError('Attribute "%s" must be %s, not %s' % (self.attrib_name, self.attrib_type.__name__, value.__class__.__name__)) + return True + +class ItemDelta(object): + pass class ScrapedItem(object): """ @@ -8,23 +62,32 @@ class ScrapedItem(object): that identifies uniquely the given scraped item. """ - def set_adaptors(self, adaptors_dict, **kwargs): - """ - Set the adaptors to use for this item. Receives a dict of the adaptors - desired for each attribute and returns the item itself. - """ - setattr(self, '_adaptors_dict', AdaptorDict(adaptors_dict)) - return self - - def set_attrib_adaptors(self, attrib, adaptors, **kwargs): - """ - Set the adaptors (from a list or tuple) to be used for a specific attribute. - """ - self._adaptors_dict[attrib] = adaptors + ATTRIBUTES = { 'guid': ItemAttribute(attrib_name='guid', attrib_type=basestring) } + _override_adaptors = { } - def attribute(self, attrname, value, override=False, add=False, debug=False): - val = self._adaptors_dict.execute(attrname, value, debug) - if val or val is False: + def __setattr__(self, attr, value): + if value is None: + self.__dict__.pop(attr, None) + return + + if attr in self.ATTRIBUTES: + self.ATTRIBUTES[attr].check(value) + self.__dict__[attr] = value + + def set_adaptor(self, attrname, adaptor): + if attrname in self.ATTRIBUTES and callable(adaptor): + self._override_adaptors[attrname] = adaptor + + def attribute(self, attrname, val, **kwargs): + if not attrname in self.ATTRIBUTES and not attrname.startswith('_'): + raise AttributeError('Attribute "%s" is not a valid attribute name. You must add it to %s.ATTRIBUTES' % (attrname, self.__class__.__name__)) + + override = kwargs.pop('override', False) + add = kwargs.pop('add', False) + adaptor = self._override_adaptors.get(attrname) or self.ATTRIBUTES[attrname].adapt + + val = adaptor(val, **kwargs) + if not val is None: curr_val = getattr(self, attrname, None) if not curr_val: setattr(self, attrname, val) @@ -40,5 +103,3 @@ class ScrapedItem(object): def __sub__(self, other): raise NotImplementedError -class ItemDelta(object): - pass