mirror of https://github.com/scrapy/scrapy.git
Added ItemAttribute objects
--HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40373
This commit is contained in:
elpolilla
parent
90d28badf6
commit
4ef7c46303
|
|
@ -1 +1 @@
|
|||
from scrapy.contrib.item.models import RobustScrapedItem, RobustItemDelta, ValidationError, ValidationPipeline, SetGUIDPipeline
|
||||
from scrapy.contrib.item.models import RobustScrapedItem, RobustItemDelta, ValidationError, ValidationPipeline
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import hashlib
|
|||
from pydispatch import dispatcher
|
||||
from pprint import PrettyPrinter
|
||||
|
||||
from scrapy.item import ScrapedItem, ItemDelta
|
||||
from scrapy.item import ScrapedItem, ItemDelta, ItemAttribute
|
||||
from scrapy.spider import spiders
|
||||
from scrapy.core import signals
|
||||
from scrapy.core.exceptions import UsageError, DropItem
|
||||
|
|
@ -30,11 +30,6 @@ class ValidationPipeline(object):
|
|||
item.validate()
|
||||
return item
|
||||
|
||||
class SetGUIDPipeline(object):
|
||||
def process_item(self, domain, response, item):
|
||||
spiders.fromdomain(domain).set_guid(item)
|
||||
return item
|
||||
|
||||
class RobustScrapedItem(ScrapedItem):
|
||||
"""
|
||||
A more robust scraped item class with a built-in validation mechanism and
|
||||
|
|
@ -42,8 +37,8 @@ class RobustScrapedItem(ScrapedItem):
|
|||
"""
|
||||
|
||||
ATTRIBUTES = {
|
||||
'guid': basestring, # a global unique identifier
|
||||
'url': basestring, # the main URL where this item was scraped from
|
||||
'guid': ItemAttribute(attrib_name='guid', attrib_type=basestring), # a global unique identifier
|
||||
'url': ItemAttribute(attrib_name='url', attrib_type=basestring), # the main URL where this item was scraped from
|
||||
}
|
||||
|
||||
def __init__(self, data=None):
|
||||
|
|
@ -64,7 +59,7 @@ class RobustScrapedItem(ScrapedItem):
|
|||
# Note that this method is called only when the attribute is not found in
|
||||
# self.__dict__ or the class/instance methods.
|
||||
if attr in self.ATTRIBUTES:
|
||||
return None
|
||||
return () if hasattr(self.ATTRIBUTES[attr].attrib_type, '__iter__') else None
|
||||
else:
|
||||
raise AttributeError(attr)
|
||||
|
||||
|
|
@ -72,30 +67,8 @@ class RobustScrapedItem(ScrapedItem):
|
|||
"""
|
||||
Set an attribute checking it matches the attribute type declared in self.ATTRIBUTES
|
||||
"""
|
||||
if not attr.startswith('_') and attr not in self.ATTRIBUTES:
|
||||
raise AttributeError('Attribute "%s" is not a valid attribute name. You must add it to %s.ATTRIBUTES' % (attr, self.__class__.__name__))
|
||||
|
||||
if value is None:
|
||||
self.__dict__.pop(attr, None)
|
||||
return
|
||||
|
||||
if attr == '_adaptors_dict':
|
||||
return object.__setattr__(self, '_adaptors_dict', value)
|
||||
|
||||
type1 = self.ATTRIBUTES[attr]
|
||||
if hasattr(type1, '__iter__'):
|
||||
if not hasattr(value, '__iter__'):
|
||||
raise TypeError('Attribute "%s" must be a sequence' % attr)
|
||||
type2 = type1[0]
|
||||
for i in value:
|
||||
if not isinstance(i, type2):
|
||||
raise TypeError('Attribute "%s" cannot contain %s, only %s' % (attr, i.__class__.__name__, type2.__name__))
|
||||
else:
|
||||
if not isinstance(value, type1):
|
||||
raise TypeError('Attribute "%s" must be %s, not %s' % (attr, type1.__name__, value.__class__.__name__))
|
||||
|
||||
self.__dict__[attr] = value
|
||||
self.__dict__['_version'] = None
|
||||
super(RobustScrapedItem, self).__setattr__(attr, value)
|
||||
|
||||
def __delattr__(self, attr):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
from scrapy.item.models import ScrapedItem, ItemDelta
|
||||
from scrapy.item.models import ScrapedItem, ItemDelta, ItemAttribute
|
||||
|
|
|
|||
|
|
@ -1,5 +1,59 @@
|
|||
from scrapy.item.adaptors import AdaptorDict
|
||||
import types
|
||||
|
||||
from traceback import format_exc
|
||||
from scrapy import log
|
||||
from scrapy.conf import settings
|
||||
from scrapy.core.exceptions import NotConfigured
|
||||
|
||||
class ItemAttribute(object):
|
||||
def __init__(self, attrib_name, attrib_type, adaptor=None):
|
||||
self.attrib_name = attrib_name
|
||||
self.attrib_type = attrib_type
|
||||
self.adaptor = adaptor
|
||||
|
||||
def adapt(self, value, **kwargs):
|
||||
debug = kwargs.get('debug') or all([settings.getbool('LOG_ENABLED'), settings.get('LOGLEVEL') == 'TRACE'])
|
||||
if not self.adaptor:
|
||||
if debug:
|
||||
log.msg('No adaptor defined for attribute %s.' % self.attrib_name, log.WARNING)
|
||||
return value
|
||||
|
||||
try:
|
||||
if debug:
|
||||
print " %07s | input >" % self.attrib_name, repr(value)
|
||||
value = self.adaptor(value, **kwargs)
|
||||
if debug:
|
||||
print " %07s | output >" % self.attrib_name, repr(value)
|
||||
|
||||
except Exception:
|
||||
print "Error in '%s' adaptor. Traceback text:" % name
|
||||
print format_exc()
|
||||
self.value = None
|
||||
return False
|
||||
|
||||
return value
|
||||
|
||||
def check(self, value):
|
||||
if not self.attrib_name:
|
||||
raise NotConfigured('You must define "attrib_name" attribute in order to use an ItemAttribute')
|
||||
|
||||
if not self.attrib_type:
|
||||
raise NotConfigured('You must define "attrib_type" attribute in order to use an ItemAttribute')
|
||||
else:
|
||||
if hasattr(self.attrib_type, '__iter__'):
|
||||
if not hasattr(value, '__iter__'):
|
||||
raise TypeError('Attribute "%s" must be a sequence' % self.attrib_name)
|
||||
iter_type = self.attrib_type[0]
|
||||
for i in value:
|
||||
if not isinstance(i, iter_type):
|
||||
raise TypeError('Attribute "%s" cannot contain %s, only %s' % (self.name, i.__class__.__name__, iter_type.__name__))
|
||||
else:
|
||||
if not isinstance(value, self.attrib_type):
|
||||
raise TypeError('Attribute "%s" must be %s, not %s' % (self.attrib_name, self.attrib_type.__name__, value.__class__.__name__))
|
||||
return True
|
||||
|
||||
class ItemDelta(object):
|
||||
pass
|
||||
|
||||
class ScrapedItem(object):
|
||||
"""
|
||||
|
|
@ -8,23 +62,32 @@ class ScrapedItem(object):
|
|||
that identifies uniquely the given scraped item.
|
||||
"""
|
||||
|
||||
def set_adaptors(self, adaptors_dict, **kwargs):
|
||||
"""
|
||||
Set the adaptors to use for this item. Receives a dict of the adaptors
|
||||
desired for each attribute and returns the item itself.
|
||||
"""
|
||||
setattr(self, '_adaptors_dict', AdaptorDict(adaptors_dict))
|
||||
return self
|
||||
|
||||
def set_attrib_adaptors(self, attrib, adaptors, **kwargs):
|
||||
"""
|
||||
Set the adaptors (from a list or tuple) to be used for a specific attribute.
|
||||
"""
|
||||
self._adaptors_dict[attrib] = adaptors
|
||||
ATTRIBUTES = { 'guid': ItemAttribute(attrib_name='guid', attrib_type=basestring) }
|
||||
_override_adaptors = { }
|
||||
|
||||
def attribute(self, attrname, value, override=False, add=False, debug=False):
|
||||
val = self._adaptors_dict.execute(attrname, value, debug)
|
||||
if val or val is False:
|
||||
def __setattr__(self, attr, value):
|
||||
if value is None:
|
||||
self.__dict__.pop(attr, None)
|
||||
return
|
||||
|
||||
if attr in self.ATTRIBUTES:
|
||||
self.ATTRIBUTES[attr].check(value)
|
||||
self.__dict__[attr] = value
|
||||
|
||||
def set_adaptor(self, attrname, adaptor):
|
||||
if attrname in self.ATTRIBUTES and callable(adaptor):
|
||||
self._override_adaptors[attrname] = adaptor
|
||||
|
||||
def attribute(self, attrname, val, **kwargs):
|
||||
if not attrname in self.ATTRIBUTES and not attrname.startswith('_'):
|
||||
raise AttributeError('Attribute "%s" is not a valid attribute name. You must add it to %s.ATTRIBUTES' % (attrname, self.__class__.__name__))
|
||||
|
||||
override = kwargs.pop('override', False)
|
||||
add = kwargs.pop('add', False)
|
||||
adaptor = self._override_adaptors.get(attrname) or self.ATTRIBUTES[attrname].adapt
|
||||
|
||||
val = adaptor(val, **kwargs)
|
||||
if not val is None:
|
||||
curr_val = getattr(self, attrname, None)
|
||||
if not curr_val:
|
||||
setattr(self, attrname, val)
|
||||
|
|
@ -40,5 +103,3 @@ class ScrapedItem(object):
|
|||
def __sub__(self, other):
|
||||
raise NotImplementedError
|
||||
|
||||
class ItemDelta(object):
|
||||
pass
|
||||
|
|
|
|||
Loading…
Reference in New Issue