mirror of https://github.com/scrapy/scrapy.git
removed obsolete scrapy.contrib.item module (RobustScrapedItem model)
This commit is contained in:
Pablo Hoffman
parent
1624981443
commit
bebc8b2027
|
|
@ -1,11 +0,0 @@
|
|||
"""
|
||||
WARNING: This module is obsolete and will be removed before the Scrapy 0.7
|
||||
release
|
||||
"""
|
||||
|
||||
import warnings
|
||||
|
||||
from scrapy.contrib.item.models import RobustScrapedItem, RobustItemDelta, ValidationError, ValidationPipeline
|
||||
|
||||
warnings.warn("scrapy.contrib.item is obsolete and will be removed soon", \
|
||||
DeprecationWarning, stacklevel=2)
|
||||
|
|
@ -1,129 +0,0 @@
|
|||
import inspect
|
||||
|
||||
from traceback import format_exc
|
||||
from scrapy.conf import settings
|
||||
|
||||
class AdaptorFunc(object):
|
||||
"""
|
||||
This is a private class that the AdaptorPipe uses to store
|
||||
its adaptation functions and their attributes.
|
||||
"""
|
||||
def __init__(self, adaptor):
|
||||
self.adaptor = adaptor
|
||||
|
||||
if inspect.isfunction(adaptor):
|
||||
self.func_args, _, _, _ = inspect.getargspec(adaptor)
|
||||
elif hasattr(adaptor, '__call__'):
|
||||
try:
|
||||
self.func_args, _, _, _ = inspect.getargspec(adaptor.__call__)
|
||||
except Exception:
|
||||
self.func_args = []
|
||||
|
||||
self.name = getattr(adaptor, 'func_name', None)
|
||||
if not self.name:
|
||||
if hasattr(adaptor, '__class__') and adaptor.__class__ is not type:
|
||||
self.name = adaptor.__class__.__name__
|
||||
else:
|
||||
self.name = adaptor.__name__
|
||||
|
||||
def __call__(self, value, kwargs):
|
||||
if 'adaptor_args' in self.func_args:
|
||||
return self.adaptor(value, kwargs)
|
||||
else:
|
||||
return self.adaptor(value)
|
||||
|
||||
def __repr__(self):
|
||||
return '<Adaptor %s >' % self.name
|
||||
|
||||
class AdaptorPipe(list):
|
||||
"""
|
||||
Class that represents an item's attribute pipeline.
|
||||
|
||||
This class is itself a list, filled by adaptors to be run
|
||||
in order to filter the input.
|
||||
"""
|
||||
def __init__(self, adaptors):
|
||||
def _filter_adaptor(adaptor):
|
||||
return AdaptorFunc(adaptor) if not isinstance(adaptor, AdaptorFunc) else adaptor
|
||||
|
||||
if not hasattr(adaptors, '__iter__'):
|
||||
raise TypeError('You must provide AdaptorPipe a list of adaptors')
|
||||
|
||||
for adaptor in adaptors:
|
||||
if not callable(adaptor):
|
||||
raise TypeError("%s is not a callable" % adaptor)
|
||||
super(AdaptorPipe, self).__init__(map(_filter_adaptor, adaptors))
|
||||
self.debug = settings.getbool('ADAPTORS_DEBUG')
|
||||
|
||||
def __call__(self, value, kwargs=None):
|
||||
"""
|
||||
Execute the adaptor pipeline for this attribute.
|
||||
"""
|
||||
|
||||
debug_padding = 0
|
||||
values = [value]
|
||||
|
||||
for adaptor in self:
|
||||
next_round = []
|
||||
for val in values:
|
||||
try:
|
||||
if self.debug:
|
||||
print "%sinput | %s <" % (' ' * debug_padding, adaptor.name), repr(val)
|
||||
|
||||
val = adaptor(val, kwargs or {})
|
||||
if isinstance(val, tuple):
|
||||
next_round.extend(val)
|
||||
elif val is not None:
|
||||
next_round.append(val)
|
||||
|
||||
if self.debug:
|
||||
print "%soutput | %s >" % (' ' * debug_padding, adaptor.name), repr(val)
|
||||
except Exception:
|
||||
print "Error in '%s' adaptor. Traceback text:" % adaptor.name
|
||||
print format_exc()
|
||||
return
|
||||
|
||||
debug_padding += 2
|
||||
values = next_round
|
||||
|
||||
return tuple(values)
|
||||
|
||||
def __add__(self, other):
|
||||
if callable(other):
|
||||
other = [other]
|
||||
elif hasattr(other, '__iter__'):
|
||||
other = list(other)
|
||||
return AdaptorPipe(super(AdaptorPipe, self).__add__(other))
|
||||
|
||||
def __repr__(self):
|
||||
return '<AdaptorPipe %s >' % super(AdaptorPipe, self).__repr__()
|
||||
|
||||
def add_adaptor(self, adaptor, position=None):
|
||||
if callable(adaptor):
|
||||
if not isinstance(adaptor, AdaptorFunc):
|
||||
adaptor = AdaptorFunc(adaptor)
|
||||
if position is None:
|
||||
self.append(adaptor)
|
||||
else:
|
||||
self.insert(position, adaptor)
|
||||
|
||||
def adaptize(adaptor, my_args=None):
|
||||
"""
|
||||
This decorator helps you add to your pipelines adaptors that are able
|
||||
to receive extra keyword arguments from the spiders.
|
||||
|
||||
If my_args is None, it'll try to guess the arguments your adaptor receives
|
||||
by instropection, and if it can't guess them, it will act as if it were a normal
|
||||
adaptor.
|
||||
To do the last, you may also call adaptize with my_args = [], or @adaptize([])
|
||||
"""
|
||||
if my_args is None:
|
||||
if inspect.isfunction(adaptor):
|
||||
my_args, _, _, _ = inspect.getargspec(adaptor)
|
||||
else:
|
||||
my_args = []
|
||||
|
||||
def _adaptor(value, adaptor_args):
|
||||
kwargs = dict((key, val) for key, val in adaptor_args.items() if key in my_args)
|
||||
return adaptor(value, **kwargs)
|
||||
return _adaptor
|
||||
|
|
@ -1,287 +0,0 @@
|
|||
"""
|
||||
This module contains some extra base models for scraped items which could be
|
||||
useful in some Scrapy implementations
|
||||
"""
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
import warnings
|
||||
|
||||
from pprint import PrettyPrinter
|
||||
|
||||
from scrapy.item import ScrapedItem
|
||||
from scrapy.contrib.item.adaptors import AdaptorPipe
|
||||
from scrapy.core.exceptions import DropItem
|
||||
from scrapy.utils.python import unique
|
||||
|
||||
class ValidationError(DropItem):
|
||||
"""Indicates a data validation error"""
|
||||
def __init__(self, problem, value=None):
|
||||
self.problem = problem
|
||||
self.value = value
|
||||
|
||||
def __str__(self):
|
||||
if self.value is not None:
|
||||
return '%s "%s"' % (self.problem, self.value)
|
||||
else:
|
||||
return '%s' % (self.problem)
|
||||
|
||||
class ValidationPipeline(object):
|
||||
def process_item(self, domain, item):
|
||||
item.validate()
|
||||
return item
|
||||
|
||||
class RobustScrapedItem(ScrapedItem):
|
||||
"""
|
||||
A more robust scraped item class with a built-in validation mechanism and
|
||||
minimal versioning support
|
||||
"""
|
||||
|
||||
ATTRIBUTES = {
|
||||
'guid': basestring, # a global unique identifier
|
||||
'url': basestring, # the main URL where this item was scraped from
|
||||
}
|
||||
|
||||
def __init__(self, data=None, adaptors_dict=None):
|
||||
super(RobustScrapedItem, self).__init__(data)
|
||||
self.__dict__['_adaptors_dict'] = adaptors_dict or {}
|
||||
self.__dict__['_version'] = None
|
||||
|
||||
def __getattr__(self, attr):
|
||||
# Return None for valid attributes not set, raise AttributeError for invalid attributes
|
||||
# Note that this method is called only when the attribute is not found in
|
||||
# self.__dict__ or the class/instance methods.
|
||||
if attr in self.ATTRIBUTES:
|
||||
return None
|
||||
else:
|
||||
raise AttributeError(attr)
|
||||
|
||||
def __setattr__(self, attr, value):
|
||||
"""
|
||||
Set an attribute checking it matches the attribute type declared in self.ATTRIBUTES
|
||||
"""
|
||||
if not attr.startswith('_') and attr not in self.ATTRIBUTES:
|
||||
raise AttributeError('Attribute "%s" is not a valid attribute name. You must add it to %s.ATTRIBUTES' % (attr, self.__class__.__name__))
|
||||
|
||||
if value is None:
|
||||
self.__dict__.pop(attr, None)
|
||||
return
|
||||
|
||||
if attr == '_adaptors_dict':
|
||||
return object.__setattr__(self, '_adaptors_dict', value)
|
||||
|
||||
type1 = self.ATTRIBUTES[attr]
|
||||
if hasattr(type1, '__iter__'):
|
||||
if not hasattr(value, '__iter__'):
|
||||
raise TypeError('Attribute "%s" must be a sequence' % attr)
|
||||
type2 = type1[0]
|
||||
for i in value:
|
||||
if not isinstance(i, type2):
|
||||
raise TypeError('Attribute "%s" cannot contain %s, only %s' % (attr, i.__class__.__name__, type2.__name__))
|
||||
else:
|
||||
if not isinstance(value, type1):
|
||||
raise TypeError('Attribute "%s" must be %s, not %s' % (attr, type1.__name__, value.__class__.__name__))
|
||||
|
||||
self.__dict__[attr] = value
|
||||
self.__dict__['_version'] = None
|
||||
|
||||
def __delattr__(self, attr):
|
||||
"""
|
||||
Delete an attribute from the ScrapedItem instance if it exists.
|
||||
If not, raise an AttributeError.
|
||||
"""
|
||||
if attr in self.__dict__:
|
||||
del self.__dict__[attr]
|
||||
self.__dict__['_version'] = None
|
||||
else:
|
||||
raise AttributeError("Attribute '%s' doesn't exist" % attr)
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, type(self)):
|
||||
return self.version == other.version
|
||||
|
||||
def __ne__(self, other):
|
||||
return self.version != other.version
|
||||
|
||||
def __sub__(self, other):
|
||||
return RobustItemDelta(other, self)
|
||||
|
||||
def __str__(self) :
|
||||
return "%s: GUID=%s, url=%s" % ( self.__class__.__name__ , self.guid, self.url )
|
||||
|
||||
def _add_single_attributes(self, attrname, attrtype, attributes):
|
||||
raise NotImplementedError('You must override _add_single_attributes method in order to join %s values into a single value.' % attrtype.__name__)
|
||||
|
||||
def attribute(self, attrname, *values, **kwargs):
|
||||
"""
|
||||
Set the provided values to the provided attribute (`attrname`) by filtering them
|
||||
through its adaptor pipeline first (if any).
|
||||
|
||||
If the attribute had been already set, it won't be overwritten unless the parameters
|
||||
`override` or `add` are True.
|
||||
|
||||
If `override` is True, the old value simply will be replaced with the new one.
|
||||
|
||||
If add is True (and there was an old value, of course), for multivalued attributes the values
|
||||
will be appended to the list of the already existing ones.
|
||||
For this same situation in single-valued attributes, the method _add_single_attributes will be
|
||||
called with the attribute's name and type, and the list of values to join as parameters.
|
||||
|
||||
The kwargs parameter is passed to the adaptors pipeline, which manages to transmit
|
||||
it to the adaptors themselves.
|
||||
"""
|
||||
def _clean_values(values):
|
||||
ret = []
|
||||
for val in values:
|
||||
if isinstance(val, tuple):
|
||||
ret.extend(val)
|
||||
elif val:
|
||||
ret.append(val)
|
||||
return ret
|
||||
|
||||
if not values:
|
||||
raise ValueError("You must specify at least one value when setting an attribute")
|
||||
if attrname not in self.ATTRIBUTES:
|
||||
raise AttributeError('Attribute "%s" is not a valid attribute name. You must add it to %s.ATTRIBUTES' % (attrname, self.__class__.__name__))
|
||||
|
||||
add = kwargs.pop('add', False)
|
||||
override = kwargs.pop('override', False)
|
||||
unique_vals = kwargs.pop('unique', False)
|
||||
old_value = getattr(self, attrname, None)
|
||||
if old_value and not any([override, add]):
|
||||
return
|
||||
|
||||
attrtype = self.ATTRIBUTES.get(attrname)
|
||||
multivalued = isinstance(attrtype, list)
|
||||
adaptors_pipe = self._adaptors_dict.get(attrname)
|
||||
new_values = [adaptors_pipe(value, kwargs) for value in values] if adaptors_pipe else [values]
|
||||
new_values = _clean_values(new_values)
|
||||
|
||||
if old_value and not override:
|
||||
if multivalued:
|
||||
new_values = old_value + new_values
|
||||
else:
|
||||
new_values.insert(0, old_value)
|
||||
|
||||
if not multivalued:
|
||||
if add and len(new_values) > 1:
|
||||
new_values = self._add_single_attributes(attrname, attrtype, new_values)
|
||||
else:
|
||||
new_values = new_values[0] if new_values else None
|
||||
elif multivalued and unique_vals:
|
||||
new_values = unique(new_values)
|
||||
|
||||
setattr(self, attrname, new_values)
|
||||
|
||||
def set_attrib_adaptors(self, attrib, pipe):
|
||||
""" Set the adaptors (from a list or tuple) to be used for a specific attribute. """
|
||||
self._adaptors_dict[attrib] = pipe if isinstance(pipe, AdaptorPipe) else AdaptorPipe(pipe)
|
||||
|
||||
def set_adaptors(self, adaptors_dict):
|
||||
"""
|
||||
Set the adaptors to use for this item. Receives a dict of the adaptors
|
||||
desired for each attribute and returns the item itself.
|
||||
"""
|
||||
self._adaptors_dict = {}
|
||||
for attrib, pipe in adaptors_dict.items():
|
||||
self.set_attrib_adaptors(attrib, pipe)
|
||||
return self
|
||||
|
||||
def add_adaptor(self, attrib, adaptor, position=None):
|
||||
"""
|
||||
Add an adaptor for the specified attribute at the given position.
|
||||
If position = None, then the adaptor is appended at the end of the pipeline.
|
||||
"""
|
||||
pipe = self._adaptors_dict.get(attrib, AdaptorPipe([]))
|
||||
pipe.add_adaptor(adaptor, position)
|
||||
self.set_attrib_adaptors(attrib, pipe)
|
||||
|
||||
def copy(self):
|
||||
"""Create a new RobustScrapedItem based on the current one"""
|
||||
return copy.deepcopy(self)
|
||||
|
||||
def validate(self):
|
||||
"""Method used to validate item attributes data"""
|
||||
raise NotImplemented
|
||||
|
||||
@property
|
||||
def version(self):
|
||||
"""
|
||||
Return a (cached) 40 char hash of all the item attributes.
|
||||
|
||||
WARNING: This cached version won't work if mutable products are
|
||||
modified directly like:
|
||||
|
||||
item.features.append('feature')
|
||||
"""
|
||||
if getattr(self, '_version', None):
|
||||
return self._version
|
||||
hash_ = hashlib.sha1()
|
||||
hash_.update("".join(["".join([n, str(v)]) for n, v in sorted(self.__dict__.iteritems())]))
|
||||
return hash_.hexdigest()
|
||||
|
||||
|
||||
class RobustItemDelta(object):
|
||||
"""
|
||||
This class represents the difference between
|
||||
a pair of RobustScrapedItems.
|
||||
"""
|
||||
|
||||
def __init__(self, old_item, new_item):
|
||||
if not isinstance(old_item, RobustScrapedItem) or \
|
||||
not isinstance(new_item, RobustScrapedItem):
|
||||
raise TypeError("Both arguments must be RobustScrapedItem instances")
|
||||
|
||||
if old_item.guid != new_item.guid:
|
||||
raise AttributeError("Item GUIDs must be equal in order to create a RobustItemDelta object")
|
||||
|
||||
self.old_item = old_item
|
||||
self.new_item = new_item
|
||||
self.diff = self.do_diff()
|
||||
|
||||
def do_diff(self):
|
||||
"""
|
||||
This method should retreive a dictionary
|
||||
containing the changes between both items
|
||||
as in this example:
|
||||
|
||||
>>> delta.do_diff()
|
||||
>>> {'attrib': {'new': 'New value', 'old': 'Old value'}, # Common attributes
|
||||
'attrib2': {'new': 'New value 2', 'old': 'Old value 2'},
|
||||
'attrib3': [{'new': 'New list value', 'old': 'Old list value'}, # List attributes
|
||||
{'new': 'New list value 2', 'old': 'Old list value 2'}]}
|
||||
"""
|
||||
|
||||
if self.old_item == self.new_item:
|
||||
return {}
|
||||
|
||||
diff = {}
|
||||
for key, value in self.old_item.__dict__.items():
|
||||
if key in self.old_item.ATTRIBUTES.keys():
|
||||
new_value = getattr(self.new_item, key)
|
||||
if value != new_value:
|
||||
diff[key] = {'new': new_value, 'old': value}
|
||||
for key, value in self.new_item.__dict__.items():
|
||||
if value and key in self.new_item.ATTRIBUTES.keys():
|
||||
if not getattr(self.old_item, key):
|
||||
diff[key] = {'new': value, 'old': None}
|
||||
return diff
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, RobustItemDelta):
|
||||
if other.old_item == self.old_item and \
|
||||
other.new_item == self.new_item and \
|
||||
other.diff == self.diff:
|
||||
return True
|
||||
return False
|
||||
|
||||
def __nonzero__(self):
|
||||
return bool(self.diff)
|
||||
|
||||
def __repr__(self):
|
||||
if self.diff:
|
||||
pp = PrettyPrinter(indent=3)
|
||||
return pp.pformat(self.diff)
|
||||
else:
|
||||
return 'No differences found between the provided items.'
|
||||
|
||||
|
|
@ -1,55 +0,0 @@
|
|||
# -*- coding: utf8 -*-
|
||||
import unittest
|
||||
|
||||
from scrapy.contrib.item import RobustScrapedItem
|
||||
|
||||
class MyItem(RobustScrapedItem):
|
||||
ATTRIBUTES = {
|
||||
'guid': basestring,
|
||||
'name': basestring,
|
||||
'children': [basestring],
|
||||
'age': int,
|
||||
'alive': bool,
|
||||
}
|
||||
|
||||
class RobustScrapedItemTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.item = MyItem()
|
||||
|
||||
def test_attribute_basic(self):
|
||||
self.assertRaises(ValueError, self.item.attribute, 'foo')
|
||||
self.assertRaises(AttributeError, self.item.attribute, 'foo', 'something')
|
||||
|
||||
self.item.attribute('name', 'John')
|
||||
self.assertEqual(self.item.name, 'John')
|
||||
|
||||
self.assertRaises(TypeError, self.item.attribute, 'children', ['Peter'])
|
||||
self.item.attribute('children', *['Peter'])
|
||||
self.assertEqual(self.item.children, ['Peter'])
|
||||
|
||||
self.item.attribute('age', 40)
|
||||
self.assertEqual(self.item.age, 40)
|
||||
|
||||
self.item.attribute('alive', False)
|
||||
self.assertEqual(self.item.alive, False)
|
||||
|
||||
def test_attribute_override(self):
|
||||
self.item.attribute('name', 'John')
|
||||
self.item.attribute('name', 'Charlie')
|
||||
self.assertEqual(self.item.name, 'John')
|
||||
|
||||
self.item.attribute('name', 'Charlie', override=True)
|
||||
self.assertEqual(self.item.name, 'Charlie')
|
||||
|
||||
def test_attribute_add(self):
|
||||
self.item.attribute('name', 'John')
|
||||
self.assertRaises(NotImplementedError, self.item.attribute, 'name', 'Doe', add=True)
|
||||
|
||||
self.item.attribute('children', *['Ken', 'Tom'])
|
||||
self.item.attribute('children', 'Bobby')
|
||||
self.item.attribute('children', 'Jimmy', add=True)
|
||||
self.assertEqual(self.item.children, ['Ken', 'Tom', 'Jimmy'])
|
||||
|
||||
self.item.attribute('children', 'Johnny', 'Rodrigo', add=True)
|
||||
self.assertEqual(self.item.children, ['Ken', 'Tom', 'Jimmy', 'Johnny', 'Rodrigo'])
|
||||
|
||||
Loading…
Reference in New Issue