diff --git a/scrapy/trunk/scrapy/contrib/adaptors/__init__.py b/scrapy/trunk/scrapy/contrib/adaptors/__init__.py
index 120bb70f7..b3951c0fc 100644
--- a/scrapy/trunk/scrapy/contrib/adaptors/__init__.py
+++ b/scrapy/trunk/scrapy/contrib/adaptors/__init__.py
@@ -1,26 +1,4 @@
-import inspect
-
from scrapy.contrib.adaptors.extraction import extract, extract_unquoted, ExtractImages
-from scrapy.contrib.adaptors.markup import remove_tags, remove_root, unquote
-from scrapy.contrib.adaptors.misc import to_unicode, clean_spaces, strip_list, drop_empty, canonicalize_urls, delist, Regex
+from scrapy.contrib.adaptors.markup import remove_tags, remove_root, Unquote
+from scrapy.contrib.adaptors.misc import to_unicode, clean_spaces, strip_list, drop_empty, canonicalize_urls, Delist, Regex
from scrapy.utils.python import unique, flatten
-
-def adaptor_gen(*args):
- subadaptors = []
- arg_mappings = []
- for subadaptor in args:
- if callable(subadaptor):
- if inspect.isfunction(subadaptor):
- func_args, varargs, varkw, defaults = inspect.getargspec(subadaptor)
- arg_mappings.append(func_args[1:])
- else:
- arg_mappings.append([])
- subadaptors.append(subadaptor)
-
- def _adaptor(value, **kwargs):
- for index, subadaptor in enumerate(subadaptors):
- adaptor_args = dict((key, val) for key, val in kwargs.items() if key in arg_mappings[index])
- value = subadaptor(value, **adaptor_args)
- return value
-
- return _adaptor
diff --git a/scrapy/trunk/scrapy/contrib/adaptors/markup.py b/scrapy/trunk/scrapy/contrib/adaptors/markup.py
index a19f1b75e..b912f73e1 100644
--- a/scrapy/trunk/scrapy/contrib/adaptors/markup.py
+++ b/scrapy/trunk/scrapy/contrib/adaptors/markup.py
@@ -21,7 +21,7 @@ def remove_root(value):
return value
return [ _remove_root(v) for v in value ]
-def unquote(value, keep_entities=None):
+class Unquote(object):
"""
Receives a list of strings, removes all of the
entities the strings may have, and returns
@@ -30,8 +30,10 @@ def unquote(value, keep_entities=None):
Input: iterable with strings
Output: list of strings
"""
- if keep_entities is None:
- keep_entities = ['lt', 'amp']
- return [ remove_entities(v, keep=keep_entities) for v in value ]
+ def __init__(self, keep=['lt', 'amp']):
+ self.keep = keep
+
+ def __call__(self, value):
+ return [ remove_entities(v, keep=self.keep) for v in value ]
diff --git a/scrapy/trunk/scrapy/contrib/item/__init__.py b/scrapy/trunk/scrapy/contrib/item/__init__.py
index a6226fec9..8f029a687 100644
--- a/scrapy/trunk/scrapy/contrib/item/__init__.py
+++ b/scrapy/trunk/scrapy/contrib/item/__init__.py
@@ -1 +1 @@
-from scrapy.contrib.item.models import RobustScrapedItem, RobustItemDelta, ValidationError, ValidationPipeline
+from scrapy.contrib.item.models import RobustScrapedItem, RobustItemDelta, ValidationError, ValidationPipeline, SetGUIDPipeline
diff --git a/scrapy/trunk/scrapy/contrib/item/models.py b/scrapy/trunk/scrapy/contrib/item/models.py
index 2832c9730..65ff653de 100644
--- a/scrapy/trunk/scrapy/contrib/item/models.py
+++ b/scrapy/trunk/scrapy/contrib/item/models.py
@@ -8,7 +8,7 @@ import hashlib
from pydispatch import dispatcher
from pprint import PrettyPrinter
-from scrapy.item import ScrapedItem, ItemDelta, ItemAttribute
+from scrapy.item import ScrapedItem, ItemDelta
from scrapy.spider import spiders
from scrapy.core import signals
from scrapy.core.exceptions import UsageError, DropItem
@@ -30,6 +30,11 @@ class ValidationPipeline(object):
item.validate()
return item
+class SetGUIDPipeline(object):
+ def process_item(self, domain, response, item):
+ spiders.fromdomain(domain).set_guid(item)
+ return item
+
class RobustScrapedItem(ScrapedItem):
"""
A more robust scraped item class with a built-in validation mechanism and
@@ -37,8 +42,8 @@ class RobustScrapedItem(ScrapedItem):
"""
ATTRIBUTES = {
- 'guid': ItemAttribute(attrib_name='guid', attrib_type=basestring), # a global unique identifier
- 'url': ItemAttribute(attrib_name='url', attrib_type=basestring), # the main URL where this item was scraped from
+ 'guid': basestring, # a global unique identifier
+ 'url': basestring, # the main URL where this item was scraped from
}
def __init__(self, data=None):
@@ -67,8 +72,30 @@ class RobustScrapedItem(ScrapedItem):
"""
Set an attribute checking it matches the attribute type declared in self.ATTRIBUTES
"""
+ if not attr.startswith('_') and attr not in self.ATTRIBUTES:
+ raise AttributeError('Attribute "%s" is not a valid attribute name. You must add it to %s.ATTRIBUTES' % (attr, self.__class__.__name__))
+
+ if value is None:
+ self.__dict__.pop(attr, None)
+ return
+
+ if attr == '_adaptors_dict':
+ return object.__setattr__(self, '_adaptors_dict', value)
+
+ type1 = self.ATTRIBUTES[attr]
+ if hasattr(type1, '__iter__'):
+ if not hasattr(value, '__iter__'):
+ raise TypeError('Attribute "%s" must be a sequence' % attr)
+ type2 = type1[0]
+ for i in value:
+ if not isinstance(i, type2):
+ raise TypeError('Attribute "%s" cannot contain %s, only %s' % (attr, i.__class__.__name__, type2.__name__))
+ else:
+ if not isinstance(value, type1):
+ raise TypeError('Attribute "%s" must be %s, not %s' % (attr, type1.__name__, value.__class__.__name__))
+
+ self.__dict__[attr] = value
self.__dict__['_version'] = None
- super(RobustScrapedItem, self).__setattr__(attr, value)
def __delattr__(self, attr):
"""
diff --git a/scrapy/trunk/scrapy/item/__init__.py b/scrapy/trunk/scrapy/item/__init__.py
index 5449a80f1..423ebfb01 100644
--- a/scrapy/trunk/scrapy/item/__init__.py
+++ b/scrapy/trunk/scrapy/item/__init__.py
@@ -1 +1 @@
-from scrapy.item.models import ScrapedItem, ItemDelta, ItemAttribute
+from scrapy.item.models import ScrapedItem, ItemDelta
diff --git a/scrapy/trunk/scrapy/item/models.py b/scrapy/trunk/scrapy/item/models.py
index c85831070..00638c977 100644
--- a/scrapy/trunk/scrapy/item/models.py
+++ b/scrapy/trunk/scrapy/item/models.py
@@ -1,59 +1,5 @@
-import types
-
-from traceback import format_exc
-from scrapy import log
+from scrapy.item.adaptors import AdaptorDict
from scrapy.conf import settings
-from scrapy.core.exceptions import NotConfigured
-
-class ItemAttribute(object):
- def __init__(self, attrib_name, attrib_type, adaptor=None):
- self.attrib_name = attrib_name
- self.attrib_type = attrib_type
- self.adaptor = adaptor
-
- def adapt(self, value, **kwargs):
- debug = kwargs.get('debug') or all([settings.getbool('LOG_ENABLED'), settings.get('LOGLEVEL') == 'TRACE'])
- if not self.adaptor:
- if debug:
- log.msg('No adaptor defined for attribute %s.' % self.attrib_name, log.WARNING)
- return value
-
- try:
- if debug:
- print " %07s | input >" % self.attrib_name, repr(value)
- value = self.adaptor(value, **kwargs)
- if debug:
- print " %07s | output >" % self.attrib_name, repr(value)
-
- except Exception:
- print "Error in '%s' adaptor. Traceback text:" % name
- print format_exc()
- self.value = None
- return False
-
- return value
-
- def check(self, value):
- if not self.attrib_name:
- raise NotConfigured('You must define "attrib_name" attribute in order to use an ItemAttribute')
-
- if not self.attrib_type:
- raise NotConfigured('You must define "attrib_type" attribute in order to use an ItemAttribute')
- else:
- if hasattr(self.attrib_type, '__iter__'):
- if not hasattr(value, '__iter__'):
- raise TypeError('Attribute "%s" must be a sequence' % self.attrib_name)
- iter_type = self.attrib_type[0]
- for i in value:
- if not isinstance(i, iter_type):
- raise TypeError('Attribute "%s" cannot contain %s, only %s' % (self.name, i.__class__.__name__, iter_type.__name__))
- else:
- if not isinstance(value, self.attrib_type):
- raise TypeError('Attribute "%s" must be %s, not %s' % (self.attrib_name, self.attrib_type.__name__, value.__class__.__name__))
- return True
-
-class ItemDelta(object):
- pass
class ScrapedItem(object):
"""
@@ -62,32 +8,23 @@ class ScrapedItem(object):
that identifies uniquely the given scraped item.
"""
- ATTRIBUTES = { 'guid': ItemAttribute(attrib_name='guid', attrib_type=basestring) }
- _override_adaptors = { }
+ def set_adaptors(self, adaptors_dict, **kwargs):
+ """
+ Set the adaptors to use for this item. Receives a dict of the adaptors
+ desired for each attribute and returns the item itself.
+ """
+ setattr(self, '_adaptors_dict', AdaptorDict(adaptors_dict))
+ return self
+
+ def set_attrib_adaptors(self, attrib, adaptors, **kwargs):
+ """
+ Set the adaptors (from a list or tuple) to be used for a specific attribute.
+ """
+ self._adaptors_dict[attrib] = adaptors
- def __setattr__(self, attr, value):
- if value is None:
- self.__dict__.pop(attr, None)
- return
-
- if attr in self.ATTRIBUTES:
- self.ATTRIBUTES[attr].check(value)
- self.__dict__[attr] = value
-
- def set_adaptor(self, attrname, adaptor):
- if attrname in self.ATTRIBUTES and callable(adaptor):
- self._override_adaptors[attrname] = adaptor
-
- def attribute(self, attrname, val, **kwargs):
- if not attrname in self.ATTRIBUTES and not attrname.startswith('_'):
- raise AttributeError('Attribute "%s" is not a valid attribute name. You must add it to %s.ATTRIBUTES' % (attrname, self.__class__.__name__))
-
- override = kwargs.pop('override', False)
- add = kwargs.pop('add', False)
- adaptor = self._override_adaptors.get(attrname) or self.ATTRIBUTES[attrname].adapt
-
- val = adaptor(val, **kwargs)
- if not val is None:
+ def attribute(self, attrname, value, override=False, add=False, debug=False):
+ val = self._adaptors_dict.execute(attrname, value, debug)
+ if val or val is False:
curr_val = getattr(self, attrname, None)
if not curr_val:
setattr(self, attrname, val)
@@ -103,3 +40,5 @@ class ScrapedItem(object):
def __sub__(self, other):
raise NotImplementedError
+class ItemDelta(object):
+ pass
diff --git a/scrapy/trunk/scrapy/tests/test_adaptors.py b/scrapy/trunk/scrapy/tests/test_adaptors.py
index 745026cd5..63b4d3a1f 100644
--- a/scrapy/trunk/scrapy/tests/test_adaptors.py
+++ b/scrapy/trunk/scrapy/tests/test_adaptors.py
@@ -52,10 +52,10 @@ class AdaptorsTestCase(unittest.TestCase):
['lala.com', 'pepe.co.uk', 'das.biz', 'lelelel.net'])
def test_unquote_all(self):
- self.assertEqual(adaptors.unquote([u'hello©&welcome', u'<br />&'], keep_entities=[]), [u'hello\xa9&welcome', u'
&'])
+ self.assertEqual(adaptors.Unquote(keep=[])([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'
&'])
def test_unquote(self):
- self.assertEqual(adaptors.unquote([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'<br />&'])
+ self.assertEqual(adaptors.Unquote()([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'<br />&'])
def test_remove_tags(self):
test_data = ['adsaas
', '']
@@ -78,7 +78,7 @@ class AdaptorsTestCase(unittest.TestCase):
[1, 2, 5, 6, 'hi'])
def test_delist(self):
- self.assertEqual(adaptors.delist(['hi', 'there', 'fellas.', 'this', 'is', 'my', 'test.']),
+ self.assertEqual(adaptors.Delist()(['hi', 'there', 'fellas.', 'this', 'is', 'my', 'test.']),
'hi there fellas. this is my test.')