mirror of https://github.com/scrapy/scrapy.git
Reverted to revision 370
--HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40376
This commit is contained in:
elpolilla
parent
fab0e6938c
commit
5f4053d9fc
|
|
@ -1,26 +1,4 @@
|
|||
import inspect
|
||||
|
||||
from scrapy.contrib.adaptors.extraction import extract, extract_unquoted, ExtractImages
|
||||
from scrapy.contrib.adaptors.markup import remove_tags, remove_root, unquote
|
||||
from scrapy.contrib.adaptors.misc import to_unicode, clean_spaces, strip_list, drop_empty, canonicalize_urls, delist, Regex
|
||||
from scrapy.contrib.adaptors.markup import remove_tags, remove_root, Unquote
|
||||
from scrapy.contrib.adaptors.misc import to_unicode, clean_spaces, strip_list, drop_empty, canonicalize_urls, Delist, Regex
|
||||
from scrapy.utils.python import unique, flatten
|
||||
|
||||
def adaptor_gen(*args):
|
||||
subadaptors = []
|
||||
arg_mappings = []
|
||||
for subadaptor in args:
|
||||
if callable(subadaptor):
|
||||
if inspect.isfunction(subadaptor):
|
||||
func_args, varargs, varkw, defaults = inspect.getargspec(subadaptor)
|
||||
arg_mappings.append(func_args[1:])
|
||||
else:
|
||||
arg_mappings.append([])
|
||||
subadaptors.append(subadaptor)
|
||||
|
||||
def _adaptor(value, **kwargs):
|
||||
for index, subadaptor in enumerate(subadaptors):
|
||||
adaptor_args = dict((key, val) for key, val in kwargs.items() if key in arg_mappings[index])
|
||||
value = subadaptor(value, **adaptor_args)
|
||||
return value
|
||||
|
||||
return _adaptor
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ def remove_root(value):
|
|||
return value
|
||||
return [ _remove_root(v) for v in value ]
|
||||
|
||||
def unquote(value, keep_entities=None):
|
||||
class Unquote(object):
|
||||
"""
|
||||
Receives a list of strings, removes all of the
|
||||
entities the strings may have, and returns
|
||||
|
|
@ -30,8 +30,10 @@ def unquote(value, keep_entities=None):
|
|||
Input: iterable with strings
|
||||
Output: list of strings
|
||||
"""
|
||||
if keep_entities is None:
|
||||
keep_entities = ['lt', 'amp']
|
||||
return [ remove_entities(v, keep=keep_entities) for v in value ]
|
||||
def __init__(self, keep=['lt', 'amp']):
|
||||
self.keep = keep
|
||||
|
||||
def __call__(self, value):
|
||||
return [ remove_entities(v, keep=self.keep) for v in value ]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
from scrapy.contrib.item.models import RobustScrapedItem, RobustItemDelta, ValidationError, ValidationPipeline
|
||||
from scrapy.contrib.item.models import RobustScrapedItem, RobustItemDelta, ValidationError, ValidationPipeline, SetGUIDPipeline
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import hashlib
|
|||
from pydispatch import dispatcher
|
||||
from pprint import PrettyPrinter
|
||||
|
||||
from scrapy.item import ScrapedItem, ItemDelta, ItemAttribute
|
||||
from scrapy.item import ScrapedItem, ItemDelta
|
||||
from scrapy.spider import spiders
|
||||
from scrapy.core import signals
|
||||
from scrapy.core.exceptions import UsageError, DropItem
|
||||
|
|
@ -30,6 +30,11 @@ class ValidationPipeline(object):
|
|||
item.validate()
|
||||
return item
|
||||
|
||||
class SetGUIDPipeline(object):
|
||||
def process_item(self, domain, response, item):
|
||||
spiders.fromdomain(domain).set_guid(item)
|
||||
return item
|
||||
|
||||
class RobustScrapedItem(ScrapedItem):
|
||||
"""
|
||||
A more robust scraped item class with a built-in validation mechanism and
|
||||
|
|
@ -37,8 +42,8 @@ class RobustScrapedItem(ScrapedItem):
|
|||
"""
|
||||
|
||||
ATTRIBUTES = {
|
||||
'guid': ItemAttribute(attrib_name='guid', attrib_type=basestring), # a global unique identifier
|
||||
'url': ItemAttribute(attrib_name='url', attrib_type=basestring), # the main URL where this item was scraped from
|
||||
'guid': basestring, # a global unique identifier
|
||||
'url': basestring, # the main URL where this item was scraped from
|
||||
}
|
||||
|
||||
def __init__(self, data=None):
|
||||
|
|
@ -67,8 +72,30 @@ class RobustScrapedItem(ScrapedItem):
|
|||
"""
|
||||
Set an attribute checking it matches the attribute type declared in self.ATTRIBUTES
|
||||
"""
|
||||
if not attr.startswith('_') and attr not in self.ATTRIBUTES:
|
||||
raise AttributeError('Attribute "%s" is not a valid attribute name. You must add it to %s.ATTRIBUTES' % (attr, self.__class__.__name__))
|
||||
|
||||
if value is None:
|
||||
self.__dict__.pop(attr, None)
|
||||
return
|
||||
|
||||
if attr == '_adaptors_dict':
|
||||
return object.__setattr__(self, '_adaptors_dict', value)
|
||||
|
||||
type1 = self.ATTRIBUTES[attr]
|
||||
if hasattr(type1, '__iter__'):
|
||||
if not hasattr(value, '__iter__'):
|
||||
raise TypeError('Attribute "%s" must be a sequence' % attr)
|
||||
type2 = type1[0]
|
||||
for i in value:
|
||||
if not isinstance(i, type2):
|
||||
raise TypeError('Attribute "%s" cannot contain %s, only %s' % (attr, i.__class__.__name__, type2.__name__))
|
||||
else:
|
||||
if not isinstance(value, type1):
|
||||
raise TypeError('Attribute "%s" must be %s, not %s' % (attr, type1.__name__, value.__class__.__name__))
|
||||
|
||||
self.__dict__[attr] = value
|
||||
self.__dict__['_version'] = None
|
||||
super(RobustScrapedItem, self).__setattr__(attr, value)
|
||||
|
||||
def __delattr__(self, attr):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
from scrapy.item.models import ScrapedItem, ItemDelta, ItemAttribute
|
||||
from scrapy.item.models import ScrapedItem, ItemDelta
|
||||
|
|
|
|||
|
|
@ -1,59 +1,5 @@
|
|||
import types
|
||||
|
||||
from traceback import format_exc
|
||||
from scrapy import log
|
||||
from scrapy.item.adaptors import AdaptorDict
|
||||
from scrapy.conf import settings
|
||||
from scrapy.core.exceptions import NotConfigured
|
||||
|
||||
class ItemAttribute(object):
|
||||
def __init__(self, attrib_name, attrib_type, adaptor=None):
|
||||
self.attrib_name = attrib_name
|
||||
self.attrib_type = attrib_type
|
||||
self.adaptor = adaptor
|
||||
|
||||
def adapt(self, value, **kwargs):
|
||||
debug = kwargs.get('debug') or all([settings.getbool('LOG_ENABLED'), settings.get('LOGLEVEL') == 'TRACE'])
|
||||
if not self.adaptor:
|
||||
if debug:
|
||||
log.msg('No adaptor defined for attribute %s.' % self.attrib_name, log.WARNING)
|
||||
return value
|
||||
|
||||
try:
|
||||
if debug:
|
||||
print " %07s | input >" % self.attrib_name, repr(value)
|
||||
value = self.adaptor(value, **kwargs)
|
||||
if debug:
|
||||
print " %07s | output >" % self.attrib_name, repr(value)
|
||||
|
||||
except Exception:
|
||||
print "Error in '%s' adaptor. Traceback text:" % name
|
||||
print format_exc()
|
||||
self.value = None
|
||||
return False
|
||||
|
||||
return value
|
||||
|
||||
def check(self, value):
|
||||
if not self.attrib_name:
|
||||
raise NotConfigured('You must define "attrib_name" attribute in order to use an ItemAttribute')
|
||||
|
||||
if not self.attrib_type:
|
||||
raise NotConfigured('You must define "attrib_type" attribute in order to use an ItemAttribute')
|
||||
else:
|
||||
if hasattr(self.attrib_type, '__iter__'):
|
||||
if not hasattr(value, '__iter__'):
|
||||
raise TypeError('Attribute "%s" must be a sequence' % self.attrib_name)
|
||||
iter_type = self.attrib_type[0]
|
||||
for i in value:
|
||||
if not isinstance(i, iter_type):
|
||||
raise TypeError('Attribute "%s" cannot contain %s, only %s' % (self.name, i.__class__.__name__, iter_type.__name__))
|
||||
else:
|
||||
if not isinstance(value, self.attrib_type):
|
||||
raise TypeError('Attribute "%s" must be %s, not %s' % (self.attrib_name, self.attrib_type.__name__, value.__class__.__name__))
|
||||
return True
|
||||
|
||||
class ItemDelta(object):
|
||||
pass
|
||||
|
||||
class ScrapedItem(object):
|
||||
"""
|
||||
|
|
@ -62,32 +8,23 @@ class ScrapedItem(object):
|
|||
that identifies uniquely the given scraped item.
|
||||
"""
|
||||
|
||||
ATTRIBUTES = { 'guid': ItemAttribute(attrib_name='guid', attrib_type=basestring) }
|
||||
_override_adaptors = { }
|
||||
def set_adaptors(self, adaptors_dict, **kwargs):
|
||||
"""
|
||||
Set the adaptors to use for this item. Receives a dict of the adaptors
|
||||
desired for each attribute and returns the item itself.
|
||||
"""
|
||||
setattr(self, '_adaptors_dict', AdaptorDict(adaptors_dict))
|
||||
return self
|
||||
|
||||
def set_attrib_adaptors(self, attrib, adaptors, **kwargs):
|
||||
"""
|
||||
Set the adaptors (from a list or tuple) to be used for a specific attribute.
|
||||
"""
|
||||
self._adaptors_dict[attrib] = adaptors
|
||||
|
||||
def __setattr__(self, attr, value):
|
||||
if value is None:
|
||||
self.__dict__.pop(attr, None)
|
||||
return
|
||||
|
||||
if attr in self.ATTRIBUTES:
|
||||
self.ATTRIBUTES[attr].check(value)
|
||||
self.__dict__[attr] = value
|
||||
|
||||
def set_adaptor(self, attrname, adaptor):
|
||||
if attrname in self.ATTRIBUTES and callable(adaptor):
|
||||
self._override_adaptors[attrname] = adaptor
|
||||
|
||||
def attribute(self, attrname, val, **kwargs):
|
||||
if not attrname in self.ATTRIBUTES and not attrname.startswith('_'):
|
||||
raise AttributeError('Attribute "%s" is not a valid attribute name. You must add it to %s.ATTRIBUTES' % (attrname, self.__class__.__name__))
|
||||
|
||||
override = kwargs.pop('override', False)
|
||||
add = kwargs.pop('add', False)
|
||||
adaptor = self._override_adaptors.get(attrname) or self.ATTRIBUTES[attrname].adapt
|
||||
|
||||
val = adaptor(val, **kwargs)
|
||||
if not val is None:
|
||||
def attribute(self, attrname, value, override=False, add=False, debug=False):
|
||||
val = self._adaptors_dict.execute(attrname, value, debug)
|
||||
if val or val is False:
|
||||
curr_val = getattr(self, attrname, None)
|
||||
if not curr_val:
|
||||
setattr(self, attrname, val)
|
||||
|
|
@ -103,3 +40,5 @@ class ScrapedItem(object):
|
|||
def __sub__(self, other):
|
||||
raise NotImplementedError
|
||||
|
||||
class ItemDelta(object):
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -52,10 +52,10 @@ class AdaptorsTestCase(unittest.TestCase):
|
|||
['lala.com', 'pepe.co.uk', 'das.biz', 'lelelel.net'])
|
||||
|
||||
def test_unquote_all(self):
|
||||
self.assertEqual(adaptors.unquote([u'hello©&welcome', u'<br />&'], keep_entities=[]), [u'hello\xa9&welcome', u'<br />&'])
|
||||
self.assertEqual(adaptors.Unquote(keep=[])([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'<br />&'])
|
||||
|
||||
def test_unquote(self):
|
||||
self.assertEqual(adaptors.unquote([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'<br />&'])
|
||||
self.assertEqual(adaptors.Unquote()([u'hello©&welcome', u'<br />&']), [u'hello\xa9&welcome', u'<br />&'])
|
||||
|
||||
def test_remove_tags(self):
|
||||
test_data = ['<a href="lala">adsaas<br /></a>', '<div id="1"><table>dsadasf</table></div>']
|
||||
|
|
@ -78,7 +78,7 @@ class AdaptorsTestCase(unittest.TestCase):
|
|||
[1, 2, 5, 6, 'hi'])
|
||||
|
||||
def test_delist(self):
|
||||
self.assertEqual(adaptors.delist(['hi', 'there', 'fellas.', 'this', 'is', 'my', 'test.']),
|
||||
self.assertEqual(adaptors.Delist()(['hi', 'there', 'fellas.', 'this', 'is', 'my', 'test.']),
|
||||
'hi there fellas. this is my test.')
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue