mirror of https://github.com/scrapy/scrapy.git
Removed AdaptorDict and added AdaptorPipe, plus the adaptize helper
--HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40404
This commit is contained in:
elpolilla
parent
cbffced594
commit
0f9eba8a97
|
|
@ -1,5 +1,6 @@
|
|||
import re
|
||||
from scrapy.utils.markup import replace_tags, remove_entities
|
||||
from scrapy.item.adaptors import adaptize
|
||||
|
||||
def remove_tags(value):
|
||||
"""
|
||||
|
|
@ -34,10 +35,10 @@ class Unquote(object):
|
|||
Input: iterable with strings
|
||||
Output: list of strings
|
||||
"""
|
||||
def __init__(self, keep=['lt', 'amp']):
|
||||
self.keep = keep
|
||||
def __init__(self, keep=None):
|
||||
self.keep = ['amp', 'lt'] if keep is None else keep
|
||||
|
||||
def __call__(self, value):
|
||||
def __call__(self, value, keep=None):
|
||||
if keep is not None:
|
||||
self.keep = keep
|
||||
return [ remove_entities(v, keep=self.keep) for v in value ]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from scrapy.xpath.selector import XPathSelector, XPathSelectorList
|
|||
from scrapy.utils.url import canonicalize_url
|
||||
from scrapy.utils.misc import extract_regex
|
||||
from scrapy.utils.python import flatten
|
||||
from scrapy.item.adaptors import adaptize
|
||||
|
||||
def to_unicode(value):
|
||||
"""
|
||||
|
|
@ -72,10 +73,10 @@ def canonicalize_urls(value):
|
|||
Output: list of unicodes(urls)
|
||||
"""
|
||||
if hasattr(value, '__iter__'):
|
||||
return [canonicalize_url(url) for url in value]
|
||||
return [canonicalize_url(str(url)) for url in value]
|
||||
elif isinstance(value, basestring):
|
||||
return canonicalize_url(value)
|
||||
return u''
|
||||
return canonicalize_url(str(value))
|
||||
return ''
|
||||
|
||||
class Delist(object):
|
||||
"""
|
||||
|
|
@ -88,7 +89,9 @@ class Delist(object):
|
|||
def __init__(self, delimiter=' '):
|
||||
self.delimiter = delimiter
|
||||
|
||||
def __call__(self, value):
|
||||
def __call__(self, value, delimiter=None):
|
||||
if delimiter is not None:
|
||||
self.delimiter = delimiter
|
||||
return self.delimiter.join(value)
|
||||
|
||||
class Regex(object):
|
||||
|
|
|
|||
|
|
@ -1,36 +1,75 @@
|
|||
import inspect
|
||||
|
||||
from traceback import format_exc
|
||||
from scrapy.conf import settings
|
||||
|
||||
class AdaptorDict(dict):
|
||||
class AdaptorPipe(list):
|
||||
"""
|
||||
Class that represents an item's attribute pipeline.
|
||||
|
||||
This class contains a dictionary of attributes, matched with a list
|
||||
of adaptors to be run for filtering the input before storing.
|
||||
This class is itself a list, filled by adaptors to be run
|
||||
in order to filter the input.
|
||||
"""
|
||||
def __init__(self, adaptors):
|
||||
super(AdaptorPipe, self).__init__([adaptor for adaptor in adaptors if callable(adaptor)])
|
||||
|
||||
def execute(self, attrname, value, debug=False):
|
||||
def __call__(self, value, **kwargs):
|
||||
"""
|
||||
Execute pipeline for attribute name "attrname" and value "value".
|
||||
Execute the adaptor pipeline for this attribute.
|
||||
"""
|
||||
debug = debug or all([settings.getbool('LOG_ENABLED'), settings.get('LOGLEVEL') == 'TRACE'])
|
||||
debug = kwargs.pop('debug', all([settings.getbool('LOG_ENABLED'), settings.get('LOGLEVEL') == 'TRACE']))
|
||||
|
||||
for adaptor in self:
|
||||
if inspect.isfunction(adaptor):
|
||||
func_args, _, _ ,_ = inspect.getargspec(adaptor)
|
||||
else:
|
||||
func_args = []
|
||||
|
||||
name = getattr(adaptor, 'func_name', None)
|
||||
if not name:
|
||||
name = adaptor.__class__.__name__ if hasattr(adaptor, '__class__') else adaptor.__name__
|
||||
|
||||
for adaptor in self.get(attrname, []):
|
||||
name = adaptor.__class__.__name__ if hasattr(adaptor, '__class__') else adaptor.__name__
|
||||
try:
|
||||
if debug:
|
||||
print " %07s | input >" % name, repr(value)
|
||||
value = adaptor(value)
|
||||
|
||||
if 'adaptor_args' in func_args:
|
||||
value = adaptor(value, kwargs)
|
||||
else:
|
||||
value = adaptor(value)
|
||||
|
||||
if debug:
|
||||
print " %07s | output >" % name, repr(value)
|
||||
|
||||
except Exception:
|
||||
print "Error in '%s' adaptor. Traceback text:" % name
|
||||
print format_exc()
|
||||
return
|
||||
|
||||
|
||||
return value
|
||||
|
||||
def __add__(self, other):
|
||||
if isinstance(other, list):
|
||||
return AdaptorPipe(super(AdaptorPipe, self).__add__(other))
|
||||
elif callable(other):
|
||||
return AdaptorPipe(self + [other])
|
||||
|
||||
def __repr__(self):
|
||||
return "<AdaptorDict [ %d attributes ] >" % len(self.keys())
|
||||
def adaptize(adaptor, my_args=None):
|
||||
"""
|
||||
This decorator helps you add to your pipelines adaptors that are able
|
||||
to receive extra keyword arguments from the spiders.
|
||||
|
||||
If my_args is None, it'll try to guess the arguments your adaptor receives
|
||||
by instropection, and if it can't guess them, it will act as if it were a normal
|
||||
adaptor.
|
||||
To do the last, you may also call adaptize with my_args = [], or @adaptize([])
|
||||
"""
|
||||
if my_args is None:
|
||||
if inspect.isfunction(adaptor):
|
||||
my_args, _, _, _ = inspect.getargspec(adaptor)
|
||||
else:
|
||||
my_args = []
|
||||
|
||||
def _adaptor(value, **adaptor_args):
|
||||
kwargs = dict((key, val) for key, val in adaptor_args.items() if key in my_args)
|
||||
return adaptor(value, **kwargs)
|
||||
return _adaptor
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from scrapy.item.adaptors import AdaptorDict
|
||||
from scrapy.item.adaptors import AdaptorPipe
|
||||
from scrapy.conf import settings
|
||||
|
||||
class ScrapedItem(object):
|
||||
|
|
@ -8,34 +8,39 @@ class ScrapedItem(object):
|
|||
that identifies uniquely the given scraped item.
|
||||
"""
|
||||
|
||||
def set_adaptors(self, adaptors_dict, **kwargs):
|
||||
def set_adaptors(self, adaptors_dict):
|
||||
"""
|
||||
Set the adaptors to use for this item. Receives a dict of the adaptors
|
||||
desired for each attribute and returns the item itself.
|
||||
"""
|
||||
setattr(self, '_adaptors_dict', AdaptorDict(adaptors_dict))
|
||||
_adaptors_dict = dict(item for item in adaptors_dict.items() if isinstance(item[1], AdaptorPipe))
|
||||
self.__dict__['_adaptors_dict'] = _adaptors_dict
|
||||
return self
|
||||
|
||||
def set_attrib_adaptors(self, attrib, adaptors, **kwargs):
|
||||
def set_attrib_adaptors(self, attrib, pipe):
|
||||
"""
|
||||
Set the adaptors (from a list or tuple) to be used for a specific attribute.
|
||||
"""
|
||||
self._adaptors_dict[attrib] = adaptors
|
||||
self._adaptors_dict[attrib] = AdaptorPipe(pipe) if hasattr(pipe, '__iter__') else None
|
||||
|
||||
def attribute(self, attrname, value, override=False, add=False, debug=False):
|
||||
val = self._adaptors_dict.execute(attrname, value, debug)
|
||||
if val or val is False:
|
||||
curr_val = getattr(self, attrname, None)
|
||||
if not curr_val:
|
||||
setattr(self, attrname, val)
|
||||
else:
|
||||
if override:
|
||||
def attribute(self, attrname, value, **kwargs):
|
||||
pipe = self._adaptors_dict.get(attrname)
|
||||
if pipe:
|
||||
val = pipe(value, **kwargs)
|
||||
if val or val is False:
|
||||
curr_val = getattr(self, attrname, None)
|
||||
if not curr_val:
|
||||
setattr(self, attrname, val)
|
||||
elif add and all(hasattr(var, '__iter__') for var in (curr_val, val)):
|
||||
newval = []
|
||||
newval.extend(curr_val)
|
||||
newval.extend(val)
|
||||
setattr(self, attrname, newval)
|
||||
else:
|
||||
if override:
|
||||
setattr(self, attrname, val)
|
||||
elif add and all(hasattr(var, '__iter__') for var in (curr_val, val)):
|
||||
newval = []
|
||||
newval.extend(curr_val)
|
||||
newval.extend(val)
|
||||
setattr(self, attrname, newval)
|
||||
elif value:
|
||||
setattr(self, attrname, value)
|
||||
|
||||
def __sub__(self, other):
|
||||
raise NotImplementedError
|
||||
|
|
|
|||
Loading…
Reference in New Issue