diff --git a/scrapy/trunk/scrapy/contrib/adaptors/markup.py b/scrapy/trunk/scrapy/contrib/adaptors/markup.py index 9b1d048f3..dab57c8d1 100644 --- a/scrapy/trunk/scrapy/contrib/adaptors/markup.py +++ b/scrapy/trunk/scrapy/contrib/adaptors/markup.py @@ -1,5 +1,6 @@ import re from scrapy.utils.markup import replace_tags, remove_entities +from scrapy.item.adaptors import adaptize def remove_tags(value): """ @@ -34,10 +35,10 @@ class Unquote(object): Input: iterable with strings Output: list of strings """ - def __init__(self, keep=['lt', 'amp']): - self.keep = keep + def __init__(self, keep=None): + self.keep = ['amp', 'lt'] if keep is None else keep - def __call__(self, value): + def __call__(self, value, keep=None): + if keep is not None: + self.keep = keep return [ remove_entities(v, keep=self.keep) for v in value ] - - diff --git a/scrapy/trunk/scrapy/contrib/adaptors/misc.py b/scrapy/trunk/scrapy/contrib/adaptors/misc.py index 71b5d04e4..a656eef3e 100644 --- a/scrapy/trunk/scrapy/contrib/adaptors/misc.py +++ b/scrapy/trunk/scrapy/contrib/adaptors/misc.py @@ -5,6 +5,7 @@ from scrapy.xpath.selector import XPathSelector, XPathSelectorList from scrapy.utils.url import canonicalize_url from scrapy.utils.misc import extract_regex from scrapy.utils.python import flatten +from scrapy.item.adaptors import adaptize def to_unicode(value): """ @@ -72,10 +73,10 @@ def canonicalize_urls(value): Output: list of unicodes(urls) """ if hasattr(value, '__iter__'): - return [canonicalize_url(url) for url in value] + return [canonicalize_url(str(url)) for url in value] elif isinstance(value, basestring): - return canonicalize_url(value) - return u'' + return canonicalize_url(str(value)) + return '' class Delist(object): """ @@ -88,7 +89,9 @@ class Delist(object): def __init__(self, delimiter=' '): self.delimiter = delimiter - def __call__(self, value): + def __call__(self, value, delimiter=None): + if delimiter is not None: + self.delimiter = delimiter return self.delimiter.join(value) class Regex(object): diff --git a/scrapy/trunk/scrapy/item/adaptors.py b/scrapy/trunk/scrapy/item/adaptors.py index 3e366cc63..5c9d9536d 100644 --- a/scrapy/trunk/scrapy/item/adaptors.py +++ b/scrapy/trunk/scrapy/item/adaptors.py @@ -1,36 +1,75 @@ +import inspect + from traceback import format_exc from scrapy.conf import settings -class AdaptorDict(dict): +class AdaptorPipe(list): """ Class that represents an item's attribute pipeline. - This class contains a dictionary of attributes, matched with a list - of adaptors to be run for filtering the input before storing. + This class is itself a list, filled by adaptors to be run + in order to filter the input. """ + def __init__(self, adaptors): + super(AdaptorPipe, self).__init__([adaptor for adaptor in adaptors if callable(adaptor)]) - def execute(self, attrname, value, debug=False): + def __call__(self, value, **kwargs): """ - Execute pipeline for attribute name "attrname" and value "value". + Execute the adaptor pipeline for this attribute. """ - debug = debug or all([settings.getbool('LOG_ENABLED'), settings.get('LOGLEVEL') == 'TRACE']) + debug = kwargs.pop('debug', all([settings.getbool('LOG_ENABLED'), settings.get('LOGLEVEL') == 'TRACE'])) + + for adaptor in self: + if inspect.isfunction(adaptor): + func_args, _, _ ,_ = inspect.getargspec(adaptor) + else: + func_args = [] + + name = getattr(adaptor, 'func_name', None) + if not name: + name = adaptor.__class__.__name__ if hasattr(adaptor, '__class__') else adaptor.__name__ - for adaptor in self.get(attrname, []): - name = adaptor.__class__.__name__ if hasattr(adaptor, '__class__') else adaptor.__name__ try: if debug: print " %07s | input >" % name, repr(value) - value = adaptor(value) + + if 'adaptor_args' in func_args: + value = adaptor(value, kwargs) + else: + value = adaptor(value) + if debug: print " %07s | output >" % name, repr(value) - except Exception: print "Error in '%s' adaptor. Traceback text:" % name print format_exc() return - + return value + def __add__(self, other): + if isinstance(other, list): + return AdaptorPipe(super(AdaptorPipe, self).__add__(other)) + elif callable(other): + return AdaptorPipe(self + [other]) - def __repr__(self): - return "" % len(self.keys()) +def adaptize(adaptor, my_args=None): + """ + This decorator helps you add to your pipelines adaptors that are able + to receive extra keyword arguments from the spiders. + + If my_args is None, it'll try to guess the arguments your adaptor receives + by instropection, and if it can't guess them, it will act as if it were a normal + adaptor. + To do the last, you may also call adaptize with my_args = [], or @adaptize([]) + """ + if my_args is None: + if inspect.isfunction(adaptor): + my_args, _, _, _ = inspect.getargspec(adaptor) + else: + my_args = [] + + def _adaptor(value, **adaptor_args): + kwargs = dict((key, val) for key, val in adaptor_args.items() if key in my_args) + return adaptor(value, **kwargs) + return _adaptor diff --git a/scrapy/trunk/scrapy/item/models.py b/scrapy/trunk/scrapy/item/models.py index 00638c977..004844f26 100644 --- a/scrapy/trunk/scrapy/item/models.py +++ b/scrapy/trunk/scrapy/item/models.py @@ -1,4 +1,4 @@ -from scrapy.item.adaptors import AdaptorDict +from scrapy.item.adaptors import AdaptorPipe from scrapy.conf import settings class ScrapedItem(object): @@ -8,34 +8,39 @@ class ScrapedItem(object): that identifies uniquely the given scraped item. """ - def set_adaptors(self, adaptors_dict, **kwargs): + def set_adaptors(self, adaptors_dict): """ Set the adaptors to use for this item. Receives a dict of the adaptors desired for each attribute and returns the item itself. """ - setattr(self, '_adaptors_dict', AdaptorDict(adaptors_dict)) + _adaptors_dict = dict(item for item in adaptors_dict.items() if isinstance(item[1], AdaptorPipe)) + self.__dict__['_adaptors_dict'] = _adaptors_dict return self - def set_attrib_adaptors(self, attrib, adaptors, **kwargs): + def set_attrib_adaptors(self, attrib, pipe): """ Set the adaptors (from a list or tuple) to be used for a specific attribute. """ - self._adaptors_dict[attrib] = adaptors + self._adaptors_dict[attrib] = AdaptorPipe(pipe) if hasattr(pipe, '__iter__') else None - def attribute(self, attrname, value, override=False, add=False, debug=False): - val = self._adaptors_dict.execute(attrname, value, debug) - if val or val is False: - curr_val = getattr(self, attrname, None) - if not curr_val: - setattr(self, attrname, val) - else: - if override: + def attribute(self, attrname, value, **kwargs): + pipe = self._adaptors_dict.get(attrname) + if pipe: + val = pipe(value, **kwargs) + if val or val is False: + curr_val = getattr(self, attrname, None) + if not curr_val: setattr(self, attrname, val) - elif add and all(hasattr(var, '__iter__') for var in (curr_val, val)): - newval = [] - newval.extend(curr_val) - newval.extend(val) - setattr(self, attrname, newval) + else: + if override: + setattr(self, attrname, val) + elif add and all(hasattr(var, '__iter__') for var in (curr_val, val)): + newval = [] + newval.extend(curr_val) + newval.extend(val) + setattr(self, attrname, newval) + elif value: + setattr(self, attrname, value) def __sub__(self, other): raise NotImplementedError