From 90d28badf696bbe161806337d515cb1d72457b55 Mon Sep 17 00:00:00 2001 From: elpolilla Date: Thu, 13 Nov 2008 10:13:11 +0000 Subject: [PATCH] Turned unquote adaptor into a function, and added adaptor factory --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40372 --- .../trunk/scrapy/contrib/adaptors/__init__.py | 26 +++++++++++++++++-- .../trunk/scrapy/contrib/adaptors/markup.py | 10 +++---- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/scrapy/trunk/scrapy/contrib/adaptors/__init__.py b/scrapy/trunk/scrapy/contrib/adaptors/__init__.py index b3951c0fc..120bb70f7 100644 --- a/scrapy/trunk/scrapy/contrib/adaptors/__init__.py +++ b/scrapy/trunk/scrapy/contrib/adaptors/__init__.py @@ -1,4 +1,26 @@ +import inspect + from scrapy.contrib.adaptors.extraction import extract, extract_unquoted, ExtractImages -from scrapy.contrib.adaptors.markup import remove_tags, remove_root, Unquote -from scrapy.contrib.adaptors.misc import to_unicode, clean_spaces, strip_list, drop_empty, canonicalize_urls, Delist, Regex +from scrapy.contrib.adaptors.markup import remove_tags, remove_root, unquote +from scrapy.contrib.adaptors.misc import to_unicode, clean_spaces, strip_list, drop_empty, canonicalize_urls, delist, Regex from scrapy.utils.python import unique, flatten + +def adaptor_gen(*args): + subadaptors = [] + arg_mappings = [] + for subadaptor in args: + if callable(subadaptor): + if inspect.isfunction(subadaptor): + func_args, varargs, varkw, defaults = inspect.getargspec(subadaptor) + arg_mappings.append(func_args[1:]) + else: + arg_mappings.append([]) + subadaptors.append(subadaptor) + + def _adaptor(value, **kwargs): + for index, subadaptor in enumerate(subadaptors): + adaptor_args = dict((key, val) for key, val in kwargs.items() if key in arg_mappings[index]) + value = subadaptor(value, **adaptor_args) + return value + + return _adaptor diff --git a/scrapy/trunk/scrapy/contrib/adaptors/markup.py b/scrapy/trunk/scrapy/contrib/adaptors/markup.py index b912f73e1..a19f1b75e 100644 --- a/scrapy/trunk/scrapy/contrib/adaptors/markup.py +++ b/scrapy/trunk/scrapy/contrib/adaptors/markup.py @@ -21,7 +21,7 @@ def remove_root(value): return value return [ _remove_root(v) for v in value ] -class Unquote(object): +def unquote(value, keep_entities=None): """ Receives a list of strings, removes all of the entities the strings may have, and returns @@ -30,10 +30,8 @@ class Unquote(object): Input: iterable with strings Output: list of strings """ - def __init__(self, keep=['lt', 'amp']): - self.keep = keep - - def __call__(self, value): - return [ remove_entities(v, keep=self.keep) for v in value ] + if keep_entities is None: + keep_entities = ['lt', 'amp'] + return [ remove_entities(v, keep=keep_entities) for v in value ]