diff --git a/docs/experimental/loaders.rst b/docs/experimental/loaders.rst index 5b9885389..ae8a10baf 100644 --- a/docs/experimental/loaders.rst +++ b/docs/experimental/loaders.rst @@ -433,6 +433,9 @@ needs. Available built-in processors ============================= +.. module:: scrapy.contrib.loader.processor + :synopsis: A collection of processors to use with Item Loaders + Even though you can use any callable function as input and output processors, Scrapy provides some commonly used processors, which are described below. Some of them, like the :class:`ApplyConcat` (which is typically used as input @@ -441,65 +444,11 @@ produce the final parsed value. Here is a list of all built-in processors: -.. _topics-loaders-applyconcat: - -ApplyConcat processor ---------------------- - -The ApplyConcat processor is the recommended processor to use if you want to -concatenate the processing of several functions in a pipeline. - -.. module:: scrapy.contrib.loader.processor - :synopsis: A collection of processors to use with Item Loaders - -.. class:: ApplyConcat(\*functions, \**default_loader_context) - - A processor which applies the given functions consecutively, in order, - concatenating their results before next function call. So each function - returns a list of values (though it could return ``None`` or a signle value - too) and the next function is called once for each of those values, - receiving one of those values as input each time. The output of each - function call (for each input value) is concatenated and each values of the - concatenation is used to call the next function, and the process repeats - until there are no functions left. - - Each function can optionally receive a ``loader_context`` parameter, which - will contain the currently active :ref:`Item Loader context - `. - - The keyword arguments passed in the consturctor are used as the default - Item Loader context values passed on each function call. However, the final - Item Loader context values passed to funtions get overriden with the - currently active Item Loader context accesible through the - :meth:`ItemLoader.context` attribute. - - Example:: - - >>> def filter_world(x): - ... return None if x == 'world' else x - ... - >>> from scrapy.contrib.loader.processor import ApplyConcat - >>> proc = ApplyConcat(filter_world, str.upper) - >>> proc(['hello', 'world', 'this', 'is', 'scrapy']) - ['HELLO, 'THIS', 'IS', 'SCRAPY'] - -.. class:: TakeFirst - - Return the first non null/empty value from the values to received, so it's - typically used as output processor of single-valued fields. It doesn't - receive any constructor arguments, nor accepts a Item Loader context. - - Example:: - - >>> from scrapy.contrib.loader.processor import TakeFirst - >>> proc = TakeFirst() - >>> proc(['', 'one', 'two', 'three']) - 'one' - .. class:: Identity - Return the original values unchanged. It doesn't receive any constructor - arguments nor accepts a Item Loader context. + The simplest processor, which doesn't do anything. It returns the original + values unchanged. It doesn't receive any constructor arguments nor accepts + Loader contexts. Example:: @@ -508,10 +457,23 @@ concatenate the processing of several functions in a pipeline. >>> proc(['one', 'two', 'three']) ['one', 'two', 'three'] +.. class:: TakeFirst + + Return the first non null/empty value from the values to received, so it's + typically used as output processor of single-valued fields. It doesn't + receive any constructor arguments, nor accepts Loader contexts. + + Example:: + + >>> from scrapy.contrib.loader.processor import TakeFirst + >>> proc = TakeFirst() + >>> proc(['', 'one', 'two', 'three']) + 'one' + .. class:: Join(separator=u' ') Return the values joined with the separator given in the constructor, which - defaults to ``u' '``. It doesn't accept a Item Loader context. + defaults to ``u' '``. It doesn't accept Loader contexts. When using the default separator, this processor is equivalent to the function: ``u' '.join`` @@ -525,3 +487,47 @@ concatenate the processing of several functions in a pipeline. >>> proc = Join('
') >>> proc(['one', 'two', 'three']) u'one
two
three' + +.. class:: Compose(\*functions, \**default_loader_context) + + A processor which is constructed from the composition of the given + functions. This means that each input value of this processor is passed to + the first function, and the result of that function is passed to the second + function, and so on, until the last function returns the output value of + this processor. + + Each function can optionally receive a ``loader_context`` parameter. For + those which does this processor will pass the currently active :ref:`Loader + context ` through that parameter. + + The keyword arguments passed in the constructor are used as the default + Loader context values passed to each function call. However, the final + Loader context values passed to functions are overridden with the currently + active Loader context accessible through the :meth:`ItemLoader.context` + attribute. + +.. class:: ApplyConcat(\*functions, \**default_loader_context) + + A processor which applies the given functions consecutively, in order, + concatenating their results before next function call. So each function + returns a list of values (though it could return ``None`` or a signle value + too) and the next function is called once for each of those values, + receiving one of those values as input each time. The output of each + function call (for each input value) is concatenated and each values of the + concatenation is used to call the next function, and the process repeats + until there are no functions left. + + As with the Compose processor, functions can receive Loader contexts, and + constructor keyword arguments are used as default context values. See + :class:`Compose` processor for more info. + + Example:: + + >>> def filter_world(x): + ... return None if x == 'world' else x + ... + >>> from scrapy.contrib.loader.processor import ApplyConcat + >>> proc = ApplyConcat(filter_world, str.upper) + >>> proc(['hello', 'world', 'this', 'is', 'scrapy']) + ['HELLO, 'THIS', 'IS', 'SCRAPY'] + diff --git a/scrapy/contrib/loader/processor.py b/scrapy/contrib/loader/processor.py index 7a4a92e19..725f6c6b8 100644 --- a/scrapy/contrib/loader/processor.py +++ b/scrapy/contrib/loader/processor.py @@ -29,17 +29,17 @@ class ApplyConcat(object): return list(values) -class Pipe(object): +class Compose(object): - def __init__(self, *functions, **default_parser_context): + def __init__(self, *functions, **default_loader_context): self.functions = functions - self.default_parser_context = default_parser_context + self.default_loader_context = default_loader_context - def __call__(self, value, parser_context=None): - if parser_context: - context = MergeDict(parser_context, self.default_parser_context) + def __call__(self, value, loader_context=None): + if loader_context: + context = MergeDict(loader_context, self.default_loader_context) else: - context = self.default_parser_context + context = self.default_loader_context wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions] for func in wrapped_funcs: value = func(value) diff --git a/scrapy/tests/test_contrib_loader.py b/scrapy/tests/test_contrib_loader.py index aa764775c..014c5c057 100644 --- a/scrapy/tests/test_contrib_loader.py +++ b/scrapy/tests/test_contrib_loader.py @@ -1,7 +1,7 @@ import unittest from scrapy.contrib.loader import ItemLoader, XPathItemLoader -from scrapy.contrib.loader.processor import ApplyConcat, Join, Identity, Pipe +from scrapy.contrib.loader.processor import ApplyConcat, Join, Identity, Compose from scrapy.newitem import Item, Field from scrapy.xpath import HtmlXPathSelector from scrapy.http import HtmlResponse @@ -244,9 +244,9 @@ class ItemLoaderTest(unittest.TestCase): ip = TestItemLoader() self.assertRaises(KeyError, ip.add_value, 'wrong_field', [u'lala', u'lolo']) - def test_pipe_pprocwssor(self): + def test_compose_processor(self): class TestItemLoader(NameItemLoader): - name_out = Pipe(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1]) + name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1]) il = TestItemLoader() il.add_value('name', [u'marta', u'other'])