diff --git a/scrapy/contrib/loader/processor.py b/scrapy/contrib/loader/processor.py index e883c219d..7a4a92e19 100644 --- a/scrapy/contrib/loader/processor.py +++ b/scrapy/contrib/loader/processor.py @@ -29,6 +29,23 @@ class ApplyConcat(object): return list(values) +class Pipe(object): + + def __init__(self, *functions, **default_parser_context): + self.functions = functions + self.default_parser_context = default_parser_context + + def __call__(self, value, parser_context=None): + if parser_context: + context = MergeDict(parser_context, self.default_parser_context) + else: + context = self.default_parser_context + wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions] + for func in wrapped_funcs: + value = func(value) + return value + + class TakeFirst(object): def __call__(self, values): diff --git a/scrapy/tests/test_contrib_loader.py b/scrapy/tests/test_contrib_loader.py index d7e36251e..0f365b194 100644 --- a/scrapy/tests/test_contrib_loader.py +++ b/scrapy/tests/test_contrib_loader.py @@ -1,7 +1,7 @@ import unittest from scrapy.contrib.loader import ItemLoader, XPathItemLoader -from scrapy.contrib.loader.processor import ApplyConcat, Join, Identity +from scrapy.contrib.loader.processor import ApplyConcat, Join, Identity, Pipe from scrapy.newitem import Item, Field from scrapy.xpath import HtmlXPathSelector from scrapy.http import HtmlResponse @@ -244,6 +244,16 @@ class ItemLoaderTest(unittest.TestCase): ip = TestItemLoader() self.assertRaises(KeyError, ip.add_value, 'wrong_field', [u'lala', u'lolo']) + def test_pipe_pprocwssor(self): + class TestItemLoader(NameItemLoader): + name_out = Pipe(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1]) + + il = TestItemLoader() + il.add_value('name', [u'marta', u'other']) + self.assertEqual(il.get_output_value('name'), u'Mart') + item = il.populate_item() + self.assertEqual(item['name'], u'Mart') + class TestXPathItemLoader(XPathItemLoader): default_item_class = TestItem