From 6059221716529927db2d34ddd813ab2ea3b95588 Mon Sep 17 00:00:00 2001 From: Ping Yin Date: Thu, 8 Apr 2010 10:59:47 +0800 Subject: [PATCH] Compose: stop process on None value by default By doing this, we can use str.lower as a processor safely without checking whether the given value is None. By passing stop_on_none=False as keyword argument, this behaviour can be changed. Signed-off-by: Ping Yin --- docs/topics/loaders.rst | 3 +++ scrapy/contrib/loader/processor.py | 3 +++ scrapy/tests/test_contrib_loader.py | 4 ++++ 3 files changed, 10 insertions(+) diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index f9d8671fc..5a0f1cef6 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -548,6 +548,9 @@ Here is a list of all built-in processors: function, and so on, until the last function returns the output value of this processor. + By default, stop process on None value. This behaviour can be changed by + passing keyword argument stop_on_none=False. + Example:: >>> from scrapy.contrib.loader.processor import Compose diff --git a/scrapy/contrib/loader/processor.py b/scrapy/contrib/loader/processor.py index 26806aeb5..bf8a355e9 100644 --- a/scrapy/contrib/loader/processor.py +++ b/scrapy/contrib/loader/processor.py @@ -33,6 +33,7 @@ class Compose(object): def __init__(self, *functions, **default_loader_context): self.functions = functions + self.stop_on_none = default_loader_context.get('stop_on_none', True) self.default_loader_context = default_loader_context def __call__(self, value, loader_context=None): @@ -42,6 +43,8 @@ class Compose(object): context = self.default_loader_context wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions] for func in wrapped_funcs: + if value is None and self.stop_on_none: + break value = func(value) return value diff --git a/scrapy/tests/test_contrib_loader.py b/scrapy/tests/test_contrib_loader.py index 0c0988756..201362140 100644 --- a/scrapy/tests/test_contrib_loader.py +++ b/scrapy/tests/test_contrib_loader.py @@ -321,6 +321,10 @@ class ProcessorsTest(unittest.TestCase): def test_compose(self): proc = Compose(lambda v: v[0], str.upper) self.assertEqual(proc(['hello', 'world']), 'HELLO') + proc = Compose(str.upper) + self.assertEqual(proc(None), None) + proc = Compose(str.upper, stop_on_none=False) + self.assertRaises(TypeError, proc, None) def test_mapcompose(self): filter_world = lambda x: None if x == 'world' else x