Compose: stop process on None value by default

By doing this, we can use str.lower as a processor safely without
checking whether the given value is None.

By passing stop_on_none=False as keyword argument, this behaviour can be changed.

Signed-off-by: Ping Yin <pkufranky@gmail.com>

docs/topics/loaders.rst

@@ -548,6 +548,9 @@ Here is a list of all built-in processors:
     function, and so on, until the last function returns the output value of
     this processor.
 
+    By default, stop process on None value. This behaviour can be changed by
+    passing keyword argument stop_on_none=False.
+
     Example::
 
         >>> from scrapy.contrib.loader.processor import Compose

scrapy/contrib/loader/processor.py

@@ -33,6 +33,7 @@ class Compose(object):
 
     def __init__(self, *functions, **default_loader_context):
         self.functions = functions
+        self.stop_on_none = default_loader_context.get('stop_on_none', True)
         self.default_loader_context = default_loader_context
     
     def __call__(self, value, loader_context=None):
@@ -42,6 +43,8 @@ class Compose(object):
             context = self.default_loader_context
         wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions]
         for func in wrapped_funcs:
+            if value is None and self.stop_on_none:
+                break
             value = func(value)
         return value
 

scrapy/tests/test_contrib_loader.py

@@ -321,6 +321,10 @@ class ProcessorsTest(unittest.TestCase):
     def test_compose(self):
         proc = Compose(lambda v: v[0], str.upper)
         self.assertEqual(proc(['hello', 'world']), 'HELLO')
+        proc = Compose(str.upper)
+        self.assertEqual(proc(None), None)
+        proc = Compose(str.upper, stop_on_none=False)
+        self.assertRaises(TypeError, proc, None)
 
     def test_mapcompose(self):
         filter_world = lambda x: None if x == 'world' else x