Merge pull request #556 from darkrho/item-loader-nones

Make `ItemLoader` ignore `None` values from processors.

scrapy/contrib/loader/__init__.py

@@ -1,18 +1,17 @@
-"""
-Item Loader
+"""Item Loader
 
 See documentation in docs/topics/loaders.rst
-"""
 
+"""
 from collections import defaultdict
-import re
 
 from scrapy.item import Item
 from scrapy.selector import Selector
-from scrapy.utils.misc import arg_to_iter, extract_regex
-from scrapy.utils.python import flatten
 from scrapy.utils.decorator import deprecated
 from scrapy.utils.deprecate import create_deprecated_class
+from scrapy.utils.misc import arg_to_iter, extract_regex
+from scrapy.utils.python import flatten
+
 from .common import wrap_loader_context
 from .processor import Identity
 
@@ -81,7 +80,9 @@ class ItemLoader(object):
     def load_item(self):
         item = self.item
         for field_name in self._values:
-            item[field_name] = self.get_output_value(field_name)
+            value = self.get_output_value(field_name)
+            if value is not None:
+                item[field_name] = value
         return item
 
     def get_output_value(self, field_name):

scrapy/tests/test_contrib_loader.py

@@ -57,6 +57,35 @@ class BasicItemLoaderTest(unittest.TestCase):
         item = il.load_item()
         self.assertEqual(item['name'], [u'Marta'])
 
+    def test_load_item_ignore_none_field_values(self):
+        def validate_sku(value):
+            # Let's assume a SKU is only digits.
+            if value.isdigit():
+                return value
+
+        class MyLoader(ItemLoader):
+            name_out = Compose(lambda vs: vs[0])  # take first which allows empty values
+            price_out = Compose(TakeFirst(), float)
+            sku_out = Compose(TakeFirst(), validate_sku)
+
+        valid_fragment = u'SKU: 1234'
+        invalid_fragment = u'SKU: not available'
+        sku_re = 'SKU: (.+)'
+
+        il = MyLoader(item={})
+        # Should not return "sku: None".
+        il.add_value('sku', [invalid_fragment], re=sku_re)
+        # Should not ignore empty values.
+        il.add_value('name', u'')
+        il.add_value('price', [u'0'])
+        self.assertEqual(il.load_item(), {
+            'name': u'',
+            'price': 0.0,
+        })
+
+        il.replace_value('sku', [valid_fragment], re=sku_re)
+        self.assertEqual(il.load_item()['sku'], u'1234')
+
     def test_add_value(self):
         il = TestItemLoader()
         il.add_value('name', u'marta')