diff --git a/scrapy/contrib/loader/__init__.py b/scrapy/contrib/loader/__init__.py index 895767269..f13bece41 100644 --- a/scrapy/contrib/loader/__init__.py +++ b/scrapy/contrib/loader/__init__.py @@ -1,18 +1,17 @@ -""" -Item Loader +"""Item Loader See documentation in docs/topics/loaders.rst -""" +""" from collections import defaultdict -import re from scrapy.item import Item from scrapy.selector import Selector -from scrapy.utils.misc import arg_to_iter, extract_regex -from scrapy.utils.python import flatten from scrapy.utils.decorator import deprecated from scrapy.utils.deprecate import create_deprecated_class +from scrapy.utils.misc import arg_to_iter, extract_regex +from scrapy.utils.python import flatten + from .common import wrap_loader_context from .processor import Identity @@ -81,7 +80,9 @@ class ItemLoader(object): def load_item(self): item = self.item for field_name in self._values: - item[field_name] = self.get_output_value(field_name) + value = self.get_output_value(field_name) + if value is not None: + item[field_name] = value return item def get_output_value(self, field_name): diff --git a/scrapy/tests/test_contrib_loader.py b/scrapy/tests/test_contrib_loader.py index 734bc31d1..3330b6105 100644 --- a/scrapy/tests/test_contrib_loader.py +++ b/scrapy/tests/test_contrib_loader.py @@ -57,6 +57,35 @@ class BasicItemLoaderTest(unittest.TestCase): item = il.load_item() self.assertEqual(item['name'], [u'Marta']) + def test_load_item_ignore_none_field_values(self): + def validate_sku(value): + # Let's assume a SKU is only digits. + if value.isdigit(): + return value + + class MyLoader(ItemLoader): + name_out = Compose(lambda vs: vs[0]) # take first which allows empty values + price_out = Compose(TakeFirst(), float) + sku_out = Compose(TakeFirst(), validate_sku) + + valid_fragment = u'SKU: 1234' + invalid_fragment = u'SKU: not available' + sku_re = 'SKU: (.+)' + + il = MyLoader(item={}) + # Should not return "sku: None". + il.add_value('sku', [invalid_fragment], re=sku_re) + # Should not ignore empty values. + il.add_value('name', u'') + il.add_value('price', [u'0']) + self.assertEqual(il.load_item(), { + 'name': u'', + 'price': 0.0, + }) + + il.replace_value('sku', [valid_fragment], re=sku_re) + self.assertEqual(il.load_item()['sku'], u'1234') + def test_add_value(self): il = TestItemLoader() il.add_value('name', u'marta')