diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 20f0f90c3..844e3828c 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -109,8 +109,14 @@ class ItemLoader(object): for proc in processors: if value is None: break + _proc = proc proc = wrap_loader_context(proc, self.context) - value = proc(value) + try: + value = proc(value) + except Exception as e: + raise ValueError("Error with processor %s value=%r error='%s: %s'" % + (_proc.__class__.__name__, value, + type(e).__name__, str(e))) return value def load_item(self): @@ -150,8 +156,15 @@ class ItemLoader(object): def _process_input_value(self, field_name, value): proc = self.get_input_processor(field_name) + _proc = proc proc = wrap_loader_context(proc, self.context) - return proc(value) + try: + return proc(value) + except Exception as e: + raise ValueError( + "Error with input processor %s: field=%r value=%r " + "error='%s: %s'" % (_proc.__class__.__name__, field_name, + value, type(e).__name__, str(e))) def _get_item_field_attr(self, field_name, key, default=None): if isinstance(self.item, Item): diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py index bf7c74bfe..468aec2cc 100644 --- a/scrapy/loader/processors.py +++ b/scrapy/loader/processors.py @@ -25,7 +25,13 @@ class MapCompose(object): for func in wrapped_funcs: next_values = [] for v in values: - next_values += arg_to_iter(func(v)) + try: + next_values += arg_to_iter(func(v)) + except Exception as e: + raise ValueError("Error in MapCompose with " + "%s value=%r error='%s: %s'" % + (str(func), value, type(e).__name__, + str(e))) values = next_values return values @@ -46,7 +52,12 @@ class Compose(object): for func in wrapped_funcs: if value is None and self.stop_on_none: break - value = func(value) + try: + value = func(value) + except Exception as e: + raise ValueError("Error in Compose with " + "%s value=%r error='%s: %s'" % + (str(func), value, type(e).__name__, str(e))) return value diff --git a/tests/test_loader.py b/tests/test_loader.py index 5a8ee1b2e..ce0fa0701 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -456,6 +456,42 @@ class BasicItemLoaderTest(unittest.TestCase): 'title': [u'Test item title 3', u'Test item 4'], }) + def test_error_input_processor(self): + class TestItem(Item): + name = Field() + + class TestItemLoader(ItemLoader): + default_item_class = TestItem + name_in = MapCompose(float) + + il = TestItemLoader() + self.assertRaises(ValueError, il.add_value, 'name', + [u'marta', u'other']) + + def test_error_output_processor(self): + class TestItem(Item): + name = Field() + + class TestItemLoader(ItemLoader): + default_item_class = TestItem + name_out = Compose(Join(), float) + + il = TestItemLoader() + il.add_value('name', u'marta') + with self.assertRaises(ValueError): + il.load_item() + + def test_error_processor_as_argument(self): + class TestItem(Item): + name = Field() + + class TestItemLoader(ItemLoader): + default_item_class = TestItem + + il = TestItemLoader() + self.assertRaises(ValueError, il.add_value, 'name', + [u'marta', u'other'], Compose(float)) + class ProcessorsTest(unittest.TestCase): @@ -482,13 +518,22 @@ class ProcessorsTest(unittest.TestCase): proc = Compose(str.upper) self.assertEqual(proc(None), None) proc = Compose(str.upper, stop_on_none=False) - self.assertRaises(TypeError, proc, None) + self.assertRaises(ValueError, proc, None) + proc = Compose(str.upper, lambda x: x + 1) + self.assertRaises(ValueError, proc, 'hello') def test_mapcompose(self): filter_world = lambda x: None if x == 'world' else x proc = MapCompose(filter_world, six.text_type.upper) self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']), [u'HELLO', u'THIS', u'IS', u'SCRAPY']) + proc = MapCompose(filter_world, six.text_type.upper) + self.assertEqual(proc(None), []) + proc = MapCompose(filter_world, six.text_type.upper) + self.assertRaises(ValueError, proc, [1]) + proc = MapCompose(filter_world, lambda x: x + 1) + self.assertRaises(ValueError, proc, 'hello') + class SelectortemLoaderTest(unittest.TestCase):