From 017fb25beef63639060485dae393ed6e244d3f70 Mon Sep 17 00:00:00 2001 From: Nikolaos-Digenis Karagiannis Date: Mon, 20 Apr 2015 11:04:31 +0300 Subject: [PATCH 1/2] loader test with processors that use item's values --- tests/test_contrib_loader.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_contrib_loader.py b/tests/test_contrib_loader.py index 6af4cf70c..fd65e8200 100644 --- a/tests/test_contrib_loader.py +++ b/tests/test_contrib_loader.py @@ -85,6 +85,27 @@ class BasicItemLoaderTest(unittest.TestCase): il.replace_value('sku', [valid_fragment], re=sku_re) self.assertEqual(il.load_item()['sku'], u'1234') + def test_self_referencing_loader(self): + class MyLoader(ItemLoader): + url_out = TakeFirst() + + def img_url_out(self, values): + return (self.get_output_value('url') or '') + values[0] + + il = MyLoader(item={}) + il.add_value('url', 'http://example.com/') + il.add_value('img_url', '1234.png') + self.assertEqual(il.load_item(), { + 'url': 'http://example.com/', + 'img_url': 'http://example.com/1234.png', + }) + + il = MyLoader(item={}) + il.add_value('img_url', '1234.png') + self.assertEqual(il.load_item(), { + 'img_url': '1234.png', + }) + def test_add_value(self): il = TestItemLoader() il.add_value('name', u'marta') From d39722f6a37bc1b1558e75b2c77542925437d3db Mon Sep 17 00:00:00 2001 From: Nikolaos-Digenis Karagiannis Date: Mon, 20 Apr 2015 11:05:56 +0300 Subject: [PATCH 2/2] ItemLoader.load_item: iterate over copy of fields --- scrapy/contrib/loader/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/contrib/loader/__init__.py b/scrapy/contrib/loader/__init__.py index 7be2524b9..c861f8a24 100644 --- a/scrapy/contrib/loader/__init__.py +++ b/scrapy/contrib/loader/__init__.py @@ -80,7 +80,7 @@ class ItemLoader(object): def load_item(self): item = self.item - for field_name in self._values: + for field_name in tuple(self._values): value = self.get_output_value(field_name) if value is not None: item[field_name] = value