From bd8a10384b462dd56b33668e8b92e4a148fd6fba Mon Sep 17 00:00:00 2001 From: Sortafreel Date: Fri, 7 Jun 2019 01:50:03 +0300 Subject: [PATCH 1/5] Add values (if there're any) when initiating items from dicts https://github.com/scrapy/scrapy/issues/3804 --- scrapy/loader/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index a7c75a46a..295a8e42d 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -35,6 +35,8 @@ class ItemLoader(object): self.parent = parent self._local_item = context['item'] = item self._local_values = defaultdict(list) + for field_name, value in item.items(): + self.add_value(field_name, value) @property def _values(self): From 754f52b02781097c8ca6835e057815c7653062d4 Mon Sep 17 00:00:00 2001 From: Sortafreel Date: Fri, 7 Jun 2019 03:20:45 +0300 Subject: [PATCH 2/5] Preprocess values if item built from dict. https://github.com/scrapy/scrapy/issues/3804 --- scrapy/loader/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 295a8e42d..7c7f66866 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -35,8 +35,9 @@ class ItemLoader(object): self.parent = parent self._local_item = context['item'] = item self._local_values = defaultdict(list) + # Preprocess values if item built from dict for field_name, value in item.items(): - self.add_value(field_name, value) + self._values[field_name] = self._process_input_value(field_name, value) @property def _values(self): From a1bca6a8e722af53241e51bbf758e7bd67671801 Mon Sep 17 00:00:00 2001 From: sortafreel Date: Tue, 11 Jun 2019 07:36:29 +0300 Subject: [PATCH 3/5] Add tests. --- scrapy/loader/__init__.py | 1 + tests/test_loader.py | 65 ++++++++++++++++++++++++++------------- 2 files changed, 45 insertions(+), 21 deletions(-) diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 7c7f66866..20f0f90c3 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -36,6 +36,7 @@ class ItemLoader(object): self._local_item = context['item'] = item self._local_values = defaultdict(list) # Preprocess values if item built from dict + # Values need to be added to item._values if added them from dict (not with add_values) for field_name, value in item.items(): self._values[field_name] = self._process_input_value(field_name, value) diff --git a/tests/test_loader.py b/tests/test_loader.py index 8b58e4dbd..eb4a01572 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -419,6 +419,29 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(item['url'], u'rabbit.hole') self.assertEqual(item['summary'], u'rabbithole') + def test_create_item_from_dict(self): + class TestItem(Item): + title = Field() + + class TestItemLoader(ItemLoader): + default_item_class = TestItem + + input_item = {'title': 'Test item title 1'} + il = TestItemLoader(item=input_item) + # Getting output value mustn't remove value from item + self.assertEqual(il.load_item(), { + 'title': 'Test item title 1', + }) + self.assertEqual(il.get_output_value('title'), 'Test item title 1') + self.assertEqual(il.load_item(), { + 'title': 'Test item title 1', + }) + + input_item = {'title': 'Test item title 2'} + il = TestItemLoader(item=input_item) + # Values from dict must be added to item _values + self.assertEqual(il._values.get('title'), 'Test item title 2') + class ProcessorsTest(unittest.TestCase): @@ -709,28 +732,28 @@ class SubselectorLoaderTest(unittest.TestCase): class SelectJmesTestCase(unittest.TestCase): - test_list_equals = { - 'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), - 'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None), - 'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}), - 'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), - 'dict': ( - 'foo.bar[*].name', - {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}}, - ['one', 'two'] - ), - 'list': ('[1]', [1, 2], 2) - } + test_list_equals = { + 'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), + 'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None), + 'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}), + 'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), + 'dict': ( + 'foo.bar[*].name', + {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}}, + ['one', 'two'] + ), + 'list': ('[1]', [1, 2], 2) + } - def test_output(self): - for l in self.test_list_equals: - expr, test_list, expected = self.test_list_equals[l] - test = SelectJmes(expr)(test_list) - self.assertEqual( - test, - expected, - msg='test "{}" got {} expected {}'.format(l, test, expected) - ) + def test_output(self): + for l in self.test_list_equals: + expr, test_list, expected = self.test_list_equals[l] + test = SelectJmes(expr)(test_list) + self.assertEqual( + test, + expected, + msg='test "{}" got {} expected {}'.format(l, test, expected) + ) if __name__ == "__main__": From 7dad2f7b130c426f2a8aee320ccbc378752a9568 Mon Sep 17 00:00:00 2001 From: sortafreel Date: Tue, 11 Jun 2019 07:43:03 +0300 Subject: [PATCH 4/5] Add more tests. --- tests/test_loader.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_loader.py b/tests/test_loader.py index eb4a01572..241630ab3 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -442,6 +442,20 @@ class BasicItemLoaderTest(unittest.TestCase): # Values from dict must be added to item _values self.assertEqual(il._values.get('title'), 'Test item title 2') + input_item = {'title': [u'Test item title 3', u'Test item 4']} + il = TestItemLoader(item=input_item) + # Same rules must work for lists + self.assertEqual(il._values.get('title'), + [u'Test item title 3', u'Test item 4']) + self.assertEqual(il.load_item(), { + 'title': [u'Test item title 3', u'Test item 4'], + }) + self.assertEqual(il.get_output_value('title'), + [u'Test item title 3', u'Test item 4']) + self.assertEqual(il.load_item(), { + 'title': [u'Test item title 3', u'Test item 4'], + }) + class ProcessorsTest(unittest.TestCase): From cdeccac6d6ccd0034a5f007ed371c1d481b32c26 Mon Sep 17 00:00:00 2001 From: sortafreel Date: Tue, 11 Jun 2019 17:38:06 +0300 Subject: [PATCH 5/5] Linting (return previous indentation). --- tests/test_loader.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/tests/test_loader.py b/tests/test_loader.py index 241630ab3..5a8ee1b2e 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -746,28 +746,28 @@ class SubselectorLoaderTest(unittest.TestCase): class SelectJmesTestCase(unittest.TestCase): - test_list_equals = { - 'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), - 'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None), - 'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}), - 'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), - 'dict': ( - 'foo.bar[*].name', - {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}}, - ['one', 'two'] - ), - 'list': ('[1]', [1, 2], 2) - } + test_list_equals = { + 'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), + 'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None), + 'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}), + 'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), + 'dict': ( + 'foo.bar[*].name', + {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}}, + ['one', 'two'] + ), + 'list': ('[1]', [1, 2], 2) + } - def test_output(self): - for l in self.test_list_equals: - expr, test_list, expected = self.test_list_equals[l] - test = SelectJmes(expr)(test_list) - self.assertEqual( - test, - expected, - msg='test "{}" got {} expected {}'.format(l, test, expected) - ) + def test_output(self): + for l in self.test_list_equals: + expr, test_list, expected = self.test_list_equals[l] + test = SelectJmes(expr)(test_list) + self.assertEqual( + test, + expected, + msg='test "{}" got {} expected {}'.format(l, test, expected) + ) if __name__ == "__main__":