From 2d4285ed8607a163fdb7cb337b2a37bccfcd4981 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Panek?= Date: Sat, 4 Apr 2020 18:52:42 +0200 Subject: [PATCH 1/6] Add support for nested Items in BaseItemExporter --- scrapy/exporters.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 0cb6cef98..36e53427b 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -52,6 +52,17 @@ class BaseItemExporter: def finish_exporting(self): pass + def _serialize_nested_items(self, value, default_value, include_empty): + if isinstance(value, BaseItem): + return dict(self._get_serialized_fields(value, default_value, include_empty)) + elif is_listlike(value): + tmp = [] + for x in value: + tmp.append(self._serialize_nested_items(x, default_value, include_empty)) + return type(value)(tmp) + else: + return value + def _get_serialized_fields(self, item, default_value=None, include_empty=None): """Return the fields to export as an iterable of tuples (name, serialized_value) @@ -72,11 +83,15 @@ class BaseItemExporter: for field_name in field_iter: if field_name in item: field = {} if isinstance(item, dict) else item.fields[field_name] - value = self.serialize_field(field, field_name, item[field_name]) + value = self._serialize_nested_items( + item[field_name], + default_value, + include_empty) + serialized_value = self.serialize_field(field, field_name, value) else: - value = default_value + serialized_value = default_value - yield field_name, value + yield field_name, serialized_value class JsonLinesItemExporter(BaseItemExporter): From fa8206ff09673bda24bcc002ffda02bf3bd47947 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Panek?= Date: Sat, 4 Apr 2020 19:14:58 +0200 Subject: [PATCH 2/6] Fix indentations --- scrapy/exporters.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 36e53427b..df0a72929 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -84,9 +84,9 @@ class BaseItemExporter: if field_name in item: field = {} if isinstance(item, dict) else item.fields[field_name] value = self._serialize_nested_items( - item[field_name], - default_value, - include_empty) + item[field_name], + default_value, + include_empty) serialized_value = self.serialize_field(field, field_name, value) else: serialized_value = default_value From 092391bc161ba81f8dd27e9249c83cac9fd6b2e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Panek?= Date: Sat, 4 Apr 2020 19:52:05 +0200 Subject: [PATCH 3/6] Fix exporting Items with nested dicts --- scrapy/exporters.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index df0a72929..a7bfbff30 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -55,6 +55,11 @@ class BaseItemExporter: def _serialize_nested_items(self, value, default_value, include_empty): if isinstance(value, BaseItem): return dict(self._get_serialized_fields(value, default_value, include_empty)) + elif isinstance(value, dict): + tmp = [] + for x in value.items(): + tmp.append(self._serialize_nested_items(x, default_value, include_empty)) + return dict(tmp) elif is_listlike(value): tmp = [] for x in value: From 947d6c8df2fa60056af7d359e843e50496ca9e34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Panek?= Date: Sat, 4 Apr 2020 19:52:47 +0200 Subject: [PATCH 4/6] Test serialization of nested items --- tests/test_exporters.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 6e2507508..f4ad6cd9e 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -23,6 +23,10 @@ class TestItem(Item): age = Field() +class TestItemWithSerializer(Item): + a = Field(serializer=str.upper) + + class BaseItemExporterTest(unittest.TestCase): def setUp(self): @@ -72,6 +76,11 @@ class BaseItemExporterTest(unittest.TestCase): res = self.ie.serialize_field(self.i.fields['age'], 'age', self.i['age']) self.assertEqual(res, u'22') + def test_serialize_nested_items(self): + i = TestItem(name=TestItemWithSerializer(a='asdf'), age=42) + res = self.ie._serialize_nested_items(i, None, None)['name']['a'] + self.assertEqual(res, u'ASDF') + def test_fields_to_export(self): ie = self._get_exporter(fields_to_export=['name']) self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', u'John\xa3')]) From f5ead7dace186dc3e751d994f165106a3a51efe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Panek?= Date: Fri, 17 Apr 2020 18:22:08 +0200 Subject: [PATCH 5/6] Code cleanup in _serialize_nested_items in BaseItemExporter --- scrapy/exporters.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index a7bfbff30..53b733260 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -53,18 +53,20 @@ class BaseItemExporter: pass def _serialize_nested_items(self, value, default_value, include_empty): + def serialize_iterable(it): + ret = [] + for x in it: + ret.append(self._serialize_nested_items(x, default_value, include_empty)) + return ret + if isinstance(value, BaseItem): return dict(self._get_serialized_fields(value, default_value, include_empty)) - elif isinstance(value, dict): - tmp = [] - for x in value.items(): - tmp.append(self._serialize_nested_items(x, default_value, include_empty)) - return dict(tmp) elif is_listlike(value): - tmp = [] - for x in value: - tmp.append(self._serialize_nested_items(x, default_value, include_empty)) - return type(value)(tmp) + if isinstance(value, dict): + serialized_list = serialize_iterable(value.items()) + else: + serialized_list = serialize_iterable(value) + return type(value)(serialized_list) else: return value From b9f9287c10790cd693eff07796274ec86d24feca Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Mon, 22 Jun 2026 12:29:15 +0200 Subject: [PATCH 6/6] Address typing issues --- scrapy/exporters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 237be23b9..d90c71665 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -74,7 +74,7 @@ class BaseItemExporter(ABC): def _serialize_nested_items( self, value: Any, default_value: Any, include_empty: bool | None ) -> Any: - def serialize_iterable(it): + def serialize_iterable(it: Iterable[Any]) -> list[Any]: return [ self._serialize_nested_items(x, default_value, include_empty) for x in it