diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 712572673..0aba1c904 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -243,12 +243,8 @@ class CsvItemExporter(BaseItemExporter):
def _write_headers_and_set_fields_to_export(self, item):
if self.include_headers_line:
if not self.fields_to_export:
- if isinstance(item, dict):
- # for dicts try using fields of the first item
- self.fields_to_export = list(item.keys())
- else:
- # use fields declared in Item
- self.fields_to_export = list(item.fields.keys())
+ # use declared field names, or keys if the item is a dict
+ self.fields_to_export = ItemAdapter(item).field_names()
row = list(self._build_row(self.fields_to_export))
self.csv_writer.writerow(row)
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index b27380309..25da54a65 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -8,6 +8,7 @@ from io import BytesIO
from datetime import datetime
import lxml.etree
+from itemadapter import ItemAdapter
from scrapy.item import Item, Field
from scrapy.utils.python import to_unicode
@@ -23,10 +24,37 @@ class TestItem(Item):
age = Field()
+def custom_serializer(value):
+ return str(int(value) + 2)
+
+
+class CustomFieldItem(Item):
+ name = Field()
+ age = Field(serializer=custom_serializer)
+
+
+try:
+ from dataclasses import make_dataclass, field
+except ImportError:
+ TestDataClass = None
+ CustomFieldDataclass = None
+else:
+ TestDataClass = make_dataclass("TestDataClass", [("name", str), ("age", int)])
+ CustomFieldDataclass = make_dataclass(
+ "CustomFieldDataclass",
+ [("name", str), ("age", int, field(metadata={"serializer": custom_serializer}))]
+ )
+
+
class BaseItemExporterTest(unittest.TestCase):
+ item_class = TestItem
+ custom_field_item_class = CustomFieldItem
+
def setUp(self):
- self.i = TestItem(name=u'John\xa3', age=u'22')
+ if self.item_class is None:
+ raise unittest.SkipTest("item class is None")
+ self.i = self.item_class(name=u'John\xa3', age=u'22')
self.output = BytesIO()
self.ie = self._get_exporter()
@@ -39,7 +67,7 @@ class BaseItemExporterTest(unittest.TestCase):
def _assert_expected_item(self, exported_dict):
for k, v in exported_dict.items():
exported_dict[k] = to_unicode(v)
- self.assertEqual(self.i, exported_dict)
+ self.assertEqual(self.i, self.item_class(**exported_dict))
def _get_nonstring_types_item(self):
return {
@@ -63,13 +91,14 @@ class BaseItemExporterTest(unittest.TestCase):
self.assertItemExportWorks(self.i)
def test_export_dict_item(self):
- self.assertItemExportWorks(dict(self.i))
+ self.assertItemExportWorks(ItemAdapter(self.i).asdict())
def test_serialize_field(self):
- res = self.ie.serialize_field(self.i.fields['name'], 'name', self.i['name'])
+ a = ItemAdapter(self.i)
+ res = self.ie.serialize_field(a.get_field_meta('name'), 'name', a['name'])
self.assertEqual(res, u'John\xa3')
- res = self.ie.serialize_field(self.i.fields['age'], 'age', self.i['age'])
+ res = self.ie.serialize_field(a.get_field_meta('age'), 'age', a['age'])
self.assertEqual(res, u'22')
def test_fields_to_export(self):
@@ -82,18 +111,16 @@ class BaseItemExporterTest(unittest.TestCase):
self.assertEqual(name, u'John\xa3')
def test_field_custom_serializer(self):
- def custom_serializer(value):
- return str(int(value) + 2)
-
- class CustomFieldItem(Item):
- name = Field()
- age = Field(serializer=custom_serializer)
-
- i = CustomFieldItem(name=u'John\xa3', age=u'22')
-
+ i = self.custom_field_item_class(name=u'John\xa3', age=u'22')
+ a = ItemAdapter(i)
ie = self._get_exporter()
- self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), u'John\xa3')
- self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24')
+ self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), u'John\xa3')
+ self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '24')
+
+
+class BaseItemExporterDataclassTest(BaseItemExporterTest):
+ item_class = TestDataClass
+ custom_field_item_class = CustomFieldDataclass
class PythonItemExporterTest(BaseItemExporterTest):
@@ -105,9 +132,9 @@ class PythonItemExporterTest(BaseItemExporterTest):
PythonItemExporter(invalid_option='something')
def test_nested_item(self):
- i1 = TestItem(name=u'Joseph', age='22')
+ i1 = self.item_class(name=u'Joseph', age='22')
i2 = dict(name=u'Maria', age=i1)
- i3 = TestItem(name=u'Jesus', age=i2)
+ i3 = self.item_class(name=u'Jesus', age=i2)
ie = self._get_exporter()
exported = ie.export_item(i3)
self.assertEqual(type(exported), dict)
@@ -119,9 +146,9 @@ class PythonItemExporterTest(BaseItemExporterTest):
self.assertEqual(type(exported['age']['age']), dict)
def test_export_list(self):
- i1 = TestItem(name=u'Joseph', age='22')
- i2 = TestItem(name=u'Maria', age=[i1])
- i3 = TestItem(name=u'Jesus', age=[i2])
+ i1 = self.item_class(name=u'Joseph', age='22')
+ i2 = self.item_class(name=u'Maria', age=[i1])
+ i3 = self.item_class(name=u'Jesus', age=[i2])
ie = self._get_exporter()
exported = ie.export_item(i3)
self.assertEqual(
@@ -132,9 +159,9 @@ class PythonItemExporterTest(BaseItemExporterTest):
self.assertEqual(type(exported['age'][0]['age'][0]), dict)
def test_export_item_dict_list(self):
- i1 = TestItem(name=u'Joseph', age='22')
+ i1 = self.item_class(name=u'Joseph', age='22')
i2 = dict(name=u'Maria', age=[i1])
- i3 = TestItem(name=u'Jesus', age=[i2])
+ i3 = self.item_class(name=u'Jesus', age=[i2])
ie = self._get_exporter()
exported = ie.export_item(i3)
self.assertEqual(
@@ -146,7 +173,7 @@ class PythonItemExporterTest(BaseItemExporterTest):
def test_export_binary(self):
exporter = PythonItemExporter(binary=True)
- value = TestItem(name=u'John\xa3', age=u'22')
+ value = self.item_class(name=u'John\xa3', age=u'22')
expected = {b'name': b'John\xc2\xa3', b'age': b'22'}
self.assertEqual(expected, exporter.export_item(value))
@@ -157,6 +184,11 @@ class PythonItemExporterTest(BaseItemExporterTest):
self.assertEqual(exported, item)
+class PythonItemExporterDataclassTest(PythonItemExporterTest):
+ item_class = TestDataClass
+ custom_field_item_class = CustomFieldDataclass
+
+
class PprintItemExporterTest(BaseItemExporterTest):
def _get_exporter(self, **kwargs):
@@ -166,6 +198,11 @@ class PprintItemExporterTest(BaseItemExporterTest):
self._assert_expected_item(eval(self.output.getvalue()))
+class PprintItemExporterDataclassTest(PprintItemExporterTest):
+ item_class = TestDataClass
+ custom_field_item_class = CustomFieldDataclass
+
+
class PickleItemExporterTest(BaseItemExporterTest):
def _get_exporter(self, **kwargs):
@@ -175,8 +212,8 @@ class PickleItemExporterTest(BaseItemExporterTest):
self._assert_expected_item(pickle.loads(self.output.getvalue()))
def test_export_multiple_items(self):
- i1 = TestItem(name='hello', age='world')
- i2 = TestItem(name='bye', age='world')
+ i1 = self.item_class(name='hello', age='world')
+ i2 = self.item_class(name='bye', age='world')
f = BytesIO()
ie = PickleItemExporter(f)
ie.start_exporting()
@@ -184,8 +221,8 @@ class PickleItemExporterTest(BaseItemExporterTest):
ie.export_item(i2)
ie.finish_exporting()
f.seek(0)
- self.assertEqual(pickle.load(f), i1)
- self.assertEqual(pickle.load(f), i2)
+ self.assertEqual(self.item_class(**pickle.load(f)), i1)
+ self.assertEqual(self.item_class(**pickle.load(f)), i2)
def test_nonstring_types_item(self):
item = self._get_nonstring_types_item()
@@ -197,6 +234,11 @@ class PickleItemExporterTest(BaseItemExporterTest):
self.assertEqual(pickle.loads(fp.getvalue()), item)
+class PickleItemExporterDataclassTest(PickleItemExporterTest):
+ item_class = TestDataClass
+ custom_field_item_class = CustomFieldDataclass
+
+
class MarshalItemExporterTest(BaseItemExporterTest):
def _get_exporter(self, **kwargs):
@@ -219,6 +261,11 @@ class MarshalItemExporterTest(BaseItemExporterTest):
self.assertEqual(marshal.load(fp), item)
+class MarshalItemExporterDataclassTest(MarshalItemExporterTest):
+ item_class = TestDataClass
+ custom_field_item_class = CustomFieldDataclass
+
+
class CsvItemExporterTest(BaseItemExporterTest):
def _get_exporter(self, **kwargs):
return CsvItemExporter(self.output, **kwargs)
@@ -245,18 +292,18 @@ class CsvItemExporterTest(BaseItemExporterTest):
def test_header_export_all(self):
self.assertExportResult(
item=self.i,
- fields_to_export=self.i.fields.keys(),
+ fields_to_export=ItemAdapter(self.i).field_names(),
expected=b'age,name\r\n22,John\xc2\xa3\r\n',
)
def test_header_export_all_dict(self):
self.assertExportResult(
- item=dict(self.i),
+ item=ItemAdapter(self.i).asdict(),
expected=b'age,name\r\n22,John\xc2\xa3\r\n',
)
def test_header_export_single_field(self):
- for item in [self.i, dict(self.i)]:
+ for item in [self.i, ItemAdapter(self.i).asdict()]:
self.assertExportResult(
item=item,
fields_to_export=['age'],
@@ -264,7 +311,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
)
def test_header_export_two_items(self):
- for item in [self.i, dict(self.i)]:
+ for item in [self.i, ItemAdapter(self.i).asdict()]:
output = BytesIO()
ie = CsvItemExporter(output)
ie.start_exporting()
@@ -275,7 +322,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
b'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n')
def test_header_no_header_line(self):
- for item in [self.i, dict(self.i)]:
+ for item in [self.i, ItemAdapter(self.i).asdict()]:
self.assertExportResult(
item=item,
include_headers_line=False,
@@ -309,6 +356,11 @@ class CsvItemExporterTest(BaseItemExporterTest):
)
+class CsvItemExporterDataclassTest(CsvItemExporterTest):
+ item_class = TestDataClass
+ custom_field_item_class = CustomFieldDataclass
+
+
class XmlItemExporterTest(BaseItemExporterTest):
def _get_exporter(self, **kwargs):
@@ -318,8 +370,7 @@ class XmlItemExporterTest(BaseItemExporterTest):
def xmltuple(elem):
children = list(elem.iterchildren())
if children:
- return [(child.tag, sorted(xmltuple(child)))
- for child in children]
+ return [(child.tag, sorted(xmltuple(child))) for child in children]
else:
return [(elem.tag, [(elem.text, ())])]
@@ -345,17 +396,21 @@ class XmlItemExporterTest(BaseItemExporterTest):
def test_multivalued_fields(self):
self.assertExportResult(
- TestItem(name=[u'John\xa3', u'Doe']),
- (
- b'\n'
- b'- John\xc2\xa3Doe
'
- )
+ self.item_class(name=[u'John\xa3', u'Doe'], age=[1, 2, 3]),
+ b"""\n
+
+ -
+ John\xc2\xa3Doe
+ 123
+
+
+ """
)
def test_nested_item(self):
- i1 = TestItem(name=u'foo\xa3hoo', age='22')
+ i1 = dict(name=u'foo\xa3hoo', age='22')
i2 = dict(name=u'bar', age=i1)
- i3 = TestItem(name=u'buz', age=i2)
+ i3 = self.item_class(name=u'buz', age=i2)
self.assertExportResult(
i3,
@@ -376,9 +431,9 @@ class XmlItemExporterTest(BaseItemExporterTest):
)
def test_nested_list_item(self):
- i1 = TestItem(name=u'foo')
+ i1 = dict(name=u'foo')
i2 = dict(name=u'bar', v2={"egg": ["spam"]})
- i3 = TestItem(name=u'buz', age=[i1, i2])
+ i3 = self.item_class(name=u'buz', age=[i1, i2])
self.assertExportResult(
i3,
@@ -412,6 +467,12 @@ class XmlItemExporterTest(BaseItemExporterTest):
)
+class XmlItemExporterDataclassTest(XmlItemExporterTest):
+
+ item_class = TestDataClass
+ custom_field_item_class = CustomFieldDataclass
+
+
class JsonLinesItemExporterTest(BaseItemExporterTest):
_expected_nested = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}}
@@ -421,12 +482,12 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
def _check_output(self):
exported = json.loads(to_unicode(self.output.getvalue().strip()))
- self.assertEqual(exported, dict(self.i))
+ self.assertEqual(exported, ItemAdapter(self.i).asdict())
def test_nested_item(self):
- i1 = TestItem(name=u'Joseph', age='22')
+ i1 = self.item_class(name=u'Joseph', age='22')
i2 = dict(name=u'Maria', age=i1)
- i3 = TestItem(name=u'Jesus', age=i2)
+ i3 = self.item_class(name=u'Jesus', age=i2)
self.ie.start_exporting()
self.ie.export_item(i3)
self.ie.finish_exporting()
@@ -449,6 +510,12 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
self.assertEqual(exported, item)
+class JsonLinesItemExporterDataclassTest(JsonLinesItemExporterTest):
+
+ item_class = TestDataClass
+ custom_field_item_class = CustomFieldDataclass
+
+
class JsonItemExporterTest(JsonLinesItemExporterTest):
_expected_nested = [JsonLinesItemExporterTest._expected_nested]
@@ -458,7 +525,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
def _check_output(self):
exported = json.loads(to_unicode(self.output.getvalue().strip()))
- self.assertEqual(exported, [dict(self.i)])
+ self.assertEqual(exported, [ItemAdapter(self.i).asdict()])
def assertTwoItemsExported(self, item):
self.ie.start_exporting()
@@ -466,28 +533,28 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
self.ie.export_item(item)
self.ie.finish_exporting()
exported = json.loads(to_unicode(self.output.getvalue()))
- self.assertEqual(exported, [dict(item), dict(item)])
+ self.assertEqual(exported, [ItemAdapter(item).asdict(), ItemAdapter(item).asdict()])
def test_two_items(self):
self.assertTwoItemsExported(self.i)
def test_two_dict_items(self):
- self.assertTwoItemsExported(dict(self.i))
+ self.assertTwoItemsExported(ItemAdapter(self.i).asdict())
def test_nested_item(self):
- i1 = TestItem(name=u'Joseph\xa3', age='22')
- i2 = TestItem(name=u'Maria', age=i1)
- i3 = TestItem(name=u'Jesus', age=i2)
+ i1 = self.item_class(name=u'Joseph\xa3', age='22')
+ i2 = self.item_class(name=u'Maria', age=i1)
+ i3 = self.item_class(name=u'Jesus', age=i2)
self.ie.start_exporting()
self.ie.export_item(i3)
self.ie.finish_exporting()
exported = json.loads(to_unicode(self.output.getvalue()))
- expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': dict(i1)}}
+ expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}}
self.assertEqual(exported, [expected])
def test_nested_dict_item(self):
i1 = dict(name=u'Joseph\xa3', age='22')
- i2 = TestItem(name=u'Maria', age=i1)
+ i2 = self.item_class(name=u'Maria', age=i1)
i3 = dict(name=u'Jesus', age=i2)
self.ie.start_exporting()
self.ie.export_item(i3)
@@ -506,7 +573,19 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
self.assertEqual(exported, [item])
-class CustomItemExporterTest(unittest.TestCase):
+class JsonItemExporterDataclassTest(JsonItemExporterTest):
+
+ item_class = TestDataClass
+ custom_field_item_class = CustomFieldDataclass
+
+
+class CustomExporterItemTest(unittest.TestCase):
+
+ item_class = TestItem
+
+ def setUp(self):
+ if self.item_class is None:
+ raise unittest.SkipTest("item class is None")
def test_exporter_custom_serializer(self):
class CustomItemExporter(BaseItemExporter):
@@ -516,16 +595,22 @@ class CustomItemExporterTest(unittest.TestCase):
else:
return super(CustomItemExporter, self).serialize_field(field, name, value)
- i = TestItem(name=u'John', age='22')
+ i = self.item_class(name=u'John', age='22')
+ a = ItemAdapter(i)
ie = CustomItemExporter()
- self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), 'John')
- self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '23')
+ self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John')
+ self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '23')
i2 = {'name': u'John', 'age': '22'}
self.assertEqual(ie.serialize_field({}, 'name', i2['name']), 'John')
self.assertEqual(ie.serialize_field({}, 'age', i2['age']), '23')
+class CustomExporterDataclassTest(CustomExporterItemTest):
+
+ item_class = TestDataClass
+
+
if __name__ == '__main__':
unittest.main()