mirror of https://github.com/scrapy/scrapy.git
Use ItemAdapter.field_names when writing header in CsvItemExporter (#4668)
This commit is contained in:
Eugenio Lacuesta
GitHub
parent
ce0c25fc85
commit
5265853937
|
|
@ -243,12 +243,8 @@ class CsvItemExporter(BaseItemExporter):
|
|||
def _write_headers_and_set_fields_to_export(self, item):
|
||||
if self.include_headers_line:
|
||||
if not self.fields_to_export:
|
||||
if isinstance(item, dict):
|
||||
# for dicts try using fields of the first item
|
||||
self.fields_to_export = list(item.keys())
|
||||
else:
|
||||
# use fields declared in Item
|
||||
self.fields_to_export = list(item.fields.keys())
|
||||
# use declared field names, or keys if the item is a dict
|
||||
self.fields_to_export = ItemAdapter(item).field_names()
|
||||
row = list(self._build_row(self.fields_to_export))
|
||||
self.csv_writer.writerow(row)
|
||||
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from io import BytesIO
|
|||
from datetime import datetime
|
||||
|
||||
import lxml.etree
|
||||
from itemadapter import ItemAdapter
|
||||
|
||||
from scrapy.item import Item, Field
|
||||
from scrapy.utils.python import to_unicode
|
||||
|
|
@ -23,10 +24,37 @@ class TestItem(Item):
|
|||
age = Field()
|
||||
|
||||
|
||||
def custom_serializer(value):
|
||||
return str(int(value) + 2)
|
||||
|
||||
|
||||
class CustomFieldItem(Item):
|
||||
name = Field()
|
||||
age = Field(serializer=custom_serializer)
|
||||
|
||||
|
||||
try:
|
||||
from dataclasses import make_dataclass, field
|
||||
except ImportError:
|
||||
TestDataClass = None
|
||||
CustomFieldDataclass = None
|
||||
else:
|
||||
TestDataClass = make_dataclass("TestDataClass", [("name", str), ("age", int)])
|
||||
CustomFieldDataclass = make_dataclass(
|
||||
"CustomFieldDataclass",
|
||||
[("name", str), ("age", int, field(metadata={"serializer": custom_serializer}))]
|
||||
)
|
||||
|
||||
|
||||
class BaseItemExporterTest(unittest.TestCase):
|
||||
|
||||
item_class = TestItem
|
||||
custom_field_item_class = CustomFieldItem
|
||||
|
||||
def setUp(self):
|
||||
self.i = TestItem(name=u'John\xa3', age=u'22')
|
||||
if self.item_class is None:
|
||||
raise unittest.SkipTest("item class is None")
|
||||
self.i = self.item_class(name=u'John\xa3', age=u'22')
|
||||
self.output = BytesIO()
|
||||
self.ie = self._get_exporter()
|
||||
|
||||
|
|
@ -39,7 +67,7 @@ class BaseItemExporterTest(unittest.TestCase):
|
|||
def _assert_expected_item(self, exported_dict):
|
||||
for k, v in exported_dict.items():
|
||||
exported_dict[k] = to_unicode(v)
|
||||
self.assertEqual(self.i, exported_dict)
|
||||
self.assertEqual(self.i, self.item_class(**exported_dict))
|
||||
|
||||
def _get_nonstring_types_item(self):
|
||||
return {
|
||||
|
|
@ -63,13 +91,14 @@ class BaseItemExporterTest(unittest.TestCase):
|
|||
self.assertItemExportWorks(self.i)
|
||||
|
||||
def test_export_dict_item(self):
|
||||
self.assertItemExportWorks(dict(self.i))
|
||||
self.assertItemExportWorks(ItemAdapter(self.i).asdict())
|
||||
|
||||
def test_serialize_field(self):
|
||||
res = self.ie.serialize_field(self.i.fields['name'], 'name', self.i['name'])
|
||||
a = ItemAdapter(self.i)
|
||||
res = self.ie.serialize_field(a.get_field_meta('name'), 'name', a['name'])
|
||||
self.assertEqual(res, u'John\xa3')
|
||||
|
||||
res = self.ie.serialize_field(self.i.fields['age'], 'age', self.i['age'])
|
||||
res = self.ie.serialize_field(a.get_field_meta('age'), 'age', a['age'])
|
||||
self.assertEqual(res, u'22')
|
||||
|
||||
def test_fields_to_export(self):
|
||||
|
|
@ -82,18 +111,16 @@ class BaseItemExporterTest(unittest.TestCase):
|
|||
self.assertEqual(name, u'John\xa3')
|
||||
|
||||
def test_field_custom_serializer(self):
|
||||
def custom_serializer(value):
|
||||
return str(int(value) + 2)
|
||||
|
||||
class CustomFieldItem(Item):
|
||||
name = Field()
|
||||
age = Field(serializer=custom_serializer)
|
||||
|
||||
i = CustomFieldItem(name=u'John\xa3', age=u'22')
|
||||
|
||||
i = self.custom_field_item_class(name=u'John\xa3', age=u'22')
|
||||
a = ItemAdapter(i)
|
||||
ie = self._get_exporter()
|
||||
self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), u'John\xa3')
|
||||
self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24')
|
||||
self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), u'John\xa3')
|
||||
self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '24')
|
||||
|
||||
|
||||
class BaseItemExporterDataclassTest(BaseItemExporterTest):
|
||||
item_class = TestDataClass
|
||||
custom_field_item_class = CustomFieldDataclass
|
||||
|
||||
|
||||
class PythonItemExporterTest(BaseItemExporterTest):
|
||||
|
|
@ -105,9 +132,9 @@ class PythonItemExporterTest(BaseItemExporterTest):
|
|||
PythonItemExporter(invalid_option='something')
|
||||
|
||||
def test_nested_item(self):
|
||||
i1 = TestItem(name=u'Joseph', age='22')
|
||||
i1 = self.item_class(name=u'Joseph', age='22')
|
||||
i2 = dict(name=u'Maria', age=i1)
|
||||
i3 = TestItem(name=u'Jesus', age=i2)
|
||||
i3 = self.item_class(name=u'Jesus', age=i2)
|
||||
ie = self._get_exporter()
|
||||
exported = ie.export_item(i3)
|
||||
self.assertEqual(type(exported), dict)
|
||||
|
|
@ -119,9 +146,9 @@ class PythonItemExporterTest(BaseItemExporterTest):
|
|||
self.assertEqual(type(exported['age']['age']), dict)
|
||||
|
||||
def test_export_list(self):
|
||||
i1 = TestItem(name=u'Joseph', age='22')
|
||||
i2 = TestItem(name=u'Maria', age=[i1])
|
||||
i3 = TestItem(name=u'Jesus', age=[i2])
|
||||
i1 = self.item_class(name=u'Joseph', age='22')
|
||||
i2 = self.item_class(name=u'Maria', age=[i1])
|
||||
i3 = self.item_class(name=u'Jesus', age=[i2])
|
||||
ie = self._get_exporter()
|
||||
exported = ie.export_item(i3)
|
||||
self.assertEqual(
|
||||
|
|
@ -132,9 +159,9 @@ class PythonItemExporterTest(BaseItemExporterTest):
|
|||
self.assertEqual(type(exported['age'][0]['age'][0]), dict)
|
||||
|
||||
def test_export_item_dict_list(self):
|
||||
i1 = TestItem(name=u'Joseph', age='22')
|
||||
i1 = self.item_class(name=u'Joseph', age='22')
|
||||
i2 = dict(name=u'Maria', age=[i1])
|
||||
i3 = TestItem(name=u'Jesus', age=[i2])
|
||||
i3 = self.item_class(name=u'Jesus', age=[i2])
|
||||
ie = self._get_exporter()
|
||||
exported = ie.export_item(i3)
|
||||
self.assertEqual(
|
||||
|
|
@ -146,7 +173,7 @@ class PythonItemExporterTest(BaseItemExporterTest):
|
|||
|
||||
def test_export_binary(self):
|
||||
exporter = PythonItemExporter(binary=True)
|
||||
value = TestItem(name=u'John\xa3', age=u'22')
|
||||
value = self.item_class(name=u'John\xa3', age=u'22')
|
||||
expected = {b'name': b'John\xc2\xa3', b'age': b'22'}
|
||||
self.assertEqual(expected, exporter.export_item(value))
|
||||
|
||||
|
|
@ -157,6 +184,11 @@ class PythonItemExporterTest(BaseItemExporterTest):
|
|||
self.assertEqual(exported, item)
|
||||
|
||||
|
||||
class PythonItemExporterDataclassTest(PythonItemExporterTest):
|
||||
item_class = TestDataClass
|
||||
custom_field_item_class = CustomFieldDataclass
|
||||
|
||||
|
||||
class PprintItemExporterTest(BaseItemExporterTest):
|
||||
|
||||
def _get_exporter(self, **kwargs):
|
||||
|
|
@ -166,6 +198,11 @@ class PprintItemExporterTest(BaseItemExporterTest):
|
|||
self._assert_expected_item(eval(self.output.getvalue()))
|
||||
|
||||
|
||||
class PprintItemExporterDataclassTest(PprintItemExporterTest):
|
||||
item_class = TestDataClass
|
||||
custom_field_item_class = CustomFieldDataclass
|
||||
|
||||
|
||||
class PickleItemExporterTest(BaseItemExporterTest):
|
||||
|
||||
def _get_exporter(self, **kwargs):
|
||||
|
|
@ -175,8 +212,8 @@ class PickleItemExporterTest(BaseItemExporterTest):
|
|||
self._assert_expected_item(pickle.loads(self.output.getvalue()))
|
||||
|
||||
def test_export_multiple_items(self):
|
||||
i1 = TestItem(name='hello', age='world')
|
||||
i2 = TestItem(name='bye', age='world')
|
||||
i1 = self.item_class(name='hello', age='world')
|
||||
i2 = self.item_class(name='bye', age='world')
|
||||
f = BytesIO()
|
||||
ie = PickleItemExporter(f)
|
||||
ie.start_exporting()
|
||||
|
|
@ -184,8 +221,8 @@ class PickleItemExporterTest(BaseItemExporterTest):
|
|||
ie.export_item(i2)
|
||||
ie.finish_exporting()
|
||||
f.seek(0)
|
||||
self.assertEqual(pickle.load(f), i1)
|
||||
self.assertEqual(pickle.load(f), i2)
|
||||
self.assertEqual(self.item_class(**pickle.load(f)), i1)
|
||||
self.assertEqual(self.item_class(**pickle.load(f)), i2)
|
||||
|
||||
def test_nonstring_types_item(self):
|
||||
item = self._get_nonstring_types_item()
|
||||
|
|
@ -197,6 +234,11 @@ class PickleItemExporterTest(BaseItemExporterTest):
|
|||
self.assertEqual(pickle.loads(fp.getvalue()), item)
|
||||
|
||||
|
||||
class PickleItemExporterDataclassTest(PickleItemExporterTest):
|
||||
item_class = TestDataClass
|
||||
custom_field_item_class = CustomFieldDataclass
|
||||
|
||||
|
||||
class MarshalItemExporterTest(BaseItemExporterTest):
|
||||
|
||||
def _get_exporter(self, **kwargs):
|
||||
|
|
@ -219,6 +261,11 @@ class MarshalItemExporterTest(BaseItemExporterTest):
|
|||
self.assertEqual(marshal.load(fp), item)
|
||||
|
||||
|
||||
class MarshalItemExporterDataclassTest(MarshalItemExporterTest):
|
||||
item_class = TestDataClass
|
||||
custom_field_item_class = CustomFieldDataclass
|
||||
|
||||
|
||||
class CsvItemExporterTest(BaseItemExporterTest):
|
||||
def _get_exporter(self, **kwargs):
|
||||
return CsvItemExporter(self.output, **kwargs)
|
||||
|
|
@ -245,18 +292,18 @@ class CsvItemExporterTest(BaseItemExporterTest):
|
|||
def test_header_export_all(self):
|
||||
self.assertExportResult(
|
||||
item=self.i,
|
||||
fields_to_export=self.i.fields.keys(),
|
||||
fields_to_export=ItemAdapter(self.i).field_names(),
|
||||
expected=b'age,name\r\n22,John\xc2\xa3\r\n',
|
||||
)
|
||||
|
||||
def test_header_export_all_dict(self):
|
||||
self.assertExportResult(
|
||||
item=dict(self.i),
|
||||
item=ItemAdapter(self.i).asdict(),
|
||||
expected=b'age,name\r\n22,John\xc2\xa3\r\n',
|
||||
)
|
||||
|
||||
def test_header_export_single_field(self):
|
||||
for item in [self.i, dict(self.i)]:
|
||||
for item in [self.i, ItemAdapter(self.i).asdict()]:
|
||||
self.assertExportResult(
|
||||
item=item,
|
||||
fields_to_export=['age'],
|
||||
|
|
@ -264,7 +311,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
|
|||
)
|
||||
|
||||
def test_header_export_two_items(self):
|
||||
for item in [self.i, dict(self.i)]:
|
||||
for item in [self.i, ItemAdapter(self.i).asdict()]:
|
||||
output = BytesIO()
|
||||
ie = CsvItemExporter(output)
|
||||
ie.start_exporting()
|
||||
|
|
@ -275,7 +322,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
|
|||
b'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n')
|
||||
|
||||
def test_header_no_header_line(self):
|
||||
for item in [self.i, dict(self.i)]:
|
||||
for item in [self.i, ItemAdapter(self.i).asdict()]:
|
||||
self.assertExportResult(
|
||||
item=item,
|
||||
include_headers_line=False,
|
||||
|
|
@ -309,6 +356,11 @@ class CsvItemExporterTest(BaseItemExporterTest):
|
|||
)
|
||||
|
||||
|
||||
class CsvItemExporterDataclassTest(CsvItemExporterTest):
|
||||
item_class = TestDataClass
|
||||
custom_field_item_class = CustomFieldDataclass
|
||||
|
||||
|
||||
class XmlItemExporterTest(BaseItemExporterTest):
|
||||
|
||||
def _get_exporter(self, **kwargs):
|
||||
|
|
@ -318,8 +370,7 @@ class XmlItemExporterTest(BaseItemExporterTest):
|
|||
def xmltuple(elem):
|
||||
children = list(elem.iterchildren())
|
||||
if children:
|
||||
return [(child.tag, sorted(xmltuple(child)))
|
||||
for child in children]
|
||||
return [(child.tag, sorted(xmltuple(child))) for child in children]
|
||||
else:
|
||||
return [(elem.tag, [(elem.text, ())])]
|
||||
|
||||
|
|
@ -345,17 +396,21 @@ class XmlItemExporterTest(BaseItemExporterTest):
|
|||
|
||||
def test_multivalued_fields(self):
|
||||
self.assertExportResult(
|
||||
TestItem(name=[u'John\xa3', u'Doe']),
|
||||
(
|
||||
b'<?xml version="1.0" encoding="utf-8"?>\n'
|
||||
b'<items><item><name><value>John\xc2\xa3</value><value>Doe</value></name></item></items>'
|
||||
)
|
||||
self.item_class(name=[u'John\xa3', u'Doe'], age=[1, 2, 3]),
|
||||
b"""<?xml version="1.0" encoding="utf-8"?>\n
|
||||
<items>
|
||||
<item>
|
||||
<name><value>John\xc2\xa3</value><value>Doe</value></name>
|
||||
<age><value>1</value><value>2</value><value>3</value></age>
|
||||
</item>
|
||||
</items>
|
||||
"""
|
||||
)
|
||||
|
||||
def test_nested_item(self):
|
||||
i1 = TestItem(name=u'foo\xa3hoo', age='22')
|
||||
i1 = dict(name=u'foo\xa3hoo', age='22')
|
||||
i2 = dict(name=u'bar', age=i1)
|
||||
i3 = TestItem(name=u'buz', age=i2)
|
||||
i3 = self.item_class(name=u'buz', age=i2)
|
||||
|
||||
self.assertExportResult(
|
||||
i3,
|
||||
|
|
@ -376,9 +431,9 @@ class XmlItemExporterTest(BaseItemExporterTest):
|
|||
)
|
||||
|
||||
def test_nested_list_item(self):
|
||||
i1 = TestItem(name=u'foo')
|
||||
i1 = dict(name=u'foo')
|
||||
i2 = dict(name=u'bar', v2={"egg": ["spam"]})
|
||||
i3 = TestItem(name=u'buz', age=[i1, i2])
|
||||
i3 = self.item_class(name=u'buz', age=[i1, i2])
|
||||
|
||||
self.assertExportResult(
|
||||
i3,
|
||||
|
|
@ -412,6 +467,12 @@ class XmlItemExporterTest(BaseItemExporterTest):
|
|||
)
|
||||
|
||||
|
||||
class XmlItemExporterDataclassTest(XmlItemExporterTest):
|
||||
|
||||
item_class = TestDataClass
|
||||
custom_field_item_class = CustomFieldDataclass
|
||||
|
||||
|
||||
class JsonLinesItemExporterTest(BaseItemExporterTest):
|
||||
|
||||
_expected_nested = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}}
|
||||
|
|
@ -421,12 +482,12 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
|
|||
|
||||
def _check_output(self):
|
||||
exported = json.loads(to_unicode(self.output.getvalue().strip()))
|
||||
self.assertEqual(exported, dict(self.i))
|
||||
self.assertEqual(exported, ItemAdapter(self.i).asdict())
|
||||
|
||||
def test_nested_item(self):
|
||||
i1 = TestItem(name=u'Joseph', age='22')
|
||||
i1 = self.item_class(name=u'Joseph', age='22')
|
||||
i2 = dict(name=u'Maria', age=i1)
|
||||
i3 = TestItem(name=u'Jesus', age=i2)
|
||||
i3 = self.item_class(name=u'Jesus', age=i2)
|
||||
self.ie.start_exporting()
|
||||
self.ie.export_item(i3)
|
||||
self.ie.finish_exporting()
|
||||
|
|
@ -449,6 +510,12 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
|
|||
self.assertEqual(exported, item)
|
||||
|
||||
|
||||
class JsonLinesItemExporterDataclassTest(JsonLinesItemExporterTest):
|
||||
|
||||
item_class = TestDataClass
|
||||
custom_field_item_class = CustomFieldDataclass
|
||||
|
||||
|
||||
class JsonItemExporterTest(JsonLinesItemExporterTest):
|
||||
|
||||
_expected_nested = [JsonLinesItemExporterTest._expected_nested]
|
||||
|
|
@ -458,7 +525,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
|
|||
|
||||
def _check_output(self):
|
||||
exported = json.loads(to_unicode(self.output.getvalue().strip()))
|
||||
self.assertEqual(exported, [dict(self.i)])
|
||||
self.assertEqual(exported, [ItemAdapter(self.i).asdict()])
|
||||
|
||||
def assertTwoItemsExported(self, item):
|
||||
self.ie.start_exporting()
|
||||
|
|
@ -466,28 +533,28 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
|
|||
self.ie.export_item(item)
|
||||
self.ie.finish_exporting()
|
||||
exported = json.loads(to_unicode(self.output.getvalue()))
|
||||
self.assertEqual(exported, [dict(item), dict(item)])
|
||||
self.assertEqual(exported, [ItemAdapter(item).asdict(), ItemAdapter(item).asdict()])
|
||||
|
||||
def test_two_items(self):
|
||||
self.assertTwoItemsExported(self.i)
|
||||
|
||||
def test_two_dict_items(self):
|
||||
self.assertTwoItemsExported(dict(self.i))
|
||||
self.assertTwoItemsExported(ItemAdapter(self.i).asdict())
|
||||
|
||||
def test_nested_item(self):
|
||||
i1 = TestItem(name=u'Joseph\xa3', age='22')
|
||||
i2 = TestItem(name=u'Maria', age=i1)
|
||||
i3 = TestItem(name=u'Jesus', age=i2)
|
||||
i1 = self.item_class(name=u'Joseph\xa3', age='22')
|
||||
i2 = self.item_class(name=u'Maria', age=i1)
|
||||
i3 = self.item_class(name=u'Jesus', age=i2)
|
||||
self.ie.start_exporting()
|
||||
self.ie.export_item(i3)
|
||||
self.ie.finish_exporting()
|
||||
exported = json.loads(to_unicode(self.output.getvalue()))
|
||||
expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': dict(i1)}}
|
||||
expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}}
|
||||
self.assertEqual(exported, [expected])
|
||||
|
||||
def test_nested_dict_item(self):
|
||||
i1 = dict(name=u'Joseph\xa3', age='22')
|
||||
i2 = TestItem(name=u'Maria', age=i1)
|
||||
i2 = self.item_class(name=u'Maria', age=i1)
|
||||
i3 = dict(name=u'Jesus', age=i2)
|
||||
self.ie.start_exporting()
|
||||
self.ie.export_item(i3)
|
||||
|
|
@ -506,7 +573,19 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
|
|||
self.assertEqual(exported, [item])
|
||||
|
||||
|
||||
class CustomItemExporterTest(unittest.TestCase):
|
||||
class JsonItemExporterDataclassTest(JsonItemExporterTest):
|
||||
|
||||
item_class = TestDataClass
|
||||
custom_field_item_class = CustomFieldDataclass
|
||||
|
||||
|
||||
class CustomExporterItemTest(unittest.TestCase):
|
||||
|
||||
item_class = TestItem
|
||||
|
||||
def setUp(self):
|
||||
if self.item_class is None:
|
||||
raise unittest.SkipTest("item class is None")
|
||||
|
||||
def test_exporter_custom_serializer(self):
|
||||
class CustomItemExporter(BaseItemExporter):
|
||||
|
|
@ -516,16 +595,22 @@ class CustomItemExporterTest(unittest.TestCase):
|
|||
else:
|
||||
return super(CustomItemExporter, self).serialize_field(field, name, value)
|
||||
|
||||
i = TestItem(name=u'John', age='22')
|
||||
i = self.item_class(name=u'John', age='22')
|
||||
a = ItemAdapter(i)
|
||||
ie = CustomItemExporter()
|
||||
|
||||
self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), 'John')
|
||||
self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '23')
|
||||
self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John')
|
||||
self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '23')
|
||||
|
||||
i2 = {'name': u'John', 'age': '22'}
|
||||
self.assertEqual(ie.serialize_field({}, 'name', i2['name']), 'John')
|
||||
self.assertEqual(ie.serialize_field({}, 'age', i2['age']), '23')
|
||||
|
||||
|
||||
class CustomExporterDataclassTest(CustomExporterItemTest):
|
||||
|
||||
item_class = TestDataClass
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Reference in New Issue