diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 923336769..3c36ef002 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -195,17 +195,25 @@ BaseItemExporter .. attribute:: fields_to_export - A list with the name of the fields that will be exported, or ``None`` if - you want to export all fields. Defaults to ``None``. + Fields to export, their order [1]_ and their output names. - Some exporters (like :class:`CsvItemExporter`) respect the order of the - fields defined in this attribute. + Possible values are: - When using :ref:`item objects ` that do not expose all their - possible fields, exporters that do not support exporting a different - subset of fields per item will only export the fields found in the first - item exported. Use ``fields_to_export`` to define all the fields to be - exported. + - ``None`` (all fields [2]_, default) + + - A list of fields:: + + ['field1', 'field2'] + + - A dict where keys are fields and values are output names:: + + {'field1': 'Field 1', 'field2': 'Field 2'} + + .. [1] Not all exporters respect the specified field order. + .. [2] When using :ref:`item objects ` that do not expose + all their possible fields, exporters that do not support exporting + a different subset of fields per item will only export the fields + found in the first item exported. .. attribute:: export_empty_fields @@ -297,8 +305,8 @@ CsvItemExporter Exports items in CSV format to the given file-like object. If the :attr:`fields_to_export` attribute is set, it will be used to define the - CSV columns and their order. The :attr:`export_empty_fields` attribute has - no effect on this exporter. + CSV columns, their order and their column names. The + :attr:`export_empty_fields` attribute has no effect on this exporter. :param file: the file-like object to use for exporting the data. Its ``write`` method should accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 398f80633..a620e2c04 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -58,7 +58,7 @@ CSV - Exporter used: :class:`~scrapy.exporters.CsvItemExporter` -- To specify columns to export and their order use +- To specify columns to export, their order and their column names, use :setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this option, but it is important for CSV because unlike many other export formats CSV uses a fixed header. @@ -522,18 +522,9 @@ FEED_EXPORT_FIELDS Default: ``None`` -A list of fields to export, optional. -Example: ``FEED_EXPORT_FIELDS = ["foo", "bar", "baz"]``. - -Use FEED_EXPORT_FIELDS option to define fields to export and their order. - -When FEED_EXPORT_FIELDS is empty or None (default), Scrapy uses the fields -defined in :ref:`item objects ` yielded by your spider. - -If an exporter requires a fixed set of fields (this is the case for -:ref:`CSV ` export format) and FEED_EXPORT_FIELDS -is empty or None, then Scrapy tries to infer field names from the -exported data - currently it uses field names from the first item. +Use the ``FEED_EXPORT_FIELDS`` setting to define the fields to export, their +order and their output names. See :attr:`BaseItemExporter.fields_to_export +` for more information. .. setting:: FEED_EXPORT_INDENT diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 1c26e81db..76cbe4d4b 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -8,6 +8,7 @@ import marshal import pickle import pprint import warnings +from collections.abc import Mapping from xml.sax.saxutils import XMLGenerator from itemadapter import is_item, ItemAdapter @@ -68,6 +69,14 @@ class BaseItemExporter: field_iter = item.field_names() else: field_iter = item.keys() + elif isinstance(self.fields_to_export, Mapping): + if include_empty: + field_iter = self.fields_to_export.items() + else: + field_iter = ( + (x, y) for x, y in self.fields_to_export.items() + if x in item + ) else: if include_empty: field_iter = self.fields_to_export @@ -75,13 +84,17 @@ class BaseItemExporter: field_iter = (x for x in self.fields_to_export if x in item) for field_name in field_iter: - if field_name in item: - field_meta = item.get_field_meta(field_name) - value = self.serialize_field(field_meta, field_name, item[field_name]) + if isinstance(field_name, str): + item_field, output_field = field_name, field_name + else: + item_field, output_field = field_name + if item_field in item: + field_meta = item.get_field_meta(item_field) + value = self.serialize_field(field_meta, output_field, item[item_field]) else: value = default_value - yield field_name, value + yield output_field, value class JsonLinesItemExporter(BaseItemExporter): @@ -246,7 +259,11 @@ class CsvItemExporter(BaseItemExporter): if not self.fields_to_export: # use declared field names, or keys if the item is a dict self.fields_to_export = ItemAdapter(item).field_names() - row = list(self._build_row(self.fields_to_export)) + if isinstance(self.fields_to_export, Mapping): + fields = self.fields_to_export.values() + else: + fields = self.fields_to_export + row = list(self._build_row(fields)) self.csv_writer.writerow(row) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 6b1ad0828..6cacc63e1 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -197,6 +197,38 @@ class BaseSettings(MutableMapping): value = json.loads(value) return dict(value) + def getdictorlist(self, name, default=None): + """Get a setting value as either a :class:`dict` or a :class:`list`. + + If the setting is already a dict or a list, a copy of it will be + returned. + + If it is a string it will be evaluated as JSON, or as a comma-separated + list of strings as a fallback. + + For example, settings populated from the command line will return: + + - ``{'key1': 'value1', 'key2': 'value2'}`` if set to + ``'{"key1": "value1", "key2": "value2"}'`` + + - ``['one', 'two']`` if set to ``'["one", "two"]'`` or ``'one,two'`` + + :param name: the setting name + :type name: string + + :param default: the value to return if no setting is found + :type default: any + """ + value = self.get(name, default) + if value is None: + return {} + if isinstance(value, str): + try: + return json.loads(value) + except ValueError: + return value.split(',') + return copy.deepcopy(value) + def getwithbase(self, name): """Get a composition of a dictionary-like setting and its `_BASE` counterpart. diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 24a6187b9..00cc53725 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -118,7 +118,7 @@ def feed_complete_default_values_from_settings(feed, settings): out = feed.copy() out.setdefault("batch_item_count", settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')) out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"]) - out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None) + out.setdefault("fields", settings.getdictorlist("FEED_EXPORT_FIELDS") or None) out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY")) out.setdefault("uri_params", settings["FEED_URI_PARAMS"]) out.setdefault("item_export_kwargs", {}) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index b263b3475..096cd3116 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -112,6 +112,14 @@ class BaseItemExporterTest(unittest.TestCase): assert isinstance(name, str) self.assertEqual(name, 'John\xa3') + ie = self._get_exporter( + fields_to_export={'name': '名稱'} + ) + self.assertEqual( + list(ie._get_serialized_fields(self.i)), + [('名稱', 'John\xa3')] + ) + def test_field_custom_serializer(self): i = self.custom_field_item_class(name='John\xa3', age='22') a = ItemAdapter(i) @@ -272,6 +280,7 @@ class MarshalItemExporterDataclassTest(MarshalItemExporterTest): class CsvItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): + self.output = tempfile.TemporaryFile() return CsvItemExporter(self.output, **kwargs) def assertCsvEqual(self, first, second, msg=None): @@ -283,7 +292,8 @@ class CsvItemExporterTest(BaseItemExporterTest): return self.assertEqual(split_csv(first), split_csv(second), msg=msg) def _check_output(self): - self.assertCsvEqual(to_unicode(self.output.getvalue()), 'age,name\r\n22,John\xa3\r\n') + self.output.seek(0) + self.assertCsvEqual(to_unicode(self.output.read()), 'age,name\r\n22,John\xa3\r\n') def assertExportResult(self, item, expected, **kwargs): fp = BytesIO() diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index f0acf1941..ec48f8d4a 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -7,6 +7,7 @@ import os import random import shutil import string +import sys import tempfile import warnings from abc import ABC, abstractmethod @@ -655,8 +656,8 @@ class FeedExportTestBase(ABC, unittest.TestCase): return data @defer.inlineCallbacks - def assertExported(self, items, header, rows, settings=None, ordered=True): - yield self.assertExportedCsv(items, header, rows, settings, ordered) + def assertExported(self, items, header, rows, settings=None): + yield self.assertExportedCsv(items, header, rows, settings) yield self.assertExportedJsonLines(items, rows, settings) yield self.assertExportedXml(items, rows, settings) yield self.assertExportedPickle(items, rows, settings) @@ -717,7 +718,7 @@ class FeedExportTest(FeedExportTestBase): return content @defer.inlineCallbacks - def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): + def assertExportedCsv(self, items, header, rows, settings=None): settings = settings or {} settings.update({ 'FEEDS': { @@ -725,15 +726,9 @@ class FeedExportTest(FeedExportTestBase): }, }) data = yield self.exported_data(items, settings) - reader = csv.DictReader(to_unicode(data['csv']).splitlines()) - got_rows = list(reader) - if ordered: - self.assertEqual(reader.fieldnames, header) - else: - self.assertEqual(set(reader.fieldnames), set(header)) - - self.assertEqual(rows, got_rows) + self.assertEqual(reader.fieldnames, list(header)) + self.assertEqual(rows, list(reader)) @defer.inlineCallbacks def assertExportedJsonLines(self, items, rows, settings=None): @@ -884,7 +879,7 @@ class FeedExportTest(FeedExportTestBase): {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'} ] header = self.MyItem.fields.keys() - yield self.assertExported(items, header, rows, ordered=False) + yield self.assertExported(items, header, rows) @defer.inlineCallbacks def test_export_no_items_not_store_empty(self): @@ -956,25 +951,72 @@ class FeedExportTest(FeedExportTestBase): {'egg': 'spam4', 'foo': '', 'baz': ''}, ] rows_jl = [dict(row) for row in items] - yield self.assertExportedCsv(items, header, rows_csv, ordered=False) + yield self.assertExportedCsv(items, header, rows_csv) yield self.assertExportedJsonLines(items, rows_jl) - # edge case: FEED_EXPORT_FIELDS==[] means the same as default None + @defer.inlineCallbacks + def test_export_items_empty_field_list(self): + # FEED_EXPORT_FIELDS==[] means the same as default None + items = [{'foo': 'bar'}] + header = ["foo"] + rows = [{'foo': 'bar'}] settings = {'FEED_EXPORT_FIELDS': []} - yield self.assertExportedCsv(items, header, rows_csv, ordered=False) - yield self.assertExportedJsonLines(items, rows_jl, settings) + yield self.assertExportedCsv(items, header, rows) + yield self.assertExportedJsonLines(items, rows, settings) - # it is possible to override fields using FEED_EXPORT_FIELDS - header = ["foo", "baz", "hello"] + @defer.inlineCallbacks + def test_export_items_field_list(self): + items = [{'foo': 'bar'}] + header = ["foo", "baz"] + rows = [{'foo': 'bar', 'baz': ''}] settings = {'FEED_EXPORT_FIELDS': header} - rows = [ - {'foo': 'bar1', 'baz': '', 'hello': ''}, - {'foo': 'bar2', 'baz': '', 'hello': 'world2'}, - {'foo': 'bar3', 'baz': 'quux3', 'hello': ''}, - {'foo': '', 'baz': '', 'hello': 'world4'}, - ] - yield self.assertExported(items, header, rows, - settings=settings, ordered=True) + yield self.assertExported(items, header, rows, settings=settings) + + @defer.inlineCallbacks + def test_export_items_comma_separated_field_list(self): + items = [{'foo': 'bar'}] + header = ["foo", "baz"] + rows = [{'foo': 'bar', 'baz': ''}] + settings = {'FEED_EXPORT_FIELDS': ",".join(header)} + yield self.assertExported(items, header, rows, settings=settings) + + @defer.inlineCallbacks + def test_export_items_json_field_list(self): + items = [{'foo': 'bar'}] + header = ["foo", "baz"] + rows = [{'foo': 'bar', 'baz': ''}] + settings = {'FEED_EXPORT_FIELDS': json.dumps(header)} + yield self.assertExported(items, header, rows, settings=settings) + + @defer.inlineCallbacks + def test_export_items_field_names(self): + items = [{'foo': 'bar'}] + header = {'foo': 'Foo'} + rows = [{'Foo': 'bar'}] + settings = {'FEED_EXPORT_FIELDS': header} + yield self.assertExported(items, list(header.values()), rows, + settings=settings) + + @defer.inlineCallbacks + def test_export_items_dict_field_names(self): + items = [{'foo': 'bar'}] + header = { + 'baz': 'Baz', + 'foo': 'Foo', + } + rows = [{'Baz': '', 'Foo': 'bar'}] + settings = {'FEED_EXPORT_FIELDS': header} + yield self.assertExported(items, ['Baz', 'Foo'], rows, + settings=settings) + + @defer.inlineCallbacks + def test_export_items_json_field_names(self): + items = [{'foo': 'bar'}] + header = {'foo': 'Foo'} + rows = [{'Foo': 'bar'}] + settings = {'FEED_EXPORT_FIELDS': json.dumps(header)} + yield self.assertExported(items, list(header.values()), rows, + settings=settings) @defer.inlineCallbacks def test_export_based_on_item_classes(self): @@ -1097,7 +1139,7 @@ class FeedExportTest(FeedExportTestBase): {'egg': 'spam', 'foo': 'bar'} ] rows_jl = items - yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv, ordered=False) + yield self.assertExportedCsv(items, ['foo', 'egg'], rows_csv) yield self.assertExportedJsonLines(items, rows_jl) @defer.inlineCallbacks @@ -1118,7 +1160,7 @@ class FeedExportTest(FeedExportTestBase): {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'} ] yield self.assertExported(items, ['foo', 'baz', 'egg'], rows, - settings=settings, ordered=True) + settings=settings) # export a subset of columns settings = {'FEED_EXPORT_FIELDS': 'egg,baz'} @@ -1127,7 +1169,7 @@ class FeedExportTest(FeedExportTestBase): {'egg': 'spam2', 'baz': 'quux2'} ] yield self.assertExported(items, ['egg', 'baz'], rows, - settings=settings, ordered=True) + settings=settings) @defer.inlineCallbacks def test_export_encoding(self): @@ -1769,7 +1811,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_lzma_plugin_filters(self): - import sys if "PyPy" in sys.version: # https://foss.heptapod.net/pypy/pypy/-/issues/3527 raise unittest.SkipTest("lzma filters doesn't work in PyPy") @@ -2016,7 +2057,7 @@ class BatchDeliveriesTest(FeedExportTestBase): self.assertEqual(expected_batch, got_batch) @defer.inlineCallbacks - def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): + def assertExportedCsv(self, items, header, rows, settings=None): settings = settings or {} settings.update({ 'FEEDS': {