diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index f5048d2da..42a93e459 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -190,14 +190,33 @@ BaseItemExporter .. attribute:: fields_to_export - A list with the name of the fields that will be exported, or None if you - want to export all fields. Defaults to None. + Fields to export, their order [1]_ and their output names. - Some exporters (like :class:`CsvItemExporter`) respect the order of the - fields defined in this attribute. + Possible values are: - Some exporters may require fields_to_export list in order to export the - data properly when spiders return dicts (not :class:`~Item` instances). + - ``None`` (all fields [2]_, default) + + - A list of fields:: + + ['field1', 'field2'] + + - A dict [3]_ where keys are fields and values are output names:: + + {'field1': 'Field 1', 'field2': 'Field 2'} + + .. [1] Not all exporters respect the specified field order. + .. [2] If you yield items as dicts (not :class:`Item` instances), + exporters that need to know the fields to export beforehand, like + :class:`CsvItemExporter`, only export the fields found in the + first item. + .. [3] Dicts preserve insertion order since `Python 3.7`_ + (`CPython 3.6`_, `PyPy 2.5`_). If you are using an older version + of Python, use an OrderedDict_ to enforce a specific field order. + + .. _Python 3.7: https://docs.python.org/whatsnew/3.7.html + .. _CPython 3.6: https://docs.python.org/whatsnew/3.6.html#new-dict-implementation + .. _PyPy 2.5: https://morepypy.blogspot.com/2015/02/pypy-250-released.html + .. _OrderedDict: https://docs.python.org/library/collections.html#collections.OrderedDict .. attribute:: export_empty_fields @@ -286,8 +305,8 @@ CsvItemExporter Exports Items in CSV format to the given file-like object. If the :attr:`fields_to_export` attribute is set, it will be used to define the - CSV columns and their order. The :attr:`export_empty_fields` attribute has - no effect on this exporter. + CSV columns, their order and their column names. The + :attr:`export_empty_fields` attribute has no effect on this exporter. :param file: the file-like object to use for exporting the data. Its ``write`` method should accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index cf70b8aca..968cb8884 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -56,7 +56,7 @@ CSV * :setting:`FEED_FORMAT`: ``csv`` * Exporter used: :class:`~scrapy.exporters.CsvItemExporter` - * To specify columns to export and their order use + * To specify columns to export, their order and their column names, use :setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this option, but it is important for CSV because unlike many other export formats CSV uses a fixed header. @@ -259,18 +259,9 @@ FEED_EXPORT_FIELDS Default: ``None`` -A list of fields to export, optional. -Example: ``FEED_EXPORT_FIELDS = ["foo", "bar", "baz"]``. - -Use FEED_EXPORT_FIELDS option to define fields to export and their order. - -When FEED_EXPORT_FIELDS is empty or None (default), Scrapy uses fields -defined in dicts or :class:`~.Item` subclasses a spider is yielding. - -If an exporter requires a fixed set of fields (this is the case for -:ref:`CSV ` export format) and FEED_EXPORT_FIELDS -is empty or None, then Scrapy tries to infer field names from the -exported data - currently it uses field names from the first item. +Use the ``FEED_EXPORT_FIELDS`` setting to define the fields to export, their +order and their output names. See :attr:`BaseItemExporter.fields_to_export +` for more information. .. setting:: FEED_EXPORT_INDENT diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 695c74fec..c05acaca5 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -2,6 +2,7 @@ Item Exporters are used to export/serialize items into different formats. """ +from collections import Mapping import csv import io import sys @@ -64,6 +65,14 @@ class BaseItemExporter(object): field_iter = six.iterkeys(item.fields) else: field_iter = six.iterkeys(item) + elif isinstance(self.fields_to_export, Mapping): + if include_empty: + field_iter = self.fields_to_export.items() + else: + field_iter = ( + (x, y) for x, y in self.fields_to_export.items() + if x in item + ) else: if include_empty: field_iter = self.fields_to_export @@ -71,13 +80,22 @@ class BaseItemExporter(object): field_iter = (x for x in self.fields_to_export if x in item) for field_name in field_iter: - if field_name in item: - field = {} if isinstance(item, dict) else item.fields[field_name] - value = self.serialize_field(field, field_name, item[field_name]) + if isinstance(field_name, six.string_types): + item_field, output_field = field_name, field_name + else: + item_field, output_field = field_name + if item_field in item: + if isinstance(item, dict): + field = {} + else: + field = item.fields[item_field] + value = self.serialize_field( + field, output_field, item[item_field] + ) else: value = default_value - yield field_name, value + yield output_field, value class JsonLinesItemExporter(BaseItemExporter): @@ -259,7 +277,11 @@ class CsvItemExporter(BaseItemExporter): else: # use fields declared in Item self.fields_to_export = list(item.fields.keys()) - row = list(self._build_row(self.fields_to_export)) + if isinstance(self.fields_to_export, Mapping): + fields = self.fields_to_export.values() + else: + fields = self.fields_to_export + row = list(self._build_row(fields)) self.csv_writer.writerow(row) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index b2f7267a2..1ed476d83 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -201,7 +201,8 @@ class FeedExporter(object): raise NotConfigured self.store_empty = settings.getbool('FEED_STORE_EMPTY') self._exporting = False - self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None + self.export_fields = settings.getdictorlist('FEED_EXPORT_FIELDS') + self.export_fields = self.export_fields or None self.indent = None if settings.get('FEED_EXPORT_INDENT') is not None: self.indent = settings.getint('FEED_EXPORT_INDENT') diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 14c93bef2..69b324e84 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -1,7 +1,7 @@ import six import json import copy -from collections import MutableMapping +from collections import MutableMapping, OrderedDict from importlib import import_module from pprint import pformat @@ -198,6 +198,39 @@ class BaseSettings(MutableMapping): value = json.loads(value) return dict(value) + def getdictorlist(self, name, default=None): + """Get a setting value as either an ``OrderedDict`` or a list. + + If the setting is already a dict or a list, a copy of it will be + returned. + + If it is a string it will be evaluated as JSON, or as a comma-separated + list of strings as a fallback. + + For example, settings populated through environment variables will + return: + + - ``OrdetedDict([('key1', 'value1'), ('key2', 'value2')])`` if set to + ``'{"key1": "value1", "key2": "value2"}'`` + + - ``['one', 'two']`` if set to ``'["one", "two"]'`` or ``'one,two'`` + + :param name: the setting name + :type name: string + + :param default: the value to return if no setting is found + :type default: any + """ + value = self.get(name, default) + if value is None: + return {} + if isinstance(value, six.string_types): + try: + return json.loads(value, object_pairs_hook=OrderedDict) + except ValueError: + return value.split(',') + return copy.deepcopy(value) + def getwithbase(self, name): """Get a composition of a dictionary-like setting and its `_BASE` counterpart. diff --git a/tests/test_exporters.py b/tests/test_exporters.py index cd72c661a..1b3dc14a1 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -1,4 +1,7 @@ +# -*- coding:utf-8 -*- + from __future__ import absolute_import +from collections import OrderedDict import re import json import marshal @@ -83,6 +86,14 @@ class BaseItemExporterTest(unittest.TestCase): assert isinstance(name, six.text_type) self.assertEqual(name, u'John\xa3') + ie = self._get_exporter( + fields_to_export=OrderedDict([('name', u'名稱')]) + ) + self.assertEqual( + list(ie._get_serialized_fields(self.i)), + [(u'名稱', u'John\xa3')] + ) + def test_field_custom_serializer(self): def custom_serializer(value): return str(int(value) + 2) @@ -214,6 +225,7 @@ class MarshalItemExporterTest(BaseItemExporterTest): class CsvItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): + self.output = tempfile.TemporaryFile() return CsvItemExporter(self.output, **kwargs) def assertCsvEqual(self, first, second, msg=None): @@ -224,7 +236,8 @@ class CsvItemExporterTest(BaseItemExporterTest): return self.assertEqual(csvsplit(first), csvsplit(second), msg) def _check_output(self): - self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n') + self.output.seek(0) + self.assertCsvEqual(to_unicode(self.output.read()), u'age,name\r\n22,John\xa3\r\n') def assertExportResult(self, item, expected, **kwargs): fp = BytesIO() diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index eef0384cf..4f72c0ff4 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1,4 +1,5 @@ from __future__ import absolute_import +from collections import OrderedDict import os import csv import json @@ -578,6 +579,62 @@ class FeedExportTest(unittest.TestCase): yield self.assertExported(items, header, rows, settings=settings, ordered=True) + # fields may be defined as a comma-separated list + header = ["foo", "baz", "hello"] + settings = {'FEED_EXPORT_FIELDS': ",".join(header)} + rows = [ + {'foo': 'bar1', 'baz': '', 'hello': ''}, + {'foo': 'bar2', 'baz': '', 'hello': 'world2'}, + {'foo': 'bar3', 'baz': 'quux3', 'hello': ''}, + {'foo': '', 'baz': '', 'hello': 'world4'}, + ] + yield self.assertExported(items, header, rows, + settings=settings, ordered=True) + + # fields may also be defined as a JSON array + header = ["foo", "baz", "hello"] + settings = {'FEED_EXPORT_FIELDS': json.dumps(header)} + rows = [ + {'foo': 'bar1', 'baz': '', 'hello': ''}, + {'foo': 'bar2', 'baz': '', 'hello': 'world2'}, + {'foo': 'bar3', 'baz': 'quux3', 'hello': ''}, + {'foo': '', 'baz': '', 'hello': 'world4'}, + ] + yield self.assertExported(items, header, rows, + settings=settings, ordered=True) + + # custom output field names can be specified + header = OrderedDict(( + ("foo", "Foo"), + ("baz", "Baz"), + ("hello", "Hello"), + )) + settings = {'FEED_EXPORT_FIELDS': header} + rows = [ + {'Foo': 'bar1', 'Baz': '', 'Hello': ''}, + {'Foo': 'bar2', 'Baz': '', 'Hello': 'world2'}, + {'Foo': 'bar3', 'Baz': 'quux3', 'Hello': ''}, + {'Foo': '', 'Baz': '', 'Hello': 'world4'}, + ] + yield self.assertExported(items, list(header.values()), rows, + settings=settings, ordered=True) + + # custom output field names can be specified as a JSON object + header = OrderedDict(( + ("foo", "Foo"), + ("baz", "Baz"), + ("hello", "Hello"), + )) + settings = {'FEED_EXPORT_FIELDS': json.dumps(header)} + rows = [ + {'Foo': 'bar1', 'Baz': '', 'Hello': ''}, + {'Foo': 'bar2', 'Baz': '', 'Hello': 'world2'}, + {'Foo': 'bar3', 'Baz': 'quux3', 'Hello': ''}, + {'Foo': '', 'Baz': '', 'Hello': 'world4'}, + ] + yield self.assertExported(items, list(header.values()), rows, + settings=settings, ordered=True) + @defer.inlineCallbacks def test_export_dicts(self): # When dicts are used, only keys from the first row are used as