RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Allow customizing export column names

This commit is contained in:
Adrián Chaves 2019-03-22 16:59:29 +01:00
parent 4196d4869b
commit 484bd0d22a
7 changed files with 165 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
docs/topics/exporters.rst
View File

@ -190,14 +190,33 @@ BaseItemExporter
.. attribute:: fields_to_export
A list with the name of the fields that will be exported, or None if you
want to export all fields. Defaults to None.
Fields to export, their order [1]_ and their output names.
Some exporters (like :class:`CsvItemExporter`) respect the order of the
fields defined in this attribute.
Possible values are:
Some exporters may require fields_to_export list in order to export the
data properly when spiders return dicts (not :class:`~Item` instances).
- ``None`` (all fields [2]_, default)
- A list of fields::
['field1', 'field2']
- A dict [3]_ where keys are fields and values are output names::
{'field1': 'Field 1', 'field2': 'Field 2'}
.. [1] Not all exporters respect the specified field order.
.. [2] If you yield items as dicts (not :class:`Item` instances),
exporters that need to know the fields to export beforehand, like
:class:`CsvItemExporter`, only export the fields found in the
first item.
.. [3] Dicts preserve insertion order since `Python 3.7`_
(`CPython 3.6`_, `PyPy 2.5`_). If you are using an older version
of Python, use an OrderedDict_ to enforce a specific field order.
.. _Python 3.7: https://docs.python.org/whatsnew/3.7.html
.. _CPython 3.6: https://docs.python.org/whatsnew/3.6.html#new-dict-implementation
.. _PyPy 2.5: https://morepypy.blogspot.com/2015/02/pypy-250-released.html
.. _OrderedDict: https://docs.python.org/library/collections.html#collections.OrderedDict
.. attribute:: export_empty_fields
@ -286,8 +305,8 @@ CsvItemExporter
Exports Items in CSV format to the given file-like object. If the
:attr:`fields_to_export` attribute is set, it will be used to define the
CSV columns and their order. The :attr:`export_empty_fields` attribute has
no effect on this exporter.
CSV columns, their order and their column names. The
:attr:`export_empty_fields` attribute has no effect on this exporter.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)

17
docs/topics/feed-exports.rst
View File

@ -56,7 +56,7 @@ CSV
* :setting:`FEED_FORMAT`: ``csv``
* Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
* To specify columns to export and their order use
* To specify columns to export, their order and their column names, use
:setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
option, but it is important for CSV because unlike many other export
formats CSV uses a fixed header.
@ -259,18 +259,9 @@ FEED_EXPORT_FIELDS
Default: ``None``
A list of fields to export, optional.
Example: ``FEED_EXPORT_FIELDS = ["foo", "bar", "baz"]``.
Use FEED_EXPORT_FIELDS option to define fields to export and their order.
When FEED_EXPORT_FIELDS is empty or None (default), Scrapy uses fields
defined in dicts or :class:`~.Item` subclasses a spider is yielding.
If an exporter requires a fixed set of fields (this is the case for
:ref:`CSV <topics-feed-format-csv>` export format) and FEED_EXPORT_FIELDS
is empty or None, then Scrapy tries to infer field names from the
exported data - currently it uses field names from the first item.
Use the ``FEED_EXPORT_FIELDS`` setting to define the fields to export, their
order and their output names. See :attr:`BaseItemExporter.fields_to_export
<scrapy.exporters.BaseItemExporter.fields_to_export>` for more information.
.. setting:: FEED_EXPORT_INDENT

32
scrapy/exporters.py
View File

@ -2,6 +2,7 @@
Item Exporters are used to export/serialize items into different formats.
"""
from collections import Mapping
import csv
import io
import sys
@ -64,6 +65,14 @@ class BaseItemExporter(object):
field_iter = six.iterkeys(item.fields)
else:
field_iter = six.iterkeys(item)
elif isinstance(self.fields_to_export, Mapping):
if include_empty:
field_iter = self.fields_to_export.items()
else:
field_iter = (
(x, y) for x, y in self.fields_to_export.items()
if x in item
)
else:
if include_empty:
field_iter = self.fields_to_export
@ -71,13 +80,22 @@ class BaseItemExporter(object):
field_iter = (x for x in self.fields_to_export if x in item)
for field_name in field_iter:
if field_name in item:
field = {} if isinstance(item, dict) else item.fields[field_name]
value = self.serialize_field(field, field_name, item[field_name])
if isinstance(field_name, six.string_types):
item_field, output_field = field_name, field_name
else:
item_field, output_field = field_name
if item_field in item:
if isinstance(item, dict):
field = {}
else:
field = item.fields[item_field]
value = self.serialize_field(
field, output_field, item[item_field]
)
else:
value = default_value
yield field_name, value
yield output_field, value
class JsonLinesItemExporter(BaseItemExporter):
@ -259,7 +277,11 @@ class CsvItemExporter(BaseItemExporter):
else:
# use fields declared in Item
self.fields_to_export = list(item.fields.keys())
row = list(self._build_row(self.fields_to_export))
if isinstance(self.fields_to_export, Mapping):
fields = self.fields_to_export.values()
else:
fields = self.fields_to_export
row = list(self._build_row(fields))
self.csv_writer.writerow(row)

3
scrapy/extensions/feedexport.py
View File

@ -201,7 +201,8 @@ class FeedExporter(object):
raise NotConfigured
self.store_empty = settings.getbool('FEED_STORE_EMPTY')
self._exporting = False
self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None
self.export_fields = settings.getdictorlist('FEED_EXPORT_FIELDS')
self.export_fields = self.export_fields or None
self.indent = None
if settings.get('FEED_EXPORT_INDENT') is not None:
self.indent = settings.getint('FEED_EXPORT_INDENT')

35
scrapy/settings/__init__.py
View File

@ -1,7 +1,7 @@
import six
import json
import copy
from collections import MutableMapping
from collections import MutableMapping, OrderedDict
from importlib import import_module
from pprint import pformat
@ -198,6 +198,39 @@ class BaseSettings(MutableMapping):
value = json.loads(value)
return dict(value)
def getdictorlist(self, name, default=None):
"""Get a setting value as either an ``OrderedDict`` or a list.
If the setting is already a dict or a list, a copy of it will be
returned.
If it is a string it will be evaluated as JSON, or as a comma-separated
list of strings as a fallback.
For example, settings populated through environment variables will
return:
- ``OrdetedDict([('key1', 'value1'), ('key2', 'value2')])`` if set to
``'{"key1": "value1", "key2": "value2"}'``
- ``['one', 'two']`` if set to ``'["one", "two"]'`` or ``'one,two'``
:param name: the setting name
:type name: string
:param default: the value to return if no setting is found
:type default: any
"""
value = self.get(name, default)
if value is None:
return {}
if isinstance(value, six.string_types):
try:
return json.loads(value, object_pairs_hook=OrderedDict)
except ValueError:
return value.split(',')
return copy.deepcopy(value)
def getwithbase(self, name):
"""Get a composition of a dictionary-like setting and its `_BASE`
counterpart.

15
tests/test_exporters.py
View File

@ -1,4 +1,7 @@
# -*- coding:utf-8 -*-
from __future__ import absolute_import
from collections import OrderedDict
import re
import json
import marshal
@ -83,6 +86,14 @@ class BaseItemExporterTest(unittest.TestCase):
assert isinstance(name, six.text_type)
self.assertEqual(name, u'John\xa3')
ie = self._get_exporter(
fields_to_export=OrderedDict([('name', u'名稱')])
)
self.assertEqual(
list(ie._get_serialized_fields(self.i)),
[(u'名稱', u'John\xa3')]
)
def test_field_custom_serializer(self):
def custom_serializer(value):
return str(int(value) + 2)
@ -214,6 +225,7 @@ class MarshalItemExporterTest(BaseItemExporterTest):
class CsvItemExporterTest(BaseItemExporterTest):
def _get_exporter(self, **kwargs):
self.output = tempfile.TemporaryFile()
return CsvItemExporter(self.output, **kwargs)
def assertCsvEqual(self, first, second, msg=None):
@ -224,7 +236,8 @@ class CsvItemExporterTest(BaseItemExporterTest):
return self.assertEqual(csvsplit(first), csvsplit(second), msg)
def _check_output(self):
self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n')
self.output.seek(0)
self.assertCsvEqual(to_unicode(self.output.read()), u'age,name\r\n22,John\xa3\r\n')
def assertExportResult(self, item, expected, **kwargs):
fp = BytesIO()

57
tests/test_feedexport.py
View File

@ -1,4 +1,5 @@
from __future__ import absolute_import
from collections import OrderedDict
import os
import csv
import json
@ -578,6 +579,62 @@ class FeedExportTest(unittest.TestCase):
yield self.assertExported(items, header, rows,
settings=settings, ordered=True)
# fields may be defined as a comma-separated list
header = ["foo", "baz", "hello"]
settings = {'FEED_EXPORT_FIELDS': ",".join(header)}
rows = [
{'foo': 'bar1', 'baz': '', 'hello': ''},
{'foo': 'bar2', 'baz': '', 'hello': 'world2'},
{'foo': 'bar3', 'baz': 'quux3', 'hello': ''},
{'foo': '', 'baz': '', 'hello': 'world4'},
]
yield self.assertExported(items, header, rows,
settings=settings, ordered=True)
# fields may also be defined as a JSON array
header = ["foo", "baz", "hello"]
settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}
rows = [
{'foo': 'bar1', 'baz': '', 'hello': ''},
{'foo': 'bar2', 'baz': '', 'hello': 'world2'},
{'foo': 'bar3', 'baz': 'quux3', 'hello': ''},
{'foo': '', 'baz': '', 'hello': 'world4'},
]
yield self.assertExported(items, header, rows,
settings=settings, ordered=True)
# custom output field names can be specified
header = OrderedDict((
("foo", "Foo"),
("baz", "Baz"),
("hello", "Hello"),
))
settings = {'FEED_EXPORT_FIELDS': header}
rows = [
{'Foo': 'bar1', 'Baz': '', 'Hello': ''},
{'Foo': 'bar2', 'Baz': '', 'Hello': 'world2'},
{'Foo': 'bar3', 'Baz': 'quux3', 'Hello': ''},
{'Foo': '', 'Baz': '', 'Hello': 'world4'},
]
yield self.assertExported(items, list(header.values()), rows,
settings=settings, ordered=True)
# custom output field names can be specified as a JSON object
header = OrderedDict((
("foo", "Foo"),
("baz", "Baz"),
("hello", "Hello"),
))
settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}
rows = [
{'Foo': 'bar1', 'Baz': '', 'Hello': ''},
{'Foo': 'bar2', 'Baz': '', 'Hello': 'world2'},
{'Foo': 'bar3', 'Baz': 'quux3', 'Hello': ''},
{'Foo': '', 'Baz': '', 'Hello': 'world4'},
]
yield self.assertExported(items, list(header.values()), rows,
settings=settings, ordered=True)
@defer.inlineCallbacks
def test_export_dicts(self):
# When dicts are used, only keys from the first row are used as