RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

TST, DOC document that Scrapy only infers field names for CSV

This commit is contained in:
Mikhail Korobov 2015-05-12 18:44:23 +05:00
parent 8d9e3b7e8d
commit e1efd19175
2 changed files with 18 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
docs/topics/feed-exports.rst
View File

@ -241,9 +241,12 @@ Example: ``FEED_EXPORT_FIELDS = ["foo", "bar", "baz"]``.
Use FEED_EXPORT_FIELDS option to define fields to export and their order.
When omitted, Scrapy uses fields defined in :class:`~.Item` subclasses a spider
is yielding. If raw dicts are used as items Scrapy tries to infer field names
from the exported data - currently it uses field names from the first item.
When omitted or empty, Scrapy uses fields defined in :class:`~.Item` subclasses
a spider is yielding. If raw dicts are used as items, FEED_EXPORT_FIELDS
is omitted and an exporter requires a fixed set of fields (this is the case
for :ref:`CSV <topics-feed-format-csv>` export format) then Scrapy tries
to infer field names from the exported data - currently it uses field names
from the first item.
.. setting:: FEED_STORE_EMPTY

18
tests/test_feedexport.py
View File

@ -185,6 +185,7 @@ class FeedExportTest(unittest.TestCase):
settings.update({'FEED_FORMAT': 'jl'})
data = yield self.exported_data(items, settings)
parsed = [json.loads(line) for line in data.splitlines()]
rows = [{k: v for k, v in row.items() if v} for row in rows]
self.assertEqual(rows, parsed)
@defer.inlineCallbacks
@ -220,15 +221,18 @@ class FeedExportTest(unittest.TestCase):
{'hello': 'world4', 'egg': 'spam4'},
]
# by default, Scrapy uses fields of the first Item
# by default, Scrapy uses fields of the first Item for CSV and
# all fields for JSON Lines
header = self.MyItem.fields.keys()
rows = [
rows_csv = [
{'egg': 'spam1', 'foo': 'bar1', 'baz': ''},
{'egg': '', 'foo': 'bar2', 'baz': ''},
{'egg': 'spam3', 'foo': 'bar3', 'baz': 'quux3'},
{'egg': 'spam4', 'foo': '', 'baz': ''},
]
yield self.assertExported(items, header, rows, ordered=False)
rows_jl = [dict(row) for row in items]
yield self.assertExportedCsv(items, header, rows_csv, ordered=False)
yield self.assertExportedJsonLines(items, rows_jl)
# but it is possible to override fields using FEED_EXPORT_FIELDS
header = ["foo", "baz", "hello"]
@ -245,16 +249,18 @@ class FeedExportTest(unittest.TestCase):
@defer.inlineCallbacks
def test_export_dicts(self):
# When dicts are used, only keys from the first row are used as
# a header.
# a header for CSV, and all fields are used for JSON Lines.
items = [
{'foo': 'bar', 'egg': 'spam'},
{'foo': 'bar', 'egg': 'spam', 'baz': 'quux'},
]
rows = [
rows_csv = [
{'egg': 'spam', 'foo': 'bar'},
{'egg': 'spam', 'foo': 'bar'}
]
yield self.assertExported(items, ['egg', 'foo'], rows, ordered=False)
rows_jl = items
yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv, ordered=False)
yield self.assertExportedJsonLines(items, rows_jl)
@defer.inlineCallbacks
def test_export_feed_export_fields(self):