From a8e895e684184e967a751ea28a7102f21dd74834 Mon Sep 17 00:00:00 2001 From: maranqz Date: Sun, 30 Aug 2020 10:57:22 +0300 Subject: [PATCH] kwargs for Item exporters classes test docs --- docs/topics/feed-exports.rst | 5 +++++ scrapy/extensions/feedexport.py | 1 + scrapy/utils/conf.py | 1 + tests/test_feedexport.py | 37 +++++++++++++++++++++++++++++++++ 4 files changed, 44 insertions(+) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 9fb2189e8..e69a64195 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -303,6 +303,9 @@ For instance:: 'store_empty': False, 'fields': None, 'indent': 4, + 'item_export_kwargs': { + 'export_empty_fields': True, + }, }, '/home/user/documents/items.xml': { 'format': 'xml', @@ -332,6 +335,8 @@ as a fallback value if that key is not provided for a specific feed definition: - ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`. +- ``item_export_kwargs``: dict with kwargs for :ref:`Item exporters ` classes. + - ``overwrite``: whether to overwrite the file if it already exists (``True``) or append to its content (``False``). diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 980825499..f32d48fa5 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -360,6 +360,7 @@ class FeedExporter: fields_to_export=feed_options['fields'], encoding=feed_options['encoding'], indent=feed_options['indent'], + **feed_options['item_export_kwargs'], ) slot = _FeedSlot( file=file, diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 4e7a9967e..b904c4a03 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -121,6 +121,7 @@ def feed_complete_default_values_from_settings(feed, settings): out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None) out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY")) out.setdefault("uri_params", settings["FEED_URI_PARAMS"]) + out.setdefault("item_export_kwargs", dict()) if settings["FEED_EXPORT_INDENT"] is None: out.setdefault("indent", None) else: diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 840e0f87b..e88e4f5ce 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1252,6 +1252,43 @@ class FeedExportTest(FeedExportTestBase): for fmt in ['json', 'xml', 'csv']: self.assertIn(f'Error storing {fmt} feed (2 items)', str(log)) + @defer.inlineCallbacks + def test_extend_kwargs(self): + items = [{'foo': 'FOO', 'bar': 'BAR'}] + + expected_with_title_csv = 'foo,bar\r\nFOO,BAR\r\n'.encode('utf-8') + expected_without_title_csv = 'FOO,BAR\r\n'.encode('utf-8') + test_cases = [ + # with title + { + 'options': { + 'format': 'csv', + 'item_export_kwargs': dict(include_headers_line=True), + }, + 'expected': expected_with_title_csv, + }, + # without title + { + 'options': { + 'format': 'csv', + 'item_export_kwargs': dict(include_headers_line=False), + }, + 'expected': expected_without_title_csv, + }, + ] + + for row in test_cases: + feed_options = row['options'] + settings = { + 'FEEDS': { + self._random_temp_filename(): feed_options, + }, + 'FEED_EXPORT_INDENT': None, + } + + data = yield self.exported_data(items, settings) + self.assertEqual(row['expected'], data[feed_options['format']]) + class BatchDeliveriesTest(FeedExportTestBase): __test__ = True