From c7bb2fa8ce2633d92a7ec2840f84b174a5494428 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 7 Mar 2017 11:55:26 -0300 Subject: [PATCH] Feed exports: consistent and backwards compatible behaviour on indent --- docs/topics/exporters.rst | 14 ++++---- docs/topics/feed-exports.rst | 6 ++-- scrapy/exporters.py | 14 +++++--- scrapy/settings/default_settings.py | 2 +- tests/test_feedexport.py | 56 ++++++++++------------------- 5 files changed, 39 insertions(+), 53 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index ad559fb35..2ad77c905 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -140,7 +140,7 @@ output examples, which assume you're exporting these two items:: BaseItemExporter ---------------- -.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=None) +.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=0) This is the (abstract) base class for all Item Exporters. It provides support for common features used by all (concrete) Item Exporters, such as @@ -218,12 +218,12 @@ BaseItemExporter .. attribute:: indent - Amount of spaces used to indent the output on each level. Defaults to ``None``, - which disables indentation. This argument behaves like ``indent`` in python's - JSON module (both for JSON and XML exporters): "If ``indent`` is a non-negative - integer, then array elements and object members will be pretty-printed with that - indent level. An indent level of 0, or negative, will only insert newlines. - ``None`` (the default) selects the most compact representation" + Amount of spaces used to indent the output on each level. Defaults to ``0``. + + * ``indent=None`` selects the most compact representation, + all items in the same line with no indentation + * ``indent<=0`` each item on it's own line, no indentation + * ``indent>0`` each item on it's own line, indentated with the provided numeric value .. highlight:: none diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index afaa972e5..e57a4e776 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -272,12 +272,12 @@ exported data - currently it uses field names from the first item. FEED_EXPORT_INDENT ------------------ -Default: ``None`` +Default: ``0`` Amount of spaces used to indent the output on each level. If ``FEED_EXPORT_INDENT`` is a non-negative integer, then array elements and object members will be pretty-printed -with that indent level. An indent level of 0, or negative, will only insert newlines. -``None`` (the default) selects the most compact representation +with that indent level. An indent level of ``0``, or negative, will put each item on a new line. +``None`` selects the most compact representation Currently used by :class:`~scrapy.exporters.JsonItemExporter` and :class:`~scrapy.exporters.XmlItemExporter` diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 1dfa2af85..e2d42b6ab 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -99,8 +99,12 @@ class JsonItemExporter(BaseItemExporter): def __init__(self, file, **kwargs): self._configure(kwargs, dont_fail=True) self.file = file + # there is a small difference between the behaviour or JsonItemExporter.indent + # and ScrapyJSONEncoder.indent. ScrapyJSONEncoder.indent=None is needed to prevent + # the addition of newlines everywhere + json_indent = self.indent if self.indent is not None and self.indent > 0 else None + kwargs.setdefault('indent', json_indent) kwargs.setdefault('ensure_ascii', not self.encoding) - kwargs.setdefault('indent', self.indent) self.encoder = ScrapyJSONEncoder(**kwargs) self.first_item = True @@ -137,8 +141,8 @@ class XmlItemExporter(BaseItemExporter): self.encoding = 'utf-8' self.xg = XMLGenerator(file, encoding=self.encoding) - def _beautify_newline(self): - if self.indent is not None: + def _beautify_newline(self, new_item=False): + if self.indent is not None and (self.indent > 0 or new_item): self._xg_characters('\n') def _beautify_indent(self, depth=1): @@ -148,7 +152,7 @@ class XmlItemExporter(BaseItemExporter): def start_exporting(self): self.xg.startDocument() self.xg.startElement(self.root_element, {}) - self._beautify_newline() + self._beautify_newline(new_item=True) def export_item(self, item): self._beautify_indent(depth=1) @@ -158,7 +162,7 @@ class XmlItemExporter(BaseItemExporter): self._export_xml_field(name, value, depth=2) self._beautify_indent(depth=1) self.xg.endElement(self.item_element) - self._beautify_newline() + self._beautify_newline(new_item=True) def finish_exporting(self): self.xg.endElement(self.root_element) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index fc265e2ba..bbc02cfdb 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -161,7 +161,7 @@ FEED_EXPORTERS_BASE = { 'marshal': 'scrapy.exporters.MarshalItemExporter', 'pickle': 'scrapy.exporters.PickleItemExporter', } -FEED_EXPORT_INDENT = None +FEED_EXPORT_INDENT = 0 FILES_STORE_S3_ACL = 'private' diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 2b82bba0c..c66c470a8 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -326,7 +326,7 @@ class FeedExportTest(unittest.TestCase): ) for fmt, expctd in formats: - settings = {'FEED_FORMAT': fmt, 'FEED_STORE_EMPTY': True} + settings = {'FEED_FORMAT': fmt, 'FEED_STORE_EMPTY': True, 'FEED_EXPORT_INDENT': None} data = yield self.exported_no_data(settings) self.assertEqual(data, expctd) @@ -451,9 +451,12 @@ class FeedExportTest(unittest.TestCase): @defer.inlineCallbacks def test_export_indentation(self): - items = [dict({'foo': ['bar']}), dict({'key': 'value'})] + items = [ + {'foo': ['bar']}, + {'key': 'value'}, + ] - output = [ + test_cases = [ # JSON { 'format': 'json', @@ -465,14 +468,8 @@ class FeedExportTest(unittest.TestCase): 'indent': -1, 'expected': b""" [ -{ -"foo": [ -"bar" -] -}, -{ -"key": "value" -} +{"foo": ["bar"]}, +{"key": "value"} ] """, }, @@ -481,14 +478,8 @@ class FeedExportTest(unittest.TestCase): 'indent': 0, 'expected': b""" [ -{ -"foo": [ -"bar" -] -}, -{ -"key": "value" -} +{"foo": ["bar"]}, +{"key": "value"} ] """, }, @@ -542,7 +533,9 @@ class FeedExportTest(unittest.TestCase): { 'format': 'xml', 'indent': None, - 'expected': b'\nbarvalue', + 'expected': b""" + +barvalue""", }, { 'format': 'xml', @@ -550,14 +543,8 @@ class FeedExportTest(unittest.TestCase): 'expected': b""" - - -bar - - - -value - +bar +value """, }, { @@ -566,14 +553,8 @@ class FeedExportTest(unittest.TestCase): 'expected': b""" - - -bar - - - -value - +bar +value """, }, { @@ -626,7 +607,8 @@ class FeedExportTest(unittest.TestCase): }, ] - for row in output: + for row in test_cases: settings = {'FEED_FORMAT': row['format'], 'FEED_EXPORT_INDENT': row['indent']} data = yield self.exported_data(items, settings) + print(row['format'], row['indent']) self.assertEqual(row['expected'].strip(), data)