Feed exports: consistent and backwards compatible behaviour on indent

docs/topics/exporters.rst

@@ -140,7 +140,7 @@ output examples, which assume you're exporting these two items::
 BaseItemExporter
 ----------------
 
-.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=None)
+.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=0)
 
    This is the (abstract) base class for all Item Exporters. It provides
    support for common features used by all (concrete) Item Exporters, such as
@@ -218,12 +218,12 @@ BaseItemExporter
 
    .. attribute:: indent
 
-      Amount of spaces used to indent the output on each level. Defaults to ``None``,
-      which disables indentation. This argument behaves like ``indent`` in python's
-      JSON module (both for JSON and XML exporters): "If ``indent`` is a non-negative
-      integer, then array elements and object members will be pretty-printed with that
-      indent level. An indent level of 0, or negative, will only insert newlines.
-      ``None`` (the default) selects the most compact representation"
+      Amount of spaces used to indent the output on each level. Defaults to ``0``.
+
+      * ``indent=None`` selects the most compact representation,
+        all items in the same line with no indentation
+      * ``indent<=0`` each item on it's own line, no indentation
+      * ``indent>0`` each item on it's own line, indentated with the provided numeric value
 
 .. highlight:: none
 

docs/topics/feed-exports.rst

@@ -272,12 +272,12 @@ exported data - currently it uses field names from the first item.
 FEED_EXPORT_INDENT
 ------------------
 
-Default: ``None``
+Default: ``0``
 
 Amount of spaces used to indent the output on each level. If ``FEED_EXPORT_INDENT``
 is a non-negative integer, then array elements and object members will be pretty-printed
-with that indent level. An indent level of 0, or negative, will only insert newlines.
-``None`` (the default) selects the most compact representation
+with that indent level. An indent level of ``0``, or negative, will put each item on a new line.
+``None`` selects the most compact representation
 
 Currently used by :class:`~scrapy.exporters.JsonItemExporter`
 and :class:`~scrapy.exporters.XmlItemExporter`

scrapy/exporters.py

@@ -99,8 +99,12 @@ class JsonItemExporter(BaseItemExporter):
     def __init__(self, file, **kwargs):
         self._configure(kwargs, dont_fail=True)
         self.file = file
+        # there is a small difference between the behaviour or JsonItemExporter.indent
+        # and ScrapyJSONEncoder.indent. ScrapyJSONEncoder.indent=None is needed to prevent
+        # the addition of newlines everywhere
+        json_indent = self.indent if self.indent is not None and self.indent > 0 else None
+        kwargs.setdefault('indent', json_indent)
         kwargs.setdefault('ensure_ascii', not self.encoding)
-        kwargs.setdefault('indent', self.indent)
         self.encoder = ScrapyJSONEncoder(**kwargs)
         self.first_item = True
 
@@ -137,8 +141,8 @@ class XmlItemExporter(BaseItemExporter):
             self.encoding = 'utf-8'
         self.xg = XMLGenerator(file, encoding=self.encoding)
 
-    def _beautify_newline(self):
-        if self.indent is not None:
+    def _beautify_newline(self, new_item=False):
+        if self.indent is not None and (self.indent > 0 or new_item):
             self._xg_characters('\n')
 
     def _beautify_indent(self, depth=1):
@@ -148,7 +152,7 @@ class XmlItemExporter(BaseItemExporter):
     def start_exporting(self):
         self.xg.startDocument()
         self.xg.startElement(self.root_element, {})
-        self._beautify_newline()
+        self._beautify_newline(new_item=True)
 
     def export_item(self, item):
         self._beautify_indent(depth=1)
@@ -158,7 +162,7 @@ class XmlItemExporter(BaseItemExporter):
             self._export_xml_field(name, value, depth=2)
         self._beautify_indent(depth=1)
         self.xg.endElement(self.item_element)
-        self._beautify_newline()
+        self._beautify_newline(new_item=True)
 
     def finish_exporting(self):
         self.xg.endElement(self.root_element)

scrapy/settings/default_settings.py

@@ -161,7 +161,7 @@ FEED_EXPORTERS_BASE = {
     'marshal': 'scrapy.exporters.MarshalItemExporter',
     'pickle': 'scrapy.exporters.PickleItemExporter',
 }
-FEED_EXPORT_INDENT = None
+FEED_EXPORT_INDENT = 0
 
 FILES_STORE_S3_ACL = 'private'
 

tests/test_feedexport.py

@@ -326,7 +326,7 @@ class FeedExportTest(unittest.TestCase):
         )
 
         for fmt, expctd in formats:
-            settings = {'FEED_FORMAT': fmt, 'FEED_STORE_EMPTY': True}
+            settings = {'FEED_FORMAT': fmt, 'FEED_STORE_EMPTY': True, 'FEED_EXPORT_INDENT': None}
             data = yield self.exported_no_data(settings)
             self.assertEqual(data, expctd)
 
@@ -451,9 +451,12 @@ class FeedExportTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_export_indentation(self):
-        items = [dict({'foo': ['bar']}), dict({'key': 'value'})]
+        items = [
+            {'foo': ['bar']},
+            {'key': 'value'},
+        ]
 
-        output = [
+        test_cases = [
             # JSON
             {
                 'format': 'json',
@@ -465,14 +468,8 @@ class FeedExportTest(unittest.TestCase):
                 'indent': -1,
                 'expected': b"""
 [
-{
-"foo": [
-"bar"
-]
-},
-{
-"key": "value"
-}
+{"foo": ["bar"]},
+{"key": "value"}
 ]
 """,
             },
@@ -481,14 +478,8 @@ class FeedExportTest(unittest.TestCase):
                 'indent': 0,
                 'expected': b"""
 [
-{
-"foo": [
-"bar"
-]
-},
-{
-"key": "value"
-}
+{"foo": ["bar"]},
+{"key": "value"}
 ]
 """,
             },
@@ -542,7 +533,9 @@ class FeedExportTest(unittest.TestCase):
             {
                 'format': 'xml',
                 'indent': None,
-                'expected': b'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo><value>bar</value></foo></item><item><key>value</key></item></items>',
+                'expected': b"""
+<?xml version="1.0" encoding="utf-8"?>
+<items><item><foo><value>bar</value></foo></item><item><key>value</key></item></items>""",
             },
             {
                 'format': 'xml',
@@ -550,14 +543,8 @@ class FeedExportTest(unittest.TestCase):
                 'expected': b"""
 <?xml version="1.0" encoding="utf-8"?>
 <items>
-<item>
-<foo>
-<value>bar</value>
-</foo>
-</item>
-<item>
-<key>value</key>
-</item>
+<item><foo><value>bar</value></foo></item>
+<item><key>value</key></item>
 </items>""",
             },
             {
@@ -566,14 +553,8 @@ class FeedExportTest(unittest.TestCase):
                 'expected': b"""
 <?xml version="1.0" encoding="utf-8"?>
 <items>
-<item>
-<foo>
-<value>bar</value>
-</foo>
-</item>
-<item>
-<key>value</key>
-</item>
+<item><foo><value>bar</value></foo></item>
+<item><key>value</key></item>
 </items>""",
             },
             {
@@ -626,7 +607,8 @@ class FeedExportTest(unittest.TestCase):
             },
         ]
 
-        for row in output:
+        for row in test_cases:
             settings = {'FEED_FORMAT': row['format'], 'FEED_EXPORT_INDENT': row['indent']}
             data = yield self.exported_data(items, settings)
+            print(row['format'], row['indent'])
             self.assertEqual(row['expected'].strip(), data)