diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 4114eda58..ad559fb35 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -140,7 +140,7 @@ output examples, which assume you're exporting these two items:: BaseItemExporter ---------------- -.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent_width=None) +.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=None) This is the (abstract) base class for all Item Exporters. It provides support for common features used by all (concrete) Item Exporters, such as @@ -149,7 +149,7 @@ BaseItemExporter These features can be configured through the constructor arguments which populate their respective instance attributes: :attr:`fields_to_export`, - :attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent_width`. + :attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent`. .. method:: export_item(item) @@ -216,10 +216,14 @@ BaseItemExporter encoding). Other value types are passed unchanged to the specific serialization library. - .. attribute:: indent_width + .. attribute:: indent - Amount of spaces used to indent the output on each level. - Defaults to ``None``, which disables indentation. + Amount of spaces used to indent the output on each level. Defaults to ``None``, + which disables indentation. This argument behaves like ``indent`` in python's + JSON module (both for JSON and XML exporters): "If ``indent`` is a non-negative + integer, then array elements and object members will be pretty-printed with that + indent level. An indent level of 0, or negative, will only insert newlines. + ``None`` (the default) selects the most compact representation" .. highlight:: none diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index ce3b5fd75..afaa972e5 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -209,7 +209,7 @@ These are the settings used for configuring the feed exports: * :setting:`FEED_STORE_EMPTY` * :setting:`FEED_EXPORT_ENCODING` * :setting:`FEED_EXPORT_FIELDS` - * :setting:`FEED_EXPORT_INDENT_WIDTH` + * :setting:`FEED_EXPORT_INDENT` .. currentmodule:: scrapy.extensions.feedexport @@ -267,15 +267,17 @@ If an exporter requires a fixed set of fields (this is the case for is empty or None, then Scrapy tries to infer field names from the exported data - currently it uses field names from the first item. -.. setting:: FEED_EXPORT_INDENT_WIDTH +.. setting:: FEED_EXPORT_INDENT -FEED_EXPORT_INDENT_WIDTH ------------------------- +FEED_EXPORT_INDENT +------------------ Default: ``None`` -Amount of spaces to indent on each level. -Set to `None` to disable indentation. +Amount of spaces used to indent the output on each level. If ``FEED_EXPORT_INDENT`` +is a non-negative integer, then array elements and object members will be pretty-printed +with that indent level. An indent level of 0, or negative, will only insert newlines. +``None`` (the default) selects the most compact representation Currently used by :class:`~scrapy.exporters.JsonItemExporter` and :class:`~scrapy.exporters.XmlItemExporter` diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 69e6c15e0..1dfa2af85 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -36,7 +36,7 @@ class BaseItemExporter(object): self.encoding = options.pop('encoding', None) self.fields_to_export = options.pop('fields_to_export', None) self.export_empty_fields = options.pop('export_empty_fields', False) - self.indent_width = options.pop('indent_width', None) + self.indent = options.pop('indent', None) if not dont_fail and options: raise TypeError("Unexpected options: %s" % ', '.join(options.keys())) @@ -100,20 +100,28 @@ class JsonItemExporter(BaseItemExporter): self._configure(kwargs, dont_fail=True) self.file = file kwargs.setdefault('ensure_ascii', not self.encoding) - self.encoder = ScrapyJSONEncoder(indent=self.indent_width, **kwargs) + kwargs.setdefault('indent', self.indent) + self.encoder = ScrapyJSONEncoder(**kwargs) self.first_item = True + def _beautify_newline(self): + if self.indent is not None: + self.file.write(b'\n') + def start_exporting(self): - self.file.write(b"[\n") + self.file.write(b"[") + self._beautify_newline() def finish_exporting(self): - self.file.write(b"\n]") + self._beautify_newline() + self.file.write(b"]") def export_item(self, item): if self.first_item: self.first_item = False else: - self.file.write(b',\n') + self.file.write(b',') + self._beautify_newline() itemdict = dict(self._get_serialized_fields(item)) data = self.encoder.encode(itemdict) self.file.write(to_bytes(data, self.encoding)) @@ -130,12 +138,12 @@ class XmlItemExporter(BaseItemExporter): self.xg = XMLGenerator(file, encoding=self.encoding) def _beautify_newline(self): - if self.indent_width: + if self.indent is not None: self._xg_characters('\n') def _beautify_indent(self, depth=1): - if self.indent_width: - self._xg_characters(' ' * self.indent_width * depth) + if self.indent: + self._xg_characters(' ' * self.indent * depth) def start_exporting(self): self.xg.startDocument() diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 26024e5e9..5f133fbde 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -172,7 +172,9 @@ class FeedExporter(object): self.store_empty = settings.getbool('FEED_STORE_EMPTY') self._exporting = False self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None - self.indent_width = settings.getint('FEED_EXPORT_INDENT_WIDTH') or None + self.indent = None + if settings.get('FEED_EXPORT_INDENT') is not None: + self.indent = settings.getint('FEED_EXPORT_INDENT') uripar = settings['FEED_URI_PARAMS'] self._uripar = load_object(uripar) if uripar else lambda x, y: None @@ -189,7 +191,7 @@ class FeedExporter(object): storage = self._get_storage(uri) file = storage.open(spider) exporter = self._get_exporter(file, fields_to_export=self.export_fields, - encoding=self.export_encoding, indent_width=self.indent_width) + encoding=self.export_encoding, indent=self.indent) if self.store_empty: exporter.start_exporting() self._exporting = True diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index cca0d3889..fc265e2ba 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -161,7 +161,7 @@ FEED_EXPORTERS_BASE = { 'marshal': 'scrapy.exporters.MarshalItemExporter', 'pickle': 'scrapy.exporters.PickleItemExporter', } -FEED_EXPORT_INDENT_WIDTH = None +FEED_EXPORT_INDENT = None FILES_STORE_S3_ACL = 'private' diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index bf002bec7..2b82bba0c 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -319,7 +319,7 @@ class FeedExportTest(unittest.TestCase): @defer.inlineCallbacks def test_export_no_items_store_empty(self): formats = ( - ('json', b'[\n\n]'), + ('json', b'[]'), ('jsonlines', b''), ('xml', b'\n'), ('csv', b''), @@ -425,25 +425,25 @@ class FeedExportTest(unittest.TestCase): header = ['foo'] formats = { - 'json': u'[\n{"foo": "Test\\u00d6"}\n]'.encode('utf-8'), + 'json': u'[{"foo": "Test\\u00d6"}]'.encode('utf-8'), 'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'), 'xml': u'\nTest\xd6'.encode('utf-8'), 'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'), } for format, expected in formats.items(): - settings = {'FEED_FORMAT': format, 'FEED_EXPORT_INDENT_WIDTH': None} + settings = {'FEED_FORMAT': format, 'FEED_EXPORT_INDENT': None} data = yield self.exported_data(items, settings) self.assertEqual(expected, data) formats = { - 'json': u'[\n{"foo": "Test\xd6"}\n]'.encode('latin-1'), + 'json': u'[{"foo": "Test\xd6"}]'.encode('latin-1'), 'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'), 'xml': u'\nTest\xd6'.encode('latin-1'), 'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'), } - settings = {'FEED_EXPORT_INDENT_WIDTH': None, 'FEED_EXPORT_ENCODING': 'latin-1'} + settings = {'FEED_EXPORT_INDENT': None, 'FEED_EXPORT_ENCODING': 'latin-1'} for format, expected in formats.items(): settings['FEED_FORMAT'] = format data = yield self.exported_data(items, settings) @@ -451,48 +451,89 @@ class FeedExportTest(unittest.TestCase): @defer.inlineCallbacks def test_export_indentation(self): - items = [dict({'foo': ['bar']})] + items = [dict({'foo': ['bar']}), dict({'key': 'value'})] output = [ # JSON { 'format': 'json', - 'indent_width': None, - 'expected': b'[\n{"foo": ["bar"]}\n]', + 'indent': None, + 'expected': b'[{"foo": ["bar"]},{"key": "value"}]', }, { 'format': 'json', - 'indent_width': 2, + 'indent': -1, + 'expected': b""" +[ +{ +"foo": [ +"bar" +] +}, +{ +"key": "value" +} +] +""", + }, + { + 'format': 'json', + 'indent': 0, + 'expected': b""" +[ +{ +"foo": [ +"bar" +] +}, +{ +"key": "value" +} +] +""", + }, + { + 'format': 'json', + 'indent': 2, 'expected': b""" [ { "foo": [ "bar" ] +}, +{ + "key": "value" } ]""", }, { 'format': 'json', - 'indent_width': 4, + 'indent': 4, 'expected': b""" [ { "foo": [ "bar" ] +}, +{ + "key": "value" } ]""", }, { 'format': 'json', - 'indent_width': 5, + 'indent': 5, 'expected': b""" [ { "foo": [ "bar" ] +}, +{ + "key": "value" } ]""", }, @@ -500,12 +541,44 @@ class FeedExportTest(unittest.TestCase): # XML { 'format': 'xml', - 'indent_width': None, - 'expected': b'\nbar', + 'indent': None, + 'expected': b'\nbarvalue', }, { 'format': 'xml', - 'indent_width': 2, + 'indent': -1, + 'expected': b""" + + + + +bar + + + +value + +""", + }, + { + 'format': 'xml', + 'indent': 0, + 'expected': b""" + + + + +bar + + + +value + +""", + }, + { + 'format': 'xml', + 'indent': 2, 'expected': b""" @@ -514,11 +587,14 @@ class FeedExportTest(unittest.TestCase): bar + + value + """, }, { 'format': 'xml', - 'indent_width': 4, + 'indent': 4, 'expected': b""" @@ -527,11 +603,14 @@ class FeedExportTest(unittest.TestCase): bar + + value + """, }, { 'format': 'xml', - 'indent_width': 5, + 'indent': 5, 'expected': b""" @@ -540,11 +619,14 @@ class FeedExportTest(unittest.TestCase): bar + + value + """, }, ] for row in output: - settings = {'FEED_FORMAT': row['format'], 'FEED_EXPORT_INDENT_WIDTH': row['indent_width']} + settings = {'FEED_FORMAT': row['format'], 'FEED_EXPORT_INDENT': row['indent']} data = yield self.exported_data(items, settings) self.assertEqual(row['expected'].strip(), data)