diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 85c73222d..4114eda58 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -140,7 +140,7 @@ output examples, which assume you're exporting these two items:: BaseItemExporter ---------------- -.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8') +.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent_width=None) This is the (abstract) base class for all Item Exporters. It provides support for common features used by all (concrete) Item Exporters, such as @@ -149,7 +149,7 @@ BaseItemExporter These features can be configured through the constructor arguments which populate their respective instance attributes: :attr:`fields_to_export`, - :attr:`export_empty_fields`, :attr:`encoding`. + :attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent_width`. .. method:: export_item(item) @@ -216,6 +216,11 @@ BaseItemExporter encoding). Other value types are passed unchanged to the specific serialization library. + .. attribute:: indent_width + + Amount of spaces used to indent the output on each level. + Defaults to ``None``, which disables indentation. + .. highlight:: none XmlItemExporter diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index efdd8c46b..ce3b5fd75 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -209,6 +209,7 @@ These are the settings used for configuring the feed exports: * :setting:`FEED_STORE_EMPTY` * :setting:`FEED_EXPORT_ENCODING` * :setting:`FEED_EXPORT_FIELDS` + * :setting:`FEED_EXPORT_INDENT_WIDTH` .. currentmodule:: scrapy.extensions.feedexport @@ -266,6 +267,19 @@ If an exporter requires a fixed set of fields (this is the case for is empty or None, then Scrapy tries to infer field names from the exported data - currently it uses field names from the first item. +.. setting:: FEED_EXPORT_INDENT_WIDTH + +FEED_EXPORT_INDENT_WIDTH +------------------------ + +Default: ``None`` + +Amount of spaces to indent on each level. +Set to `None` to disable indentation. + +Currently used by :class:`~scrapy.exporters.JsonItemExporter` +and :class:`~scrapy.exporters.XmlItemExporter` + .. setting:: FEED_STORE_EMPTY FEED_STORE_EMPTY diff --git a/scrapy/exporters.py b/scrapy/exporters.py index c4b1b3476..69e6c15e0 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -36,6 +36,7 @@ class BaseItemExporter(object): self.encoding = options.pop('encoding', None) self.fields_to_export = options.pop('fields_to_export', None) self.export_empty_fields = options.pop('export_empty_fields', False) + self.indent_width = options.pop('indent_width', None) if not dont_fail and options: raise TypeError("Unexpected options: %s" % ', '.join(options.keys())) @@ -99,7 +100,7 @@ class JsonItemExporter(BaseItemExporter): self._configure(kwargs, dont_fail=True) self.file = file kwargs.setdefault('ensure_ascii', not self.encoding) - self.encoder = ScrapyJSONEncoder(**kwargs) + self.encoder = ScrapyJSONEncoder(indent=self.indent_width, **kwargs) self.first_item = True def start_exporting(self): @@ -128,33 +129,52 @@ class XmlItemExporter(BaseItemExporter): self.encoding = 'utf-8' self.xg = XMLGenerator(file, encoding=self.encoding) + def _beautify_newline(self): + if self.indent_width: + self._xg_characters('\n') + + def _beautify_indent(self, depth=1): + if self.indent_width: + self._xg_characters(' ' * self.indent_width * depth) + def start_exporting(self): self.xg.startDocument() self.xg.startElement(self.root_element, {}) + self._beautify_newline() def export_item(self, item): + self._beautify_indent(depth=1) self.xg.startElement(self.item_element, {}) + self._beautify_newline() for name, value in self._get_serialized_fields(item, default_value=''): - self._export_xml_field(name, value) + self._export_xml_field(name, value, depth=2) + self._beautify_indent(depth=1) self.xg.endElement(self.item_element) + self._beautify_newline() def finish_exporting(self): self.xg.endElement(self.root_element) self.xg.endDocument() - def _export_xml_field(self, name, serialized_value): + def _export_xml_field(self, name, serialized_value, depth): + self._beautify_indent(depth=depth) self.xg.startElement(name, {}) if hasattr(serialized_value, 'items'): + self._beautify_newline() for subname, value in serialized_value.items(): - self._export_xml_field(subname, value) + self._export_xml_field(subname, value, depth=depth+1) + self._beautify_indent(depth=depth) elif is_listlike(serialized_value): + self._beautify_newline() for value in serialized_value: - self._export_xml_field('value', value) + self._export_xml_field('value', value, depth=depth+1) + self._beautify_indent(depth=depth) elif isinstance(serialized_value, six.text_type): self._xg_characters(serialized_value) else: self._xg_characters(str(serialized_value)) self.xg.endElement(name) + self._beautify_newline() # Workaround for http://bugs.python.org/issue17606 # Before Python 2.7.4 xml.sax.saxutils required bytes; diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 85d328528..26024e5e9 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -172,6 +172,7 @@ class FeedExporter(object): self.store_empty = settings.getbool('FEED_STORE_EMPTY') self._exporting = False self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None + self.indent_width = settings.getint('FEED_EXPORT_INDENT_WIDTH') or None uripar = settings['FEED_URI_PARAMS'] self._uripar = load_object(uripar) if uripar else lambda x, y: None @@ -188,7 +189,7 @@ class FeedExporter(object): storage = self._get_storage(uri) file = storage.open(spider) exporter = self._get_exporter(file, fields_to_export=self.export_fields, - encoding=self.export_encoding) + encoding=self.export_encoding, indent_width=self.indent_width) if self.store_empty: exporter.start_exporting() self._exporting = True diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index d73c595d2..cca0d3889 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -161,6 +161,7 @@ FEED_EXPORTERS_BASE = { 'marshal': 'scrapy.exporters.MarshalItemExporter', 'pickle': 'scrapy.exporters.PickleItemExporter', } +FEED_EXPORT_INDENT_WIDTH = None FILES_STORE_S3_ACL = 'private' diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 2d137edf4..bf002bec7 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -431,10 +431,10 @@ class FeedExportTest(unittest.TestCase): 'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'), } - for format in formats: - settings = {'FEED_FORMAT': format} + for format, expected in formats.items(): + settings = {'FEED_FORMAT': format, 'FEED_EXPORT_INDENT_WIDTH': None} data = yield self.exported_data(items, settings) - self.assertEqual(formats[format], data) + self.assertEqual(expected, data) formats = { 'json': u'[\n{"foo": "Test\xd6"}\n]'.encode('latin-1'), @@ -443,7 +443,108 @@ class FeedExportTest(unittest.TestCase): 'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'), } - for format in formats: - settings = {'FEED_FORMAT': format, 'FEED_EXPORT_ENCODING': 'latin-1'} + settings = {'FEED_EXPORT_INDENT_WIDTH': None, 'FEED_EXPORT_ENCODING': 'latin-1'} + for format, expected in formats.items(): + settings['FEED_FORMAT'] = format data = yield self.exported_data(items, settings) - self.assertEqual(formats[format], data) + self.assertEqual(expected, data) + + @defer.inlineCallbacks + def test_export_indentation(self): + items = [dict({'foo': ['bar']})] + + output = [ + # JSON + { + 'format': 'json', + 'indent_width': None, + 'expected': b'[\n{"foo": ["bar"]}\n]', + }, + { + 'format': 'json', + 'indent_width': 2, + 'expected': b""" +[ +{ + "foo": [ + "bar" + ] +} +]""", + }, + { + 'format': 'json', + 'indent_width': 4, + 'expected': b""" +[ +{ + "foo": [ + "bar" + ] +} +]""", + }, + { + 'format': 'json', + 'indent_width': 5, + 'expected': b""" +[ +{ + "foo": [ + "bar" + ] +} +]""", + }, + + # XML + { + 'format': 'xml', + 'indent_width': None, + 'expected': b'\nbar', + }, + { + 'format': 'xml', + 'indent_width': 2, + 'expected': b""" + + + + + bar + + +""", + }, + { + 'format': 'xml', + 'indent_width': 4, + 'expected': b""" + + + + + bar + + +""", + }, + { + 'format': 'xml', + 'indent_width': 5, + 'expected': b""" + + + + + bar + + +""", + }, + ] + + for row in output: + settings = {'FEED_FORMAT': row['format'], 'FEED_EXPORT_INDENT_WIDTH': row['indent_width']} + data = yield self.exported_data(items, settings) + self.assertEqual(row['expected'].strip(), data)