RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Feed exports: beautify JSON and XML

This commit is contained in:
Eugenio Lacuesta 2016-12-19 10:43:04 -03:00
parent 802ed30e8e
commit 7e9153b38d
6 changed files with 156 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
docs/topics/exporters.rst
View File

@ -140,7 +140,7 @@ output examples, which assume you're exporting these two items::
BaseItemExporter
----------------
.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8')
.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent_width=None)
This is the (abstract) base class for all Item Exporters. It provides
support for common features used by all (concrete) Item Exporters, such as
@ -149,7 +149,7 @@ BaseItemExporter
These features can be configured through the constructor arguments which
populate their respective instance attributes: :attr:`fields_to_export`,
:attr:`export_empty_fields`, :attr:`encoding`.
:attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent_width`.
.. method:: export_item(item)
@ -216,6 +216,11 @@ BaseItemExporter
encoding). Other value types are passed unchanged to the specific
serialization library.
.. attribute:: indent_width
Amount of spaces used to indent the output on each level.
Defaults to ``None``, which disables indentation.
.. highlight:: none
XmlItemExporter

14
docs/topics/feed-exports.rst
View File

@ -209,6 +209,7 @@ These are the settings used for configuring the feed exports:
* :setting:`FEED_STORE_EMPTY`
* :setting:`FEED_EXPORT_ENCODING`
* :setting:`FEED_EXPORT_FIELDS`
* :setting:`FEED_EXPORT_INDENT_WIDTH`
.. currentmodule:: scrapy.extensions.feedexport
@ -266,6 +267,19 @@ If an exporter requires a fixed set of fields (this is the case for
is empty or None, then Scrapy tries to infer field names from the
exported data - currently it uses field names from the first item.
.. setting:: FEED_EXPORT_INDENT_WIDTH
FEED_EXPORT_INDENT_WIDTH
------------------------
Default: ``None``
Amount of spaces to indent on each level.
Set to `None` to disable indentation.
Currently used by :class:`~scrapy.exporters.JsonItemExporter`
and :class:`~scrapy.exporters.XmlItemExporter`
.. setting:: FEED_STORE_EMPTY
FEED_STORE_EMPTY

30
scrapy/exporters.py
View File

@ -36,6 +36,7 @@ class BaseItemExporter(object):
self.encoding = options.pop('encoding', None)
self.fields_to_export = options.pop('fields_to_export', None)
self.export_empty_fields = options.pop('export_empty_fields', False)
self.indent_width = options.pop('indent_width', None)
if not dont_fail and options:
raise TypeError("Unexpected options: %s" % ', '.join(options.keys()))
@ -99,7 +100,7 @@ class JsonItemExporter(BaseItemExporter):
self._configure(kwargs, dont_fail=True)
self.file = file
kwargs.setdefault('ensure_ascii', not self.encoding)
self.encoder = ScrapyJSONEncoder(**kwargs)
self.encoder = ScrapyJSONEncoder(indent=self.indent_width, **kwargs)
self.first_item = True
def start_exporting(self):
@ -128,33 +129,52 @@ class XmlItemExporter(BaseItemExporter):
self.encoding = 'utf-8'
self.xg = XMLGenerator(file, encoding=self.encoding)
def _beautify_newline(self):
if self.indent_width:
self._xg_characters('\n')
def _beautify_indent(self, depth=1):
if self.indent_width:
self._xg_characters(' ' * self.indent_width * depth)
def start_exporting(self):
self.xg.startDocument()
self.xg.startElement(self.root_element, {})
self._beautify_newline()
def export_item(self, item):
self._beautify_indent(depth=1)
self.xg.startElement(self.item_element, {})
self._beautify_newline()
for name, value in self._get_serialized_fields(item, default_value=''):
self._export_xml_field(name, value)
self._export_xml_field(name, value, depth=2)
self._beautify_indent(depth=1)
self.xg.endElement(self.item_element)
self._beautify_newline()
def finish_exporting(self):
self.xg.endElement(self.root_element)
self.xg.endDocument()
def _export_xml_field(self, name, serialized_value):
def _export_xml_field(self, name, serialized_value, depth):
self._beautify_indent(depth=depth)
self.xg.startElement(name, {})
if hasattr(serialized_value, 'items'):
self._beautify_newline()
for subname, value in serialized_value.items():
self._export_xml_field(subname, value)
self._export_xml_field(subname, value, depth=depth+1)
self._beautify_indent(depth=depth)
elif is_listlike(serialized_value):
self._beautify_newline()
for value in serialized_value:
self._export_xml_field('value', value)
self._export_xml_field('value', value, depth=depth+1)
self._beautify_indent(depth=depth)
elif isinstance(serialized_value, six.text_type):
self._xg_characters(serialized_value)
else:
self._xg_characters(str(serialized_value))
self.xg.endElement(name)
self._beautify_newline()
# Workaround for http://bugs.python.org/issue17606
# Before Python 2.7.4 xml.sax.saxutils required bytes;

3
scrapy/extensions/feedexport.py
View File

@ -172,6 +172,7 @@ class FeedExporter(object):
self.store_empty = settings.getbool('FEED_STORE_EMPTY')
self._exporting = False
self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None
self.indent_width = settings.getint('FEED_EXPORT_INDENT_WIDTH') or None
uripar = settings['FEED_URI_PARAMS']
self._uripar = load_object(uripar) if uripar else lambda x, y: None
@ -188,7 +189,7 @@ class FeedExporter(object):
storage = self._get_storage(uri)
file = storage.open(spider)
exporter = self._get_exporter(file, fields_to_export=self.export_fields,
encoding=self.export_encoding)
encoding=self.export_encoding, indent_width=self.indent_width)
if self.store_empty:
exporter.start_exporting()
self._exporting = True

1
scrapy/settings/default_settings.py
View File

@ -161,6 +161,7 @@ FEED_EXPORTERS_BASE = {
'marshal': 'scrapy.exporters.MarshalItemExporter',
'pickle': 'scrapy.exporters.PickleItemExporter',
}
FEED_EXPORT_INDENT_WIDTH = None
FILES_STORE_S3_ACL = 'private'

113
tests/test_feedexport.py
View File

@ -431,10 +431,10 @@ class FeedExportTest(unittest.TestCase):
'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'),
}
for format in formats:
settings = {'FEED_FORMAT': format}
for format, expected in formats.items():
settings = {'FEED_FORMAT': format, 'FEED_EXPORT_INDENT_WIDTH': None}
data = yield self.exported_data(items, settings)
self.assertEqual(formats[format], data)
self.assertEqual(expected, data)
formats = {
'json': u'[\n{"foo": "Test\xd6"}\n]'.encode('latin-1'),
@ -443,7 +443,108 @@ class FeedExportTest(unittest.TestCase):
'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'),
}
for format in formats:
settings = {'FEED_FORMAT': format, 'FEED_EXPORT_ENCODING': 'latin-1'}
settings = {'FEED_EXPORT_INDENT_WIDTH': None, 'FEED_EXPORT_ENCODING': 'latin-1'}
for format, expected in formats.items():
settings['FEED_FORMAT'] = format
data = yield self.exported_data(items, settings)
self.assertEqual(formats[format], data)
self.assertEqual(expected, data)
@defer.inlineCallbacks
def test_export_indentation(self):
items = [dict({'foo': ['bar']})]
output = [
# JSON
{
'format': 'json',
'indent_width': None,
'expected': b'[\n{"foo": ["bar"]}\n]',
},
{
'format': 'json',
'indent_width': 2,
'expected': b"""
[
{
"foo": [
"bar"
]
}
]""",
},
{
'format': 'json',
'indent_width': 4,
'expected': b"""
[
{
"foo": [
"bar"
]
}
]""",
},
{
'format': 'json',
'indent_width': 5,
'expected': b"""
[
{
"foo": [
"bar"
]
}
]""",
},
# XML
{
'format': 'xml',
'indent_width': None,
'expected': b'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo><value>bar</value></foo></item></items>',
},
{
'format': 'xml',
'indent_width': 2,
'expected': b"""
<?xml version="1.0" encoding="utf-8"?>
<items>
<item>
<foo>
<value>bar</value>
</foo>
</item>
</items>""",
},
{
'format': 'xml',
'indent_width': 4,
'expected': b"""
<?xml version="1.0" encoding="utf-8"?>
<items>
<item>
<foo>
<value>bar</value>
</foo>
</item>
</items>""",
},
{
'format': 'xml',
'indent_width': 5,
'expected': b"""
<?xml version="1.0" encoding="utf-8"?>
<items>
<item>
<foo>
<value>bar</value>
</foo>
</item>
</items>""",
},
]
for row in output:
settings = {'FEED_FORMAT': row['format'], 'FEED_EXPORT_INDENT_WIDTH': row['indent_width']}
data = yield self.exported_data(items, settings)
self.assertEqual(row['expected'].strip(), data)