RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Feed exports: enforce difference between None and 0 on indent 

Also rename params and settings from "indent_width" to just "indent"
This commit is contained in:
Eugenio Lacuesta 2017-02-23 10:21:33 -03:00
parent 7e9153b38d
commit 766b2c8453
6 changed files with 137 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
docs/topics/exporters.rst
View File

@ -140,7 +140,7 @@ output examples, which assume you're exporting these two items::
BaseItemExporter
----------------
.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent_width=None)
.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=None)
This is the (abstract) base class for all Item Exporters. It provides
support for common features used by all (concrete) Item Exporters, such as
@ -149,7 +149,7 @@ BaseItemExporter
These features can be configured through the constructor arguments which
populate their respective instance attributes: :attr:`fields_to_export`,
:attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent_width`.
:attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent`.
.. method:: export_item(item)
@ -216,10 +216,14 @@ BaseItemExporter
encoding). Other value types are passed unchanged to the specific
serialization library.
.. attribute:: indent_width
.. attribute:: indent
Amount of spaces used to indent the output on each level.
Defaults to ``None``, which disables indentation.
Amount of spaces used to indent the output on each level. Defaults to ``None``,
which disables indentation. This argument behaves like ``indent`` in python's
JSON module (both for JSON and XML exporters): "If ``indent`` is a non-negative
integer, then array elements and object members will be pretty-printed with that
indent level. An indent level of 0, or negative, will only insert newlines.
``None`` (the default) selects the most compact representation"
.. highlight:: none

14
docs/topics/feed-exports.rst
View File

@ -209,7 +209,7 @@ These are the settings used for configuring the feed exports:
* :setting:`FEED_STORE_EMPTY`
* :setting:`FEED_EXPORT_ENCODING`
* :setting:`FEED_EXPORT_FIELDS`
* :setting:`FEED_EXPORT_INDENT_WIDTH`
* :setting:`FEED_EXPORT_INDENT`
.. currentmodule:: scrapy.extensions.feedexport
@ -267,15 +267,17 @@ If an exporter requires a fixed set of fields (this is the case for
is empty or None, then Scrapy tries to infer field names from the
exported data - currently it uses field names from the first item.
.. setting:: FEED_EXPORT_INDENT_WIDTH
.. setting:: FEED_EXPORT_INDENT
FEED_EXPORT_INDENT_WIDTH
------------------------
FEED_EXPORT_INDENT
------------------
Default: ``None``
Amount of spaces to indent on each level.
Set to `None` to disable indentation.
Amount of spaces used to indent the output on each level. If ``FEED_EXPORT_INDENT``
is a non-negative integer, then array elements and object members will be pretty-printed
with that indent level. An indent level of 0, or negative, will only insert newlines.
``None`` (the default) selects the most compact representation
Currently used by :class:`~scrapy.exporters.JsonItemExporter`
and :class:`~scrapy.exporters.XmlItemExporter`

24
scrapy/exporters.py
View File

@ -36,7 +36,7 @@ class BaseItemExporter(object):
self.encoding = options.pop('encoding', None)
self.fields_to_export = options.pop('fields_to_export', None)
self.export_empty_fields = options.pop('export_empty_fields', False)
self.indent_width = options.pop('indent_width', None)
self.indent = options.pop('indent', None)
if not dont_fail and options:
raise TypeError("Unexpected options: %s" % ', '.join(options.keys()))
@ -100,20 +100,28 @@ class JsonItemExporter(BaseItemExporter):
self._configure(kwargs, dont_fail=True)
self.file = file
kwargs.setdefault('ensure_ascii', not self.encoding)
self.encoder = ScrapyJSONEncoder(indent=self.indent_width, **kwargs)
kwargs.setdefault('indent', self.indent)
self.encoder = ScrapyJSONEncoder(**kwargs)
self.first_item = True
def _beautify_newline(self):
if self.indent is not None:
self.file.write(b'\n')
def start_exporting(self):
self.file.write(b"[\n")
self.file.write(b"[")
self._beautify_newline()
def finish_exporting(self):
self.file.write(b"\n]")
self._beautify_newline()
self.file.write(b"]")
def export_item(self, item):
if self.first_item:
self.first_item = False
else:
self.file.write(b',\n')
self.file.write(b',')
self._beautify_newline()
itemdict = dict(self._get_serialized_fields(item))
data = self.encoder.encode(itemdict)
self.file.write(to_bytes(data, self.encoding))
@ -130,12 +138,12 @@ class XmlItemExporter(BaseItemExporter):
self.xg = XMLGenerator(file, encoding=self.encoding)
def _beautify_newline(self):
if self.indent_width:
if self.indent is not None:
self._xg_characters('\n')
def _beautify_indent(self, depth=1):
if self.indent_width:
self._xg_characters(' ' * self.indent_width * depth)
if self.indent:
self._xg_characters(' ' * self.indent * depth)
def start_exporting(self):
self.xg.startDocument()

6
scrapy/extensions/feedexport.py
View File

@ -172,7 +172,9 @@ class FeedExporter(object):
self.store_empty = settings.getbool('FEED_STORE_EMPTY')
self._exporting = False
self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None
self.indent_width = settings.getint('FEED_EXPORT_INDENT_WIDTH') or None
self.indent = None
if settings.get('FEED_EXPORT_INDENT') is not None:
self.indent = settings.getint('FEED_EXPORT_INDENT')
uripar = settings['FEED_URI_PARAMS']
self._uripar = load_object(uripar) if uripar else lambda x, y: None
@ -189,7 +191,7 @@ class FeedExporter(object):
storage = self._get_storage(uri)
file = storage.open(spider)
exporter = self._get_exporter(file, fields_to_export=self.export_fields,
encoding=self.export_encoding, indent_width=self.indent_width)
encoding=self.export_encoding, indent=self.indent)
if self.store_empty:
exporter.start_exporting()
self._exporting = True

2
scrapy/settings/default_settings.py
View File

@ -161,7 +161,7 @@ FEED_EXPORTERS_BASE = {
'marshal': 'scrapy.exporters.MarshalItemExporter',
'pickle': 'scrapy.exporters.PickleItemExporter',
}
FEED_EXPORT_INDENT_WIDTH = None
FEED_EXPORT_INDENT = None
FILES_STORE_S3_ACL = 'private'

116
tests/test_feedexport.py
View File

@ -319,7 +319,7 @@ class FeedExportTest(unittest.TestCase):
@defer.inlineCallbacks
def test_export_no_items_store_empty(self):
formats = (
('json', b'[\n\n]'),
('json', b'[]'),
('jsonlines', b''),
('xml', b'<?xml version="1.0" encoding="utf-8"?>\n<items></items>'),
('csv', b''),
@ -425,25 +425,25 @@ class FeedExportTest(unittest.TestCase):
header = ['foo']
formats = {
'json': u'[\n{"foo": "Test\\u00d6"}\n]'.encode('utf-8'),
'json': u'[{"foo": "Test\\u00d6"}]'.encode('utf-8'),
'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'),
'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('utf-8'),
'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'),
}
for format, expected in formats.items():
settings = {'FEED_FORMAT': format, 'FEED_EXPORT_INDENT_WIDTH': None}
settings = {'FEED_FORMAT': format, 'FEED_EXPORT_INDENT': None}
data = yield self.exported_data(items, settings)
self.assertEqual(expected, data)
formats = {
'json': u'[\n{"foo": "Test\xd6"}\n]'.encode('latin-1'),
'json': u'[{"foo": "Test\xd6"}]'.encode('latin-1'),
'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'),
'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('latin-1'),
'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'),
}
settings = {'FEED_EXPORT_INDENT_WIDTH': None, 'FEED_EXPORT_ENCODING': 'latin-1'}
settings = {'FEED_EXPORT_INDENT': None, 'FEED_EXPORT_ENCODING': 'latin-1'}
for format, expected in formats.items():
settings['FEED_FORMAT'] = format
data = yield self.exported_data(items, settings)
@ -451,48 +451,89 @@ class FeedExportTest(unittest.TestCase):
@defer.inlineCallbacks
def test_export_indentation(self):
items = [dict({'foo': ['bar']})]
items = [dict({'foo': ['bar']}), dict({'key': 'value'})]
output = [
# JSON
{
'format': 'json',
'indent_width': None,
'expected': b'[\n{"foo": ["bar"]}\n]',
'indent': None,
'expected': b'[{"foo": ["bar"]},{"key": "value"}]',
},
{
'format': 'json',
'indent_width': 2,
'indent': -1,
'expected': b"""
[
{
"foo": [
"bar"
]
},
{
"key": "value"
}
]
""",
},
{
'format': 'json',
'indent': 0,
'expected': b"""
[
{
"foo": [
"bar"
]
},
{
"key": "value"
}
]
""",
},
{
'format': 'json',
'indent': 2,
'expected': b"""
[
{
"foo": [
"bar"
]
},
{
"key": "value"
}
]""",
},
{
'format': 'json',
'indent_width': 4,
'indent': 4,
'expected': b"""
[
{
"foo": [
"bar"
]
},
{
"key": "value"
}
]""",
},
{
'format': 'json',
'indent_width': 5,
'indent': 5,
'expected': b"""
[
{
"foo": [
"bar"
]
},
{
"key": "value"
}
]""",
},
@ -500,12 +541,44 @@ class FeedExportTest(unittest.TestCase):
# XML
{
'format': 'xml',
'indent_width': None,
'expected': b'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo><value>bar</value></foo></item></items>',
'indent': None,
'expected': b'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo><value>bar</value></foo></item><item><key>value</key></item></items>',
},
{
'format': 'xml',
'indent_width': 2,
'indent': -1,
'expected': b"""
<?xml version="1.0" encoding="utf-8"?>
<items>
<item>
<foo>
<value>bar</value>
</foo>
</item>
<item>
<key>value</key>
</item>
</items>""",
},
{
'format': 'xml',
'indent': 0,
'expected': b"""
<?xml version="1.0" encoding="utf-8"?>
<items>
<item>
<foo>
<value>bar</value>
</foo>
</item>
<item>
<key>value</key>
</item>
</items>""",
},
{
'format': 'xml',
'indent': 2,
'expected': b"""
<?xml version="1.0" encoding="utf-8"?>
<items>
@ -514,11 +587,14 @@ class FeedExportTest(unittest.TestCase):
<value>bar</value>
</foo>
</item>
<item>
<key>value</key>
</item>
</items>""",
},
{
'format': 'xml',
'indent_width': 4,
'indent': 4,
'expected': b"""
<?xml version="1.0" encoding="utf-8"?>
<items>
@ -527,11 +603,14 @@ class FeedExportTest(unittest.TestCase):
<value>bar</value>
</foo>
</item>
<item>
<key>value</key>
</item>
</items>""",
},
{
'format': 'xml',
'indent_width': 5,
'indent': 5,
'expected': b"""
<?xml version="1.0" encoding="utf-8"?>
<items>
@ -540,11 +619,14 @@ class FeedExportTest(unittest.TestCase):
<value>bar</value>
</foo>
</item>
<item>
<key>value</key>
</item>
</items>""",
},
]
for row in output:
settings = {'FEED_FORMAT': row['format'], 'FEED_EXPORT_INDENT_WIDTH': row['indent_width']}
settings = {'FEED_FORMAT': row['format'], 'FEED_EXPORT_INDENT': row['indent']}
data = yield self.exported_data(items, settings)
self.assertEqual(row['expected'].strip(), data)