RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

PY3: ported json(lines), xml exporters

This commit is contained in:
Elias Dorneles 2016-01-21 18:24:06 -02:00
parent b6ef1f19fd
commit c76190d491
2 changed files with 64 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
scrapy/exporters.py
View File

@ -11,7 +11,10 @@ from six.moves import cPickle as pickle
from xml.sax.saxutils import XMLGenerator
from scrapy.utils.serialize import ScrapyJSONEncoder
from scrapy.utils.python import to_bytes, to_unicode
from scrapy.item import BaseItem
import warnings
__all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter',
'CsvItemExporter', 'XmlItemExporter', 'JsonLinesItemExporter',
@ -83,7 +86,7 @@ class JsonLinesItemExporter(BaseItemExporter):
def export_item(self, item):
itemdict = dict(self._get_serialized_fields(item))
self.file.write(self.encoder.encode(itemdict) + '\n')
self.file.write(to_bytes(self.encoder.encode(itemdict) + '\n'))
class JsonItemExporter(BaseItemExporter):
@ -95,18 +98,18 @@ class JsonItemExporter(BaseItemExporter):
self.first_item = True
def start_exporting(self):
self.file.write("[")
self.file.write(b"[")
def finish_exporting(self):
self.file.write("]")
self.file.write(b"]")
def export_item(self, item):
if self.first_item:
self.first_item = False
else:
self.file.write(',\n')
self.file.write(b',\n')
itemdict = dict(self._get_serialized_fields(item))
self.file.write(self.encoder.encode(itemdict))
self.file.write(to_bytes(self.encoder.encode(itemdict)))
class XmlItemExporter(BaseItemExporter):
@ -136,8 +139,9 @@ class XmlItemExporter(BaseItemExporter):
if hasattr(serialized_value, 'items'):
for subname, value in serialized_value.items():
self._export_xml_field(subname, value)
elif hasattr(serialized_value, '__iter__'):
for value in serialized_value:
elif (hasattr(serialized_value, '__iter__')
and not isinstance(serialized_value, six.string_types)):
for value in serialized_value:
self._export_xml_field('value', value)
else:
self._xg_characters(serialized_value)
@ -150,7 +154,7 @@ class XmlItemExporter(BaseItemExporter):
# and Python 3.x will require unicode, so ">= 2.7.4" should be fine.
if sys.version_info[:3] >= (2, 7, 4):
def _xg_characters(self, serialized_value):
if not isinstance(serialized_value, unicode):
if not isinstance(serialized_value, six.text_type):
serialized_value = serialized_value.decode(self.encoding)
return self.xg.characters(serialized_value)
else:
@ -177,7 +181,7 @@ class CsvItemExporter(BaseItemExporter):
value = self._join_multivalued.join(value)
except TypeError: # list in value may not contain strings
pass
return value.encode(self.encoding) if isinstance(value, unicode) else value
return value.encode(self.encoding) if isinstance(value, six.text_type) else value
def export_item(self, item):
if self._headers_not_written:
@ -231,7 +235,7 @@ class PprintItemExporter(BaseItemExporter):
def export_item(self, item):
itemdict = dict(self._get_serialized_fields(item))
self.file.write(pprint.pformat(itemdict) + '\n')
self.file.write(to_bytes(pprint.pformat(itemdict) + '\n'))
class PythonItemExporter(BaseItemExporter):
@ -240,6 +244,13 @@ class PythonItemExporter(BaseItemExporter):
json, msgpack, binc, etc) can be used on top of it. Its main goal is to
seamless support what BaseItemExporter does plus nested items.
"""
def _configure(self, options, dont_fail=False):
self.binary = options.pop('binary', True)
super(PythonItemExporter, self)._configure(options, dont_fail)
if self.binary:
warnings.warn(
"PythonItemExporter will drop support for binary export in the future",
PendingDeprecationWarning)
def serialize_field(self, field, name, value):
serializer = field.get('serializer', self._serialize_value)
@ -250,9 +261,13 @@ class PythonItemExporter(BaseItemExporter):
return self.export_item(value)
if isinstance(value, dict):
return dict(self._serialize_dict(value))
if hasattr(value, '__iter__'):
if hasattr(value, '__iter__') \
and not isinstance(value, six.string_types):
return [self._serialize_value(v) for v in value]
return value.encode(self.encoding) if isinstance(value, unicode) else value
if self.binary:
return to_bytes(value, encoding=self.encoding)
else:
return to_unicode(value, encoding=self.encoding)
def _serialize_dict(self, value):
for key, val in six.iteritems(value):

72
tests/test_exporters.py
View File

@ -3,6 +3,7 @@ import re
import json
import unittest
from io import BytesIO
import six
from six.moves import cPickle as pickle
import lxml.etree
@ -80,7 +81,7 @@ class BaseItemExporterTest(unittest.TestCase):
self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24')
class MidRefactoringBaseItemExporterTest(BaseItemExporterTest):
class IntermediateRefactoringBaseItemExporterTest(BaseItemExporterTest):
"""Class introduced just to keep old behavior of BaseItemExporterTest for the
test cases that inherit from it while we make changes to exporters one by
one -- a needed refactoring trick because the test cases are quite coupled.
@ -127,9 +128,9 @@ class MidRefactoringBaseItemExporterTest(BaseItemExporterTest):
self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24')
class PythonItemExporterTest(MidRefactoringBaseItemExporterTest):
class PythonItemExporterTest(BaseItemExporterTest):
def _get_exporter(self, **kwargs):
return PythonItemExporter(**kwargs)
return PythonItemExporter(binary=False, **kwargs)
def test_nested_item(self):
i1 = TestItem(name=u'Joseph', age='22')
@ -194,7 +195,8 @@ class PickleItemExporterTest(BaseItemExporterTest):
self.assertEqual(pickle.load(f), i2)
class CsvItemExporterTest(MidRefactoringBaseItemExporterTest):
@unittest.skipUnless(six.PY2, "TODO")
class CsvItemExporterTest(IntermediateRefactoringBaseItemExporterTest):
def _get_exporter(self, **kwargs):
return CsvItemExporter(self.output, **kwargs)
@ -294,13 +296,13 @@ class XmlItemExporterTest(BaseItemExporterTest):
self.assertXmlEquivalent(fp.getvalue(), expected_value)
def _check_output(self):
expected_value = '<?xml version="1.0" encoding="utf-8"?>\n<items><item><age>22</age><name>John\xc2\xa3</name></item></items>'
expected_value = u'<?xml version="1.0" ?>\n<items><item><age>22</age><name>John\xa3</name></item></items>'
self.assertXmlEquivalent(self.output.getvalue(), expected_value)
def test_multivalued_fields(self):
self.assertExportResult(
TestItem(name=[u'John\xa3', u'Doe']),
'<?xml version="1.0" encoding="utf-8"?>\n<items><item><name><value>John\xc2\xa3</value><value>Doe</value></name></item></items>'
u'<?xml version="1.0" ?>\n<items><item><name><value>John\xa3</value><value>Doe</value></name></item></items>'
)
def test_nested_item(self):
@ -309,19 +311,19 @@ class XmlItemExporterTest(BaseItemExporterTest):
i3 = TestItem(name=u'buz', age=i2)
self.assertExportResult(i3,
'<?xml version="1.0" encoding="utf-8"?>\n'
'<items>'
'<item>'
'<age>'
'<age>'
'<age>22</age>'
'<name>foo\xc2\xa3hoo</name>'
'</age>'
'<name>bar</name>'
'</age>'
'<name>buz</name>'
'</item>'
'</items>'
u'<?xml version="1.0" ?>\n'
u'<items>'
u'<item>'
u'<age>'
u'<age>'
u'<age>22</age>'
u'<name>foo\xa3hoo</name>'
u'</age>'
u'<name>bar</name>'
u'</age>'
u'<name>buz</name>'
u'</item>'
u'</items>'
)
def test_nested_list_item(self):
@ -330,16 +332,16 @@ class XmlItemExporterTest(BaseItemExporterTest):
i3 = TestItem(name=u'buz', age=[i1, i2])
self.assertExportResult(i3,
'<?xml version="1.0" encoding="utf-8"?>\n'
'<items>'
'<item>'
'<age>'
'<value><name>foo</name></value>'
'<value><name>bar</name><v2><egg><value>spam</value></egg></v2></value>'
'</age>'
'<name>buz</name>'
'</item>'
'</items>'
u'<?xml version="1.0" ?>\n'
u'<items>'
u'<item>'
u'<age>'
u'<value><name>foo</name></value>'
u'<value><name>bar</name><v2><egg><value>spam</value></egg></v2></value>'
u'</age>'
u'<name>buz</name>'
u'</item>'
u'</items>'
)
@ -351,7 +353,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
return JsonLinesItemExporter(self.output, **kwargs)
def _check_output(self):
exported = json.loads(self.output.getvalue().strip())
exported = json.loads(to_unicode(self.output.getvalue().strip()))
self.assertEqual(exported, dict(self.i))
def test_nested_item(self):
@ -361,7 +363,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
self.ie.start_exporting()
self.ie.export_item(i3)
self.ie.finish_exporting()
exported = json.loads(self.output.getvalue())
exported = json.loads(to_unicode(self.output.getvalue()))
self.assertEqual(exported, self._expected_nested)
def test_extra_keywords(self):
@ -379,7 +381,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
return JsonItemExporter(self.output, **kwargs)
def _check_output(self):
exported = json.loads(self.output.getvalue().strip())
exported = json.loads(to_unicode(self.output.getvalue().strip()))
self.assertEqual(exported, [dict(self.i)])
def assertTwoItemsExported(self, item):
@ -387,7 +389,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
self.ie.export_item(item)
self.ie.export_item(item)
self.ie.finish_exporting()
exported = json.loads(self.output.getvalue())
exported = json.loads(to_unicode(self.output.getvalue()))
self.assertEqual(exported, [dict(item), dict(item)])
def test_two_items(self):
@ -403,7 +405,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
self.ie.start_exporting()
self.ie.export_item(i3)
self.ie.finish_exporting()
exported = json.loads(self.output.getvalue())
exported = json.loads(to_unicode(self.output.getvalue()))
expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': dict(i1)}}
self.assertEqual(exported, [expected])
@ -414,7 +416,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
self.ie.start_exporting()
self.ie.export_item(i3)
self.ie.finish_exporting()
exported = json.loads(self.output.getvalue())
exported = json.loads(to_unicode(self.output.getvalue()))
expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': i1}}
self.assertEqual(exported, [expected])