diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 6f679480d..4138f6192 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -11,7 +11,10 @@ from six.moves import cPickle as pickle
from xml.sax.saxutils import XMLGenerator
from scrapy.utils.serialize import ScrapyJSONEncoder
+from scrapy.utils.python import to_bytes, to_unicode
from scrapy.item import BaseItem
+import warnings
+
__all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter',
'CsvItemExporter', 'XmlItemExporter', 'JsonLinesItemExporter',
@@ -83,7 +86,7 @@ class JsonLinesItemExporter(BaseItemExporter):
def export_item(self, item):
itemdict = dict(self._get_serialized_fields(item))
- self.file.write(self.encoder.encode(itemdict) + '\n')
+ self.file.write(to_bytes(self.encoder.encode(itemdict) + '\n'))
class JsonItemExporter(BaseItemExporter):
@@ -95,18 +98,18 @@ class JsonItemExporter(BaseItemExporter):
self.first_item = True
def start_exporting(self):
- self.file.write("[")
+ self.file.write(b"[")
def finish_exporting(self):
- self.file.write("]")
+ self.file.write(b"]")
def export_item(self, item):
if self.first_item:
self.first_item = False
else:
- self.file.write(',\n')
+ self.file.write(b',\n')
itemdict = dict(self._get_serialized_fields(item))
- self.file.write(self.encoder.encode(itemdict))
+ self.file.write(to_bytes(self.encoder.encode(itemdict)))
class XmlItemExporter(BaseItemExporter):
@@ -136,8 +139,9 @@ class XmlItemExporter(BaseItemExporter):
if hasattr(serialized_value, 'items'):
for subname, value in serialized_value.items():
self._export_xml_field(subname, value)
- elif hasattr(serialized_value, '__iter__'):
- for value in serialized_value:
+ elif (hasattr(serialized_value, '__iter__')
+ and not isinstance(serialized_value, six.string_types)):
+ for value in serialized_value:
self._export_xml_field('value', value)
else:
self._xg_characters(serialized_value)
@@ -150,7 +154,7 @@ class XmlItemExporter(BaseItemExporter):
# and Python 3.x will require unicode, so ">= 2.7.4" should be fine.
if sys.version_info[:3] >= (2, 7, 4):
def _xg_characters(self, serialized_value):
- if not isinstance(serialized_value, unicode):
+ if not isinstance(serialized_value, six.text_type):
serialized_value = serialized_value.decode(self.encoding)
return self.xg.characters(serialized_value)
else:
@@ -177,7 +181,7 @@ class CsvItemExporter(BaseItemExporter):
value = self._join_multivalued.join(value)
except TypeError: # list in value may not contain strings
pass
- return value.encode(self.encoding) if isinstance(value, unicode) else value
+ return value.encode(self.encoding) if isinstance(value, six.text_type) else value
def export_item(self, item):
if self._headers_not_written:
@@ -231,7 +235,7 @@ class PprintItemExporter(BaseItemExporter):
def export_item(self, item):
itemdict = dict(self._get_serialized_fields(item))
- self.file.write(pprint.pformat(itemdict) + '\n')
+ self.file.write(to_bytes(pprint.pformat(itemdict) + '\n'))
class PythonItemExporter(BaseItemExporter):
@@ -240,6 +244,13 @@ class PythonItemExporter(BaseItemExporter):
json, msgpack, binc, etc) can be used on top of it. Its main goal is to
seamless support what BaseItemExporter does plus nested items.
"""
+ def _configure(self, options, dont_fail=False):
+ self.binary = options.pop('binary', True)
+ super(PythonItemExporter, self)._configure(options, dont_fail)
+ if self.binary:
+ warnings.warn(
+ "PythonItemExporter will drop support for binary export in the future",
+ PendingDeprecationWarning)
def serialize_field(self, field, name, value):
serializer = field.get('serializer', self._serialize_value)
@@ -250,9 +261,13 @@ class PythonItemExporter(BaseItemExporter):
return self.export_item(value)
if isinstance(value, dict):
return dict(self._serialize_dict(value))
- if hasattr(value, '__iter__'):
+ if hasattr(value, '__iter__') \
+ and not isinstance(value, six.string_types):
return [self._serialize_value(v) for v in value]
- return value.encode(self.encoding) if isinstance(value, unicode) else value
+ if self.binary:
+ return to_bytes(value, encoding=self.encoding)
+ else:
+ return to_unicode(value, encoding=self.encoding)
def _serialize_dict(self, value):
for key, val in six.iteritems(value):
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index c84fb978a..05374e617 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -3,6 +3,7 @@ import re
import json
import unittest
from io import BytesIO
+import six
from six.moves import cPickle as pickle
import lxml.etree
@@ -80,7 +81,7 @@ class BaseItemExporterTest(unittest.TestCase):
self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24')
-class MidRefactoringBaseItemExporterTest(BaseItemExporterTest):
+class IntermediateRefactoringBaseItemExporterTest(BaseItemExporterTest):
"""Class introduced just to keep old behavior of BaseItemExporterTest for the
test cases that inherit from it while we make changes to exporters one by
one -- a needed refactoring trick because the test cases are quite coupled.
@@ -127,9 +128,9 @@ class MidRefactoringBaseItemExporterTest(BaseItemExporterTest):
self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24')
-class PythonItemExporterTest(MidRefactoringBaseItemExporterTest):
+class PythonItemExporterTest(BaseItemExporterTest):
def _get_exporter(self, **kwargs):
- return PythonItemExporter(**kwargs)
+ return PythonItemExporter(binary=False, **kwargs)
def test_nested_item(self):
i1 = TestItem(name=u'Joseph', age='22')
@@ -194,7 +195,8 @@ class PickleItemExporterTest(BaseItemExporterTest):
self.assertEqual(pickle.load(f), i2)
-class CsvItemExporterTest(MidRefactoringBaseItemExporterTest):
+@unittest.skipUnless(six.PY2, "TODO")
+class CsvItemExporterTest(IntermediateRefactoringBaseItemExporterTest):
def _get_exporter(self, **kwargs):
return CsvItemExporter(self.output, **kwargs)
@@ -294,13 +296,13 @@ class XmlItemExporterTest(BaseItemExporterTest):
self.assertXmlEquivalent(fp.getvalue(), expected_value)
def _check_output(self):
- expected_value = '\n- 22John\xc2\xa3
'
+ expected_value = u'\n- 22John\xa3
'
self.assertXmlEquivalent(self.output.getvalue(), expected_value)
def test_multivalued_fields(self):
self.assertExportResult(
TestItem(name=[u'John\xa3', u'Doe']),
- '\n- John\xc2\xa3Doe
'
+ u'\n- John\xa3Doe
'
)
def test_nested_item(self):
@@ -309,19 +311,19 @@ class XmlItemExporterTest(BaseItemExporterTest):
i3 = TestItem(name=u'buz', age=i2)
self.assertExportResult(i3,
- '\n'
- ''
- '- '
- ''
- ''
- '22'
- 'foo\xc2\xa3hoo'
- ''
- 'bar'
- ''
- 'buz'
- '
'
- ''
+ u'\n'
+ u''
+ u'- '
+ u''
+ u''
+ u'22'
+ u'foo\xa3hoo'
+ u''
+ u'bar'
+ u''
+ u'buz'
+ u'
'
+ u''
)
def test_nested_list_item(self):
@@ -330,16 +332,16 @@ class XmlItemExporterTest(BaseItemExporterTest):
i3 = TestItem(name=u'buz', age=[i1, i2])
self.assertExportResult(i3,
- '\n'
- ''
- '- '
- ''
- 'foo'
- 'barspam'
- ''
- 'buz'
- '
'
- ''
+ u'\n'
+ u''
+ u'- '
+ u''
+ u'foo'
+ u'barspam'
+ u''
+ u'buz'
+ u'
'
+ u''
)
@@ -351,7 +353,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
return JsonLinesItemExporter(self.output, **kwargs)
def _check_output(self):
- exported = json.loads(self.output.getvalue().strip())
+ exported = json.loads(to_unicode(self.output.getvalue().strip()))
self.assertEqual(exported, dict(self.i))
def test_nested_item(self):
@@ -361,7 +363,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
self.ie.start_exporting()
self.ie.export_item(i3)
self.ie.finish_exporting()
- exported = json.loads(self.output.getvalue())
+ exported = json.loads(to_unicode(self.output.getvalue()))
self.assertEqual(exported, self._expected_nested)
def test_extra_keywords(self):
@@ -379,7 +381,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
return JsonItemExporter(self.output, **kwargs)
def _check_output(self):
- exported = json.loads(self.output.getvalue().strip())
+ exported = json.loads(to_unicode(self.output.getvalue().strip()))
self.assertEqual(exported, [dict(self.i)])
def assertTwoItemsExported(self, item):
@@ -387,7 +389,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
self.ie.export_item(item)
self.ie.export_item(item)
self.ie.finish_exporting()
- exported = json.loads(self.output.getvalue())
+ exported = json.loads(to_unicode(self.output.getvalue()))
self.assertEqual(exported, [dict(item), dict(item)])
def test_two_items(self):
@@ -403,7 +405,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
self.ie.start_exporting()
self.ie.export_item(i3)
self.ie.finish_exporting()
- exported = json.loads(self.output.getvalue())
+ exported = json.loads(to_unicode(self.output.getvalue()))
expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': dict(i1)}}
self.assertEqual(exported, [expected])
@@ -414,7 +416,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
self.ie.start_exporting()
self.ie.export_item(i3)
self.ie.finish_exporting()
- exported = json.loads(self.output.getvalue())
+ exported = json.loads(to_unicode(self.output.getvalue()))
expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': i1}}
self.assertEqual(exported, [expected])