diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 6f679480d..4138f6192 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -11,7 +11,10 @@ from six.moves import cPickle as pickle from xml.sax.saxutils import XMLGenerator from scrapy.utils.serialize import ScrapyJSONEncoder +from scrapy.utils.python import to_bytes, to_unicode from scrapy.item import BaseItem +import warnings + __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter', 'CsvItemExporter', 'XmlItemExporter', 'JsonLinesItemExporter', @@ -83,7 +86,7 @@ class JsonLinesItemExporter(BaseItemExporter): def export_item(self, item): itemdict = dict(self._get_serialized_fields(item)) - self.file.write(self.encoder.encode(itemdict) + '\n') + self.file.write(to_bytes(self.encoder.encode(itemdict) + '\n')) class JsonItemExporter(BaseItemExporter): @@ -95,18 +98,18 @@ class JsonItemExporter(BaseItemExporter): self.first_item = True def start_exporting(self): - self.file.write("[") + self.file.write(b"[") def finish_exporting(self): - self.file.write("]") + self.file.write(b"]") def export_item(self, item): if self.first_item: self.first_item = False else: - self.file.write(',\n') + self.file.write(b',\n') itemdict = dict(self._get_serialized_fields(item)) - self.file.write(self.encoder.encode(itemdict)) + self.file.write(to_bytes(self.encoder.encode(itemdict))) class XmlItemExporter(BaseItemExporter): @@ -136,8 +139,9 @@ class XmlItemExporter(BaseItemExporter): if hasattr(serialized_value, 'items'): for subname, value in serialized_value.items(): self._export_xml_field(subname, value) - elif hasattr(serialized_value, '__iter__'): - for value in serialized_value: + elif (hasattr(serialized_value, '__iter__') + and not isinstance(serialized_value, six.string_types)): + for value in serialized_value: self._export_xml_field('value', value) else: self._xg_characters(serialized_value) @@ -150,7 +154,7 @@ class XmlItemExporter(BaseItemExporter): # and Python 3.x will require unicode, so ">= 2.7.4" should be fine. if sys.version_info[:3] >= (2, 7, 4): def _xg_characters(self, serialized_value): - if not isinstance(serialized_value, unicode): + if not isinstance(serialized_value, six.text_type): serialized_value = serialized_value.decode(self.encoding) return self.xg.characters(serialized_value) else: @@ -177,7 +181,7 @@ class CsvItemExporter(BaseItemExporter): value = self._join_multivalued.join(value) except TypeError: # list in value may not contain strings pass - return value.encode(self.encoding) if isinstance(value, unicode) else value + return value.encode(self.encoding) if isinstance(value, six.text_type) else value def export_item(self, item): if self._headers_not_written: @@ -231,7 +235,7 @@ class PprintItemExporter(BaseItemExporter): def export_item(self, item): itemdict = dict(self._get_serialized_fields(item)) - self.file.write(pprint.pformat(itemdict) + '\n') + self.file.write(to_bytes(pprint.pformat(itemdict) + '\n')) class PythonItemExporter(BaseItemExporter): @@ -240,6 +244,13 @@ class PythonItemExporter(BaseItemExporter): json, msgpack, binc, etc) can be used on top of it. Its main goal is to seamless support what BaseItemExporter does plus nested items. """ + def _configure(self, options, dont_fail=False): + self.binary = options.pop('binary', True) + super(PythonItemExporter, self)._configure(options, dont_fail) + if self.binary: + warnings.warn( + "PythonItemExporter will drop support for binary export in the future", + PendingDeprecationWarning) def serialize_field(self, field, name, value): serializer = field.get('serializer', self._serialize_value) @@ -250,9 +261,13 @@ class PythonItemExporter(BaseItemExporter): return self.export_item(value) if isinstance(value, dict): return dict(self._serialize_dict(value)) - if hasattr(value, '__iter__'): + if hasattr(value, '__iter__') \ + and not isinstance(value, six.string_types): return [self._serialize_value(v) for v in value] - return value.encode(self.encoding) if isinstance(value, unicode) else value + if self.binary: + return to_bytes(value, encoding=self.encoding) + else: + return to_unicode(value, encoding=self.encoding) def _serialize_dict(self, value): for key, val in six.iteritems(value): diff --git a/tests/test_exporters.py b/tests/test_exporters.py index c84fb978a..05374e617 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -3,6 +3,7 @@ import re import json import unittest from io import BytesIO +import six from six.moves import cPickle as pickle import lxml.etree @@ -80,7 +81,7 @@ class BaseItemExporterTest(unittest.TestCase): self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24') -class MidRefactoringBaseItemExporterTest(BaseItemExporterTest): +class IntermediateRefactoringBaseItemExporterTest(BaseItemExporterTest): """Class introduced just to keep old behavior of BaseItemExporterTest for the test cases that inherit from it while we make changes to exporters one by one -- a needed refactoring trick because the test cases are quite coupled. @@ -127,9 +128,9 @@ class MidRefactoringBaseItemExporterTest(BaseItemExporterTest): self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24') -class PythonItemExporterTest(MidRefactoringBaseItemExporterTest): +class PythonItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): - return PythonItemExporter(**kwargs) + return PythonItemExporter(binary=False, **kwargs) def test_nested_item(self): i1 = TestItem(name=u'Joseph', age='22') @@ -194,7 +195,8 @@ class PickleItemExporterTest(BaseItemExporterTest): self.assertEqual(pickle.load(f), i2) -class CsvItemExporterTest(MidRefactoringBaseItemExporterTest): +@unittest.skipUnless(six.PY2, "TODO") +class CsvItemExporterTest(IntermediateRefactoringBaseItemExporterTest): def _get_exporter(self, **kwargs): return CsvItemExporter(self.output, **kwargs) @@ -294,13 +296,13 @@ class XmlItemExporterTest(BaseItemExporterTest): self.assertXmlEquivalent(fp.getvalue(), expected_value) def _check_output(self): - expected_value = '\n22John\xc2\xa3' + expected_value = u'\n22John\xa3' self.assertXmlEquivalent(self.output.getvalue(), expected_value) def test_multivalued_fields(self): self.assertExportResult( TestItem(name=[u'John\xa3', u'Doe']), - '\nJohn\xc2\xa3Doe' + u'\nJohn\xa3Doe' ) def test_nested_item(self): @@ -309,19 +311,19 @@ class XmlItemExporterTest(BaseItemExporterTest): i3 = TestItem(name=u'buz', age=i2) self.assertExportResult(i3, - '\n' - '' - '' - '' - '' - '22' - 'foo\xc2\xa3hoo' - '' - 'bar' - '' - 'buz' - '' - '' + u'\n' + u'' + u'' + u'' + u'' + u'22' + u'foo\xa3hoo' + u'' + u'bar' + u'' + u'buz' + u'' + u'' ) def test_nested_list_item(self): @@ -330,16 +332,16 @@ class XmlItemExporterTest(BaseItemExporterTest): i3 = TestItem(name=u'buz', age=[i1, i2]) self.assertExportResult(i3, - '\n' - '' - '' - '' - 'foo' - 'barspam' - '' - 'buz' - '' - '' + u'\n' + u'' + u'' + u'' + u'foo' + u'barspam' + u'' + u'buz' + u'' + u'' ) @@ -351,7 +353,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): return JsonLinesItemExporter(self.output, **kwargs) def _check_output(self): - exported = json.loads(self.output.getvalue().strip()) + exported = json.loads(to_unicode(self.output.getvalue().strip())) self.assertEqual(exported, dict(self.i)) def test_nested_item(self): @@ -361,7 +363,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() - exported = json.loads(self.output.getvalue()) + exported = json.loads(to_unicode(self.output.getvalue())) self.assertEqual(exported, self._expected_nested) def test_extra_keywords(self): @@ -379,7 +381,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): return JsonItemExporter(self.output, **kwargs) def _check_output(self): - exported = json.loads(self.output.getvalue().strip()) + exported = json.loads(to_unicode(self.output.getvalue().strip())) self.assertEqual(exported, [dict(self.i)]) def assertTwoItemsExported(self, item): @@ -387,7 +389,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.export_item(item) self.ie.export_item(item) self.ie.finish_exporting() - exported = json.loads(self.output.getvalue()) + exported = json.loads(to_unicode(self.output.getvalue())) self.assertEqual(exported, [dict(item), dict(item)]) def test_two_items(self): @@ -403,7 +405,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() - exported = json.loads(self.output.getvalue()) + exported = json.loads(to_unicode(self.output.getvalue())) expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': dict(i1)}} self.assertEqual(exported, [expected]) @@ -414,7 +416,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() - exported = json.loads(self.output.getvalue()) + exported = json.loads(to_unicode(self.output.getvalue())) expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': i1}} self.assertEqual(exported, [expected])