diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 7e1d01a0a..145468dbe 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -3,6 +3,7 @@ Item Exporters are used to export/serialize items into different formats. """ import csv +import io import sys import pprint import marshal @@ -11,7 +12,11 @@ from six.moves import cPickle as pickle from xml.sax.saxutils import XMLGenerator from scrapy.utils.serialize import ScrapyJSONEncoder +from scrapy.utils.python import to_bytes, to_unicode, to_native_str, is_listlike from scrapy.item import BaseItem +from scrapy.exceptions import ScrapyDeprecationWarning +import warnings + __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter', 'CsvItemExporter', 'XmlItemExporter', 'JsonLinesItemExporter', @@ -38,7 +43,7 @@ class BaseItemExporter(object): raise NotImplementedError def serialize_field(self, field, name, value): - serializer = field.get('serializer', self._to_str_if_unicode) + serializer = field.get('serializer', lambda x: x) return serializer(value) def start_exporting(self): @@ -47,9 +52,6 @@ class BaseItemExporter(object): def finish_exporting(self): pass - def _to_str_if_unicode(self, value): - return value.encode(self.encoding) if isinstance(value, unicode) else value - def _get_serialized_fields(self, item, default_value=None, include_empty=None): """Return the fields to export as an iterable of tuples (name, serialized_value) @@ -86,10 +88,10 @@ class JsonLinesItemExporter(BaseItemExporter): def export_item(self, item): itemdict = dict(self._get_serialized_fields(item)) - self.file.write(self.encoder.encode(itemdict) + '\n') + self.file.write(to_bytes(self.encoder.encode(itemdict) + '\n')) -class JsonItemExporter(JsonLinesItemExporter): +class JsonItemExporter(BaseItemExporter): def __init__(self, file, **kwargs): self._configure(kwargs, dont_fail=True) @@ -98,18 +100,18 @@ class JsonItemExporter(JsonLinesItemExporter): self.first_item = True def start_exporting(self): - self.file.write("[") + self.file.write(b"[") def finish_exporting(self): - self.file.write("]") + self.file.write(b"]") def export_item(self, item): if self.first_item: self.first_item = False else: - self.file.write(',\n') + self.file.write(b',\n') itemdict = dict(self._get_serialized_fields(item)) - self.file.write(self.encoder.encode(itemdict)) + self.file.write(to_bytes(self.encoder.encode(itemdict))) class XmlItemExporter(BaseItemExporter): @@ -139,7 +141,7 @@ class XmlItemExporter(BaseItemExporter): if hasattr(serialized_value, 'items'): for subname, value in serialized_value.items(): self._export_xml_field(subname, value) - elif hasattr(serialized_value, '__iter__'): + elif is_listlike(serialized_value): for value in serialized_value: self._export_xml_field('value', value) else: @@ -153,10 +155,10 @@ class XmlItemExporter(BaseItemExporter): # and Python 3.x will require unicode, so ">= 2.7.4" should be fine. if sys.version_info[:3] >= (2, 7, 4): def _xg_characters(self, serialized_value): - if not isinstance(serialized_value, unicode): + if not isinstance(serialized_value, six.text_type): serialized_value = serialized_value.decode(self.encoding) return self.xg.characters(serialized_value) - else: + else: # pragma: no cover def _xg_characters(self, serialized_value): return self.xg.characters(serialized_value) @@ -166,17 +168,22 @@ class CsvItemExporter(BaseItemExporter): def __init__(self, file, include_headers_line=True, join_multivalued=',', **kwargs): self._configure(kwargs, dont_fail=True) self.include_headers_line = include_headers_line + file = file if six.PY2 else io.TextIOWrapper(file, line_buffering=True) self.csv_writer = csv.writer(file, **kwargs) self._headers_not_written = True self._join_multivalued = join_multivalued - def _to_str_if_unicode(self, value): + def serialize_field(self, field, name, value): + serializer = field.get('serializer', self._join_if_needed) + return serializer(value) + + def _join_if_needed(self, value): if isinstance(value, (list, tuple)): try: - value = self._join_multivalued.join(value) + return self._join_multivalued.join(value) except TypeError: # list in value may not contain strings pass - return super(CsvItemExporter, self)._to_str_if_unicode(value) + return value def export_item(self, item): if self._headers_not_written: @@ -185,9 +192,16 @@ class CsvItemExporter(BaseItemExporter): fields = self._get_serialized_fields(item, default_value='', include_empty=True) - values = [x[1] for x in fields] + values = list(self._build_row(x for _, x in fields)) self.csv_writer.writerow(values) + def _build_row(self, values): + for s in values: + try: + yield to_native_str(s) + except TypeError: + yield to_native_str(repr(s)) + def _write_headers_and_set_fields_to_export(self, item): if self.include_headers_line: if not self.fields_to_export: @@ -197,7 +211,8 @@ class CsvItemExporter(BaseItemExporter): else: # use fields declared in Item self.fields_to_export = list(item.fields.keys()) - self.csv_writer.writerow(self.fields_to_export) + row = list(self._build_row(self.fields_to_export)) + self.csv_writer.writerow(row) class PickleItemExporter(BaseItemExporter): @@ -230,7 +245,7 @@ class PprintItemExporter(BaseItemExporter): def export_item(self, item): itemdict = dict(self._get_serialized_fields(item)) - self.file.write(pprint.pformat(itemdict) + '\n') + self.file.write(to_bytes(pprint.pformat(itemdict) + '\n')) class PythonItemExporter(BaseItemExporter): @@ -239,6 +254,13 @@ class PythonItemExporter(BaseItemExporter): json, msgpack, binc, etc) can be used on top of it. Its main goal is to seamless support what BaseItemExporter does plus nested items. """ + def _configure(self, options, dont_fail=False): + self.binary = options.pop('binary', True) + super(PythonItemExporter, self)._configure(options, dont_fail) + if self.binary: + warnings.warn( + "PythonItemExporter will drop support for binary export in the future", + ScrapyDeprecationWarning) def serialize_field(self, field, name, value): serializer = field.get('serializer', self._serialize_value) @@ -249,13 +271,20 @@ class PythonItemExporter(BaseItemExporter): return self.export_item(value) if isinstance(value, dict): return dict(self._serialize_dict(value)) - if hasattr(value, '__iter__'): + if is_listlike(value): return [self._serialize_value(v) for v in value] - return self._to_str_if_unicode(value) + if self.binary: + return to_bytes(value, encoding=self.encoding) + else: + return to_unicode(value, encoding=self.encoding) def _serialize_dict(self, value): for key, val in six.iteritems(value): + key = to_bytes(key) if self.binary else key yield key, self._serialize_value(val) def export_item(self, item): - return dict(self._get_serialized_fields(item)) + result = dict(self._get_serialized_fields(item)) + if self.binary: + result = dict(self._serialize_dict(result)) + return result diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index fdf077413..6385ad3b1 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -1,4 +1,3 @@ -tests/test_exporters.py tests/test_linkextractors_deprecated.py tests/test_proxy_connect.py diff --git a/tests/test_exporters.py b/tests/test_exporters.py index b24633959..1633e1039 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -1,17 +1,21 @@ from __future__ import absolute_import import re import json +import marshal +import tempfile import unittest from io import BytesIO from six.moves import cPickle as pickle import lxml.etree +import six from scrapy.item import Item, Field from scrapy.utils.python import to_unicode from scrapy.exporters import ( BaseItemExporter, PprintItemExporter, PickleItemExporter, CsvItemExporter, - XmlItemExporter, JsonLinesItemExporter, JsonItemExporter, PythonItemExporter + XmlItemExporter, JsonLinesItemExporter, JsonItemExporter, + PythonItemExporter, MarshalItemExporter ) @@ -23,7 +27,7 @@ class TestItem(Item): class BaseItemExporterTest(unittest.TestCase): def setUp(self): - self.i = TestItem(name=u'John\xa3', age='22') + self.i = TestItem(name=u'John\xa3', age=u'22') self.output = BytesIO() self.ie = self._get_exporter() @@ -56,19 +60,19 @@ class BaseItemExporterTest(unittest.TestCase): def test_serialize_field(self): res = self.ie.serialize_field(self.i.fields['name'], 'name', self.i['name']) - self.assertEqual(res, 'John\xc2\xa3') + self.assertEqual(res, u'John\xa3') res = self.ie.serialize_field(self.i.fields['age'], 'age', self.i['age']) - self.assertEqual(res, '22') + self.assertEqual(res, u'22') def test_fields_to_export(self): ie = self._get_exporter(fields_to_export=['name']) - self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', 'John\xc2\xa3')]) + self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', u'John\xa3')]) ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1') - name = list(ie._get_serialized_fields(self.i))[0][1] - assert isinstance(name, str) - self.assertEqual(name, 'John\xa3') + _, name = list(ie._get_serialized_fields(self.i))[0] + assert isinstance(name, six.text_type) + self.assertEqual(name, u'John\xa3') def test_field_custom_serializer(self): def custom_serializer(value): @@ -78,16 +82,20 @@ class BaseItemExporterTest(unittest.TestCase): name = Field() age = Field(serializer=custom_serializer) - i = CustomFieldItem(name=u'John\xa3', age='22') + i = CustomFieldItem(name=u'John\xa3', age=u'22') ie = self._get_exporter() - self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), 'John\xc2\xa3') + self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), u'John\xa3') self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24') class PythonItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): - return PythonItemExporter(**kwargs) + return PythonItemExporter(binary=False, **kwargs) + + def test_invalid_option(self): + with self.assertRaisesRegexp(TypeError, "Unexpected options: invalid_option"): + PythonItemExporter(invalid_option='something') def test_nested_item(self): i1 = TestItem(name=u'Joseph', age='22') @@ -120,6 +128,12 @@ class PythonItemExporterTest(BaseItemExporterTest): self.assertEqual(type(exported['age'][0]), dict) self.assertEqual(type(exported['age'][0]['age'][0]), dict) + def test_export_binary(self): + exporter = PythonItemExporter(binary=True) + value = TestItem(name=u'John\xa3', age=u'22') + expected = {b'name': b'John\xc2\xa3', b'age': b'22'} + self.assertEqual(expected, exporter.export_item(value)) + class PprintItemExporterTest(BaseItemExporterTest): @@ -152,18 +166,30 @@ class PickleItemExporterTest(BaseItemExporterTest): self.assertEqual(pickle.load(f), i2) -class CsvItemExporterTest(BaseItemExporterTest): +class MarshalItemExporterTest(BaseItemExporterTest): + def _get_exporter(self, **kwargs): + self.output = tempfile.TemporaryFile() + return MarshalItemExporter(self.output, **kwargs) + + def _check_output(self): + self.output.seek(0) + self._assert_expected_item(marshal.load(self.output)) + + +class CsvItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): return CsvItemExporter(self.output, **kwargs) def assertCsvEqual(self, first, second, msg=None): + first = to_unicode(first) + second = to_unicode(second) csvsplit = lambda csv: [sorted(re.split(r'(,|\s+)', line)) for line in csv.splitlines(True)] return self.assertEqual(csvsplit(first), csvsplit(second), msg) def _check_output(self): - self.assertCsvEqual(self.output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n') + self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n') def assertExportResult(self, item, expected, **kwargs): fp = BytesIO() @@ -177,13 +203,13 @@ class CsvItemExporterTest(BaseItemExporterTest): self.assertExportResult( item=self.i, fields_to_export=self.i.fields.keys(), - expected='age,name\r\n22,John\xc2\xa3\r\n', + expected=b'age,name\r\n22,John\xc2\xa3\r\n', ) def test_header_export_all_dict(self): self.assertExportResult( item=dict(self.i), - expected='age,name\r\n22,John\xc2\xa3\r\n', + expected=b'age,name\r\n22,John\xc2\xa3\r\n', ) def test_header_export_single_field(self): @@ -191,7 +217,7 @@ class CsvItemExporterTest(BaseItemExporterTest): self.assertExportResult( item=item, fields_to_export=['age'], - expected='age\r\n22\r\n', + expected=b'age\r\n22\r\n', ) def test_header_export_two_items(self): @@ -202,14 +228,15 @@ class CsvItemExporterTest(BaseItemExporterTest): ie.export_item(item) ie.export_item(item) ie.finish_exporting() - self.assertCsvEqual(output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n') + self.assertCsvEqual(output.getvalue(), + b'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n') def test_header_no_header_line(self): for item in [self.i, dict(self.i)]: self.assertExportResult( item=item, include_headers_line=False, - expected='22,John\xc2\xa3\r\n', + expected=b'22,John\xc2\xa3\r\n', ) def test_join_multivalue(self): @@ -224,6 +251,13 @@ class CsvItemExporterTest(BaseItemExporterTest): expected='"Mary,Paul",John\r\n', ) + def test_join_multivalue_not_strings(self): + self.assertExportResult( + item=dict(name='John', friends=[4, 8]), + include_headers_line=False, + expected='"[4, 8]",John\r\n', + ) + class XmlItemExporterTest(BaseItemExporterTest): @@ -252,13 +286,13 @@ class XmlItemExporterTest(BaseItemExporterTest): self.assertXmlEquivalent(fp.getvalue(), expected_value) def _check_output(self): - expected_value = '\n22John\xc2\xa3' + expected_value = b'\n22John\xc2\xa3' self.assertXmlEquivalent(self.output.getvalue(), expected_value) def test_multivalued_fields(self): self.assertExportResult( TestItem(name=[u'John\xa3', u'Doe']), - '\nJohn\xc2\xa3Doe' + b'\nJohn\xc2\xa3Doe' ) def test_nested_item(self): @@ -267,19 +301,19 @@ class XmlItemExporterTest(BaseItemExporterTest): i3 = TestItem(name=u'buz', age=i2) self.assertExportResult(i3, - '\n' - '' - '' - '' - '' - '22' - 'foo\xc2\xa3hoo' - '' - 'bar' - '' - 'buz' - '' - '' + b'\n' + b'' + b'' + b'' + b'' + b'22' + b'foo\xc2\xa3hoo' + b'' + b'bar' + b'' + b'buz' + b'' + b'' ) def test_nested_list_item(self): @@ -288,16 +322,16 @@ class XmlItemExporterTest(BaseItemExporterTest): i3 = TestItem(name=u'buz', age=[i1, i2]) self.assertExportResult(i3, - '\n' - '' - '' - '' - 'foo' - 'barspam' - '' - 'buz' - '' - '' + b'\n' + b'' + b'' + b'' + b'foo' + b'barspam' + b'' + b'buz' + b'' + b'' ) @@ -309,7 +343,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): return JsonLinesItemExporter(self.output, **kwargs) def _check_output(self): - exported = json.loads(self.output.getvalue().strip()) + exported = json.loads(to_unicode(self.output.getvalue().strip())) self.assertEqual(exported, dict(self.i)) def test_nested_item(self): @@ -319,7 +353,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() - exported = json.loads(self.output.getvalue()) + exported = json.loads(to_unicode(self.output.getvalue())) self.assertEqual(exported, self._expected_nested) def test_extra_keywords(self): @@ -337,7 +371,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): return JsonItemExporter(self.output, **kwargs) def _check_output(self): - exported = json.loads(self.output.getvalue().strip()) + exported = json.loads(to_unicode(self.output.getvalue().strip())) self.assertEqual(exported, [dict(self.i)]) def assertTwoItemsExported(self, item): @@ -345,7 +379,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.export_item(item) self.ie.export_item(item) self.ie.finish_exporting() - exported = json.loads(self.output.getvalue()) + exported = json.loads(to_unicode(self.output.getvalue())) self.assertEqual(exported, [dict(item), dict(item)]) def test_two_items(self): @@ -361,7 +395,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() - exported = json.loads(self.output.getvalue()) + exported = json.loads(to_unicode(self.output.getvalue())) expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': dict(i1)}} self.assertEqual(exported, [expected]) @@ -372,7 +406,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() - exported = json.loads(self.output.getvalue()) + exported = json.loads(to_unicode(self.output.getvalue())) expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': i1}} self.assertEqual(exported, [expected]) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index d6c96ca74..176fd93e3 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -5,7 +5,6 @@ import json from io import BytesIO import tempfile import shutil -import six from six.moves.urllib.parse import urlparse from zope.interface.verify import verifyObject @@ -22,6 +21,7 @@ from scrapy.extensions.feedexport import ( S3FeedStorage, StdoutFeedStorage ) from scrapy.utils.test import assert_aws_environ +from scrapy.utils.python import to_native_str class FileFeedStorageTest(unittest.TestCase): @@ -120,8 +120,6 @@ class StdoutFeedStorageTest(unittest.TestCase): class FeedExportTest(unittest.TestCase): - skip = not six.PY2 - class MyItem(scrapy.Item): foo = scrapy.Field() egg = scrapy.Field() @@ -170,7 +168,7 @@ class FeedExportTest(unittest.TestCase): settings.update({'FEED_FORMAT': 'csv'}) data = yield self.exported_data(items, settings) - reader = csv.DictReader(data.splitlines()) + reader = csv.DictReader(to_native_str(data).splitlines()) got_rows = list(reader) if ordered: self.assertEqual(reader.fieldnames, header) @@ -184,14 +182,57 @@ class FeedExportTest(unittest.TestCase): settings = settings or {} settings.update({'FEED_FORMAT': 'jl'}) data = yield self.exported_data(items, settings) - parsed = [json.loads(line) for line in data.splitlines()] + parsed = [json.loads(to_native_str(line)) for line in data.splitlines()] rows = [{k: v for k, v in row.items() if v} for row in rows] self.assertEqual(rows, parsed) + @defer.inlineCallbacks + def assertExportedXml(self, items, rows, settings=None): + settings = settings or {} + settings.update({'FEED_FORMAT': 'xml'}) + data = yield self.exported_data(items, settings) + rows = [{k: v for k, v in row.items() if v} for row in rows] + import lxml.etree + root = lxml.etree.fromstring(data) + got_rows = [{e.tag: e.text for e in it} for it in root.findall('item')] + self.assertEqual(rows, got_rows) + + def _load_until_eof(self, data, load_func): + bytes_output = BytesIO(data) + result = [] + while True: + try: + result.append(load_func(bytes_output)) + except EOFError: + break + return result + + @defer.inlineCallbacks + def assertExportedPickle(self, items, rows, settings=None): + settings = settings or {} + settings.update({'FEED_FORMAT': 'pickle'}) + data = yield self.exported_data(items, settings) + expected = [{k: v for k, v in row.items() if v} for row in rows] + import pickle + result = self._load_until_eof(data, load_func=pickle.load) + self.assertEqual(expected, result) + + @defer.inlineCallbacks + def assertExportedMarshal(self, items, rows, settings=None): + settings = settings or {} + settings.update({'FEED_FORMAT': 'marshal'}) + data = yield self.exported_data(items, settings) + expected = [{k: v for k, v in row.items() if v} for row in rows] + import marshal + result = self._load_until_eof(data, load_func=marshal.load) + self.assertEqual(expected, result) + @defer.inlineCallbacks def assertExported(self, items, header, rows, settings=None, ordered=True): yield self.assertExportedCsv(items, header, rows, settings, ordered) yield self.assertExportedJsonLines(items, rows, settings) + yield self.assertExportedXml(items, rows, settings) + yield self.assertExportedPickle(items, rows, settings) @defer.inlineCallbacks def test_export_items(self):