From b746d85f4ca11f7f0149d06f7a4b58501ad3ee23 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Thu, 21 Jan 2016 21:12:43 -0200 Subject: [PATCH] PY3 port csv exporter --- scrapy/exporters.py | 17 ++++++----- tests/test_exporters.py | 67 ++++++----------------------------------- 2 files changed, 20 insertions(+), 64 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index c029ac473..8d7ffbc71 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -3,6 +3,7 @@ Item Exporters are used to export/serialize items into different formats. """ import csv +import io import sys import pprint import marshal @@ -11,7 +12,7 @@ from six.moves import cPickle as pickle from xml.sax.saxutils import XMLGenerator from scrapy.utils.serialize import ScrapyJSONEncoder -from scrapy.utils.python import to_bytes, to_unicode, is_listlike +from scrapy.utils.python import to_bytes, to_unicode, to_native_str, is_listlike from scrapy.item import BaseItem import warnings @@ -166,21 +167,22 @@ class CsvItemExporter(BaseItemExporter): def __init__(self, file, include_headers_line=True, join_multivalued=',', **kwargs): self._configure(kwargs, dont_fail=True) self.include_headers_line = include_headers_line + file = file if six.PY2 else io.TextIOWrapper(file, line_buffering=True) self.csv_writer = csv.writer(file, **kwargs) self._headers_not_written = True self._join_multivalued = join_multivalued def serialize_field(self, field, name, value): - serializer = field.get('serializer', self._to_str_if_unicode) + serializer = field.get('serializer', self._join_if_needed) return serializer(value) - def _to_str_if_unicode(self, value): + def _join_if_needed(self, value): if isinstance(value, (list, tuple)): try: - value = self._join_multivalued.join(value) + return self._join_multivalued.join(value) except TypeError: # list in value may not contain strings pass - return value.encode(self.encoding) if isinstance(value, six.text_type) else value + return value def export_item(self, item): if self._headers_not_written: @@ -189,7 +191,7 @@ class CsvItemExporter(BaseItemExporter): fields = self._get_serialized_fields(item, default_value='', include_empty=True) - values = [x[1] for x in fields] + values = [to_native_str(x) for _, x in fields] self.csv_writer.writerow(values) def _write_headers_and_set_fields_to_export(self, item): @@ -201,7 +203,8 @@ class CsvItemExporter(BaseItemExporter): else: # use fields declared in Item self.fields_to_export = list(item.fields.keys()) - self.csv_writer.writerow(self.fields_to_export) + row = [to_native_str(s) for s in self.fields_to_export] + self.csv_writer.writerow(row) class PickleItemExporter(BaseItemExporter): diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 05374e617..39e996062 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -3,7 +3,6 @@ import re import json import unittest from io import BytesIO -import six from six.moves import cPickle as pickle import lxml.etree @@ -81,53 +80,6 @@ class BaseItemExporterTest(unittest.TestCase): self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24') -class IntermediateRefactoringBaseItemExporterTest(BaseItemExporterTest): - """Class introduced just to keep old behavior of BaseItemExporterTest for the - test cases that inherit from it while we make changes to exporters one by - one -- a needed refactoring trick because the test cases are quite coupled. - - When we're done with the changes, we'll have ditched this class. - """ - def test_serialize_field(self): - if self.ie.__class__ is BaseItemExporter: - return - - res = self.ie.serialize_field(self.i.fields['name'], 'name', self.i['name']) - self.assertEqual(res, 'John\xc2\xa3') - - res = self.ie.serialize_field(self.i.fields['age'], 'age', self.i['age']) - self.assertEqual(res, '22') - - def test_fields_to_export(self): - if self.ie.__class__ is BaseItemExporter: - return - - ie = self._get_exporter(fields_to_export=['name']) - self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', 'John\xc2\xa3')]) - - ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1') - name = list(ie._get_serialized_fields(self.i))[0][1] - assert isinstance(name, str) - self.assertEqual(name, 'John\xa3') - - def test_field_custom_serializer(self): - if self.ie.__class__ is BaseItemExporter: - return - - def custom_serializer(value): - return str(int(value) + 2) - - class CustomFieldItem(Item): - name = Field() - age = Field(serializer=custom_serializer) - - i = CustomFieldItem(name=u'John\xa3', age='22') - - ie = self._get_exporter() - self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), 'John\xc2\xa3') - self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24') - - class PythonItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): return PythonItemExporter(binary=False, **kwargs) @@ -195,19 +147,19 @@ class PickleItemExporterTest(BaseItemExporterTest): self.assertEqual(pickle.load(f), i2) -@unittest.skipUnless(six.PY2, "TODO") -class CsvItemExporterTest(IntermediateRefactoringBaseItemExporterTest): - +class CsvItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): return CsvItemExporter(self.output, **kwargs) def assertCsvEqual(self, first, second, msg=None): + first = to_unicode(first) + second = to_unicode(second) csvsplit = lambda csv: [sorted(re.split(r'(,|\s+)', line)) for line in csv.splitlines(True)] return self.assertEqual(csvsplit(first), csvsplit(second), msg) def _check_output(self): - self.assertCsvEqual(self.output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n') + self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n') def assertExportResult(self, item, expected, **kwargs): fp = BytesIO() @@ -221,13 +173,13 @@ class CsvItemExporterTest(IntermediateRefactoringBaseItemExporterTest): self.assertExportResult( item=self.i, fields_to_export=self.i.fields.keys(), - expected='age,name\r\n22,John\xc2\xa3\r\n', + expected=b'age,name\r\n22,John\xc2\xa3\r\n', ) def test_header_export_all_dict(self): self.assertExportResult( item=dict(self.i), - expected='age,name\r\n22,John\xc2\xa3\r\n', + expected=b'age,name\r\n22,John\xc2\xa3\r\n', ) def test_header_export_single_field(self): @@ -235,7 +187,7 @@ class CsvItemExporterTest(IntermediateRefactoringBaseItemExporterTest): self.assertExportResult( item=item, fields_to_export=['age'], - expected='age\r\n22\r\n', + expected=b'age\r\n22\r\n', ) def test_header_export_two_items(self): @@ -246,14 +198,15 @@ class CsvItemExporterTest(IntermediateRefactoringBaseItemExporterTest): ie.export_item(item) ie.export_item(item) ie.finish_exporting() - self.assertCsvEqual(output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n') + self.assertCsvEqual(output.getvalue(), + b'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n') def test_header_no_header_line(self): for item in [self.i, dict(self.i)]: self.assertExportResult( item=item, include_headers_line=False, - expected='22,John\xc2\xa3\r\n', + expected=b'22,John\xc2\xa3\r\n', ) def test_join_multivalue(self):