mirror of https://github.com/scrapy/scrapy.git
PY3 port csv exporter
This commit is contained in:
Elias Dorneles
parent
9f35c28643
commit
b746d85f4c
|
|
@ -3,6 +3,7 @@ Item Exporters are used to export/serialize items into different formats.
|
|||
"""
|
||||
|
||||
import csv
|
||||
import io
|
||||
import sys
|
||||
import pprint
|
||||
import marshal
|
||||
|
|
@ -11,7 +12,7 @@ from six.moves import cPickle as pickle
|
|||
from xml.sax.saxutils import XMLGenerator
|
||||
|
||||
from scrapy.utils.serialize import ScrapyJSONEncoder
|
||||
from scrapy.utils.python import to_bytes, to_unicode, is_listlike
|
||||
from scrapy.utils.python import to_bytes, to_unicode, to_native_str, is_listlike
|
||||
from scrapy.item import BaseItem
|
||||
import warnings
|
||||
|
||||
|
|
@ -166,21 +167,22 @@ class CsvItemExporter(BaseItemExporter):
|
|||
def __init__(self, file, include_headers_line=True, join_multivalued=',', **kwargs):
|
||||
self._configure(kwargs, dont_fail=True)
|
||||
self.include_headers_line = include_headers_line
|
||||
file = file if six.PY2 else io.TextIOWrapper(file, line_buffering=True)
|
||||
self.csv_writer = csv.writer(file, **kwargs)
|
||||
self._headers_not_written = True
|
||||
self._join_multivalued = join_multivalued
|
||||
|
||||
def serialize_field(self, field, name, value):
|
||||
serializer = field.get('serializer', self._to_str_if_unicode)
|
||||
serializer = field.get('serializer', self._join_if_needed)
|
||||
return serializer(value)
|
||||
|
||||
def _to_str_if_unicode(self, value):
|
||||
def _join_if_needed(self, value):
|
||||
if isinstance(value, (list, tuple)):
|
||||
try:
|
||||
value = self._join_multivalued.join(value)
|
||||
return self._join_multivalued.join(value)
|
||||
except TypeError: # list in value may not contain strings
|
||||
pass
|
||||
return value.encode(self.encoding) if isinstance(value, six.text_type) else value
|
||||
return value
|
||||
|
||||
def export_item(self, item):
|
||||
if self._headers_not_written:
|
||||
|
|
@ -189,7 +191,7 @@ class CsvItemExporter(BaseItemExporter):
|
|||
|
||||
fields = self._get_serialized_fields(item, default_value='',
|
||||
include_empty=True)
|
||||
values = [x[1] for x in fields]
|
||||
values = [to_native_str(x) for _, x in fields]
|
||||
self.csv_writer.writerow(values)
|
||||
|
||||
def _write_headers_and_set_fields_to_export(self, item):
|
||||
|
|
@ -201,7 +203,8 @@ class CsvItemExporter(BaseItemExporter):
|
|||
else:
|
||||
# use fields declared in Item
|
||||
self.fields_to_export = list(item.fields.keys())
|
||||
self.csv_writer.writerow(self.fields_to_export)
|
||||
row = [to_native_str(s) for s in self.fields_to_export]
|
||||
self.csv_writer.writerow(row)
|
||||
|
||||
|
||||
class PickleItemExporter(BaseItemExporter):
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ import re
|
|||
import json
|
||||
import unittest
|
||||
from io import BytesIO
|
||||
import six
|
||||
from six.moves import cPickle as pickle
|
||||
|
||||
import lxml.etree
|
||||
|
|
@ -81,53 +80,6 @@ class BaseItemExporterTest(unittest.TestCase):
|
|||
self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24')
|
||||
|
||||
|
||||
class IntermediateRefactoringBaseItemExporterTest(BaseItemExporterTest):
|
||||
"""Class introduced just to keep old behavior of BaseItemExporterTest for the
|
||||
test cases that inherit from it while we make changes to exporters one by
|
||||
one -- a needed refactoring trick because the test cases are quite coupled.
|
||||
|
||||
When we're done with the changes, we'll have ditched this class.
|
||||
"""
|
||||
def test_serialize_field(self):
|
||||
if self.ie.__class__ is BaseItemExporter:
|
||||
return
|
||||
|
||||
res = self.ie.serialize_field(self.i.fields['name'], 'name', self.i['name'])
|
||||
self.assertEqual(res, 'John\xc2\xa3')
|
||||
|
||||
res = self.ie.serialize_field(self.i.fields['age'], 'age', self.i['age'])
|
||||
self.assertEqual(res, '22')
|
||||
|
||||
def test_fields_to_export(self):
|
||||
if self.ie.__class__ is BaseItemExporter:
|
||||
return
|
||||
|
||||
ie = self._get_exporter(fields_to_export=['name'])
|
||||
self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', 'John\xc2\xa3')])
|
||||
|
||||
ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1')
|
||||
name = list(ie._get_serialized_fields(self.i))[0][1]
|
||||
assert isinstance(name, str)
|
||||
self.assertEqual(name, 'John\xa3')
|
||||
|
||||
def test_field_custom_serializer(self):
|
||||
if self.ie.__class__ is BaseItemExporter:
|
||||
return
|
||||
|
||||
def custom_serializer(value):
|
||||
return str(int(value) + 2)
|
||||
|
||||
class CustomFieldItem(Item):
|
||||
name = Field()
|
||||
age = Field(serializer=custom_serializer)
|
||||
|
||||
i = CustomFieldItem(name=u'John\xa3', age='22')
|
||||
|
||||
ie = self._get_exporter()
|
||||
self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), 'John\xc2\xa3')
|
||||
self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24')
|
||||
|
||||
|
||||
class PythonItemExporterTest(BaseItemExporterTest):
|
||||
def _get_exporter(self, **kwargs):
|
||||
return PythonItemExporter(binary=False, **kwargs)
|
||||
|
|
@ -195,19 +147,19 @@ class PickleItemExporterTest(BaseItemExporterTest):
|
|||
self.assertEqual(pickle.load(f), i2)
|
||||
|
||||
|
||||
@unittest.skipUnless(six.PY2, "TODO")
|
||||
class CsvItemExporterTest(IntermediateRefactoringBaseItemExporterTest):
|
||||
|
||||
class CsvItemExporterTest(BaseItemExporterTest):
|
||||
def _get_exporter(self, **kwargs):
|
||||
return CsvItemExporter(self.output, **kwargs)
|
||||
|
||||
def assertCsvEqual(self, first, second, msg=None):
|
||||
first = to_unicode(first)
|
||||
second = to_unicode(second)
|
||||
csvsplit = lambda csv: [sorted(re.split(r'(,|\s+)', line))
|
||||
for line in csv.splitlines(True)]
|
||||
return self.assertEqual(csvsplit(first), csvsplit(second), msg)
|
||||
|
||||
def _check_output(self):
|
||||
self.assertCsvEqual(self.output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n')
|
||||
self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n')
|
||||
|
||||
def assertExportResult(self, item, expected, **kwargs):
|
||||
fp = BytesIO()
|
||||
|
|
@ -221,13 +173,13 @@ class CsvItemExporterTest(IntermediateRefactoringBaseItemExporterTest):
|
|||
self.assertExportResult(
|
||||
item=self.i,
|
||||
fields_to_export=self.i.fields.keys(),
|
||||
expected='age,name\r\n22,John\xc2\xa3\r\n',
|
||||
expected=b'age,name\r\n22,John\xc2\xa3\r\n',
|
||||
)
|
||||
|
||||
def test_header_export_all_dict(self):
|
||||
self.assertExportResult(
|
||||
item=dict(self.i),
|
||||
expected='age,name\r\n22,John\xc2\xa3\r\n',
|
||||
expected=b'age,name\r\n22,John\xc2\xa3\r\n',
|
||||
)
|
||||
|
||||
def test_header_export_single_field(self):
|
||||
|
|
@ -235,7 +187,7 @@ class CsvItemExporterTest(IntermediateRefactoringBaseItemExporterTest):
|
|||
self.assertExportResult(
|
||||
item=item,
|
||||
fields_to_export=['age'],
|
||||
expected='age\r\n22\r\n',
|
||||
expected=b'age\r\n22\r\n',
|
||||
)
|
||||
|
||||
def test_header_export_two_items(self):
|
||||
|
|
@ -246,14 +198,15 @@ class CsvItemExporterTest(IntermediateRefactoringBaseItemExporterTest):
|
|||
ie.export_item(item)
|
||||
ie.export_item(item)
|
||||
ie.finish_exporting()
|
||||
self.assertCsvEqual(output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n')
|
||||
self.assertCsvEqual(output.getvalue(),
|
||||
b'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n')
|
||||
|
||||
def test_header_no_header_line(self):
|
||||
for item in [self.i, dict(self.i)]:
|
||||
self.assertExportResult(
|
||||
item=item,
|
||||
include_headers_line=False,
|
||||
expected='22,John\xc2\xa3\r\n',
|
||||
expected=b'22,John\xc2\xa3\r\n',
|
||||
)
|
||||
|
||||
def test_join_multivalue(self):
|
||||
|
|
|
|||
Loading…
Reference in New Issue