From f1d971a5c0cdfe0f4fe5619146cd6818324fc98e Mon Sep 17 00:00:00 2001 From: stummjr Date: Wed, 27 Jan 2016 14:34:46 -0200 Subject: [PATCH 1/3] fix PythonItemExporter for non-string types --- scrapy/exporters.py | 8 ++++---- tests/test_exporters.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 145468dbe..c7c78d054 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -273,10 +273,10 @@ class PythonItemExporter(BaseItemExporter): return dict(self._serialize_dict(value)) if is_listlike(value): return [self._serialize_value(v) for v in value] - if self.binary: - return to_bytes(value, encoding=self.encoding) - else: - return to_unicode(value, encoding=self.encoding) + encode_func = to_bytes if self.binary else to_unicode + if isinstance(value, (six.text_type, bytes)): + return encode_func(value, encoding=self.encoding) + return value def _serialize_dict(self, value): for key, val in six.iteritems(value): diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 1633e1039..662f8ec5c 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -134,6 +134,19 @@ class PythonItemExporterTest(BaseItemExporterTest): expected = {b'name': b'John\xc2\xa3', b'age': b'22'} self.assertEqual(expected, exporter.export_item(value)) + def test_other_python_types_item(self): + from datetime import datetime + now = datetime.now() + item = { + 'boolean': False, + 'number': 22, + 'time': now, + 'float': 3.14, + } + ie = self._get_exporter() + exported = ie.export_item(item) + self.assertEqual(exported, item) + class PprintItemExporterTest(BaseItemExporterTest): From c55ff110a34d39be27bbd3d03fbf52caa271b4c9 Mon Sep 17 00:00:00 2001 From: stummjr Date: Wed, 27 Jan 2016 15:43:17 -0200 Subject: [PATCH 2/3] Fix CSV exporter for non string Python types. --- scrapy/exporters.py | 2 +- tests/test_exporters.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index c7c78d054..55d74332b 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -200,7 +200,7 @@ class CsvItemExporter(BaseItemExporter): try: yield to_native_str(s) except TypeError: - yield to_native_str(repr(s)) + yield to_native_str(str(s)) def _write_headers_and_set_fields_to_export(self, item): if self.include_headers_line: diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 662f8ec5c..97c09a495 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -271,6 +271,21 @@ class CsvItemExporterTest(BaseItemExporterTest): expected='"[4, 8]",John\r\n', ) + def test_other_python_types_item(self): + from datetime import datetime + now = datetime(2015, 1, 1, 1, 1, 1) + item = { + 'boolean': False, + 'number': 22, + 'time': now, + 'float': 3.14, + } + self.assertExportResult( + item=item, + include_headers_line=False, + expected='22,False,3.14,2015-01-01 01:01:01\r\n' + ) + class XmlItemExporterTest(BaseItemExporterTest): From 27758f60ada4791c044bfe8bc86d267aa930c744 Mon Sep 17 00:00:00 2001 From: stummjr Date: Wed, 27 Jan 2016 16:28:01 -0200 Subject: [PATCH 3/3] Changes fallback for CSVItemExporter, avoiding to call to_native_str(str()). --- scrapy/exporters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 55d74332b..35f50838b 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -200,7 +200,7 @@ class CsvItemExporter(BaseItemExporter): try: yield to_native_str(s) except TypeError: - yield to_native_str(str(s)) + yield s def _write_headers_and_set_fields_to_export(self, item): if self.include_headers_line: