From a2d6fa5adc17035cebb8dad4a3bd2020236b4f71 Mon Sep 17 00:00:00 2001 From: maranqz Date: Tue, 25 Aug 2020 13:34:43 +0300 Subject: [PATCH 1/3] Add errors parameter for CsvItemExporter with tests --- scrapy/exporters.py | 5 +++-- tests/test_exporters.py | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 95518b3ac..8cd2077b6 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -195,7 +195,7 @@ class XmlItemExporter(BaseItemExporter): class CsvItemExporter(BaseItemExporter): - def __init__(self, file, include_headers_line=True, join_multivalued=',', **kwargs): + def __init__(self, file, include_headers_line=True, join_multivalued=',', errors=None, **kwargs): super().__init__(dont_fail=True, **kwargs) if not self.encoding: self.encoding = 'utf-8' @@ -205,7 +205,8 @@ class CsvItemExporter(BaseItemExporter): line_buffering=False, write_through=True, encoding=self.encoding, - newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034 + newline='', # Windows needs this https://github.com/scrapy/scrapy/issues/3034 + errors=errors, ) self.csv_writer = csv.writer(self.stream, **self._kwargs) self._headers_not_written = True diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 6c25a0064..ebc477e74 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -355,6 +355,23 @@ class CsvItemExporterTest(BaseItemExporterTest): expected='22,False,3.14,2015-01-01 01:01:01\r\n' ) + def test_errors_default(self): + with self.assertRaises(UnicodeEncodeError): + self.assertExportResult( + item=dict(text=u'W\u0275\u200Brd'), + expected=None, + encoding='windows-1251', + ) + + def test_errors_xmlcharrefreplace(self): + self.assertExportResult( + item=dict(text=u'W\u0275\u200Brd'), + include_headers_line=False, + expected='Wɵ​rd\r\n', + encoding='windows-1251', + errors='xmlcharrefreplace', + ) + class CsvItemExporterDataclassTest(CsvItemExporterTest): item_class = TestDataClass From 560c335c0782ec9a834a83abfa9dfbaa8fd67fed Mon Sep 17 00:00:00 2001 From: maranqz Date: Wed, 26 Aug 2020 14:00:51 +0300 Subject: [PATCH 2/3] Add errors parameter in documentation. --- docs/topics/exporters.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 11ef5b2a6..3f8906326 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -292,7 +292,7 @@ XmlItemExporter CsvItemExporter --------------- -.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', **kwargs) +.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', errors=None, **kwargs) Exports items in CSV format to the given file-like object. If the :attr:`fields_to_export` attribute is set, it will be used to define the @@ -311,6 +311,10 @@ CsvItemExporter multi-valued fields, if found. :type include_headers_line: str + :param errors: The optional string that specifies how encoding and decoding errors are to be handled. + For more information see `documentation `_. + :type errors: str + The additional keyword arguments of this ``__init__`` method are passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the :func:`csv.writer` function, so you can use any :func:`csv.writer` function From 29725e4b58bfe417388b5ce2c2b4f1c587a465da Mon Sep 17 00:00:00 2001 From: maranqz Date: Wed, 26 Aug 2020 14:38:37 +0300 Subject: [PATCH 3/3] Fix tabulation of rst and change documentation link. --- docs/topics/exporters.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 3f8906326..0203def74 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -311,9 +311,10 @@ CsvItemExporter multi-valued fields, if found. :type include_headers_line: str - :param errors: The optional string that specifies how encoding and decoding errors are to be handled. - For more information see `documentation `_. - :type errors: str + :param errors: The optional string that specifies how encoding and decoding + errors are to be handled. For more information see + :class:`io.TextIOWrapper`. + :type errors: str The additional keyword arguments of this ``__init__`` method are passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the