Merge pull request #4755 from maranqz/csv-item-exporter-errors

Add errors parameter for CsvItemExporter with tests

docs/topics/exporters.rst

@@ -292,7 +292,7 @@ XmlItemExporter
 CsvItemExporter
 ---------------
 
-.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', **kwargs)
+.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', errors=None, **kwargs)
 
    Exports items in CSV format to the given file-like object. If the
    :attr:`fields_to_export` attribute is set, it will be used to define the
@@ -311,6 +311,11 @@ CsvItemExporter
       multi-valued fields, if found.
    :type include_headers_line: str
 
+   :param errors: The optional string that specifies how encoding and decoding
+      errors are to be handled. For more information see
+      :class:`io.TextIOWrapper`.
+   :type errors: str
+
    The additional keyword arguments of this ``__init__`` method are passed to the
    :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the
    :func:`csv.writer` function, so you can use any :func:`csv.writer` function

scrapy/exporters.py

@@ -195,7 +195,7 @@ class XmlItemExporter(BaseItemExporter):
 
 class CsvItemExporter(BaseItemExporter):
 
-    def __init__(self, file, include_headers_line=True, join_multivalued=',', **kwargs):
+    def __init__(self, file, include_headers_line=True, join_multivalued=',', errors=None, **kwargs):
         super().__init__(dont_fail=True, **kwargs)
         if not self.encoding:
             self.encoding = 'utf-8'
@@ -205,7 +205,8 @@ class CsvItemExporter(BaseItemExporter):
             line_buffering=False,
             write_through=True,
             encoding=self.encoding,
-            newline=''  # Windows needs this https://github.com/scrapy/scrapy/issues/3034
+            newline='',  # Windows needs this https://github.com/scrapy/scrapy/issues/3034
+            errors=errors,
         )
         self.csv_writer = csv.writer(self.stream, **self._kwargs)
         self._headers_not_written = True

tests/test_exporters.py

@@ -355,6 +355,23 @@ class CsvItemExporterTest(BaseItemExporterTest):
             expected='22,False,3.14,2015-01-01 01:01:01\r\n'
         )
 
+    def test_errors_default(self):
+        with self.assertRaises(UnicodeEncodeError):
+            self.assertExportResult(
+                item=dict(text=u'W\u0275\u200Brd'),
+                expected=None,
+                encoding='windows-1251',
+            )
+
+    def test_errors_xmlcharrefreplace(self):
+        self.assertExportResult(
+            item=dict(text=u'W\u0275\u200Brd'),
+            include_headers_line=False,
+            expected='Wɵ​rd\r\n',
+            encoding='windows-1251',
+            errors='xmlcharrefreplace',
+        )
+
 
 class CsvItemExporterDataclassTest(CsvItemExporterTest):
     item_class = TestDataClass