Fix silent data loss in CsvItemExporter (closes #4002) (#7651)

Adds state tracking to CsvItemExporter to detect when auto-detected fields drop data on items. Injects a warning instructing users to configure FEED_EXPORT_FIELDS. Includes unit tests. Corrects the false positive logic in stalled PR #7613.

scrapy/exporters.py

@@ -5,6 +5,7 @@ Item Exporters are used to export/serialize items into different formats.
 from __future__ import annotations
 
 import csv
+import logging
 import marshal
 import pickle
 import pprint
@@ -24,6 +25,8 @@ from scrapy.utils.serialize import ScrapyJSONEncoder
 if TYPE_CHECKING:
     from json import JSONEncoder
 
+logger = logging.getLogger(__name__)
+
 __all__ = [
     "BaseItemExporter",
     "CsvItemExporter",
@@ -254,6 +257,8 @@ class CsvItemExporter(BaseItemExporter):
         self.csv_writer = csv.writer(self.stream, **self._kwargs)
         self._headers_not_written = True
         self._join_multivalued = join_multivalued
+        self._autodetected_fields = False
+        self._data_loss_warned = False
 
     def serialize_field(
         self, field: Mapping[str, Any] | Field, name: str, value: Any
@@ -274,6 +279,22 @@ class CsvItemExporter(BaseItemExporter):
             self._headers_not_written = False
             self._write_headers_and_set_fields_to_export(item)
 
+        if (
+            self._autodetected_fields
+            and self.fields_to_export is not None
+            and not self._data_loss_warned
+        ):
+            item_fields = ItemAdapter(item).field_names()
+            dropped_fields = set(item_fields) - set(self.fields_to_export)
+
+            if dropped_fields:
+                dropped_fields_display = sorted(dropped_fields)
+                logger.warning(
+                    f"CSVExporter dropped fields {dropped_fields_display}. "
+                    f"To avoid this, fully configure your FEED_EXPORT_FIELDS setting. "
+                    f"See: https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-fields",
+                )
+                self._data_loss_warned = True
         fields = self._get_serialized_fields(item, default_value="", include_empty=True)
         values = list(self._build_row(x for _, x in fields))
         self.csv_writer.writerow(values)
@@ -293,6 +314,7 @@ class CsvItemExporter(BaseItemExporter):
             if not self.fields_to_export:
                 # use declared field names, or keys if the item is a dict
                 self.fields_to_export = ItemAdapter(item).field_names()
+                self._autodetected_fields = True
             fields: Iterable[str]
             if isinstance(self.fields_to_export, Mapping):
                 fields = self.fields_to_export.values()

tests/test_exporters.py

@@ -384,6 +384,20 @@ class TestCsvItemExporter(TestBaseItemExporter):
             errors="xmlcharrefreplace",
         )
 
+    def test_csv_dropped_fields_warning(self, caplog):
+        out = BytesIO()
+        exporter = CsvItemExporter(out)
+        exporter.start_exporting()
+
+        exporter.export_item({"name": "Apple"})
+
+        with caplog.at_level("WARNING", logger="scrapy.exporters"):
+            exporter.export_item({"name": "Banana", "price": 2.00})
+
+        assert len(caplog.records) == 1
+        assert "CSVExporter dropped fields" in caplog.text
+        assert "price" in caplog.text
+
 
 class TestCsvItemExporterDataclass(TestCsvItemExporter):
     item_class = MyDataClass