diff --git a/scrapy/exporters.py b/scrapy/exporters.py index c3540d724..5a11df833 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -5,6 +5,7 @@ Item Exporters are used to export/serialize items into different formats. from __future__ import annotations import csv +import logging import marshal import pickle import pprint @@ -24,6 +25,8 @@ from scrapy.utils.serialize import ScrapyJSONEncoder if TYPE_CHECKING: from json import JSONEncoder +logger = logging.getLogger(__name__) + __all__ = [ "BaseItemExporter", "CsvItemExporter", @@ -254,6 +257,8 @@ class CsvItemExporter(BaseItemExporter): self.csv_writer = csv.writer(self.stream, **self._kwargs) self._headers_not_written = True self._join_multivalued = join_multivalued + self._autodetected_fields = False + self._data_loss_warned = False def serialize_field( self, field: Mapping[str, Any] | Field, name: str, value: Any @@ -274,6 +279,22 @@ class CsvItemExporter(BaseItemExporter): self._headers_not_written = False self._write_headers_and_set_fields_to_export(item) + if ( + self._autodetected_fields + and self.fields_to_export is not None + and not self._data_loss_warned + ): + item_fields = ItemAdapter(item).field_names() + dropped_fields = set(item_fields) - set(self.fields_to_export) + + if dropped_fields: + dropped_fields_display = sorted(dropped_fields) + logger.warning( + f"CSVExporter dropped fields {dropped_fields_display}. " + f"To avoid this, fully configure your FEED_EXPORT_FIELDS setting. " + f"See: https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-fields", + ) + self._data_loss_warned = True fields = self._get_serialized_fields(item, default_value="", include_empty=True) values = list(self._build_row(x for _, x in fields)) self.csv_writer.writerow(values) @@ -293,6 +314,7 @@ class CsvItemExporter(BaseItemExporter): if not self.fields_to_export: # use declared field names, or keys if the item is a dict self.fields_to_export = ItemAdapter(item).field_names() + self._autodetected_fields = True fields: Iterable[str] if isinstance(self.fields_to_export, Mapping): fields = self.fields_to_export.values() diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 877e0708b..3b997767e 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -384,6 +384,20 @@ class TestCsvItemExporter(TestBaseItemExporter): errors="xmlcharrefreplace", ) + def test_csv_dropped_fields_warning(self, caplog): + out = BytesIO() + exporter = CsvItemExporter(out) + exporter.start_exporting() + + exporter.export_item({"name": "Apple"}) + + with caplog.at_level("WARNING", logger="scrapy.exporters"): + exporter.export_item({"name": "Banana", "price": 2.00}) + + assert len(caplog.records) == 1 + assert "CSVExporter dropped fields" in caplog.text + assert "price" in caplog.text + class TestCsvItemExporterDataclass(TestCsvItemExporter): item_class = MyDataClass