Document how to write a custom item exporter (#7931)

docs/topics/exporters.rst

@@ -136,6 +136,70 @@ Example:
                   return f"$ {str(value)}"
               return super().serialize_field(field, name, value)
 
+.. _custom-exporters:
+
+Writing your own item exporter
+==============================
+
+To write an item exporter, subclass :class:`BaseItemExporter` and implement
+:meth:`~BaseItemExporter.export_item`, where
+:meth:`~BaseItemExporter.get_serialized_fields` gives you the ``(name, value)``
+pairs to export.
+
+To make your exporter available to the :ref:`feed exports
+<topics-feed-exports>`, list it in the :setting:`FEED_EXPORTERS` setting. Feed
+exports :ref:`build <from-crawler>` it with the output file as the first
+positional argument, and with the ``fields``, ``encoding`` and ``indent``
+:ref:`feed options <feed-options>` and every key of ``item_export_kwargs`` as
+keyword arguments, so your ``__init__`` method must forward unknown keyword
+arguments to :class:`BaseItemExporter`.
+
+The file object belongs to whoever opened it, i.e. to the feed storage in the
+case of feed exports, which also closes it. If you need a text file, for
+example to use :func:`csv.writer` or another Python API that does not accept a
+binary file, wrap it with :class:`io.TextIOWrapper` and call
+:meth:`~io.TextIOBase.detach` on the wrapper in
+:meth:`~BaseItemExporter.finish_exporting`; otherwise the wrapper closes the
+underlying file when it is garbage-collected.
+
+For example, the following item exporter writes items as blocks of
+``name: value`` lines:
+
+.. code-block:: python
+
+    from io import TextIOWrapper
+
+    from scrapy.exporters import BaseItemExporter
+
+
+    class TextItemExporter(BaseItemExporter):
+        def __init__(self, file, item_separator="\n", **kwargs):
+            super().__init__(**kwargs)
+            self.item_separator = item_separator
+            self.stream = TextIOWrapper(
+                file, encoding=self.encoding or "utf-8", write_through=True
+            )
+
+        def export_item(self, item):
+            for name, value in self.get_serialized_fields(item):
+                print(f"{name}: {value}", file=self.stream)
+            self.stream.write(self.item_separator)
+
+        def finish_exporting(self):
+            self.stream.detach()
+
+To use it as the ``txt`` feed format:
+
+.. code-block:: python
+
+    FEED_EXPORTERS = {"txt": "myproject.exporters.TextItemExporter"}
+    FEEDS = {
+        "items.txt": {
+            "format": "txt",
+            "item_export_kwargs": {"item_separator": "---\n"},
+        },
+    }
+
 .. _topics-exporters-reference:
 
 Built-in Item Exporters reference
@@ -168,6 +232,8 @@ BaseItemExporter
 
       Exports the given item. This method must be implemented in subclasses.
 
+   .. automethod:: BaseItemExporter.get_serialized_fields
+
    .. method:: serialize_field(field, name, value)
 
       Return the serialized value for the given field. You can override this

scrapy/exporters.py

@@ -85,11 +85,16 @@ class BaseItemExporter(ABC):
         declared = (name for name in adapter.field_names() if name in populated)
         return dict.fromkeys([*declared, *adapter.keys()])
 
-    def _get_serialized_fields(
+    def get_serialized_fields(
         self, item: Any, default_value: Any = None, include_empty: bool | None = None
     ) -> Iterable[tuple[str, Any]]:
-        """Return the fields to export as an iterable of tuples
-        (name, serialized_value)
+        """Return the fields of *item* to export, as an iterable of
+        ``(name, serialized_value)`` tuples, taking :attr:`fields_to_export`
+        into account and applying :meth:`serialize_field` to every value.
+
+        Fields missing from *item* are exported with *default_value*.
+
+        *include_empty* overrides :attr:`export_empty_fields`.
         """
         item = ItemAdapter(item)
 
@@ -136,7 +141,7 @@ class JsonLinesItemExporter(BaseItemExporter):
         self.encoder: JSONEncoder = ScrapyJSONEncoder(**self._kwargs)
 
     def export_item(self, item: Any) -> None:
-        itemdict = dict(self._get_serialized_fields(item))
+        itemdict = dict(self.get_serialized_fields(item))
         data = self.encoder.encode(itemdict) + "\n"
         self.file.write(to_bytes(data, self.encoding))
 
@@ -176,7 +181,7 @@ class JsonItemExporter(BaseItemExporter):
         self.file.write(b"]")
 
     def export_item(self, item: Any) -> None:
-        itemdict = dict(self._get_serialized_fields(item))
+        itemdict = dict(self.get_serialized_fields(item))
         data = to_bytes(self.encoder.encode(itemdict), self.encoding)
         self._add_comma_after_first()
         self.file.write(data)
@@ -216,7 +221,7 @@ class XmlItemExporter(BaseItemExporter):
         self._beautify_indent(depth=1)
         self.xg.startElement(self.item_element, AttributesImpl({}))
         self._beautify_newline()
-        for name, value in self._get_serialized_fields(item, default_value=""):
+        for name, value in self.get_serialized_fields(item, default_value=""):
             self._export_xml_field(name, value, depth=2)
         self._beautify_indent(depth=1)
         self.xg.endElement(self.item_element)
@@ -310,7 +315,7 @@ class CsvItemExporter(BaseItemExporter):
                     f"See: https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-fields",
                 )
                 self._data_loss_warned = True
-        fields = self._get_serialized_fields(item, default_value="", include_empty=True)
+        fields = self.get_serialized_fields(item, default_value="", include_empty=True)
         values = list(self._build_row(x for _, x in fields))
         self.csv_writer.writerow(values)
 
@@ -347,7 +352,7 @@ class PickleItemExporter(BaseItemExporter):
         self.protocol: int = protocol
 
     def export_item(self, item: Any) -> None:
-        d = dict(self._get_serialized_fields(item))
+        d = dict(self.get_serialized_fields(item))
         pickle.dump(d, self.file, self.protocol)
 
 
@@ -365,7 +370,7 @@ class MarshalItemExporter(BaseItemExporter):
         self.file: BytesIO = file
 
     def export_item(self, item: Any) -> None:
-        marshal.dump(dict(self._get_serialized_fields(item)), self.file)
+        marshal.dump(dict(self.get_serialized_fields(item)), self.file)
 
 
 class PprintItemExporter(BaseItemExporter):
@@ -374,7 +379,7 @@ class PprintItemExporter(BaseItemExporter):
         self.file: BytesIO = file
 
     def export_item(self, item: Any) -> None:
-        itemdict = dict(self._get_serialized_fields(item))
+        itemdict = dict(self.get_serialized_fields(item))
         self.file.write(to_bytes(pprint.pformat(itemdict) + "\n"))
 
 
@@ -417,5 +422,5 @@ class PythonItemExporter(BaseItemExporter):
             yield key, self._serialize_value(value)
 
     def export_item(self, item: Any) -> dict[str | bytes, Any]:  # type: ignore[override]
-        result: dict[str | bytes, Any] = dict(self._get_serialized_fields(item))
+        result: dict[str | bytes, Any] = dict(self.get_serialized_fields(item))
         return result

tests/test_exporters.py

@@ -108,26 +108,26 @@ class TestBaseItemExporter(ABC):
 
     def test_fields_to_export(self):
         ie = self._get_exporter(fields_to_export=["name"])
-        assert list(ie._get_serialized_fields(self.i)) == [("name", "John\xa3")]
+        assert list(ie.get_serialized_fields(self.i)) == [("name", "John\xa3")]
 
         ie = self._get_exporter(fields_to_export=["name"], encoding="latin-1")
-        _, name = next(iter(ie._get_serialized_fields(self.i)))
+        _, name = next(iter(ie.get_serialized_fields(self.i)))
         assert isinstance(name, str)
         assert name == "John\xa3"
 
         ie = self._get_exporter(fields_to_export={"name": "名稱"})
-        assert list(ie._get_serialized_fields(self.i)) == [("名稱", "John\xa3")]
+        assert list(ie.get_serialized_fields(self.i)) == [("名稱", "John\xa3")]
 
     def test_field_order(self):
         item = self.item_class(age="22", name="John\xa3")
         ie = self._get_exporter()
-        assert [name for name, _ in ie._get_serialized_fields(item)] == ["name", "age"]
+        assert [name for name, _ in ie.get_serialized_fields(item)] == ["name", "age"]
 
     def test_field_order_dict_item(self):
         ie = self._get_exporter()
-        assert [name for name, _ in ie._get_serialized_fields({"age": "22"})] == ["age"]
+        assert [name for name, _ in ie.get_serialized_fields({"age": "22"})] == ["age"]
         assert [
-            name for name, _ in ie._get_serialized_fields({"age": "22", "name": "John"})
+            name for name, _ in ie.get_serialized_fields({"age": "22", "name": "John"})
         ] == ["age", "name"]
 
     def test_field_custom_serializer(self):