From 786ab1049431bd890bbf7b0b35551037b385d04a Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 19:31:53 +0200 Subject: [PATCH] Document how to write a custom item exporter (#7931) --- docs/topics/exporters.rst | 66 +++++++++++++++++++++++++++++++++++++++ scrapy/exporters.py | 27 +++++++++------- tests/test_exporters.py | 12 +++---- 3 files changed, 88 insertions(+), 17 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index c43b7e20f..56b995e18 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -136,6 +136,70 @@ Example: return f"$ {str(value)}" return super().serialize_field(field, name, value) +.. _custom-exporters: + +Writing your own item exporter +============================== + +To write an item exporter, subclass :class:`BaseItemExporter` and implement +:meth:`~BaseItemExporter.export_item`, where +:meth:`~BaseItemExporter.get_serialized_fields` gives you the ``(name, value)`` +pairs to export. + +To make your exporter available to the :ref:`feed exports +`, list it in the :setting:`FEED_EXPORTERS` setting. Feed +exports :ref:`build ` it with the output file as the first +positional argument, and with the ``fields``, ``encoding`` and ``indent`` +:ref:`feed options ` and every key of ``item_export_kwargs`` as +keyword arguments, so your ``__init__`` method must forward unknown keyword +arguments to :class:`BaseItemExporter`. + +The file object belongs to whoever opened it, i.e. to the feed storage in the +case of feed exports, which also closes it. If you need a text file, for +example to use :func:`csv.writer` or another Python API that does not accept a +binary file, wrap it with :class:`io.TextIOWrapper` and call +:meth:`~io.TextIOBase.detach` on the wrapper in +:meth:`~BaseItemExporter.finish_exporting`; otherwise the wrapper closes the +underlying file when it is garbage-collected. + +For example, the following item exporter writes items as blocks of +``name: value`` lines: + +.. code-block:: python + + from io import TextIOWrapper + + from scrapy.exporters import BaseItemExporter + + + class TextItemExporter(BaseItemExporter): + def __init__(self, file, item_separator="\n", **kwargs): + super().__init__(**kwargs) + self.item_separator = item_separator + self.stream = TextIOWrapper( + file, encoding=self.encoding or "utf-8", write_through=True + ) + + def export_item(self, item): + for name, value in self.get_serialized_fields(item): + print(f"{name}: {value}", file=self.stream) + self.stream.write(self.item_separator) + + def finish_exporting(self): + self.stream.detach() + +To use it as the ``txt`` feed format: + +.. code-block:: python + + FEED_EXPORTERS = {"txt": "myproject.exporters.TextItemExporter"} + FEEDS = { + "items.txt": { + "format": "txt", + "item_export_kwargs": {"item_separator": "---\n"}, + }, + } + .. _topics-exporters-reference: Built-in Item Exporters reference @@ -168,6 +232,8 @@ BaseItemExporter Exports the given item. This method must be implemented in subclasses. + .. automethod:: BaseItemExporter.get_serialized_fields + .. method:: serialize_field(field, name, value) Return the serialized value for the given field. You can override this diff --git a/scrapy/exporters.py b/scrapy/exporters.py index ea600d1a8..7f8aaf059 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -85,11 +85,16 @@ class BaseItemExporter(ABC): declared = (name for name in adapter.field_names() if name in populated) return dict.fromkeys([*declared, *adapter.keys()]) - def _get_serialized_fields( + def get_serialized_fields( self, item: Any, default_value: Any = None, include_empty: bool | None = None ) -> Iterable[tuple[str, Any]]: - """Return the fields to export as an iterable of tuples - (name, serialized_value) + """Return the fields of *item* to export, as an iterable of + ``(name, serialized_value)`` tuples, taking :attr:`fields_to_export` + into account and applying :meth:`serialize_field` to every value. + + Fields missing from *item* are exported with *default_value*. + + *include_empty* overrides :attr:`export_empty_fields`. """ item = ItemAdapter(item) @@ -136,7 +141,7 @@ class JsonLinesItemExporter(BaseItemExporter): self.encoder: JSONEncoder = ScrapyJSONEncoder(**self._kwargs) def export_item(self, item: Any) -> None: - itemdict = dict(self._get_serialized_fields(item)) + itemdict = dict(self.get_serialized_fields(item)) data = self.encoder.encode(itemdict) + "\n" self.file.write(to_bytes(data, self.encoding)) @@ -176,7 +181,7 @@ class JsonItemExporter(BaseItemExporter): self.file.write(b"]") def export_item(self, item: Any) -> None: - itemdict = dict(self._get_serialized_fields(item)) + itemdict = dict(self.get_serialized_fields(item)) data = to_bytes(self.encoder.encode(itemdict), self.encoding) self._add_comma_after_first() self.file.write(data) @@ -216,7 +221,7 @@ class XmlItemExporter(BaseItemExporter): self._beautify_indent(depth=1) self.xg.startElement(self.item_element, AttributesImpl({})) self._beautify_newline() - for name, value in self._get_serialized_fields(item, default_value=""): + for name, value in self.get_serialized_fields(item, default_value=""): self._export_xml_field(name, value, depth=2) self._beautify_indent(depth=1) self.xg.endElement(self.item_element) @@ -310,7 +315,7 @@ class CsvItemExporter(BaseItemExporter): f"See: https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-fields", ) self._data_loss_warned = True - fields = self._get_serialized_fields(item, default_value="", include_empty=True) + fields = self.get_serialized_fields(item, default_value="", include_empty=True) values = list(self._build_row(x for _, x in fields)) self.csv_writer.writerow(values) @@ -347,7 +352,7 @@ class PickleItemExporter(BaseItemExporter): self.protocol: int = protocol def export_item(self, item: Any) -> None: - d = dict(self._get_serialized_fields(item)) + d = dict(self.get_serialized_fields(item)) pickle.dump(d, self.file, self.protocol) @@ -365,7 +370,7 @@ class MarshalItemExporter(BaseItemExporter): self.file: BytesIO = file def export_item(self, item: Any) -> None: - marshal.dump(dict(self._get_serialized_fields(item)), self.file) + marshal.dump(dict(self.get_serialized_fields(item)), self.file) class PprintItemExporter(BaseItemExporter): @@ -374,7 +379,7 @@ class PprintItemExporter(BaseItemExporter): self.file: BytesIO = file def export_item(self, item: Any) -> None: - itemdict = dict(self._get_serialized_fields(item)) + itemdict = dict(self.get_serialized_fields(item)) self.file.write(to_bytes(pprint.pformat(itemdict) + "\n")) @@ -417,5 +422,5 @@ class PythonItemExporter(BaseItemExporter): yield key, self._serialize_value(value) def export_item(self, item: Any) -> dict[str | bytes, Any]: # type: ignore[override] - result: dict[str | bytes, Any] = dict(self._get_serialized_fields(item)) + result: dict[str | bytes, Any] = dict(self.get_serialized_fields(item)) return result diff --git a/tests/test_exporters.py b/tests/test_exporters.py index b857728ba..99afb8beb 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -108,26 +108,26 @@ class TestBaseItemExporter(ABC): def test_fields_to_export(self): ie = self._get_exporter(fields_to_export=["name"]) - assert list(ie._get_serialized_fields(self.i)) == [("name", "John\xa3")] + assert list(ie.get_serialized_fields(self.i)) == [("name", "John\xa3")] ie = self._get_exporter(fields_to_export=["name"], encoding="latin-1") - _, name = next(iter(ie._get_serialized_fields(self.i))) + _, name = next(iter(ie.get_serialized_fields(self.i))) assert isinstance(name, str) assert name == "John\xa3" ie = self._get_exporter(fields_to_export={"name": "名稱"}) - assert list(ie._get_serialized_fields(self.i)) == [("名稱", "John\xa3")] + assert list(ie.get_serialized_fields(self.i)) == [("名稱", "John\xa3")] def test_field_order(self): item = self.item_class(age="22", name="John\xa3") ie = self._get_exporter() - assert [name for name, _ in ie._get_serialized_fields(item)] == ["name", "age"] + assert [name for name, _ in ie.get_serialized_fields(item)] == ["name", "age"] def test_field_order_dict_item(self): ie = self._get_exporter() - assert [name for name, _ in ie._get_serialized_fields({"age": "22"})] == ["age"] + assert [name for name, _ in ie.get_serialized_fields({"age": "22"})] == ["age"] assert [ - name for name, _ in ie._get_serialized_fields({"age": "22", "name": "John"}) + name for name, _ in ie.get_serialized_fields({"age": "22", "name": "John"}) ] == ["age", "name"] def test_field_custom_serializer(self):