From 65b37286cc56b6a87c4d1eca6bdff79f658f09d2 Mon Sep 17 00:00:00 2001 From: Adrian Date: Wed, 12 Aug 2026 17:34:25 +0200 Subject: [PATCH] Document the ITEM_PROCESSOR setting (#7983) --- docs/topics/settings.rst | 14 ++++++++++++++ scrapy/pipelines/__init__.py | 19 ++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 27ef3f7ef..010c9d1c6 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1529,6 +1529,20 @@ Default: ``{}`` A dict containing the pipelines enabled by default in Scrapy. You should never modify this setting in your project, modify :setting:`ITEM_PIPELINES` instead. +.. setting:: ITEM_PROCESSOR + +ITEM_PROCESSOR +-------------- + +Default: ``"scrapy.pipelines.ItemPipelineManager"`` + +The :ref:`component ` that builds the :ref:`item pipeline +` from :setting:`ITEM_PIPELINES` and runs scraped items +through it. It must implement :class:`~scrapy.pipelines.ItemProcessorProtocol`. + +.. autoclass:: scrapy.pipelines.ItemProcessorProtocol + :members: + .. setting:: JOBDIR diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index 383c461e6..e6483d939 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -8,7 +8,7 @@ from __future__ import annotations import asyncio import warnings -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any, Protocol, cast from twisted.internet.defer import Deferred, DeferredList, FirstError @@ -28,6 +28,23 @@ if TYPE_CHECKING: from scrapy.settings import Settings +class ItemProcessorProtocol(Protocol): + """Protocol for item processor implementations. + + See :setting:`ITEM_PROCESSOR`. + """ + + async def open_spider_async(self) -> None: + """Get the item processor ready to process items.""" + + async def process_item_async(self, item: Any) -> Any: + """Return the processed *item*, or raise + :exc:`~scrapy.exceptions.DropItem` to drop it.""" + + async def close_spider_async(self) -> None: + """Release any resource that the item processor is using.""" + + class ItemPipelineManager(MiddlewareManager): component_name = "item pipeline"