Document the ITEM_PROCESSOR setting (#7983)

This commit is contained in:
Adrian 2026-08-12 17:34:25 +02:00 committed by GitHub
parent 52cc2da72d
commit 65b37286cc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 32 additions and 1 deletions

View File

@ -1529,6 +1529,20 @@ Default: ``{}``
A dict containing the pipelines enabled by default in Scrapy. You should never
modify this setting in your project, modify :setting:`ITEM_PIPELINES` instead.
.. setting:: ITEM_PROCESSOR
ITEM_PROCESSOR
--------------
Default: ``"scrapy.pipelines.ItemPipelineManager"``
The :ref:`component <topics-components>` that builds the :ref:`item pipeline
<topics-item-pipeline>` from :setting:`ITEM_PIPELINES` and runs scraped items
through it. It must implement :class:`~scrapy.pipelines.ItemProcessorProtocol`.
.. autoclass:: scrapy.pipelines.ItemProcessorProtocol
:members:
.. setting:: JOBDIR

View File

@ -8,7 +8,7 @@ from __future__ import annotations
import asyncio
import warnings
from typing import TYPE_CHECKING, Any, cast
from typing import TYPE_CHECKING, Any, Protocol, cast
from twisted.internet.defer import Deferred, DeferredList, FirstError
@ -28,6 +28,23 @@ if TYPE_CHECKING:
from scrapy.settings import Settings
class ItemProcessorProtocol(Protocol):
"""Protocol for item processor implementations.
See :setting:`ITEM_PROCESSOR`.
"""
async def open_spider_async(self) -> None:
"""Get the item processor ready to process items."""
async def process_item_async(self, item: Any) -> Any:
"""Return the processed *item*, or raise
:exc:`~scrapy.exceptions.DropItem` to drop it."""
async def close_spider_async(self) -> None:
"""Release any resource that the item processor is using."""
class ItemPipelineManager(MiddlewareManager):
component_name = "item pipeline"