mirror of https://github.com/scrapy/scrapy.git
Document the ITEM_PROCESSOR setting (#7983)
This commit is contained in:
parent
52cc2da72d
commit
65b37286cc
|
|
@ -1529,6 +1529,20 @@ Default: ``{}``
|
|||
A dict containing the pipelines enabled by default in Scrapy. You should never
|
||||
modify this setting in your project, modify :setting:`ITEM_PIPELINES` instead.
|
||||
|
||||
.. setting:: ITEM_PROCESSOR
|
||||
|
||||
ITEM_PROCESSOR
|
||||
--------------
|
||||
|
||||
Default: ``"scrapy.pipelines.ItemPipelineManager"``
|
||||
|
||||
The :ref:`component <topics-components>` that builds the :ref:`item pipeline
|
||||
<topics-item-pipeline>` from :setting:`ITEM_PIPELINES` and runs scraped items
|
||||
through it. It must implement :class:`~scrapy.pipelines.ItemProcessorProtocol`.
|
||||
|
||||
.. autoclass:: scrapy.pipelines.ItemProcessorProtocol
|
||||
:members:
|
||||
|
||||
|
||||
.. setting:: JOBDIR
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ from __future__ import annotations
|
|||
|
||||
import asyncio
|
||||
import warnings
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
from typing import TYPE_CHECKING, Any, Protocol, cast
|
||||
|
||||
from twisted.internet.defer import Deferred, DeferredList, FirstError
|
||||
|
||||
|
|
@ -28,6 +28,23 @@ if TYPE_CHECKING:
|
|||
from scrapy.settings import Settings
|
||||
|
||||
|
||||
class ItemProcessorProtocol(Protocol):
|
||||
"""Protocol for item processor implementations.
|
||||
|
||||
See :setting:`ITEM_PROCESSOR`.
|
||||
"""
|
||||
|
||||
async def open_spider_async(self) -> None:
|
||||
"""Get the item processor ready to process items."""
|
||||
|
||||
async def process_item_async(self, item: Any) -> Any:
|
||||
"""Return the processed *item*, or raise
|
||||
:exc:`~scrapy.exceptions.DropItem` to drop it."""
|
||||
|
||||
async def close_spider_async(self) -> None:
|
||||
"""Release any resource that the item processor is using."""
|
||||
|
||||
|
||||
class ItemPipelineManager(MiddlewareManager):
|
||||
component_name = "item pipeline"
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue