RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge ea88269210 into e28e56aa61

This commit is contained in:
Adrian 2026-08-15 11:16:48 -05:00 committed by GitHub
parent e28e56aa61 ea88269210
commit 759d3b7587
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 128 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
docs/topics/spider-middleware.rst
View File

@ -122,14 +122,20 @@ one or more of these methods:
This method is an :term:`asynchronous generator` called with the
results from the spider after the spider has processed the response.
.. versionchanged:: VERSION
It is also called with the output of a request :ref:`errback
<errbacks>`, with *response* set to ``None``, when the errback runs
because a download failed.
*result* is lazy: a generator callback runs as *result* is iterated, so
code that runs before that iteration runs before the callback body.
.. seealso:: :ref:`universal-spider-middleware`.
:param response: the response which generated this output from the
spider
:type response: :class:`~scrapy.http.Response` object
spider, or ``None`` if the output comes from an errback called
because a download failed
:type response: :class:`~scrapy.http.Response` object or ``None``
:param result: the results from the spider
:type result: an :term:`asynchronous iterable` of
@ -145,9 +151,14 @@ one or more of these methods:
.. method:: process_spider_exception(response, exception)
This method is called when a spider callback or a
:meth:`process_spider_output` method (from a previous spider
middleware) raises an exception.
This method is called when a spider callback, a request :ref:`errback
<errbacks>` or a :meth:`process_spider_output` method (from a previous
spider middleware) raises an exception.
.. versionchanged:: VERSION
It is also called for exceptions raised while iterating the output
of an errback that runs because a download failed, with *response*
set to ``None``.
:meth:`process_spider_exception` should return either ``None`` or an
iterable of :class:`~scrapy.Request` or :ref:`item <topics-items>`
@ -163,8 +174,8 @@ one or more of these methods:
:meth:`process_spider_exception` will be called.
:param response: the response being processed when the exception was
raised
:type response: :class:`~scrapy.http.Response` object
raised, or ``None`` if there was no response
:type response: :class:`~scrapy.http.Response` object or ``None``
:param exception: the exception raised
:type exception: :exc:`Exception` object

7
scrapy/core/scraper.py
View File

@ -265,8 +265,11 @@ class Scraper:
return
try:
# call the request errback with the downloader error
output = await self.call_spider_async(result, request)
# call the request errback with the downloader error and the spider
# middlewares with its output
output = await self.spidermw._scrape_failure_async(
self.call_spider_async, result, request
)
except Exception as spider_exc:
# the errback didn't silence the exception
assert self.crawler.spider

22
scrapy/core/spidermw.py
View File

@ -99,7 +99,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
async def _evaluate_iterable(
self,
response: Response,
response: Response | None,
iterable: AsyncIterator[_T],
exception_processor_index: int,
recover_to: MutableAsyncChain[_T],
@ -115,7 +115,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
def _process_spider_exception(
self,
response: Response,
response: Response | None,
exception: Exception,
start_index: int = 0,
) -> MutableAsyncChain[_T]:
@ -151,7 +151,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
def _process_spider_output(
self,
response: Response,
response: Response | None,
result: AsyncIterator[_T],
start_index: int = 0,
) -> MutableAsyncChain[_T]:
@ -172,7 +172,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
return MutableAsyncChain(result, recovered)
async def _process_callback_output(
self, response: Response, result: AsyncIterator[_T]
self, response: Response | None, result: AsyncIterator[_T]
) -> MutableAsyncChain[_T]:
recovered: MutableAsyncChain[_T] = MutableAsyncChain()
result = self._evaluate_iterable(response, result, 0, recovered)
@ -227,6 +227,20 @@ class SpiderMiddlewareManager(MiddlewareManager):
await _defer_sleep_async()
return self._process_spider_exception(response, ex)
async def _scrape_failure_async(
self,
scrape_func: ScrapeFunc[_T],
failure: Failure,
request: Request,
) -> MutableAsyncChain[_T]:
# There is no response to run the process_spider_input chain on, so the
# errback output enters the process_spider_output chain with None as the
# response. Exceptions from the errback itself are left to the caller,
# which tells apart the download error from a different one.
it: Iterable[_T] | AsyncIterator[_T] = await scrape_func(failure, request)
ait = it if isinstance(it, AsyncIterator) else as_async_generator(it)
return await self._process_callback_output(None, ait)
async def process_start(
self, spider: Spider | None = None
) -> AsyncIterator[Any] | None:

10
scrapy/spidermiddlewares/base.py
View File

@ -86,8 +86,9 @@ class BaseSpiderMiddleware:
:type request: :class:`~scrapy.Request` object
:param response: the response being processed
:type response: :class:`~scrapy.http.Response` object or ``None`` for
start requests
:type response: :class:`~scrapy.http.Response` object, or ``None`` for
start requests and for the output of a request errback called
because a download failed
:return: the processed request or ``None``
"""
@ -104,8 +105,9 @@ class BaseSpiderMiddleware:
:type item: item object
:param response: the response being processed
:type response: :class:`~scrapy.http.Response` object or ``None`` for
start items
:type response: :class:`~scrapy.http.Response` object, or ``None`` for
start items and for the output of a request errback called because
a download failed
:return: the processed item or ``None``
"""

18
scrapy/spidermiddlewares/start.py
View File

@ -1,12 +1,13 @@
from __future__ import annotations
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
from scrapy import Request
from .base import BaseSpiderMiddleware
if TYPE_CHECKING:
from scrapy.http import Request
from scrapy.http.response import Response
from collections.abc import AsyncIterator
class StartSpiderMiddleware(BaseSpiderMiddleware):
@ -23,9 +24,8 @@ class StartSpiderMiddleware(BaseSpiderMiddleware):
<topics-downloader-middleware>`.
"""
def get_processed_request(
self, request: Request, response: Response | None
) -> Request | None:
if response is None:
request.meta.setdefault("is_start_request", True)
return request
async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]:
async for o in start:
if isinstance(o, Request):
o.meta.setdefault("is_start_request", True)
yield o

72
tests/test_spidermiddleware_output_chain.py
View File

@ -243,6 +243,51 @@ class GeneratorOutputChainSpider(Spider):
yield {"processed": ["parse-second-item"]}
# ================================================================================
# (5) errback output after a download error
class LogOutputMiddleware(_BaseSpiderMiddleware):
async def process_spider_output(self, response, result):
async for o in result:
self.crawler.spider.logger.info(
f"Middleware: output {o} with response {response}"
)
yield o
def process_spider_exception(self, response, exception):
self.crawler.spider.logger.info(
f"Middleware: {exception.__class__.__name__} exception caught"
f" with response {response}"
)
return []
class DownloadErrorSpider(Spider):
name = "DownloadErrorSpider"
custom_settings = {
"SPIDER_MIDDLEWARES": {
LogOutputMiddleware: 10,
},
}
async def start(self):
yield Request(self.mockserver.url("/drop?abort=1"), errback=self.errback)
def errback(self, failure):
yield {"from": "errback"}
yield Request(self.mockserver.url("/status?n=200"), callback=self.parse)
def parse(self, response):
self.logger.info(f"is_start_request: {response.meta.get('is_start_request')}")
class DownloadErrorFailSpider(DownloadErrorSpider):
name = "DownloadErrorFailSpider"
def errback(self, failure):
yield {"from": "errback"}
raise LookupError
# ================================================================================
class TestSpiderMiddleware:
mockserver: MockServer
@ -426,3 +471,30 @@ class TestSpiderMiddleware:
assert str(item_from_callback) in log4
assert str(item_recovered) in log4
assert "parse-second-item" not in log4
@coroutine_test
async def test_download_error_errback_output(
self, caplog: pytest.LogCaptureFixture
) -> None:
"""
(5.1) The output of an errback called because a download failed goes
through the process_spider_output chain, with None as the response.
"""
log5 = await self.crawl_log(DownloadErrorSpider, caplog)
assert "Middleware: output {'from': 'errback'} with response None" in log5
assert "'item_scraped_count': 1" in log5
assert "Crawled (200)" in log5
assert "is_start_request: None" in log5
@coroutine_test
async def test_download_error_errback_exception(
self, caplog: pytest.LogCaptureFixture
) -> None:
"""
(5.2) An exception from such an errback goes through the
process_spider_exception chain.
"""
log5 = await self.crawl_log(DownloadErrorFailSpider, caplog)
assert "Middleware: output {'from': 'errback'} with response None" in log5
assert "Middleware: LookupError exception caught with response None" in log5
assert "'item_scraped_count': 1" in log5