RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Log an error but move on if Spider.iter_seeds is not an async generator

This commit is contained in:
Adrián Chaves 2025-03-14 23:54:07 +01:00
parent 05a21285f4
commit 8ccb674f33
12 changed files with 66 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/topics/spider-middleware.rst
View File

@ -70,7 +70,7 @@ one or more of these methods:
.. class:: SpiderMiddleware
.. method:: process_seeds(seeds: AsyncIterable[Any], /) -> AsyncIterable[Any]
.. method:: process_seeds(seeds: AsyncIterator[Any], /) -> AsyncIterator[Any]
:async:
Iterate over the output of :meth:`~scrapy.Spider.yield_seeds` or that

4
scrapy/commands/bench.py
View File

@ -13,7 +13,7 @@ from scrapy.linkextractors import LinkExtractor
if TYPE_CHECKING:
import argparse
from collections.abc import AsyncIterable
from collections.abc import AsyncIterator
class Command(ScrapyCommand):
@ -59,7 +59,7 @@ class _BenchSpider(scrapy.Spider):
baseurl = "http://localhost:8998"
link_extractor = LinkExtractor()
async def yield_seeds(self) -> AsyncIterable[Any]:
async def yield_seeds(self) -> AsyncIterator[Any]:
qargs = {"total": self.total, "show": self.show}
url = f"{self.baseurl}?{urlencode(qargs, doseq=True)}"
yield scrapy.Request(url, dont_filter=True)

4
scrapy/commands/parse.py
View File

@ -22,7 +22,7 @@ from scrapy.utils.spider import spidercls_for_request
if TYPE_CHECKING:
import argparse
from collections.abc import AsyncGenerator, AsyncIterable, Coroutine, Iterable
from collections.abc import AsyncGenerator, AsyncIterator, Coroutine, Iterable
from twisted.python.failure import Failure
@ -258,7 +258,7 @@ class Command(BaseRunSpiderCommand):
if not self.spidercls:
logger.error("Unable to find spider for: %(url)s", {"url": url})
async def yield_seeds(spider: Spider) -> AsyncIterable[Any]:
async def yield_seeds(spider: Spider) -> AsyncIterator[Any]:
yield self.prepare_request(spider, Request(url), opts)
if self.spidercls:

4
scrapy/core/engine.py
View File

@ -27,7 +27,7 @@ from scrapy.utils.reactor import CallLaterOnce
from ._seeding import SeedingPolicy
if TYPE_CHECKING:
from collections.abc import AsyncIterable, Callable, Generator
from collections.abc import AsyncIterator, Callable, Generator
from scrapy.core.downloader import Downloader
from scrapy.core.scheduler import BaseScheduler
@ -110,7 +110,7 @@ class ExecutionEngine:
)
self.start_time: float | None = None
self._load_seeding_policy()
self._seeds: AsyncIterable[Any] | None = None
self._seeds: AsyncIterator[Any] | None = None
self._waiting_for_seed: bool = False
def _load_seeding_policy(self) -> None:

24
scrapy/core/spidermw.py
View File

@ -7,8 +7,8 @@ See documentation in docs/topics/spider-middleware.rst
from __future__ import annotations
import logging
from collections.abc import AsyncIterable, Callable, Iterable
from inspect import isasyncgenfunction, iscoroutine, iscoroutinefunction
from collections.abc import AsyncIterable, AsyncIterator, Callable, Iterable
from inspect import isasyncgenfunction, iscoroutine
from itertools import islice
from typing import TYPE_CHECKING, Any, TypeVar, Union, cast
from warnings import warn
@ -371,8 +371,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
@inlineCallbacks
def process_seeds(
self, spider: Spider
) -> Generator[Deferred[Any], Any, AsyncIterable[Any]]:
) -> Generator[Deferred[Any], Any, AsyncIterator[Any] | None]:
self._check_deprecated_start_requests_use(spider)
seeds: AsyncIterator[Any]
if self._use_start_requests:
sync_seeds = iter(spider.start_requests())
sync_seeds = yield self._process_chain(
@ -380,7 +381,14 @@ class SpiderMiddlewareManager(MiddlewareManager):
)
seeds = as_async_generator(sync_seeds)
else:
seeds = yield self._iter_seeds(spider)
if not isasyncgenfunction(spider.yield_seeds):
logger.error(
f"{global_object_name(spider.yield_seeds)} must be an "
f"async generator function, i.e. an async def function "
f"with yield statements."
)
return None
seeds = spider.yield_seeds()
seeds = yield self._process_chain("process_seeds", seeds)
return seeds
@ -454,14 +462,6 @@ class SpiderMiddlewareManager(MiddlewareManager):
f"https://docs.scrapy.org/en/VERSION/news.html"
)
@staticmethod
def _iter_seeds(spider: Spider):
fn = spider.yield_seeds
if isasyncgenfunction(fn):
return fn().__aiter__()
assert iscoroutinefunction(fn)
return deferred_from_coro(fn())
# This method is only needed until _async compatibility methods are removed.
@staticmethod
def _get_async_method_pair(

4
scrapy/spiders/__init__.py
View File

@ -15,7 +15,7 @@ from scrapy.utils.trackref import object_ref
from scrapy.utils.url import url_is_from_spider
if TYPE_CHECKING:
from collections.abc import AsyncIterable, Iterable
from collections.abc import AsyncIterator, Iterable
from twisted.internet.defer import Deferred
@ -78,7 +78,7 @@ class Spider(object_ref):
self.settings: BaseSettings = crawler.settings
crawler.signals.connect(self.close, signals.spider_closed)
async def yield_seeds(self) -> AsyncIterable[Any]:
async def yield_seeds(self) -> AsyncIterator[Any]:
"""Yield the initial :class:`~scrapy.Request` objects to send.
.. versionadded:: VERSION

4
scrapy/spiders/init.py
View File

@ -1,7 +1,7 @@
from __future__ import annotations
import warnings
from collections.abc import AsyncIterable, Iterable
from collections.abc import AsyncIterator, Iterable
from typing import TYPE_CHECKING, Any, cast
from scrapy import Request
@ -29,7 +29,7 @@ class InitSpider(Spider):
stacklevel=2,
)
async def yield_seeds(self) -> AsyncIterable[Any]:
async def yield_seeds(self) -> AsyncIterator[Any]:
for seed in self.start_requests():
yield seed

4
scrapy/spiders/sitemap.py
View File

@ -4,7 +4,7 @@ import logging
import re
# Iterable is needed at the run time for the SitemapSpider._parse_sitemap() annotation
from collections.abc import AsyncIterable, Iterable, Sequence # noqa: TC003
from collections.abc import AsyncIterator, Iterable, Sequence # noqa: TC003
from typing import TYPE_CHECKING, Any, cast
from scrapy.http import Request, Response, XmlResponse
@ -53,7 +53,7 @@ class SitemapSpider(Spider):
self._cbs.append((regex(r), c))
self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow]
async def yield_seeds(self) -> AsyncIterable[Any]:
async def yield_seeds(self) -> AsyncIterator[Any]:
for seed in self.start_requests():
yield seed

19
tests/test_commands.py
View File

@ -676,15 +676,6 @@ class MySpider(scrapy.Spider):
yield
"""
badspider = """
import scrapy
class BadSpider(scrapy.Spider):
name = "bad"
async def yield_seeds(self):
raise Exception("oops!")
"""
@contextmanager
def _create_file(self, content: str, name: str | None = None) -> Iterator[str]:
with TemporaryDirectory() as tmpdir:
@ -772,11 +763,6 @@ class MySpider(scrapy.Spider):
log = self.get_log("", name="myspider.txt")
assert "Unable to load" in log
def test_yield_seeds_errors(self):
log = self.get_log(self.badspider, name="badspider.py")
assert "yield_seeds" in log
assert "badspider.py" in log
def test_asyncio_enabled_true(self):
log = self.get_log(
self.debug_log_spider,
@ -1007,11 +993,6 @@ class TestWindowsRunSpiderCommand(TestRunSpiderCommand):
raise unittest.SkipTest("Windows required for .pyw files")
return super().setUp()
def test_yield_seeds_errors(self):
log = self.get_log(self.badspider, name="badspider.pyw")
assert "yield_seeds" in log
assert "badspider.pyw" in log
def test_runspider_unable_to_load(self):
raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ")

21
tests/test_crawl.py
View File

@ -361,27 +361,6 @@ with multiples lines
assert s["engine.spider.name"] == crawler.spider.name
assert s["len(engine.scraper.slot.active)"] == "1"
@defer.inlineCallbacks
def test_graceful_crawl_error_handling(self):
"""
Test whether errors happening anywhere in Crawler.crawl() are properly
reported (and not somehow swallowed) after a graceful engine shutdown.
The errors should not come from within Scrapy's core but from within
spiders/middlewares/etc., e.g. raised in Spider.test_yield_seeds(),
SpiderMiddleware.process_test_yield_seeds(), etc.
"""
class TestError(Exception):
pass
class FaultySpider(SimpleSpider):
async def yield_seeds(self):
raise TestError
crawler = get_crawler(FaultySpider)
yield self.assertFailure(crawler.crawl(mockserver=self.mockserver), TestError)
assert not crawler.crawling
@defer.inlineCallbacks
def test_open_spider_error_on_faulty_pipeline(self):
settings = {

62
tests/test_spider_yield_seeds.py
View File

@ -1,6 +1,7 @@
from asyncio import sleep
import pytest
from testfixtures import LogCapture
from twisted.internet.defer import Deferred
from twisted.trial.unittest import TestCase
@ -30,6 +31,8 @@ def twisted_sleep(seconds):
class MainTestCase(TestCase):
# Utility methods
async def _test_spider(self, spider, expected_items=None):
actual_items = []
expected_items = [] if expected_items is None else expected_items
@ -43,6 +46,15 @@ class MainTestCase(TestCase):
assert crawler.stats.get_value("finish_reason") == "finished"
assert actual_items == expected_items
async def _test_yield_seeds(self, yield_seeds_, expected_items=None):
class TestSpider(Spider):
name = "test"
yield_seeds = yield_seeds_
await self._test_spider(TestSpider, expected_items)
# Basic usage
@deferred_f_from_coro_f
async def test_start_urls(self):
class TestSpider(Spider):
@ -55,7 +67,7 @@ class MainTestCase(TestCase):
await self._test_spider(TestSpider, [ITEM_A])
@deferred_f_from_coro_f
async def test_yield_seeds(self):
async def test_main(self):
class TestSpider(Spider):
name = "test"
@ -64,16 +76,7 @@ class MainTestCase(TestCase):
await self._test_spider(TestSpider, [ITEM_A])
@deferred_f_from_coro_f
async def test_yield_seeds_subclass(self):
class BaseSpider(Spider):
async def yield_seeds(self):
yield ITEM_A
class TestSpider(BaseSpider):
name = "test"
await self._test_spider(TestSpider, [ITEM_A])
# Deprecation of start_requests and universal implementation support.
@deferred_f_from_coro_f
async def test_deprecated(self):
@ -112,26 +115,7 @@ class MainTestCase(TestCase):
await self._test_spider(TestSpider, [ITEM_A])
@deferred_f_from_coro_f
async def test_universal_subclass(self):
class BaseSpider(Spider):
async def yield_seeds(self):
yield ITEM_A
def start_requests(self):
yield ITEM_B
class TestSpider(BaseSpider):
name = "test"
await self._test_spider(TestSpider, [ITEM_A])
async def _test_yield_seeds(self, yield_seeds_, expected_items=None):
class TestSpider(Spider):
name = "test"
yield_seeds = yield_seeds_
await self._test_spider(TestSpider, expected_items)
# Delays.
@pytest.mark.only_asyncio
@deferred_f_from_coro_f
@ -151,3 +135,19 @@ class MainTestCase(TestCase):
yield ITEM_A
await self._test_yield_seeds(yield_seeds, [ITEM_A])
# Exceptions
@deferred_f_from_coro_f
async def test_non_generator_async_def(self):
class TestSpider(Spider):
name = "test"
async def yield_seeds(self):
return
crawler = get_crawler(TestSpider)
with LogCapture() as log:
await maybe_deferred_to_future(crawler.crawl())
assert crawler.stats.get_value("finish_reason") == "finished"
assert "TestSpider.yield_seeds must be an async generator function" in str(log)

15
tests/test_spidermiddleware_process_seeds.py
View File

@ -1,6 +1,7 @@
from asyncio import sleep
import pytest
from testfixtures import LogCapture
from twisted.trial.unittest import TestCase
from scrapy import Spider, signals
@ -117,7 +118,9 @@ class MainTestCase(TestCase):
assert actual_items == expected_items, f"{actual_items=} != {expected_items=}"
async def _test_wrap(self, spider_middleware, spider_cls, expected_items=None):
expected_items = expected_items or [ITEM_A, ITEM_B, ITEM_C]
expected_items = (
[ITEM_A, ITEM_B, ITEM_C] if expected_items is None else expected_items
)
await self._test([spider_middleware], spider_cls, expected_items)
@deferred_f_from_coro_f
@ -153,14 +156,16 @@ class MainTestCase(TestCase):
@deferred_f_from_coro_f
async def test_deprecated_mw_modern_spider(self):
with (
LogCapture() as log,
pytest.warns(
ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"
),
pytest.raises(
ValueError, match=r"only compatible with \(deprecated\) spiders"
),
):
await self._test_wrap(DeprecatedWrapSpiderMiddleware, ModernWrapSpider)
await self._test_wrap(
DeprecatedWrapSpiderMiddleware, ModernWrapSpider, expected_items=[]
)
assert "only compatible with (deprecated) spiders" in str(log)
@deferred_f_from_coro_f
async def test_deprecated_mw_universal_spider(self):