Revert "Support defining Spider.start() as a sync generator"

This reverts commit debab6cff6.
This commit is contained in:
Adrián Chaves 2025-03-20 12:09:25 +01:00
parent 28b78457aa
commit 0867ecc0c7
2 changed files with 10 additions and 32 deletions

View File

@ -8,12 +8,7 @@ from __future__ import annotations
import logging
from collections.abc import AsyncIterable, Callable, Iterable
from functools import wraps
from inspect import (
isasyncgenfunction,
iscoroutine,
isgeneratorfunction,
)
from inspect import isasyncgenfunction, iscoroutine, iscoroutinefunction
from itertools import islice
from typing import TYPE_CHECKING, Any, TypeVar, Union, cast
from warnings import warn
@ -54,21 +49,6 @@ def _isiterable(o: Any) -> bool:
return isinstance(o, (Iterable, AsyncIterable))
def _sync_generator_to_async(f: Callable) -> Callable:
@wraps(f)
async def wrapper(*args, **kwargs):
for item in f(*args, **kwargs):
yield item
return wrapper
def _maybe_sync_generator_to_async(f: Callable) -> Callable:
if isgeneratorfunction(f):
return _sync_generator_to_async(f)
return f
class SpiderMiddlewareManager(MiddlewareManager):
component_name = "spider middleware"
@ -400,7 +380,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
)
start = as_async_generator(sync_start)
else:
start = yield _maybe_sync_generator_to_async(spider.start)()
start = yield self._iter_seeds(spider)
start = yield self._process_chain("process_start", start)
return start
@ -474,6 +454,14 @@ class SpiderMiddlewareManager(MiddlewareManager):
f"https://docs.scrapy.org/en/VERSION/news.html"
)
@staticmethod
def _iter_seeds(spider: Spider):
fn = spider.start
if isasyncgenfunction(fn):
return fn().__aiter__()
assert iscoroutinefunction(fn)
return deferred_from_coro(fn())
# This method is only needed until _async compatibility methods are removed.
@staticmethod
def _get_async_method_pair(

View File

@ -75,16 +75,6 @@ class MainTestCase(TestCase):
await self._test_spider(TestSpider, [ITEM_A])
@deferred_f_from_coro_f
async def test_start_sync(self):
class TestSpider(Spider):
name = "test"
def start(self):
yield ITEM_A
await self._test_spider(TestSpider, [ITEM_A])
@deferred_f_from_coro_f
async def test_deprecated(self):
class TestSpider(Spider):