diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index d5ad4bc8b..7b3826d21 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -8,12 +8,7 @@ from __future__ import annotations import logging from collections.abc import AsyncIterable, Callable, Iterable -from functools import wraps -from inspect import ( - isasyncgenfunction, - iscoroutine, - isgeneratorfunction, -) +from inspect import isasyncgenfunction, iscoroutine, iscoroutinefunction from itertools import islice from typing import TYPE_CHECKING, Any, TypeVar, Union, cast from warnings import warn @@ -54,21 +49,6 @@ def _isiterable(o: Any) -> bool: return isinstance(o, (Iterable, AsyncIterable)) -def _sync_generator_to_async(f: Callable) -> Callable: - @wraps(f) - async def wrapper(*args, **kwargs): - for item in f(*args, **kwargs): - yield item - - return wrapper - - -def _maybe_sync_generator_to_async(f: Callable) -> Callable: - if isgeneratorfunction(f): - return _sync_generator_to_async(f) - return f - - class SpiderMiddlewareManager(MiddlewareManager): component_name = "spider middleware" @@ -400,7 +380,7 @@ class SpiderMiddlewareManager(MiddlewareManager): ) start = as_async_generator(sync_start) else: - start = yield _maybe_sync_generator_to_async(spider.start)() + start = yield self._iter_seeds(spider) start = yield self._process_chain("process_start", start) return start @@ -474,6 +454,14 @@ class SpiderMiddlewareManager(MiddlewareManager): f"https://docs.scrapy.org/en/VERSION/news.html" ) + @staticmethod + def _iter_seeds(spider: Spider): + fn = spider.start + if isasyncgenfunction(fn): + return fn().__aiter__() + assert iscoroutinefunction(fn) + return deferred_from_coro(fn()) + # This method is only needed until _async compatibility methods are removed. @staticmethod def _get_async_method_pair( diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py index 5daa7d209..34f1d5e66 100644 --- a/tests/test_spider_start.py +++ b/tests/test_spider_start.py @@ -75,16 +75,6 @@ class MainTestCase(TestCase): await self._test_spider(TestSpider, [ITEM_A]) - @deferred_f_from_coro_f - async def test_start_sync(self): - class TestSpider(Spider): - name = "test" - - def start(self): - yield ITEM_A - - await self._test_spider(TestSpider, [ITEM_A]) - @deferred_f_from_coro_f async def test_deprecated(self): class TestSpider(Spider):