From f3868e11fb0ef3b0aeae3ebf64a78b1deb733fed Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 4 May 2026 17:34:24 +0500 Subject: [PATCH] Drop start_requests() support. (#7490) --- extras/qpsclient.py | 4 - scrapy/core/spidermw.py | 151 +----------- scrapy/spidermiddlewares/base.py | 7 - scrapy/spiders/__init__.py | 22 +- scrapy/spiders/sitemap.py | 4 - tests/test_spider_start.py | 99 -------- tests/test_spidermiddleware_base.py | 22 +- tests/test_spidermiddleware_process_start.py | 236 ------------------- tests/test_spidermiddleware_start.py | 17 -- 9 files changed, 18 insertions(+), 544 deletions(-) diff --git a/extras/qpsclient.py b/extras/qpsclient.py index 269b27336..8e5001c1d 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -35,10 +35,6 @@ class QPSSpider(Spider): self.download_delay = float(self.download_delay) async def start(self): - for item_or_request in self.start_requests(): - yield item_or_request - - def start_requests(self): url = self.benchurl if self.latency is not None: url += f"?latency={self.latency}" diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 67342099d..790317357 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -33,7 +33,6 @@ from scrapy.utils.python import MutableAsyncChain, MutableChain, global_object_n if TYPE_CHECKING: from collections.abc import Generator - from scrapy.crawler import Crawler from scrapy.settings import BaseSettings @@ -60,76 +59,12 @@ class SpiderMiddlewareManager(MiddlewareManager): settings.get_component_priority_dict_with_base("SPIDER_MIDDLEWARES") ) - def __init__(self, *middlewares: Any, crawler: Crawler | None = None) -> None: - self._check_deprecated_process_start_requests_use(middlewares) - super().__init__(*middlewares, crawler=crawler) - - def _check_deprecated_process_start_requests_use( - self, middlewares: tuple[Any, ...] - ) -> None: - deprecated_middlewares = [ - middleware - for middleware in middlewares - if hasattr(middleware, "process_start_requests") - and not hasattr(middleware, "process_start") - ] - modern_middlewares = [ - middleware - for middleware in middlewares - if not hasattr(middleware, "process_start_requests") - and hasattr(middleware, "process_start") - ] - if deprecated_middlewares and modern_middlewares: - raise ValueError( - "You are trying to combine spider middlewares that only " - "define the deprecated process_start_requests() method () " - "with spider middlewares that only define the " - "process_start() method (). This is not possible. You must " - "either disable or make universal 1 of those 2 sets of " - "spider middlewares. Making a spider middleware universal " - "means having it define both methods. See the release notes " - "of Scrapy 2.13 for details: " - "https://docs.scrapy.org/en/2.13/news.html" - ) - - self._use_start_requests = bool(deprecated_middlewares) - if self._use_start_requests: - deprecated_middleware_list = ", ".join( - global_object_name(middleware.__class__) - for middleware in deprecated_middlewares - ) - warn( - f"The following enabled spider middlewares, directly or " - f"through their parent classes, define the deprecated " - f"process_start_requests() method: " - f"{deprecated_middleware_list}. process_start_requests() has " - f"been deprecated in favor of a new method, process_start(), " - f"to support asynchronous code execution. " - f"process_start_requests() will stop being called in a future " - f"version of Scrapy. If you use Scrapy 2.13 or higher " - f"only, replace process_start_requests() with " - f"process_start(); note that process_start() is a coroutine " - f"(async def). If you need to maintain compatibility with " - f"lower Scrapy versions, when defining " - f"process_start_requests() in a spider middleware class, " - f"define process_start() as well. See the release notes of " - f"Scrapy 2.13 for details: " - f"https://docs.scrapy.org/en/2.13/news.html", - ScrapyDeprecationWarning, - stacklevel=2, - ) - def _add_middleware(self, mw: Any) -> None: if hasattr(mw, "process_spider_input"): self.methods["process_spider_input"].append(mw.process_spider_input) self._check_mw_method_spider_arg(mw.process_spider_input) - if self._use_start_requests: - if hasattr(mw, "process_start_requests"): - self.methods["process_start_requests"].appendleft( - mw.process_start_requests - ) - elif hasattr(mw, "process_start"): + if hasattr(mw, "process_start"): self.methods["process_start"].appendleft(mw.process_start) process_spider_output = self._get_async_method_pair(mw, "process_spider_output") @@ -451,88 +386,8 @@ class SpiderMiddlewareManager(MiddlewareManager): ) warn(msg, category=ScrapyDeprecationWarning, stacklevel=2) self._set_compat_spider(spider) - self._check_deprecated_start_requests_use() - if self._use_start_requests: - sync_start = iter(self._spider.start_requests()) - sync_start = await self._process_chain( - "process_start_requests", sync_start, always_add_spider=True - ) - start: AsyncIterator[Any] = as_async_generator(sync_start) - else: - start = self._spider.start() - start = await self._process_chain("process_start", start) - return start - - def _check_deprecated_start_requests_use(self) -> None: - start_requests_cls = None - start_cls = None - spidercls = self._spider.__class__ - mro = spidercls.__mro__ - - for cls in mro: - cls_dict = cls.__dict__ - if start_requests_cls is None and "start_requests" in cls_dict: - start_requests_cls = cls - if start_cls is None and "start" in cls_dict: - start_cls = cls - if start_requests_cls is not None and start_cls is not None: - break - - # Spider defines both, start_requests and start. - assert start_requests_cls is not None - assert start_cls is not None - - if ( - start_requests_cls is not Spider - and start_cls is not start_requests_cls - and mro.index(start_requests_cls) < mro.index(start_cls) - ): - src = global_object_name(start_requests_cls) - if start_requests_cls is not spidercls: - src += f" (inherited by {global_object_name(spidercls)})" - warn( - f"{src} defines the deprecated start_requests() method. " - f"start_requests() has been deprecated in favor of a new " - f"method, start(), to support asynchronous code " - f"execution. start_requests() will stop being called in a " - f"future version of Scrapy. If you use Scrapy 2.13 or " - f"higher only, replace start_requests() with start(); " - f"note that start() is a coroutine (async def). If you " - f"need to maintain compatibility with lower Scrapy versions, " - f"when overriding start_requests() in a spider class, " - f"override start() as well; you can use super() to " - f"reuse the inherited start() implementation without " - f"copy-pasting. See the release notes of Scrapy 2.13 for " - f"details: https://docs.scrapy.org/en/2.13/news.html", - ScrapyDeprecationWarning, - stacklevel=2, - ) - - if ( - self._use_start_requests - and start_cls is not Spider - and start_requests_cls is not start_cls - and mro.index(start_cls) < mro.index(start_requests_cls) - ): - src = global_object_name(start_cls) - if start_cls is not spidercls: - src += f" (inherited by {global_object_name(spidercls)})" - raise ValueError( - f"{src} does not define the deprecated start_requests() " - f"method. However, one or more of your enabled spider " - f"middlewares (reported in an earlier deprecation warning) " - f"define the process_start_requests() method, and not the " - f"process_start() method, making them only compatible with " - f"(deprecated) spiders that define the start_requests() " - f"method. To solve this issue, disable the offending spider " - f"middlewares, upgrade them as described in that earlier " - f"deprecation warning, or make your spider compatible with " - f"deprecated spider middlewares (and earlier Scrapy versions) " - f"by defining a sync start_requests() method that works " - f"similarly to its existing start() method. See the " - f"release notes of Scrapy 2.13 for details: " - f"https://docs.scrapy.org/en/2.13/news.html" - ) + start = self._spider.start() + return await self._process_chain("process_start", start) # This method is only needed until _async compatibility methods are removed. @staticmethod diff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.py index 889fc6df1..e09f2d10e 100644 --- a/scrapy/spidermiddlewares/base.py +++ b/scrapy/spidermiddlewares/base.py @@ -41,13 +41,6 @@ class BaseSpiderMiddleware: def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def process_start_requests( - self, start: Iterable[Any], spider: Spider - ) -> Iterable[Any]: - for o in start: - if (o := self._get_processed(o, None)) is not None: - yield o - async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]: async for o in start: if (o := self._get_processed(o, None)) is not None: diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 018e510c5..299a5d43f 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -7,17 +7,15 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import logging -import warnings from typing import TYPE_CHECKING, Any, cast from scrapy import signals -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider if TYPE_CHECKING: - from collections.abc import AsyncIterator, Iterable + from collections.abc import AsyncIterator from twisted.internet.defer import Deferred @@ -127,24 +125,6 @@ class Spider(object_ref): .. seealso:: :ref:`start-requests` """ - with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", category=ScrapyDeprecationWarning, module=r"^scrapy\.spiders$" - ) - for item_or_request in self.start_requests(): - yield item_or_request - - def start_requests(self) -> Iterable[Any]: - warnings.warn( - ( - "The Spider.start_requests() method is deprecated, use " - "Spider.start() instead. If you are calling " - "super().start_requests() from a Spider.start() override, " - "iterate super().start() instead." - ), - ScrapyDeprecationWarning, - stacklevel=2, - ) if not self.start_urls and hasattr(self, "start_url"): raise AttributeError( "Crawling could not start: 'start_urls' not found " diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index d87ffccaf..2a80b8d24 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -54,10 +54,6 @@ class SitemapSpider(Spider): self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow] async def start(self) -> AsyncIterator[Any]: - for item_or_request in self.start_requests(): - yield item_or_request - - def start_requests(self) -> Iterable[Request]: for url in self.sitemap_urls: yield Request(url, self._parse_sitemap) diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py index 4e359fd33..aef2093ac 100644 --- a/tests/test_spider_start.py +++ b/tests/test_spider_start.py @@ -1,15 +1,12 @@ from __future__ import annotations -import re import warnings from asyncio import sleep from typing import Any import pytest -from testfixtures import LogCapture from scrapy import Spider, signals -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.test import get_crawler @@ -77,81 +74,6 @@ class TestMain: warnings.simplefilter("error") await self._test_spider(TestSpider, [ITEM_A]) - @coroutine_test - async def test_deprecated(self): - class TestSpider(Spider): - name = "test" - - def start_requests(self): - yield ITEM_A - - with pytest.warns(ScrapyDeprecationWarning): - await self._test_spider(TestSpider, [ITEM_A]) - - @coroutine_test - async def test_deprecated_subclass(self): - class BaseSpider(Spider): - def start_requests(self): - yield ITEM_A - - class TestSpider(BaseSpider): - name = "test" - - # The warning must be about the base class and not the subclass. - with pytest.warns(ScrapyDeprecationWarning, match="BaseSpider"): - await self._test_spider(TestSpider, [ITEM_A]) - - @coroutine_test - async def test_universal(self): - class TestSpider(Spider): - name = "test" - - async def start(self): - yield ITEM_A - - def start_requests(self): - yield ITEM_B - - with warnings.catch_warnings(): - warnings.simplefilter("error") - await self._test_spider(TestSpider, [ITEM_A]) - - @coroutine_test - async def test_universal_subclass(self): - class BaseSpider(Spider): - async def start(self): - yield ITEM_A - - def start_requests(self): - yield ITEM_B - - class TestSpider(BaseSpider): - name = "test" - - with warnings.catch_warnings(): - warnings.simplefilter("error") - await self._test_spider(TestSpider, [ITEM_A]) - - @coroutine_test - async def test_start_deprecated_super(self): - class TestSpider(Spider): - name = "test" - - async def start(self): - for item_or_request in super().start_requests(): - yield item_or_request - - msg = "use Spider.start() instead" - with pytest.warns(ScrapyDeprecationWarning, match=re.escape(msg)) as ws: - await self._test_spider(TestSpider, []) - - for w in ws: - if isinstance(w.message, ScrapyDeprecationWarning) and msg in str( - w.message - ): - assert w.filename.endswith("test_spider_start.py") - break - async def _test_start(self, start_, expected_items=None): class TestSpider(Spider): name = "test" @@ -176,24 +98,3 @@ class TestMain: yield ITEM_A await self._test_start(start, [ITEM_A]) - - # Exceptions - - @coroutine_test - async def test_deprecated_non_generator_exception(self): - class TestSpider(Spider): - name = "test" - - def start_requests(self): - raise RuntimeError - - with ( - LogCapture() as log, - pytest.warns( - ScrapyDeprecationWarning, - match=r"defines the deprecated start_requests\(\) method", - ), - ): - await self._test_spider(TestSpider, []) - - assert "in start_requests\n raise RuntimeError" in str(log) diff --git a/tests/test_spidermiddleware_base.py b/tests/test_spidermiddleware_base.py index da570904d..70326f3f1 100644 --- a/tests/test_spidermiddleware_base.py +++ b/tests/test_spidermiddleware_base.py @@ -7,7 +7,9 @@ import pytest from scrapy import Request, Spider from scrapy.http import Response from scrapy.spidermiddlewares.base import BaseSpiderMiddleware +from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.test import get_crawler +from tests.utils.decorators import coroutine_test if TYPE_CHECKING: from scrapy.crawler import Crawler @@ -18,7 +20,8 @@ def crawler() -> Crawler: return get_crawler(Spider) -def test_trivial(crawler: Crawler) -> None: +@coroutine_test +async def test_trivial(crawler: Crawler) -> None: class TrivialSpiderMiddleware(BaseSpiderMiddleware): pass @@ -29,12 +32,13 @@ def test_trivial(crawler: Crawler) -> None: spider_output = [test_req, {"foo": "bar"}] for processed in [ list(mw.process_spider_output(Response("data:,"), spider_output)), - list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type] + await collect_asyncgen(mw.process_start(as_async_generator(spider_output))), ]: assert processed == [test_req, {"foo": "bar"}] -def test_processed_request(crawler: Crawler) -> None: +@coroutine_test +async def test_processed_request(crawler: Crawler) -> None: class ProcessReqSpiderMiddleware(BaseSpiderMiddleware): def get_processed_request( self, request: Request, response: Response | None @@ -52,7 +56,7 @@ def test_processed_request(crawler: Crawler) -> None: spider_output = [test_req1, {"foo": "bar"}, test_req2, test_req3] for processed in [ list(mw.process_spider_output(Response("data:,"), spider_output)), - list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type] + await collect_asyncgen(mw.process_start(as_async_generator(spider_output))), ]: assert len(processed) == 3 assert isinstance(processed[0], Request) @@ -62,7 +66,8 @@ def test_processed_request(crawler: Crawler) -> None: assert processed[2].url == "data:30," -def test_processed_item(crawler: Crawler) -> None: +@coroutine_test +async def test_processed_item(crawler: Crawler) -> None: class ProcessItemSpiderMiddleware(BaseSpiderMiddleware): def get_processed_item(self, item: Any, response: Response | None) -> Any: if item["foo"] == 2: @@ -76,12 +81,13 @@ def test_processed_item(crawler: Crawler) -> None: spider_output = [{"foo": 1}, {"foo": 2}, test_req, {"foo": 3}] for processed in [ list(mw.process_spider_output(Response("data:,"), spider_output)), - list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type] + await collect_asyncgen(mw.process_start(as_async_generator(spider_output))), ]: assert processed == [{"foo": 1}, test_req, {"foo": 30}] -def test_processed_both(crawler: Crawler) -> None: +@coroutine_test +async def test_processed_both(crawler: Crawler) -> None: class ProcessBothSpiderMiddleware(BaseSpiderMiddleware): def get_processed_request( self, request: Request, response: Response | None @@ -113,7 +119,7 @@ def test_processed_both(crawler: Crawler) -> None: ] for processed in [ list(mw.process_spider_output(Response("data:,"), spider_output)), - list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type] + await collect_asyncgen(mw.process_start(as_async_generator(spider_output))), ]: assert len(processed) == 4 assert isinstance(processed[0], Request) diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py index ddece52c4..c907c6d73 100644 --- a/tests/test_spidermiddleware_process_start.py +++ b/tests/test_spidermiddleware_process_start.py @@ -4,7 +4,6 @@ from asyncio import sleep import pytest from scrapy import Spider, signals -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.test import get_crawler from tests.test_spider_start import SLEEP_SECONDS @@ -38,15 +37,6 @@ class TwistedSleepSpiderMiddleware: yield item_or_request -class UniversalSpiderMiddleware: - async def process_start(self, start): - async for item_or_request in start: - yield item_or_request - - def process_start_requests(self, start_requests, spider): - raise NotImplementedError - - # Spiders and spider middlewares for TestMain._test_wrap @@ -61,23 +51,6 @@ class ModernWrapSpiderSubclass(ModernWrapSpider): name = "test" -class UniversalWrapSpider(Spider): - name = "test" - - async def start(self): - yield ITEM_B - - def start_requests(self): - yield ITEM_D - - -class DeprecatedWrapSpider(Spider): - name = "test" - - def start_requests(self): - yield ITEM_B - - class ModernWrapSpiderMiddleware: async def process_start(self, start): yield ITEM_A @@ -86,26 +59,6 @@ class ModernWrapSpiderMiddleware: yield ITEM_C -class UniversalWrapSpiderMiddleware: - async def process_start(self, start): - yield ITEM_A - async for item_or_request in start: - yield item_or_request - yield ITEM_C - - def process_start_requests(self, start, spider): - yield ITEM_A - yield from start - yield ITEM_C - - -class DeprecatedWrapSpiderMiddleware: - def process_start_requests(self, start, spider): - yield ITEM_A - yield from start - yield ITEM_C - - class TestMain: async def _test(self, spider_middlewares, spider_cls, expected_items): actual_items = [] @@ -136,195 +89,6 @@ class TestMain: warnings.simplefilter("error") await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider) - @coroutine_test - async def test_modern_mw_universal_spider(self): - with warnings.catch_warnings(): - warnings.simplefilter("error") - await self._test_wrap(ModernWrapSpiderMiddleware, UniversalWrapSpider) - - @coroutine_test - async def test_modern_mw_deprecated_spider(self): - with pytest.warns( - ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" - ): - await self._test_wrap(ModernWrapSpiderMiddleware, DeprecatedWrapSpider) - - @coroutine_test - async def test_universal_mw_modern_spider(self): - with warnings.catch_warnings(): - warnings.simplefilter("error") - await self._test_wrap(UniversalWrapSpiderMiddleware, ModernWrapSpider) - - @coroutine_test - async def test_universal_mw_universal_spider(self): - with warnings.catch_warnings(): - warnings.simplefilter("error") - await self._test_wrap(UniversalWrapSpiderMiddleware, UniversalWrapSpider) - - @coroutine_test - async def test_universal_mw_deprecated_spider(self): - with pytest.warns( - ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" - ): - await self._test_wrap(UniversalWrapSpiderMiddleware, DeprecatedWrapSpider) - - @coroutine_test - async def test_deprecated_mw_modern_spider(self): - with ( - pytest.warns( - ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" - ), - pytest.raises( - ValueError, match=r"only compatible with \(deprecated\) spiders" - ), - ): - await self._test_wrap(DeprecatedWrapSpiderMiddleware, ModernWrapSpider) - - @coroutine_test - async def test_deprecated_mw_modern_spider_subclass(self): - with ( - pytest.warns( - ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" - ), - pytest.raises( - ValueError, - match=r"^\S+?\.ModernWrapSpider \(inherited by \S+?.ModernWrapSpiderSubclass\) .*? only compatible with \(deprecated\) spiders", - ), - ): - await self._test_wrap( - DeprecatedWrapSpiderMiddleware, ModernWrapSpiderSubclass - ) - - @coroutine_test - async def test_deprecated_mw_universal_spider(self): - with pytest.warns( - ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" - ): - await self._test_wrap( - DeprecatedWrapSpiderMiddleware, - UniversalWrapSpider, - [ITEM_A, ITEM_D, ITEM_C], - ) - - @coroutine_test - async def test_deprecated_mw_deprecated_spider(self): - with ( - pytest.warns( - ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" - ), - pytest.warns( - ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" - ), - ): - await self._test_wrap(DeprecatedWrapSpiderMiddleware, DeprecatedWrapSpider) - - @coroutine_test - async def test_modern_mw_universal_mw_modern_spider(self): - with warnings.catch_warnings(): - warnings.simplefilter("error") - await self._test_douple_wrap( - ModernWrapSpiderMiddleware, - UniversalWrapSpiderMiddleware, - ModernWrapSpider, - ) - - @coroutine_test - async def test_modern_mw_deprecated_mw_modern_spider(self): - with pytest.raises(ValueError, match=r"trying to combine spider middlewares"): - await self._test_douple_wrap( - ModernWrapSpiderMiddleware, - DeprecatedWrapSpiderMiddleware, - ModernWrapSpider, - ) - - @coroutine_test - async def test_universal_mw_deprecated_mw_modern_spider(self): - with ( - pytest.warns( - ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" - ), - pytest.raises( - ValueError, match=r"only compatible with \(deprecated\) spiders" - ), - ): - await self._test_douple_wrap( - UniversalWrapSpiderMiddleware, - DeprecatedWrapSpiderMiddleware, - ModernWrapSpider, - ) - - @coroutine_test - async def test_modern_mw_universal_mw_universal_spider(self): - with warnings.catch_warnings(): - warnings.simplefilter("error") - await self._test_douple_wrap( - ModernWrapSpiderMiddleware, - UniversalWrapSpiderMiddleware, - UniversalWrapSpider, - ) - - @coroutine_test - async def test_modern_mw_deprecated_mw_universal_spider(self): - with pytest.raises(ValueError, match=r"trying to combine spider middlewares"): - await self._test_douple_wrap( - ModernWrapSpiderMiddleware, - DeprecatedWrapSpiderMiddleware, - UniversalWrapSpider, - ) - - @coroutine_test - async def test_universal_mw_deprecated_mw_universal_spider(self): - with pytest.warns( - ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" - ): - await self._test_douple_wrap( - UniversalWrapSpiderMiddleware, - DeprecatedWrapSpiderMiddleware, - UniversalWrapSpider, - [ITEM_A, ITEM_A, ITEM_D, ITEM_C, ITEM_C], - ) - - @coroutine_test - async def test_modern_mw_universal_mw_deprecated_spider(self): - with pytest.warns( - ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" - ): - await self._test_douple_wrap( - ModernWrapSpiderMiddleware, - UniversalWrapSpiderMiddleware, - DeprecatedWrapSpider, - ) - - @coroutine_test - async def test_modern_mw_deprecated_mw_deprecated_spider(self): - with pytest.raises(ValueError, match=r"trying to combine spider middlewares"): - await self._test_douple_wrap( - ModernWrapSpiderMiddleware, - DeprecatedWrapSpiderMiddleware, - DeprecatedWrapSpider, - ) - - @coroutine_test - async def test_universal_mw_deprecated_mw_deprecated_spider(self): - with ( - pytest.warns( - ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" - ), - pytest.warns( - ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" - ), - ): - await self._test_douple_wrap( - UniversalWrapSpiderMiddleware, - DeprecatedWrapSpiderMiddleware, - DeprecatedWrapSpider, - ) - - @coroutine_test - async def test_universal_mw_uses_process_start(self): - """Test that process_start_requests() isn't used when process_start() exists.""" - await self._test([UniversalSpiderMiddleware], ModernWrapSpider, [ITEM_B]) - async def _test_sleep(self, spider_middlewares): class TestSpider(Spider): name = "test" diff --git a/tests/test_spidermiddleware_start.py b/tests/test_spidermiddleware_start.py index 76976d962..26397d37c 100644 --- a/tests/test_spidermiddleware_start.py +++ b/tests/test_spidermiddleware_start.py @@ -23,20 +23,3 @@ class TestMiddleware: async for request in mw.process_start(start()) ] assert result == [True, True, False, "foo"] - - @coroutine_test - async def test_sync(self): - crawler = get_crawler(Spider) - mw = build_from_crawler(StartSpiderMiddleware, crawler) - - def start(): - yield Request("data:,1") - yield Request("data:,2", meta={"is_start_request": True}) - yield Request("data:,2", meta={"is_start_request": False}) - yield Request("data:,2", meta={"is_start_request": "foo"}) - - result = [ - request.meta["is_start_request"] - for request in mw.process_start_requests(start(), Spider("test")) - ] - assert result == [True, True, False, "foo"]