mirror of https://github.com/scrapy/scrapy.git
Drop start_requests() support. (#7490)
This commit is contained in:
Andrey Rakhmatullin
GitHub
parent
9ca206da64
commit
f3868e11fb
|
|
@ -35,10 +35,6 @@ class QPSSpider(Spider):
|
|||
self.download_delay = float(self.download_delay)
|
||||
|
||||
async def start(self):
|
||||
for item_or_request in self.start_requests():
|
||||
yield item_or_request
|
||||
|
||||
def start_requests(self):
|
||||
url = self.benchurl
|
||||
if self.latency is not None:
|
||||
url += f"?latency={self.latency}"
|
||||
|
|
|
|||
|
|
@ -33,7 +33,6 @@ from scrapy.utils.python import MutableAsyncChain, MutableChain, global_object_n
|
|||
if TYPE_CHECKING:
|
||||
from collections.abc import Generator
|
||||
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.settings import BaseSettings
|
||||
|
||||
|
||||
|
|
@ -60,76 +59,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
settings.get_component_priority_dict_with_base("SPIDER_MIDDLEWARES")
|
||||
)
|
||||
|
||||
def __init__(self, *middlewares: Any, crawler: Crawler | None = None) -> None:
|
||||
self._check_deprecated_process_start_requests_use(middlewares)
|
||||
super().__init__(*middlewares, crawler=crawler)
|
||||
|
||||
def _check_deprecated_process_start_requests_use(
|
||||
self, middlewares: tuple[Any, ...]
|
||||
) -> None:
|
||||
deprecated_middlewares = [
|
||||
middleware
|
||||
for middleware in middlewares
|
||||
if hasattr(middleware, "process_start_requests")
|
||||
and not hasattr(middleware, "process_start")
|
||||
]
|
||||
modern_middlewares = [
|
||||
middleware
|
||||
for middleware in middlewares
|
||||
if not hasattr(middleware, "process_start_requests")
|
||||
and hasattr(middleware, "process_start")
|
||||
]
|
||||
if deprecated_middlewares and modern_middlewares:
|
||||
raise ValueError(
|
||||
"You are trying to combine spider middlewares that only "
|
||||
"define the deprecated process_start_requests() method () "
|
||||
"with spider middlewares that only define the "
|
||||
"process_start() method (). This is not possible. You must "
|
||||
"either disable or make universal 1 of those 2 sets of "
|
||||
"spider middlewares. Making a spider middleware universal "
|
||||
"means having it define both methods. See the release notes "
|
||||
"of Scrapy 2.13 for details: "
|
||||
"https://docs.scrapy.org/en/2.13/news.html"
|
||||
)
|
||||
|
||||
self._use_start_requests = bool(deprecated_middlewares)
|
||||
if self._use_start_requests:
|
||||
deprecated_middleware_list = ", ".join(
|
||||
global_object_name(middleware.__class__)
|
||||
for middleware in deprecated_middlewares
|
||||
)
|
||||
warn(
|
||||
f"The following enabled spider middlewares, directly or "
|
||||
f"through their parent classes, define the deprecated "
|
||||
f"process_start_requests() method: "
|
||||
f"{deprecated_middleware_list}. process_start_requests() has "
|
||||
f"been deprecated in favor of a new method, process_start(), "
|
||||
f"to support asynchronous code execution. "
|
||||
f"process_start_requests() will stop being called in a future "
|
||||
f"version of Scrapy. If you use Scrapy 2.13 or higher "
|
||||
f"only, replace process_start_requests() with "
|
||||
f"process_start(); note that process_start() is a coroutine "
|
||||
f"(async def). If you need to maintain compatibility with "
|
||||
f"lower Scrapy versions, when defining "
|
||||
f"process_start_requests() in a spider middleware class, "
|
||||
f"define process_start() as well. See the release notes of "
|
||||
f"Scrapy 2.13 for details: "
|
||||
f"https://docs.scrapy.org/en/2.13/news.html",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
def _add_middleware(self, mw: Any) -> None:
|
||||
if hasattr(mw, "process_spider_input"):
|
||||
self.methods["process_spider_input"].append(mw.process_spider_input)
|
||||
self._check_mw_method_spider_arg(mw.process_spider_input)
|
||||
|
||||
if self._use_start_requests:
|
||||
if hasattr(mw, "process_start_requests"):
|
||||
self.methods["process_start_requests"].appendleft(
|
||||
mw.process_start_requests
|
||||
)
|
||||
elif hasattr(mw, "process_start"):
|
||||
if hasattr(mw, "process_start"):
|
||||
self.methods["process_start"].appendleft(mw.process_start)
|
||||
|
||||
process_spider_output = self._get_async_method_pair(mw, "process_spider_output")
|
||||
|
|
@ -451,88 +386,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
)
|
||||
warn(msg, category=ScrapyDeprecationWarning, stacklevel=2)
|
||||
self._set_compat_spider(spider)
|
||||
self._check_deprecated_start_requests_use()
|
||||
if self._use_start_requests:
|
||||
sync_start = iter(self._spider.start_requests())
|
||||
sync_start = await self._process_chain(
|
||||
"process_start_requests", sync_start, always_add_spider=True
|
||||
)
|
||||
start: AsyncIterator[Any] = as_async_generator(sync_start)
|
||||
else:
|
||||
start = self._spider.start()
|
||||
start = await self._process_chain("process_start", start)
|
||||
return start
|
||||
|
||||
def _check_deprecated_start_requests_use(self) -> None:
|
||||
start_requests_cls = None
|
||||
start_cls = None
|
||||
spidercls = self._spider.__class__
|
||||
mro = spidercls.__mro__
|
||||
|
||||
for cls in mro:
|
||||
cls_dict = cls.__dict__
|
||||
if start_requests_cls is None and "start_requests" in cls_dict:
|
||||
start_requests_cls = cls
|
||||
if start_cls is None and "start" in cls_dict:
|
||||
start_cls = cls
|
||||
if start_requests_cls is not None and start_cls is not None:
|
||||
break
|
||||
|
||||
# Spider defines both, start_requests and start.
|
||||
assert start_requests_cls is not None
|
||||
assert start_cls is not None
|
||||
|
||||
if (
|
||||
start_requests_cls is not Spider
|
||||
and start_cls is not start_requests_cls
|
||||
and mro.index(start_requests_cls) < mro.index(start_cls)
|
||||
):
|
||||
src = global_object_name(start_requests_cls)
|
||||
if start_requests_cls is not spidercls:
|
||||
src += f" (inherited by {global_object_name(spidercls)})"
|
||||
warn(
|
||||
f"{src} defines the deprecated start_requests() method. "
|
||||
f"start_requests() has been deprecated in favor of a new "
|
||||
f"method, start(), to support asynchronous code "
|
||||
f"execution. start_requests() will stop being called in a "
|
||||
f"future version of Scrapy. If you use Scrapy 2.13 or "
|
||||
f"higher only, replace start_requests() with start(); "
|
||||
f"note that start() is a coroutine (async def). If you "
|
||||
f"need to maintain compatibility with lower Scrapy versions, "
|
||||
f"when overriding start_requests() in a spider class, "
|
||||
f"override start() as well; you can use super() to "
|
||||
f"reuse the inherited start() implementation without "
|
||||
f"copy-pasting. See the release notes of Scrapy 2.13 for "
|
||||
f"details: https://docs.scrapy.org/en/2.13/news.html",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
if (
|
||||
self._use_start_requests
|
||||
and start_cls is not Spider
|
||||
and start_requests_cls is not start_cls
|
||||
and mro.index(start_cls) < mro.index(start_requests_cls)
|
||||
):
|
||||
src = global_object_name(start_cls)
|
||||
if start_cls is not spidercls:
|
||||
src += f" (inherited by {global_object_name(spidercls)})"
|
||||
raise ValueError(
|
||||
f"{src} does not define the deprecated start_requests() "
|
||||
f"method. However, one or more of your enabled spider "
|
||||
f"middlewares (reported in an earlier deprecation warning) "
|
||||
f"define the process_start_requests() method, and not the "
|
||||
f"process_start() method, making them only compatible with "
|
||||
f"(deprecated) spiders that define the start_requests() "
|
||||
f"method. To solve this issue, disable the offending spider "
|
||||
f"middlewares, upgrade them as described in that earlier "
|
||||
f"deprecation warning, or make your spider compatible with "
|
||||
f"deprecated spider middlewares (and earlier Scrapy versions) "
|
||||
f"by defining a sync start_requests() method that works "
|
||||
f"similarly to its existing start() method. See the "
|
||||
f"release notes of Scrapy 2.13 for details: "
|
||||
f"https://docs.scrapy.org/en/2.13/news.html"
|
||||
)
|
||||
start = self._spider.start()
|
||||
return await self._process_chain("process_start", start)
|
||||
|
||||
# This method is only needed until _async compatibility methods are removed.
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -41,13 +41,6 @@ class BaseSpiderMiddleware:
|
|||
def from_crawler(cls, crawler: Crawler) -> Self:
|
||||
return cls(crawler)
|
||||
|
||||
def process_start_requests(
|
||||
self, start: Iterable[Any], spider: Spider
|
||||
) -> Iterable[Any]:
|
||||
for o in start:
|
||||
if (o := self._get_processed(o, None)) is not None:
|
||||
yield o
|
||||
|
||||
async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]:
|
||||
async for o in start:
|
||||
if (o := self._get_processed(o, None)) is not None:
|
||||
|
|
|
|||
|
|
@ -7,17 +7,15 @@ See documentation in docs/topics/spiders.rst
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import warnings
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.utils.trackref import object_ref
|
||||
from scrapy.utils.url import url_is_from_spider
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import AsyncIterator, Iterable
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from twisted.internet.defer import Deferred
|
||||
|
||||
|
|
@ -127,24 +125,6 @@ class Spider(object_ref):
|
|||
|
||||
.. seealso:: :ref:`start-requests`
|
||||
"""
|
||||
with warnings.catch_warnings():
|
||||
warnings.filterwarnings(
|
||||
"ignore", category=ScrapyDeprecationWarning, module=r"^scrapy\.spiders$"
|
||||
)
|
||||
for item_or_request in self.start_requests():
|
||||
yield item_or_request
|
||||
|
||||
def start_requests(self) -> Iterable[Any]:
|
||||
warnings.warn(
|
||||
(
|
||||
"The Spider.start_requests() method is deprecated, use "
|
||||
"Spider.start() instead. If you are calling "
|
||||
"super().start_requests() from a Spider.start() override, "
|
||||
"iterate super().start() instead."
|
||||
),
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
if not self.start_urls and hasattr(self, "start_url"):
|
||||
raise AttributeError(
|
||||
"Crawling could not start: 'start_urls' not found "
|
||||
|
|
|
|||
|
|
@ -54,10 +54,6 @@ class SitemapSpider(Spider):
|
|||
self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow]
|
||||
|
||||
async def start(self) -> AsyncIterator[Any]:
|
||||
for item_or_request in self.start_requests():
|
||||
yield item_or_request
|
||||
|
||||
def start_requests(self) -> Iterable[Request]:
|
||||
for url in self.sitemap_urls:
|
||||
yield Request(url, self._parse_sitemap)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,15 +1,12 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import warnings
|
||||
from asyncio import sleep
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from testfixtures import LogCapture
|
||||
|
||||
from scrapy import Spider, signals
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.utils.defer import maybe_deferred_to_future
|
||||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
|
@ -77,81 +74,6 @@ class TestMain:
|
|||
warnings.simplefilter("error")
|
||||
await self._test_spider(TestSpider, [ITEM_A])
|
||||
|
||||
@coroutine_test
|
||||
async def test_deprecated(self):
|
||||
class TestSpider(Spider):
|
||||
name = "test"
|
||||
|
||||
def start_requests(self):
|
||||
yield ITEM_A
|
||||
|
||||
with pytest.warns(ScrapyDeprecationWarning):
|
||||
await self._test_spider(TestSpider, [ITEM_A])
|
||||
|
||||
@coroutine_test
|
||||
async def test_deprecated_subclass(self):
|
||||
class BaseSpider(Spider):
|
||||
def start_requests(self):
|
||||
yield ITEM_A
|
||||
|
||||
class TestSpider(BaseSpider):
|
||||
name = "test"
|
||||
|
||||
# The warning must be about the base class and not the subclass.
|
||||
with pytest.warns(ScrapyDeprecationWarning, match="BaseSpider"):
|
||||
await self._test_spider(TestSpider, [ITEM_A])
|
||||
|
||||
@coroutine_test
|
||||
async def test_universal(self):
|
||||
class TestSpider(Spider):
|
||||
name = "test"
|
||||
|
||||
async def start(self):
|
||||
yield ITEM_A
|
||||
|
||||
def start_requests(self):
|
||||
yield ITEM_B
|
||||
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error")
|
||||
await self._test_spider(TestSpider, [ITEM_A])
|
||||
|
||||
@coroutine_test
|
||||
async def test_universal_subclass(self):
|
||||
class BaseSpider(Spider):
|
||||
async def start(self):
|
||||
yield ITEM_A
|
||||
|
||||
def start_requests(self):
|
||||
yield ITEM_B
|
||||
|
||||
class TestSpider(BaseSpider):
|
||||
name = "test"
|
||||
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error")
|
||||
await self._test_spider(TestSpider, [ITEM_A])
|
||||
|
||||
@coroutine_test
|
||||
async def test_start_deprecated_super(self):
|
||||
class TestSpider(Spider):
|
||||
name = "test"
|
||||
|
||||
async def start(self):
|
||||
for item_or_request in super().start_requests():
|
||||
yield item_or_request
|
||||
|
||||
msg = "use Spider.start() instead"
|
||||
with pytest.warns(ScrapyDeprecationWarning, match=re.escape(msg)) as ws:
|
||||
await self._test_spider(TestSpider, [])
|
||||
|
||||
for w in ws:
|
||||
if isinstance(w.message, ScrapyDeprecationWarning) and msg in str(
|
||||
w.message
|
||||
):
|
||||
assert w.filename.endswith("test_spider_start.py")
|
||||
break
|
||||
|
||||
async def _test_start(self, start_, expected_items=None):
|
||||
class TestSpider(Spider):
|
||||
name = "test"
|
||||
|
|
@ -176,24 +98,3 @@ class TestMain:
|
|||
yield ITEM_A
|
||||
|
||||
await self._test_start(start, [ITEM_A])
|
||||
|
||||
# Exceptions
|
||||
|
||||
@coroutine_test
|
||||
async def test_deprecated_non_generator_exception(self):
|
||||
class TestSpider(Spider):
|
||||
name = "test"
|
||||
|
||||
def start_requests(self):
|
||||
raise RuntimeError
|
||||
|
||||
with (
|
||||
LogCapture() as log,
|
||||
pytest.warns(
|
||||
ScrapyDeprecationWarning,
|
||||
match=r"defines the deprecated start_requests\(\) method",
|
||||
),
|
||||
):
|
||||
await self._test_spider(TestSpider, [])
|
||||
|
||||
assert "in start_requests\n raise RuntimeError" in str(log)
|
||||
|
|
|
|||
|
|
@ -7,7 +7,9 @@ import pytest
|
|||
from scrapy import Request, Spider
|
||||
from scrapy.http import Response
|
||||
from scrapy.spidermiddlewares.base import BaseSpiderMiddleware
|
||||
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
|
||||
from scrapy.utils.test import get_crawler
|
||||
from tests.utils.decorators import coroutine_test
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scrapy.crawler import Crawler
|
||||
|
|
@ -18,7 +20,8 @@ def crawler() -> Crawler:
|
|||
return get_crawler(Spider)
|
||||
|
||||
|
||||
def test_trivial(crawler: Crawler) -> None:
|
||||
@coroutine_test
|
||||
async def test_trivial(crawler: Crawler) -> None:
|
||||
class TrivialSpiderMiddleware(BaseSpiderMiddleware):
|
||||
pass
|
||||
|
||||
|
|
@ -29,12 +32,13 @@ def test_trivial(crawler: Crawler) -> None:
|
|||
spider_output = [test_req, {"foo": "bar"}]
|
||||
for processed in [
|
||||
list(mw.process_spider_output(Response("data:,"), spider_output)),
|
||||
list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type]
|
||||
await collect_asyncgen(mw.process_start(as_async_generator(spider_output))),
|
||||
]:
|
||||
assert processed == [test_req, {"foo": "bar"}]
|
||||
|
||||
|
||||
def test_processed_request(crawler: Crawler) -> None:
|
||||
@coroutine_test
|
||||
async def test_processed_request(crawler: Crawler) -> None:
|
||||
class ProcessReqSpiderMiddleware(BaseSpiderMiddleware):
|
||||
def get_processed_request(
|
||||
self, request: Request, response: Response | None
|
||||
|
|
@ -52,7 +56,7 @@ def test_processed_request(crawler: Crawler) -> None:
|
|||
spider_output = [test_req1, {"foo": "bar"}, test_req2, test_req3]
|
||||
for processed in [
|
||||
list(mw.process_spider_output(Response("data:,"), spider_output)),
|
||||
list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type]
|
||||
await collect_asyncgen(mw.process_start(as_async_generator(spider_output))),
|
||||
]:
|
||||
assert len(processed) == 3
|
||||
assert isinstance(processed[0], Request)
|
||||
|
|
@ -62,7 +66,8 @@ def test_processed_request(crawler: Crawler) -> None:
|
|||
assert processed[2].url == "data:30,"
|
||||
|
||||
|
||||
def test_processed_item(crawler: Crawler) -> None:
|
||||
@coroutine_test
|
||||
async def test_processed_item(crawler: Crawler) -> None:
|
||||
class ProcessItemSpiderMiddleware(BaseSpiderMiddleware):
|
||||
def get_processed_item(self, item: Any, response: Response | None) -> Any:
|
||||
if item["foo"] == 2:
|
||||
|
|
@ -76,12 +81,13 @@ def test_processed_item(crawler: Crawler) -> None:
|
|||
spider_output = [{"foo": 1}, {"foo": 2}, test_req, {"foo": 3}]
|
||||
for processed in [
|
||||
list(mw.process_spider_output(Response("data:,"), spider_output)),
|
||||
list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type]
|
||||
await collect_asyncgen(mw.process_start(as_async_generator(spider_output))),
|
||||
]:
|
||||
assert processed == [{"foo": 1}, test_req, {"foo": 30}]
|
||||
|
||||
|
||||
def test_processed_both(crawler: Crawler) -> None:
|
||||
@coroutine_test
|
||||
async def test_processed_both(crawler: Crawler) -> None:
|
||||
class ProcessBothSpiderMiddleware(BaseSpiderMiddleware):
|
||||
def get_processed_request(
|
||||
self, request: Request, response: Response | None
|
||||
|
|
@ -113,7 +119,7 @@ def test_processed_both(crawler: Crawler) -> None:
|
|||
]
|
||||
for processed in [
|
||||
list(mw.process_spider_output(Response("data:,"), spider_output)),
|
||||
list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type]
|
||||
await collect_asyncgen(mw.process_start(as_async_generator(spider_output))),
|
||||
]:
|
||||
assert len(processed) == 4
|
||||
assert isinstance(processed[0], Request)
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ from asyncio import sleep
|
|||
import pytest
|
||||
|
||||
from scrapy import Spider, signals
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.utils.defer import maybe_deferred_to_future
|
||||
from scrapy.utils.test import get_crawler
|
||||
from tests.test_spider_start import SLEEP_SECONDS
|
||||
|
|
@ -38,15 +37,6 @@ class TwistedSleepSpiderMiddleware:
|
|||
yield item_or_request
|
||||
|
||||
|
||||
class UniversalSpiderMiddleware:
|
||||
async def process_start(self, start):
|
||||
async for item_or_request in start:
|
||||
yield item_or_request
|
||||
|
||||
def process_start_requests(self, start_requests, spider):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
# Spiders and spider middlewares for TestMain._test_wrap
|
||||
|
||||
|
||||
|
|
@ -61,23 +51,6 @@ class ModernWrapSpiderSubclass(ModernWrapSpider):
|
|||
name = "test"
|
||||
|
||||
|
||||
class UniversalWrapSpider(Spider):
|
||||
name = "test"
|
||||
|
||||
async def start(self):
|
||||
yield ITEM_B
|
||||
|
||||
def start_requests(self):
|
||||
yield ITEM_D
|
||||
|
||||
|
||||
class DeprecatedWrapSpider(Spider):
|
||||
name = "test"
|
||||
|
||||
def start_requests(self):
|
||||
yield ITEM_B
|
||||
|
||||
|
||||
class ModernWrapSpiderMiddleware:
|
||||
async def process_start(self, start):
|
||||
yield ITEM_A
|
||||
|
|
@ -86,26 +59,6 @@ class ModernWrapSpiderMiddleware:
|
|||
yield ITEM_C
|
||||
|
||||
|
||||
class UniversalWrapSpiderMiddleware:
|
||||
async def process_start(self, start):
|
||||
yield ITEM_A
|
||||
async for item_or_request in start:
|
||||
yield item_or_request
|
||||
yield ITEM_C
|
||||
|
||||
def process_start_requests(self, start, spider):
|
||||
yield ITEM_A
|
||||
yield from start
|
||||
yield ITEM_C
|
||||
|
||||
|
||||
class DeprecatedWrapSpiderMiddleware:
|
||||
def process_start_requests(self, start, spider):
|
||||
yield ITEM_A
|
||||
yield from start
|
||||
yield ITEM_C
|
||||
|
||||
|
||||
class TestMain:
|
||||
async def _test(self, spider_middlewares, spider_cls, expected_items):
|
||||
actual_items = []
|
||||
|
|
@ -136,195 +89,6 @@ class TestMain:
|
|||
warnings.simplefilter("error")
|
||||
await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider)
|
||||
|
||||
@coroutine_test
|
||||
async def test_modern_mw_universal_spider(self):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error")
|
||||
await self._test_wrap(ModernWrapSpiderMiddleware, UniversalWrapSpider)
|
||||
|
||||
@coroutine_test
|
||||
async def test_modern_mw_deprecated_spider(self):
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)"
|
||||
):
|
||||
await self._test_wrap(ModernWrapSpiderMiddleware, DeprecatedWrapSpider)
|
||||
|
||||
@coroutine_test
|
||||
async def test_universal_mw_modern_spider(self):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error")
|
||||
await self._test_wrap(UniversalWrapSpiderMiddleware, ModernWrapSpider)
|
||||
|
||||
@coroutine_test
|
||||
async def test_universal_mw_universal_spider(self):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error")
|
||||
await self._test_wrap(UniversalWrapSpiderMiddleware, UniversalWrapSpider)
|
||||
|
||||
@coroutine_test
|
||||
async def test_universal_mw_deprecated_spider(self):
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)"
|
||||
):
|
||||
await self._test_wrap(UniversalWrapSpiderMiddleware, DeprecatedWrapSpider)
|
||||
|
||||
@coroutine_test
|
||||
async def test_deprecated_mw_modern_spider(self):
|
||||
with (
|
||||
pytest.warns(
|
||||
ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"
|
||||
),
|
||||
pytest.raises(
|
||||
ValueError, match=r"only compatible with \(deprecated\) spiders"
|
||||
),
|
||||
):
|
||||
await self._test_wrap(DeprecatedWrapSpiderMiddleware, ModernWrapSpider)
|
||||
|
||||
@coroutine_test
|
||||
async def test_deprecated_mw_modern_spider_subclass(self):
|
||||
with (
|
||||
pytest.warns(
|
||||
ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"
|
||||
),
|
||||
pytest.raises(
|
||||
ValueError,
|
||||
match=r"^\S+?\.ModernWrapSpider \(inherited by \S+?.ModernWrapSpiderSubclass\) .*? only compatible with \(deprecated\) spiders",
|
||||
),
|
||||
):
|
||||
await self._test_wrap(
|
||||
DeprecatedWrapSpiderMiddleware, ModernWrapSpiderSubclass
|
||||
)
|
||||
|
||||
@coroutine_test
|
||||
async def test_deprecated_mw_universal_spider(self):
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"
|
||||
):
|
||||
await self._test_wrap(
|
||||
DeprecatedWrapSpiderMiddleware,
|
||||
UniversalWrapSpider,
|
||||
[ITEM_A, ITEM_D, ITEM_C],
|
||||
)
|
||||
|
||||
@coroutine_test
|
||||
async def test_deprecated_mw_deprecated_spider(self):
|
||||
with (
|
||||
pytest.warns(
|
||||
ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"
|
||||
),
|
||||
pytest.warns(
|
||||
ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)"
|
||||
),
|
||||
):
|
||||
await self._test_wrap(DeprecatedWrapSpiderMiddleware, DeprecatedWrapSpider)
|
||||
|
||||
@coroutine_test
|
||||
async def test_modern_mw_universal_mw_modern_spider(self):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error")
|
||||
await self._test_douple_wrap(
|
||||
ModernWrapSpiderMiddleware,
|
||||
UniversalWrapSpiderMiddleware,
|
||||
ModernWrapSpider,
|
||||
)
|
||||
|
||||
@coroutine_test
|
||||
async def test_modern_mw_deprecated_mw_modern_spider(self):
|
||||
with pytest.raises(ValueError, match=r"trying to combine spider middlewares"):
|
||||
await self._test_douple_wrap(
|
||||
ModernWrapSpiderMiddleware,
|
||||
DeprecatedWrapSpiderMiddleware,
|
||||
ModernWrapSpider,
|
||||
)
|
||||
|
||||
@coroutine_test
|
||||
async def test_universal_mw_deprecated_mw_modern_spider(self):
|
||||
with (
|
||||
pytest.warns(
|
||||
ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"
|
||||
),
|
||||
pytest.raises(
|
||||
ValueError, match=r"only compatible with \(deprecated\) spiders"
|
||||
),
|
||||
):
|
||||
await self._test_douple_wrap(
|
||||
UniversalWrapSpiderMiddleware,
|
||||
DeprecatedWrapSpiderMiddleware,
|
||||
ModernWrapSpider,
|
||||
)
|
||||
|
||||
@coroutine_test
|
||||
async def test_modern_mw_universal_mw_universal_spider(self):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error")
|
||||
await self._test_douple_wrap(
|
||||
ModernWrapSpiderMiddleware,
|
||||
UniversalWrapSpiderMiddleware,
|
||||
UniversalWrapSpider,
|
||||
)
|
||||
|
||||
@coroutine_test
|
||||
async def test_modern_mw_deprecated_mw_universal_spider(self):
|
||||
with pytest.raises(ValueError, match=r"trying to combine spider middlewares"):
|
||||
await self._test_douple_wrap(
|
||||
ModernWrapSpiderMiddleware,
|
||||
DeprecatedWrapSpiderMiddleware,
|
||||
UniversalWrapSpider,
|
||||
)
|
||||
|
||||
@coroutine_test
|
||||
async def test_universal_mw_deprecated_mw_universal_spider(self):
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"
|
||||
):
|
||||
await self._test_douple_wrap(
|
||||
UniversalWrapSpiderMiddleware,
|
||||
DeprecatedWrapSpiderMiddleware,
|
||||
UniversalWrapSpider,
|
||||
[ITEM_A, ITEM_A, ITEM_D, ITEM_C, ITEM_C],
|
||||
)
|
||||
|
||||
@coroutine_test
|
||||
async def test_modern_mw_universal_mw_deprecated_spider(self):
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)"
|
||||
):
|
||||
await self._test_douple_wrap(
|
||||
ModernWrapSpiderMiddleware,
|
||||
UniversalWrapSpiderMiddleware,
|
||||
DeprecatedWrapSpider,
|
||||
)
|
||||
|
||||
@coroutine_test
|
||||
async def test_modern_mw_deprecated_mw_deprecated_spider(self):
|
||||
with pytest.raises(ValueError, match=r"trying to combine spider middlewares"):
|
||||
await self._test_douple_wrap(
|
||||
ModernWrapSpiderMiddleware,
|
||||
DeprecatedWrapSpiderMiddleware,
|
||||
DeprecatedWrapSpider,
|
||||
)
|
||||
|
||||
@coroutine_test
|
||||
async def test_universal_mw_deprecated_mw_deprecated_spider(self):
|
||||
with (
|
||||
pytest.warns(
|
||||
ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"
|
||||
),
|
||||
pytest.warns(
|
||||
ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)"
|
||||
),
|
||||
):
|
||||
await self._test_douple_wrap(
|
||||
UniversalWrapSpiderMiddleware,
|
||||
DeprecatedWrapSpiderMiddleware,
|
||||
DeprecatedWrapSpider,
|
||||
)
|
||||
|
||||
@coroutine_test
|
||||
async def test_universal_mw_uses_process_start(self):
|
||||
"""Test that process_start_requests() isn't used when process_start() exists."""
|
||||
await self._test([UniversalSpiderMiddleware], ModernWrapSpider, [ITEM_B])
|
||||
|
||||
async def _test_sleep(self, spider_middlewares):
|
||||
class TestSpider(Spider):
|
||||
name = "test"
|
||||
|
|
|
|||
|
|
@ -23,20 +23,3 @@ class TestMiddleware:
|
|||
async for request in mw.process_start(start())
|
||||
]
|
||||
assert result == [True, True, False, "foo"]
|
||||
|
||||
@coroutine_test
|
||||
async def test_sync(self):
|
||||
crawler = get_crawler(Spider)
|
||||
mw = build_from_crawler(StartSpiderMiddleware, crawler)
|
||||
|
||||
def start():
|
||||
yield Request("data:,1")
|
||||
yield Request("data:,2", meta={"is_start_request": True})
|
||||
yield Request("data:,2", meta={"is_start_request": False})
|
||||
yield Request("data:,2", meta={"is_start_request": "foo"})
|
||||
|
||||
result = [
|
||||
request.meta["is_start_request"]
|
||||
for request in mw.process_start_requests(start(), Spider("test"))
|
||||
]
|
||||
assert result == [True, True, False, "foo"]
|
||||
|
|
|
|||
Loading…
Reference in New Issue