Remove InitSpider.

This commit is contained in:
Andrey Rakhmatullin 2026-05-03 14:12:16 +05:00
parent 14f49ab63c
commit 8ecfd20fcd
3 changed files with 1 additions and 89 deletions

View File

@ -126,7 +126,6 @@ ignore_errors = true
[[tool.mypy.overrides]]
module = [
"scrapy.core.downloader.webclient",
"scrapy.spiders.init",
"tests.test_webclient",
]
allow_any_generics = true
@ -138,7 +137,6 @@ warn_return_any = false
# usually no type hints
[[tool.mypy.overrides]]
module = [
# "IPython.*",
"bpython",
"brotli",
"brotlicffi",

View File

@ -1,64 +0,0 @@
from __future__ import annotations
import warnings
from typing import TYPE_CHECKING, Any, cast
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.spiders import Spider
from scrapy.utils.spider import iterate_spider_output
if TYPE_CHECKING:
from collections.abc import AsyncIterator, Iterable
from scrapy import Request
from scrapy.http import Response
class InitSpider(Spider):
"""Base Spider with initialization facilities
.. warning:: This class is deprecated. Copy its code into your project if needed.
It will be removed in a future Scrapy version.
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
warnings.warn(
"InitSpider is deprecated. Copy its code from Scrapy's source if needed. "
"Will be removed in a future version.",
ScrapyDeprecationWarning,
stacklevel=2,
)
async def start(self) -> AsyncIterator[Any]:
with warnings.catch_warnings():
warnings.filterwarnings(
"ignore", category=ScrapyDeprecationWarning, module=r"^scrapy\.spiders$"
)
for item_or_request in self.start_requests():
yield item_or_request
def start_requests(self) -> Iterable[Request]:
self._postinit_reqs: Iterable[Request] = super().start_requests()
return cast("Iterable[Request]", iterate_spider_output(self.init_request()))
def initialized(self, response: Response | None = None) -> Any:
"""This method must be set as the callback of your last initialization
request. See self.init_request() docstring for more info.
"""
return self.__dict__.pop("_postinit_reqs")
def init_request(self) -> Any:
"""This function should return one initialization request, with the
self.initialized method as callback. When the self.initialized method
is called this spider is considered initialized. If you need to perform
several requests for initializing your spider, you can do so by using
different callbacks. The only requirement is that the final callback
(of the last initialization request) must be self.initialized.
The default implementation calls self.initialized immediately, and
means that no initialization is needed. This method should be
overridden only when you need to perform requests to initialize your
spider
"""
return self.initialized()

View File

@ -11,10 +11,9 @@ from scrapy.crawler import Crawler
from scrapy.http import Response, TextResponse, XmlResponse
from scrapy.settings import Settings
from scrapy.spiders import CSVFeedSpider, Spider, XMLFeedSpider
from scrapy.spiders.init import InitSpider
from scrapy.utils.test import get_crawler, get_reactor_settings
from tests import get_testdata
from tests.utils.decorators import coroutine_test, inline_callbacks_test
from tests.utils.decorators import inline_callbacks_test
class TestSpider:
@ -122,27 +121,6 @@ class TestSpider:
mock_logger.log.assert_called_once_with("INFO", "test log msg")
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
class TestInitSpider(TestSpider):
spider_class = InitSpider
@coroutine_test
async def test_start_urls(self):
responses = []
class TestSpider(self.spider_class):
name = "test"
start_urls = ["data:,"]
async def parse(self, response):
responses.append(response)
crawler = get_crawler(TestSpider)
await crawler.crawl_async()
assert len(responses) == 1
assert responses[0].url == "data:,"
class TestXMLFeedSpider(TestSpider):
spider_class = XMLFeedSpider