diff --git a/pyproject.toml b/pyproject.toml index d35b333ff..ea7893e2a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -126,7 +126,6 @@ ignore_errors = true [[tool.mypy.overrides]] module = [ "scrapy.core.downloader.webclient", - "scrapy.spiders.init", "tests.test_webclient", ] allow_any_generics = true @@ -138,7 +137,6 @@ warn_return_any = false # usually no type hints [[tool.mypy.overrides]] module = [ -# "IPython.*", "bpython", "brotli", "brotlicffi", diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py deleted file mode 100644 index 957bfffd3..000000000 --- a/scrapy/spiders/init.py +++ /dev/null @@ -1,64 +0,0 @@ -from __future__ import annotations - -import warnings -from typing import TYPE_CHECKING, Any, cast - -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.spiders import Spider -from scrapy.utils.spider import iterate_spider_output - -if TYPE_CHECKING: - from collections.abc import AsyncIterator, Iterable - - from scrapy import Request - from scrapy.http import Response - - -class InitSpider(Spider): - """Base Spider with initialization facilities - - .. warning:: This class is deprecated. Copy its code into your project if needed. - It will be removed in a future Scrapy version. - """ - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - warnings.warn( - "InitSpider is deprecated. Copy its code from Scrapy's source if needed. " - "Will be removed in a future version.", - ScrapyDeprecationWarning, - stacklevel=2, - ) - - async def start(self) -> AsyncIterator[Any]: - with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", category=ScrapyDeprecationWarning, module=r"^scrapy\.spiders$" - ) - for item_or_request in self.start_requests(): - yield item_or_request - - def start_requests(self) -> Iterable[Request]: - self._postinit_reqs: Iterable[Request] = super().start_requests() - return cast("Iterable[Request]", iterate_spider_output(self.init_request())) - - def initialized(self, response: Response | None = None) -> Any: - """This method must be set as the callback of your last initialization - request. See self.init_request() docstring for more info. - """ - return self.__dict__.pop("_postinit_reqs") - - def init_request(self) -> Any: - """This function should return one initialization request, with the - self.initialized method as callback. When the self.initialized method - is called this spider is considered initialized. If you need to perform - several requests for initializing your spider, you can do so by using - different callbacks. The only requirement is that the final callback - (of the last initialization request) must be self.initialized. - - The default implementation calls self.initialized immediately, and - means that no initialization is needed. This method should be - overridden only when you need to perform requests to initialize your - spider - """ - return self.initialized() diff --git a/tests/test_spider.py b/tests/test_spider.py index ff9aed74f..23efed77a 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -11,10 +11,9 @@ from scrapy.crawler import Crawler from scrapy.http import Response, TextResponse, XmlResponse from scrapy.settings import Settings from scrapy.spiders import CSVFeedSpider, Spider, XMLFeedSpider -from scrapy.spiders.init import InitSpider from scrapy.utils.test import get_crawler, get_reactor_settings from tests import get_testdata -from tests.utils.decorators import coroutine_test, inline_callbacks_test +from tests.utils.decorators import inline_callbacks_test class TestSpider: @@ -122,27 +121,6 @@ class TestSpider: mock_logger.log.assert_called_once_with("INFO", "test log msg") -@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestInitSpider(TestSpider): - spider_class = InitSpider - - @coroutine_test - async def test_start_urls(self): - responses = [] - - class TestSpider(self.spider_class): - name = "test" - start_urls = ["data:,"] - - async def parse(self, response): - responses.append(response) - - crawler = get_crawler(TestSpider) - await crawler.crawl_async() - assert len(responses) == 1 - assert responses[0].url == "data:," - - class TestXMLFeedSpider(TestSpider): spider_class = XMLFeedSpider