mirror of https://github.com/scrapy/scrapy.git
Remove InitSpider.
This commit is contained in:
parent
14f49ab63c
commit
8ecfd20fcd
|
|
@ -126,7 +126,6 @@ ignore_errors = true
|
|||
[[tool.mypy.overrides]]
|
||||
module = [
|
||||
"scrapy.core.downloader.webclient",
|
||||
"scrapy.spiders.init",
|
||||
"tests.test_webclient",
|
||||
]
|
||||
allow_any_generics = true
|
||||
|
|
@ -138,7 +137,6 @@ warn_return_any = false
|
|||
# usually no type hints
|
||||
[[tool.mypy.overrides]]
|
||||
module = [
|
||||
# "IPython.*",
|
||||
"bpython",
|
||||
"brotli",
|
||||
"brotlicffi",
|
||||
|
|
|
|||
|
|
@ -1,64 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import warnings
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.spider import iterate_spider_output
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import AsyncIterator, Iterable
|
||||
|
||||
from scrapy import Request
|
||||
from scrapy.http import Response
|
||||
|
||||
|
||||
class InitSpider(Spider):
|
||||
"""Base Spider with initialization facilities
|
||||
|
||||
.. warning:: This class is deprecated. Copy its code into your project if needed.
|
||||
It will be removed in a future Scrapy version.
|
||||
"""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
warnings.warn(
|
||||
"InitSpider is deprecated. Copy its code from Scrapy's source if needed. "
|
||||
"Will be removed in a future version.",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
async def start(self) -> AsyncIterator[Any]:
|
||||
with warnings.catch_warnings():
|
||||
warnings.filterwarnings(
|
||||
"ignore", category=ScrapyDeprecationWarning, module=r"^scrapy\.spiders$"
|
||||
)
|
||||
for item_or_request in self.start_requests():
|
||||
yield item_or_request
|
||||
|
||||
def start_requests(self) -> Iterable[Request]:
|
||||
self._postinit_reqs: Iterable[Request] = super().start_requests()
|
||||
return cast("Iterable[Request]", iterate_spider_output(self.init_request()))
|
||||
|
||||
def initialized(self, response: Response | None = None) -> Any:
|
||||
"""This method must be set as the callback of your last initialization
|
||||
request. See self.init_request() docstring for more info.
|
||||
"""
|
||||
return self.__dict__.pop("_postinit_reqs")
|
||||
|
||||
def init_request(self) -> Any:
|
||||
"""This function should return one initialization request, with the
|
||||
self.initialized method as callback. When the self.initialized method
|
||||
is called this spider is considered initialized. If you need to perform
|
||||
several requests for initializing your spider, you can do so by using
|
||||
different callbacks. The only requirement is that the final callback
|
||||
(of the last initialization request) must be self.initialized.
|
||||
|
||||
The default implementation calls self.initialized immediately, and
|
||||
means that no initialization is needed. This method should be
|
||||
overridden only when you need to perform requests to initialize your
|
||||
spider
|
||||
"""
|
||||
return self.initialized()
|
||||
|
|
@ -11,10 +11,9 @@ from scrapy.crawler import Crawler
|
|||
from scrapy.http import Response, TextResponse, XmlResponse
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.spiders import CSVFeedSpider, Spider, XMLFeedSpider
|
||||
from scrapy.spiders.init import InitSpider
|
||||
from scrapy.utils.test import get_crawler, get_reactor_settings
|
||||
from tests import get_testdata
|
||||
from tests.utils.decorators import coroutine_test, inline_callbacks_test
|
||||
from tests.utils.decorators import inline_callbacks_test
|
||||
|
||||
|
||||
class TestSpider:
|
||||
|
|
@ -122,27 +121,6 @@ class TestSpider:
|
|||
mock_logger.log.assert_called_once_with("INFO", "test log msg")
|
||||
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
class TestInitSpider(TestSpider):
|
||||
spider_class = InitSpider
|
||||
|
||||
@coroutine_test
|
||||
async def test_start_urls(self):
|
||||
responses = []
|
||||
|
||||
class TestSpider(self.spider_class):
|
||||
name = "test"
|
||||
start_urls = ["data:,"]
|
||||
|
||||
async def parse(self, response):
|
||||
responses.append(response)
|
||||
|
||||
crawler = get_crawler(TestSpider)
|
||||
await crawler.crawl_async()
|
||||
assert len(responses) == 1
|
||||
assert responses[0].url == "data:,"
|
||||
|
||||
|
||||
class TestXMLFeedSpider(TestSpider):
|
||||
spider_class = XMLFeedSpider
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue