mirror of https://github.com/scrapy/scrapy.git
Split out test_crawler_runners.py.
This commit is contained in:
Andrey Rakhmatullin
parent
368d429c8c
commit
273951902a
|
|
@ -14,14 +14,13 @@ from twisted.internet.ssl import Certificate
|
|||
from twisted.python.failure import Failure
|
||||
|
||||
from scrapy import Spider, signals
|
||||
from scrapy.crawler import AsyncCrawlerRunner, Crawler, CrawlerRunner
|
||||
from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, StopDownload
|
||||
from scrapy.http import Request
|
||||
from scrapy.http.response import Response
|
||||
from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future
|
||||
from scrapy.utils.defer import maybe_deferred_to_future
|
||||
from scrapy.utils.engine import format_engine_status, get_engine_status
|
||||
from scrapy.utils.python import to_unicode
|
||||
from scrapy.utils.test import get_crawler, get_reactor_settings
|
||||
from scrapy.utils.test import get_crawler
|
||||
from tests import NON_EXISTING_RESOLVABLE
|
||||
from tests.spiders import (
|
||||
AsyncDefAsyncioGenComplexSpider,
|
||||
|
|
@ -430,50 +429,6 @@ with multiples lines
|
|||
)
|
||||
assert not crawler.crawling
|
||||
|
||||
@coroutine_test
|
||||
async def test_crawlerrunner_accepts_crawler(
|
||||
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
|
||||
) -> None:
|
||||
crawler = Crawler(SimpleSpider, get_reactor_settings())
|
||||
runner = CrawlerRunner()
|
||||
with caplog.at_level(logging.DEBUG):
|
||||
await maybe_deferred_to_future(
|
||||
runner.crawl(
|
||||
crawler,
|
||||
mockserver.url("/status?n=200"),
|
||||
mockserver=mockserver,
|
||||
)
|
||||
)
|
||||
assert "Got response 200" in caplog.text
|
||||
|
||||
@coroutine_test
|
||||
async def test_crawl_multiple(
|
||||
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
|
||||
) -> None:
|
||||
settings_dict = get_reactor_settings()
|
||||
runner_cls = (
|
||||
CrawlerRunner
|
||||
if settings_dict.get("TWISTED_REACTOR_ENABLED", True)
|
||||
else AsyncCrawlerRunner
|
||||
)
|
||||
runner = runner_cls(settings_dict)
|
||||
runner.crawl(
|
||||
SimpleSpider,
|
||||
mockserver.url("/status?n=200"),
|
||||
mockserver=mockserver,
|
||||
)
|
||||
runner.crawl(
|
||||
SimpleSpider,
|
||||
mockserver.url("/status?n=503"),
|
||||
mockserver=mockserver,
|
||||
)
|
||||
|
||||
with caplog.at_level(logging.DEBUG):
|
||||
await ensure_awaitable(runner.join())
|
||||
|
||||
self._assert_retried(caplog.text)
|
||||
assert "Got response 200" in caplog.text
|
||||
|
||||
@coroutine_test
|
||||
async def test_unknown_url_scheme(self, caplog: pytest.LogCaptureFixture) -> None:
|
||||
crawler = get_crawler(SimpleSpider)
|
||||
|
|
|
|||
|
|
@ -10,22 +10,14 @@ from typing import TYPE_CHECKING, Any, ClassVar
|
|||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from zope.interface.exceptions import MultipleInvalid
|
||||
|
||||
import scrapy
|
||||
from scrapy import Spider
|
||||
from scrapy.crawler import (
|
||||
AsyncCrawlerProcess,
|
||||
AsyncCrawlerRunner,
|
||||
Crawler,
|
||||
CrawlerProcess,
|
||||
CrawlerRunner,
|
||||
CrawlerRunnerBase,
|
||||
)
|
||||
from scrapy.crawler import AsyncCrawlerProcess, Crawler, CrawlerProcess
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.extensions.throttle import AutoThrottle
|
||||
from scrapy.settings import Settings, default_settings
|
||||
from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future
|
||||
from scrapy.utils.defer import maybe_deferred_to_future
|
||||
from scrapy.utils.log import (
|
||||
_uninstall_scrapy_root_handler,
|
||||
configure_logging,
|
||||
|
|
@ -33,7 +25,8 @@ from scrapy.utils.log import (
|
|||
)
|
||||
from scrapy.utils.spider import DefaultSpider
|
||||
from scrapy.utils.test import get_crawler, get_reactor_settings
|
||||
from tests.spiders import ExceptionSpider, NoRequestsSpider
|
||||
from tests.spiders import NoRequestsSpider
|
||||
from tests.utils import assert_option_is_default
|
||||
from tests.utils.decorators import coroutine_test
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -53,14 +46,7 @@ def get_raw_crawler(
|
|||
return Crawler(spidercls or DefaultSpider, settings)
|
||||
|
||||
|
||||
class TestBaseCrawler:
|
||||
@staticmethod
|
||||
def assertOptionIsDefault(settings: Settings, key: str) -> None:
|
||||
assert isinstance(settings, Settings)
|
||||
assert settings[key] == getattr(default_settings, key)
|
||||
|
||||
|
||||
class TestCrawler(TestBaseCrawler):
|
||||
class TestCrawler:
|
||||
def test_populate_spidercls_settings(self) -> None:
|
||||
spider_settings: dict[str, Any] = {
|
||||
"TEST1": "spider",
|
||||
|
|
@ -91,11 +77,11 @@ class TestCrawler(TestBaseCrawler):
|
|||
def test_crawler_accepts_dict(self) -> None:
|
||||
crawler = get_crawler(DefaultSpider, {"foo": "bar"})
|
||||
assert crawler.settings["foo"] == "bar"
|
||||
self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED")
|
||||
assert_option_is_default(crawler.settings, "RETRY_ENABLED")
|
||||
|
||||
def test_crawler_accepts_None(self) -> None:
|
||||
crawler = Crawler(DefaultSpider)
|
||||
self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED")
|
||||
assert_option_is_default(crawler.settings, "RETRY_ENABLED")
|
||||
|
||||
def test_crawler_rejects_spider_objects(self) -> None:
|
||||
with pytest.raises(ValueError, match="spidercls argument must be a class"):
|
||||
|
|
@ -587,78 +573,6 @@ class TestCrawlerLogging:
|
|||
assert "debug message" in logged
|
||||
|
||||
|
||||
class SpiderLoaderWithWrongInterface:
|
||||
def unneeded_method(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
class TestCrawlerRunner(TestBaseCrawler):
|
||||
def test_spider_manager_verify_interface(self) -> None:
|
||||
settings = Settings(
|
||||
{
|
||||
"SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
|
||||
}
|
||||
)
|
||||
with pytest.raises(MultipleInvalid):
|
||||
CrawlerRunner(settings)
|
||||
|
||||
def test_crawler_runner_accepts_dict(self) -> None:
|
||||
runner = CrawlerRunner({"foo": "bar"})
|
||||
assert runner.settings["foo"] == "bar"
|
||||
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
def test_crawler_runner_accepts_None(self) -> None:
|
||||
runner = CrawlerRunner()
|
||||
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
|
||||
class TestAsyncCrawlerRunner(TestBaseCrawler):
|
||||
def test_spider_manager_verify_interface(self) -> None:
|
||||
settings = Settings(
|
||||
{
|
||||
"SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
|
||||
}
|
||||
)
|
||||
with pytest.raises(MultipleInvalid):
|
||||
AsyncCrawlerRunner(settings)
|
||||
|
||||
def test_crawler_runner_accepts_dict(self) -> None:
|
||||
runner = AsyncCrawlerRunner({"foo": "bar"})
|
||||
assert runner.settings["foo"] == "bar"
|
||||
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
def test_crawler_runner_accepts_None(self) -> None:
|
||||
runner = AsyncCrawlerRunner()
|
||||
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
|
||||
class TestCrawlerProcess(TestBaseCrawler):
|
||||
def test_crawler_process_accepts_dict(self) -> None:
|
||||
runner = CrawlerProcess({"foo": "bar"}, install_root_handler=False)
|
||||
assert runner.settings["foo"] == "bar"
|
||||
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
def test_crawler_process_accepts_None(self) -> None:
|
||||
runner = CrawlerProcess(install_root_handler=False)
|
||||
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
class TestAsyncCrawlerProcess(TestBaseCrawler):
|
||||
def test_crawler_process_accepts_dict(self, reactor_pytest: str) -> None:
|
||||
runner = AsyncCrawlerProcess(
|
||||
{"foo": "bar", "TWISTED_REACTOR_ENABLED": reactor_pytest != "none"},
|
||||
install_root_handler=False,
|
||||
)
|
||||
assert runner.settings["foo"] == "bar"
|
||||
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
@pytest.mark.requires_reactor # can't pass TWISTED_REACTOR_ENABLED=False
|
||||
def test_crawler_process_accepts_None(self) -> None:
|
||||
runner = AsyncCrawlerProcess(install_root_handler=False)
|
||||
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
|
||||
class TestAsyncCrawlerProcessReactorlessHelpers:
|
||||
"""Unit tests for the reactorless shutdown helpers of AsyncCrawlerProcess.
|
||||
|
||||
|
|
@ -798,145 +712,6 @@ class TestAsyncCrawlerProcessReactorlessHelpers:
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
|
||||
def test_runner_settings_applied_to_crawler_instance(
|
||||
runner_cls: type[CrawlerRunnerBase],
|
||||
) -> None:
|
||||
runner = runner_cls({"FOO": "runner"})
|
||||
crawler = Crawler(DefaultSpider)
|
||||
result = runner.create_crawler(crawler)
|
||||
assert result is crawler
|
||||
assert result.settings["FOO"] == "runner"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
|
||||
def test_spider_custom_settings_override_runner(
|
||||
runner_cls: type[CrawlerRunnerBase],
|
||||
) -> None:
|
||||
class MySpider(DefaultSpider):
|
||||
custom_settings = {"FOO": "spider"}
|
||||
|
||||
runner = runner_cls({"FOO": "runner"})
|
||||
crawler = Crawler(MySpider)
|
||||
runner.create_crawler(crawler)
|
||||
assert crawler.settings["FOO"] == "spider"
|
||||
|
||||
|
||||
def test_create_crawler_instance_consistent_with_spider_class() -> None:
|
||||
runner = AsyncCrawlerRunner({"FOO": "runner"})
|
||||
|
||||
crawler_from_class = runner.create_crawler(DefaultSpider)
|
||||
|
||||
pre_built = Crawler(DefaultSpider)
|
||||
runner.create_crawler(pre_built)
|
||||
|
||||
assert crawler_from_class.settings["FOO"] == "runner"
|
||||
assert pre_built.settings["FOO"] == "runner"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
|
||||
def test_create_crawler_rejects_spider_object(
|
||||
runner_cls: type[CrawlerRunnerBase],
|
||||
) -> None:
|
||||
runner = runner_cls()
|
||||
with pytest.raises(ValueError, match="cannot be a spider object"):
|
||||
runner.create_crawler(DefaultSpider()) # type: ignore[arg-type]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
|
||||
def test_crawl_rejects_spider_object(runner_cls: type[CrawlerRunnerBase]) -> None:
|
||||
runner = runner_cls()
|
||||
with pytest.raises(ValueError, match="cannot be a spider object"):
|
||||
runner.crawl(DefaultSpider()) # type: ignore[arg-type]
|
||||
|
||||
|
||||
@pytest.mark.requires_reactor # CrawlerRunner requires a reactor
|
||||
class TestCrawlerRunnerHasSpider:
|
||||
@pytest.fixture
|
||||
def runner(self) -> CrawlerRunnerBase:
|
||||
return CrawlerRunner(get_reactor_settings())
|
||||
|
||||
@staticmethod
|
||||
async def _crawl(runner: CrawlerRunnerBase, spider: type[Spider]) -> None:
|
||||
await ensure_awaitable(runner.crawl(spider))
|
||||
|
||||
@coroutine_test
|
||||
async def test_crawler_runner_bootstrap_successful(
|
||||
self, runner: CrawlerRunnerBase
|
||||
) -> None:
|
||||
await self._crawl(runner, NoRequestsSpider)
|
||||
assert not runner.bootstrap_failed
|
||||
|
||||
@coroutine_test
|
||||
async def test_crawler_runner_bootstrap_successful_for_several(
|
||||
self, runner: CrawlerRunnerBase
|
||||
) -> None:
|
||||
await self._crawl(runner, NoRequestsSpider)
|
||||
await self._crawl(runner, NoRequestsSpider)
|
||||
assert not runner.bootstrap_failed
|
||||
|
||||
@coroutine_test
|
||||
async def test_crawler_runner_bootstrap_failed(
|
||||
self, runner: CrawlerRunnerBase
|
||||
) -> None:
|
||||
try:
|
||||
await self._crawl(runner, ExceptionSpider)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
pytest.fail("Exception should be raised from spider")
|
||||
|
||||
assert runner.bootstrap_failed
|
||||
|
||||
@coroutine_test
|
||||
async def test_crawler_runner_bootstrap_failed_for_several(
|
||||
self, runner: CrawlerRunnerBase
|
||||
) -> None:
|
||||
try:
|
||||
await self._crawl(runner, ExceptionSpider)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
pytest.fail("Exception should be raised from spider")
|
||||
|
||||
await self._crawl(runner, NoRequestsSpider)
|
||||
|
||||
assert runner.bootstrap_failed
|
||||
|
||||
@coroutine_test
|
||||
async def test_crawler_runner_asyncio_enabled_true(
|
||||
self, reactor_pytest: str
|
||||
) -> None:
|
||||
if reactor_pytest != "asyncio":
|
||||
runner = CrawlerRunner(
|
||||
settings={
|
||||
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
|
||||
}
|
||||
)
|
||||
with pytest.raises(
|
||||
Exception,
|
||||
match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)",
|
||||
):
|
||||
await self._crawl(runner, NoRequestsSpider)
|
||||
else:
|
||||
runner = CrawlerRunner(
|
||||
settings={
|
||||
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
|
||||
}
|
||||
)
|
||||
await self._crawl(runner, NoRequestsSpider)
|
||||
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider):
|
||||
@pytest.fixture
|
||||
def runner(self) -> CrawlerRunnerBase:
|
||||
return AsyncCrawlerRunner(get_reactor_settings())
|
||||
|
||||
def test_crawler_runner_asyncio_enabled_true(self) -> None: # type: ignore[override]
|
||||
pytest.skip("This test is only for CrawlerRunner")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("settings", "items"),
|
||||
[
|
||||
|
|
|
|||
|
|
@ -0,0 +1,284 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
from zope.interface.exceptions import MultipleInvalid
|
||||
|
||||
from scrapy.crawler import (
|
||||
AsyncCrawlerProcess,
|
||||
AsyncCrawlerRunner,
|
||||
Crawler,
|
||||
CrawlerProcess,
|
||||
CrawlerRunner,
|
||||
CrawlerRunnerBase,
|
||||
)
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future
|
||||
from scrapy.utils.spider import DefaultSpider
|
||||
from scrapy.utils.test import get_reactor_settings
|
||||
from tests.spiders import ExceptionSpider, NoRequestsSpider, SimpleSpider
|
||||
from tests.utils import assert_option_is_default
|
||||
from tests.utils.decorators import coroutine_test
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scrapy import Spider
|
||||
from tests.mockserver.http import MockServer
|
||||
|
||||
|
||||
class SpiderLoaderWithWrongInterface:
|
||||
def unneeded_method(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
class TestCrawlerRunner:
|
||||
def test_spider_manager_verify_interface(self) -> None:
|
||||
settings = Settings(
|
||||
{
|
||||
"SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
|
||||
}
|
||||
)
|
||||
with pytest.raises(MultipleInvalid):
|
||||
CrawlerRunner(settings)
|
||||
|
||||
def test_crawler_runner_accepts_dict(self) -> None:
|
||||
runner = CrawlerRunner({"foo": "bar"})
|
||||
assert runner.settings["foo"] == "bar"
|
||||
assert_option_is_default(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
def test_crawler_runner_accepts_None(self) -> None:
|
||||
runner = CrawlerRunner()
|
||||
assert_option_is_default(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
|
||||
class TestAsyncCrawlerRunner:
|
||||
def test_spider_manager_verify_interface(self) -> None:
|
||||
settings = Settings(
|
||||
{
|
||||
"SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
|
||||
}
|
||||
)
|
||||
with pytest.raises(MultipleInvalid):
|
||||
AsyncCrawlerRunner(settings)
|
||||
|
||||
def test_crawler_runner_accepts_dict(self) -> None:
|
||||
runner = AsyncCrawlerRunner({"foo": "bar"})
|
||||
assert runner.settings["foo"] == "bar"
|
||||
assert_option_is_default(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
def test_crawler_runner_accepts_None(self) -> None:
|
||||
runner = AsyncCrawlerRunner()
|
||||
assert_option_is_default(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
|
||||
class TestCrawlerProcess:
|
||||
def test_crawler_process_accepts_dict(self) -> None:
|
||||
runner = CrawlerProcess({"foo": "bar"}, install_root_handler=False)
|
||||
assert runner.settings["foo"] == "bar"
|
||||
assert_option_is_default(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
def test_crawler_process_accepts_None(self) -> None:
|
||||
runner = CrawlerProcess(install_root_handler=False)
|
||||
assert_option_is_default(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
class TestAsyncCrawlerProcess:
|
||||
def test_crawler_process_accepts_dict(self, reactor_pytest: str) -> None:
|
||||
runner = AsyncCrawlerProcess(
|
||||
{"foo": "bar", "TWISTED_REACTOR_ENABLED": reactor_pytest != "none"},
|
||||
install_root_handler=False,
|
||||
)
|
||||
assert runner.settings["foo"] == "bar"
|
||||
assert_option_is_default(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
@pytest.mark.requires_reactor # can't pass TWISTED_REACTOR_ENABLED=False
|
||||
def test_crawler_process_accepts_None(self) -> None:
|
||||
runner = AsyncCrawlerProcess(install_root_handler=False)
|
||||
assert_option_is_default(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
|
||||
@pytest.mark.requires_reactor # CrawlerRunner requires a reactor
|
||||
class TestCrawlerRunnerHasSpider:
|
||||
@pytest.fixture
|
||||
def runner(self) -> CrawlerRunnerBase:
|
||||
return CrawlerRunner(get_reactor_settings())
|
||||
|
||||
@staticmethod
|
||||
async def _crawl(runner: CrawlerRunnerBase, spider: type[Spider]) -> None:
|
||||
await ensure_awaitable(runner.crawl(spider))
|
||||
|
||||
@coroutine_test
|
||||
async def test_crawler_runner_bootstrap_successful(
|
||||
self, runner: CrawlerRunnerBase
|
||||
) -> None:
|
||||
await self._crawl(runner, NoRequestsSpider)
|
||||
assert not runner.bootstrap_failed
|
||||
|
||||
@coroutine_test
|
||||
async def test_crawler_runner_bootstrap_successful_for_several(
|
||||
self, runner: CrawlerRunnerBase
|
||||
) -> None:
|
||||
await self._crawl(runner, NoRequestsSpider)
|
||||
await self._crawl(runner, NoRequestsSpider)
|
||||
assert not runner.bootstrap_failed
|
||||
|
||||
@coroutine_test
|
||||
async def test_crawler_runner_bootstrap_failed(
|
||||
self, runner: CrawlerRunnerBase
|
||||
) -> None:
|
||||
try:
|
||||
await self._crawl(runner, ExceptionSpider)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
pytest.fail("Exception should be raised from spider")
|
||||
|
||||
assert runner.bootstrap_failed
|
||||
|
||||
@coroutine_test
|
||||
async def test_crawler_runner_bootstrap_failed_for_several(
|
||||
self, runner: CrawlerRunnerBase
|
||||
) -> None:
|
||||
try:
|
||||
await self._crawl(runner, ExceptionSpider)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
pytest.fail("Exception should be raised from spider")
|
||||
|
||||
await self._crawl(runner, NoRequestsSpider)
|
||||
|
||||
assert runner.bootstrap_failed
|
||||
|
||||
@coroutine_test
|
||||
async def test_crawler_runner_asyncio_enabled_true(
|
||||
self, reactor_pytest: str
|
||||
) -> None:
|
||||
if reactor_pytest != "asyncio":
|
||||
runner = CrawlerRunner(
|
||||
settings={
|
||||
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
|
||||
}
|
||||
)
|
||||
with pytest.raises(
|
||||
Exception,
|
||||
match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)",
|
||||
):
|
||||
await self._crawl(runner, NoRequestsSpider)
|
||||
else:
|
||||
runner = CrawlerRunner(
|
||||
settings={
|
||||
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
|
||||
}
|
||||
)
|
||||
await self._crawl(runner, NoRequestsSpider)
|
||||
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider):
|
||||
@pytest.fixture
|
||||
def runner(self) -> CrawlerRunnerBase:
|
||||
return AsyncCrawlerRunner(get_reactor_settings())
|
||||
|
||||
def test_crawler_runner_asyncio_enabled_true(self) -> None: # type: ignore[override]
|
||||
pytest.skip("This test is only for CrawlerRunner")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
|
||||
def test_runner_settings_applied_to_crawler_instance(
|
||||
runner_cls: type[CrawlerRunnerBase],
|
||||
) -> None:
|
||||
runner = runner_cls({"FOO": "runner"})
|
||||
crawler = Crawler(DefaultSpider)
|
||||
result = runner.create_crawler(crawler)
|
||||
assert result is crawler
|
||||
assert result.settings["FOO"] == "runner"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
|
||||
def test_spider_custom_settings_override_runner(
|
||||
runner_cls: type[CrawlerRunnerBase],
|
||||
) -> None:
|
||||
class MySpider(DefaultSpider):
|
||||
custom_settings = {"FOO": "spider"}
|
||||
|
||||
runner = runner_cls({"FOO": "runner"})
|
||||
crawler = Crawler(MySpider)
|
||||
runner.create_crawler(crawler)
|
||||
assert crawler.settings["FOO"] == "spider"
|
||||
|
||||
|
||||
def test_create_crawler_instance_consistent_with_spider_class() -> None:
|
||||
runner = AsyncCrawlerRunner({"FOO": "runner"})
|
||||
|
||||
crawler_from_class = runner.create_crawler(DefaultSpider)
|
||||
|
||||
pre_built = Crawler(DefaultSpider)
|
||||
runner.create_crawler(pre_built)
|
||||
|
||||
assert crawler_from_class.settings["FOO"] == "runner"
|
||||
assert pre_built.settings["FOO"] == "runner"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
|
||||
def test_create_crawler_rejects_spider_object(
|
||||
runner_cls: type[CrawlerRunnerBase],
|
||||
) -> None:
|
||||
runner = runner_cls()
|
||||
with pytest.raises(ValueError, match="cannot be a spider object"):
|
||||
runner.create_crawler(DefaultSpider()) # type: ignore[arg-type]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
|
||||
def test_crawl_rejects_spider_object(runner_cls: type[CrawlerRunnerBase]) -> None:
|
||||
runner = runner_cls()
|
||||
with pytest.raises(ValueError, match="cannot be a spider object"):
|
||||
runner.crawl(DefaultSpider()) # type: ignore[arg-type]
|
||||
|
||||
|
||||
@coroutine_test
|
||||
async def test_crawlerrunner_accepts_crawler(
|
||||
caplog: pytest.LogCaptureFixture, mockserver: MockServer
|
||||
) -> None:
|
||||
crawler = Crawler(SimpleSpider, get_reactor_settings())
|
||||
runner = CrawlerRunner()
|
||||
with caplog.at_level(logging.DEBUG):
|
||||
await maybe_deferred_to_future(
|
||||
runner.crawl(
|
||||
crawler,
|
||||
mockserver.url("/status?n=200"),
|
||||
mockserver=mockserver,
|
||||
)
|
||||
)
|
||||
assert "Got response 200" in caplog.text
|
||||
|
||||
|
||||
@coroutine_test
|
||||
async def test_crawl_multiple(
|
||||
caplog: pytest.LogCaptureFixture, mockserver: MockServer
|
||||
) -> None:
|
||||
settings_dict = get_reactor_settings()
|
||||
runner_cls = (
|
||||
CrawlerRunner
|
||||
if settings_dict.get("TWISTED_REACTOR_ENABLED", True)
|
||||
else AsyncCrawlerRunner
|
||||
)
|
||||
runner = runner_cls(settings_dict)
|
||||
runner.crawl(
|
||||
SimpleSpider,
|
||||
mockserver.url("/status?n=200"),
|
||||
mockserver=mockserver,
|
||||
)
|
||||
runner.crawl(
|
||||
SimpleSpider,
|
||||
mockserver.url("/status?n=503"),
|
||||
mockserver=mockserver,
|
||||
)
|
||||
|
||||
with caplog.at_level(logging.DEBUG):
|
||||
await ensure_awaitable(runner.join())
|
||||
|
||||
assert "Got response 200" in caplog.text
|
||||
assert "Gave up retrying" in caplog.text
|
||||
|
|
@ -1,13 +1,19 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from twisted.internet.defer import Deferred
|
||||
|
||||
from scrapy.settings import Settings, default_settings
|
||||
from scrapy.utils.asyncio import is_asyncio_available
|
||||
from scrapy.utils.defer import maybe_deferred_to_future
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
|
||||
def twisted_sleep(seconds: float):
|
||||
from twisted.internet import reactor
|
||||
|
|
@ -48,3 +54,8 @@ class OneShotLoop:
|
|||
|
||||
def stop(self) -> None:
|
||||
self.running = False
|
||||
|
||||
|
||||
def assert_option_is_default(settings: Settings, key: str) -> None:
|
||||
assert isinstance(settings, Settings)
|
||||
assert settings[key] == getattr(default_settings, key)
|
||||
|
|
|
|||
Loading…
Reference in New Issue