diff --git a/tests/AsyncCrawlerProcess/dns_resolver_deprecated.py b/tests/AsyncCrawlerProcess/dns_resolver_deprecated.py new file mode 100644 index 000000000..8c8df96bb --- /dev/null +++ b/tests/AsyncCrawlerProcess/dns_resolver_deprecated.py @@ -0,0 +1,31 @@ +import sys + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess +from scrapy.settings import Settings + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +settings = Settings() +# The deprecated DNS_RESOLVER setting, set above its default priority so that +# AsyncCrawlerProcess._setup_reactor() emits the deprecation warning. +settings.set("DNS_RESOLVER", "scrapy.resolver.CachingThreadedResolver", priority=10) +if len(sys.argv) > 1 and sys.argv[1] == "twisted-wins": + # TWISTED_DNS_RESOLVER at a higher priority takes precedence over the + # deprecated DNS_RESOLVER setting. + settings.set( + "TWISTED_DNS_RESOLVER", + "scrapy.resolver.CachingThreadedResolver", + priority=20, + ) + +process = AsyncCrawlerProcess(settings) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/CrawlerProcess/dns_resolver_deprecated.py b/tests/CrawlerProcess/dns_resolver_deprecated.py new file mode 100644 index 000000000..b8cd24325 --- /dev/null +++ b/tests/CrawlerProcess/dns_resolver_deprecated.py @@ -0,0 +1,31 @@ +import sys + +import scrapy +from scrapy.crawler import CrawlerProcess +from scrapy.settings import Settings + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +settings = Settings() +# The deprecated DNS_RESOLVER setting, set above its default priority so that +# CrawlerProcess._setup_reactor() emits the deprecation warning. +settings.set("DNS_RESOLVER", "scrapy.resolver.CachingThreadedResolver", priority=10) +if len(sys.argv) > 1 and sys.argv[1] == "twisted-wins": + # TWISTED_DNS_RESOLVER at a higher priority takes precedence over the + # deprecated DNS_RESOLVER setting. + settings.set( + "TWISTED_DNS_RESOLVER", + "scrapy.resolver.CachingThreadedResolver", + priority=20, + ) + +process = CrawlerProcess(settings) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 0cddfd0ed..c342e81d6 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -687,6 +687,22 @@ def test_create_crawler_instance_consistent_with_spider_class() -> None: assert pre_built.settings["FOO"] == "runner" +@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner]) +def test_create_crawler_rejects_spider_object( + runner_cls: type[CrawlerRunnerBase], +) -> None: + runner = runner_cls() + with pytest.raises(ValueError, match="cannot be a spider object"): + runner.create_crawler(DefaultSpider()) + + +@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner]) +def test_crawl_rejects_spider_object(runner_cls: type[CrawlerRunnerBase]) -> None: + runner = runner_cls() + with pytest.raises(ValueError, match="cannot be a spider object"): + runner.crawl(DefaultSpider()) + + class ExceptionSpider(scrapy.Spider): name = "exception" diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py index 146d94ecd..4b0e8e54b 100644 --- a/tests/test_crawler_subprocess.py +++ b/tests/test_crawler_subprocess.py @@ -126,6 +126,16 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): assert "TimeoutError" not in log assert "scrapy.exceptions.CannotResolveHostError" not in log + def test_dns_resolver_deprecated(self) -> None: + log = self.run_script("dns_resolver_deprecated.py") + assert "Spider closed (finished)" in log + assert "The DNS_RESOLVER setting is deprecated" in log + + def test_dns_resolver_deprecated_twisted_dns_resolver(self) -> None: + log = self.run_script("dns_resolver_deprecated.py", "twisted-wins") + assert "Spider closed (finished)" in log + assert "The DNS_RESOLVER setting is deprecated" in log + def test_twisted_reactor_asyncio(self) -> None: log = self.run_script("twisted_reactor_asyncio.py") assert "Spider closed (finished)" in log