diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 8e3ae7879..9828fe6f5 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -363,9 +363,12 @@ class CrawlerRunnerBase(ABC): """ Return a :class:`~scrapy.crawler.Crawler` object. - * If ``crawler_or_spidercls`` is a Crawler, it is returned as-is. + * If ``crawler_or_spidercls`` is a Crawler, the runner's settings are + merged into it as defaults: for each setting, the runner's value + is applied only if the Crawler does not already have that setting at + an equal or higher priority. The Crawler is then returned. * If ``crawler_or_spidercls`` is a Spider subclass, a new Crawler - is constructed for it. + is constructed for it using this runner's settings. * If ``crawler_or_spidercls`` is a string, this function finds a spider with this name in a Scrapy project (using spider loader), then creates a Crawler instance for it. @@ -376,6 +379,7 @@ class CrawlerRunnerBase(ABC): "it must be a spider class (or a Crawler object)" ) if isinstance(crawler_or_spidercls, Crawler): + crawler_or_spidercls.settings.update(self.settings) return crawler_or_spidercls return self._create_crawler(crawler_or_spidercls) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 66b15bfdd..be6c80429 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -14,7 +14,7 @@ from twisted.internet.ssl import Certificate from twisted.python.failure import Failure from scrapy import Spider, signals -from scrapy.crawler import AsyncCrawlerRunner, CrawlerRunner +from scrapy.crawler import AsyncCrawlerRunner, Crawler, CrawlerRunner from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, StopDownload from scrapy.http import Request from scrapy.http.response import Response @@ -432,7 +432,7 @@ with multiples lines async def test_crawlerrunner_accepts_crawler( self, caplog: pytest.LogCaptureFixture, mockserver: MockServer ) -> None: - crawler = get_crawler(SimpleSpider) + crawler = Crawler(SimpleSpider, get_reactor_settings()) runner = CrawlerRunner() with caplog.at_level(logging.DEBUG): await maybe_deferred_to_future( diff --git a/tests/test_crawler.py b/tests/test_crawler.py index c646aea4b..0cddfd0ed 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -651,6 +651,42 @@ class TestAsyncCrawlerProcess(TestBaseCrawler): self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") +@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner]) +def test_runner_settings_applied_to_crawler_instance( + runner_cls: type[CrawlerRunnerBase], +) -> None: + runner = runner_cls({"FOO": "runner"}) + crawler = Crawler(DefaultSpider) + result = runner.create_crawler(crawler) + assert result is crawler + assert result.settings["FOO"] == "runner" + + +@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner]) +def test_spider_custom_settings_override_runner( + runner_cls: type[CrawlerRunnerBase], +) -> None: + class MySpider(DefaultSpider): + custom_settings = {"FOO": "spider"} + + runner = runner_cls({"FOO": "runner"}) + crawler = Crawler(MySpider) + runner.create_crawler(crawler) + assert crawler.settings["FOO"] == "spider" + + +def test_create_crawler_instance_consistent_with_spider_class() -> None: + runner = AsyncCrawlerRunner({"FOO": "runner"}) + + crawler_from_class = runner.create_crawler(DefaultSpider) + + pre_built = Crawler(DefaultSpider) + runner.create_crawler(pre_built) + + assert crawler_from_class.settings["FOO"] == "runner" + assert pre_built.settings["FOO"] == "runner" + + class ExceptionSpider(scrapy.Spider): name = "exception"