Do not ignore CrawlerProcess settings (#7647)

* Do not ignore CrawlerProcess settings

* Update test_crawlerrunner_accepts_crawler

scrapy/crawler.py

@@ -363,9 +363,12 @@ class CrawlerRunnerBase(ABC):
         """
         Return a :class:`~scrapy.crawler.Crawler` object.
 
-        * If ``crawler_or_spidercls`` is a Crawler, it is returned as-is.
+        * If ``crawler_or_spidercls`` is a Crawler, the runner's settings are
+          merged into it as defaults: for each setting, the runner's value
+          is applied only if the Crawler does not already have that setting at
+          an equal or higher priority. The Crawler is then returned.
         * If ``crawler_or_spidercls`` is a Spider subclass, a new Crawler
-          is constructed for it.
+          is constructed for it using this runner's settings.
         * If ``crawler_or_spidercls`` is a string, this function finds
           a spider with this name in a Scrapy project (using spider loader),
           then creates a Crawler instance for it.
@@ -376,6 +379,7 @@ class CrawlerRunnerBase(ABC):
                 "it must be a spider class (or a Crawler object)"
             )
         if isinstance(crawler_or_spidercls, Crawler):
+            crawler_or_spidercls.settings.update(self.settings)
             return crawler_or_spidercls
         return self._create_crawler(crawler_or_spidercls)
 

tests/test_crawl.py

@@ -14,7 +14,7 @@ from twisted.internet.ssl import Certificate
 from twisted.python.failure import Failure
 
 from scrapy import Spider, signals
-from scrapy.crawler import AsyncCrawlerRunner, CrawlerRunner
+from scrapy.crawler import AsyncCrawlerRunner, Crawler, CrawlerRunner
 from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, StopDownload
 from scrapy.http import Request
 from scrapy.http.response import Response
@@ -432,7 +432,7 @@ with multiples lines
     async def test_crawlerrunner_accepts_crawler(
         self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
     ) -> None:
-        crawler = get_crawler(SimpleSpider)
+        crawler = Crawler(SimpleSpider, get_reactor_settings())
         runner = CrawlerRunner()
         with caplog.at_level(logging.DEBUG):
             await maybe_deferred_to_future(

tests/test_crawler.py

@@ -651,6 +651,42 @@ class TestAsyncCrawlerProcess(TestBaseCrawler):
         self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
 
 
+@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
+def test_runner_settings_applied_to_crawler_instance(
+    runner_cls: type[CrawlerRunnerBase],
+) -> None:
+    runner = runner_cls({"FOO": "runner"})
+    crawler = Crawler(DefaultSpider)
+    result = runner.create_crawler(crawler)
+    assert result is crawler
+    assert result.settings["FOO"] == "runner"
+
+
+@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
+def test_spider_custom_settings_override_runner(
+    runner_cls: type[CrawlerRunnerBase],
+) -> None:
+    class MySpider(DefaultSpider):
+        custom_settings = {"FOO": "spider"}
+
+    runner = runner_cls({"FOO": "runner"})
+    crawler = Crawler(MySpider)
+    runner.create_crawler(crawler)
+    assert crawler.settings["FOO"] == "spider"
+
+
+def test_create_crawler_instance_consistent_with_spider_class() -> None:
+    runner = AsyncCrawlerRunner({"FOO": "runner"})
+
+    crawler_from_class = runner.create_crawler(DefaultSpider)
+
+    pre_built = Crawler(DefaultSpider)
+    runner.create_crawler(pre_built)
+
+    assert crawler_from_class.settings["FOO"] == "runner"
+    assert pre_built.settings["FOO"] == "runner"
+
+
 class ExceptionSpider(scrapy.Spider):
     name = "exception"