Let scrapy settings --get support add-ons and spiders

docs/topics/commands.rst

@@ -563,6 +563,17 @@ Example usage::
     $ scrapy settings --get DOWNLOAD_DELAY
     0
 
+The settings of enabled :ref:`add-ons <topics-addons>` are taken into account.
+:ref:`Spider settings <spider-settings>` are only taken into account if you name
+a spider with ``--spider``::
+
+    $ scrapy settings --get DOWNLOAD_DELAY --spider myspider
+    2.0
+
+.. versionchanged:: VERSION
+   Add-on settings are now taken into account, and the ``--spider`` option was
+   added.
+
 .. command:: runspider
 
 runspider

scrapy/commands/settings.py

@@ -1,9 +1,20 @@
-import argparse
-import json
-from typing import Any, ClassVar
+from __future__ import annotations
 
+import json
+from typing import TYPE_CHECKING, Any, ClassVar
+
+from scrapy import Spider
+from scrapy.addons import AddonManager
 from scrapy.commands import ScrapyCommand
+from scrapy.crawler import Crawler
+from scrapy.exceptions import UsageError
 from scrapy.settings import BaseSettings
+from scrapy.spiderloader import get_spider_loader
+
+if TYPE_CHECKING:
+    import argparse
+
+    from scrapy.settings import Settings
 
 
 class Command(ScrapyCommand):
@@ -45,10 +56,33 @@ class Command(ScrapyCommand):
             metavar="SETTING",
             help="print setting value, interpreted as a list",
         )
+        parser.add_argument(
+            "--spider",
+            dest="spider",
+            metavar="SPIDER",
+            help="also apply the settings of this spider",
+        )
+
+    def _build_settings(self, spider_name: str | None) -> Settings:
+        """Return the settings as they would be during a crawl."""
+        assert self.settings is not None
+        # Must run before get_spider_loader(), because add-ons may change
+        # SPIDER_MODULES and other pre-crawler settings that it relies on.
+        AddonManager.load_pre_crawler_settings(self.settings)
+        spidercls: type[Spider] = Spider
+        if spider_name is not None:
+            try:
+                spidercls = get_spider_loader(self.settings).load(spider_name)
+            except KeyError:
+                raise UsageError(f"Unable to find spider: {spider_name}") from None
+        # Building a Crawler applies the spider settings and does not install a
+        # reactor or build any component.
+        crawler = Crawler(spidercls, self.settings)
+        crawler.addons.load_settings(crawler.settings)
+        return crawler.settings
 
     def run(self, args: list[str], opts: argparse.Namespace) -> None:
-        assert self.settings is not None
-        settings = self.settings
+        settings = self._build_settings(opts.spider)
         if opts.get:
             s = settings.get(opts.get)
             if isinstance(s, BaseSettings):

tests/test_commands.py

@@ -558,6 +558,80 @@ class TestExecute:
         assert capsys.readouterr().out.strip() == "scrapybot"
 
 
+class TestSettingsCommand(TestProjectBase):
+    @pytest.fixture(autouse=True)
+    def create_files(self, proj_path: Path) -> None:
+        proj_mod_path = proj_path / self.project_name
+        (proj_mod_path / "addons.py").write_text("""
+class MyAddon:
+    def update_settings(self, settings):
+        settings.set("FROM_ADDON", "addon", priority="addon")
+        settings.set("SPIDER_WINS", "addon", priority="addon")
+        settings.set("SEEN_BY_ADDON", settings.get("FROM_SPIDER"), priority="addon")
+
+    @classmethod
+    def update_pre_crawler_settings(cls, settings):
+        settings.set("FROM_ADDON_PRE_CRAWLER", "addon", priority="addon")
+""")
+        (proj_mod_path / "spiders" / "sp.py").write_text("""
+import scrapy
+
+class MySpider(scrapy.Spider):
+    name = "sp"
+    custom_settings = {
+        "FROM_SPIDER": "spider",
+        "SPIDER_WINS": "spider",
+    }
+""")
+        self._append_settings(
+            proj_mod_path,
+            f'ADDONS = {{"{self.project_name}.addons.MyAddon": 100}}\n',
+        )
+
+    @staticmethod
+    def _get(proj_path: Path, setting: str, *args: str) -> str:
+        returncode, out, err = proc("settings", "--get", setting, *args, cwd=proj_path)
+        assert returncode == 0, err
+        return out.strip()
+
+    def test_project_setting(self, proj_path: Path) -> None:
+        assert self._get(proj_path, "BOT_NAME") == self.project_name
+
+    def test_addon(self, proj_path: Path) -> None:
+        assert self._get(proj_path, "FROM_ADDON") == "addon"
+
+    def test_addon_pre_crawler(self, proj_path: Path) -> None:
+        assert self._get(proj_path, "FROM_ADDON_PRE_CRAWLER") == "addon"
+
+    def test_cmdline_beats_addon(self, proj_path: Path) -> None:
+        assert (
+            self._get(proj_path, "FROM_ADDON", "-s", "FROM_ADDON=cmdline") == "cmdline"
+        )
+
+    def test_no_spider(self, proj_path: Path) -> None:
+        assert self._get(proj_path, "FROM_SPIDER") == "None"
+        assert self._get(proj_path, "SPIDER_WINS") == "addon"
+
+    def test_spider(self, proj_path: Path) -> None:
+        assert self._get(proj_path, "FROM_SPIDER", "--spider", "sp") == "spider"
+        assert self._get(proj_path, "FROM_ADDON", "--spider", "sp") == "addon"
+
+    def test_spider_beats_addon(self, proj_path: Path) -> None:
+        assert self._get(proj_path, "SPIDER_WINS", "--spider", "sp") == "spider"
+
+    def test_addons_see_spider_settings(self, proj_path: Path) -> None:
+        """Add-ons are loaded after spider settings are applied, as in a crawl."""
+        assert self._get(proj_path, "SEEN_BY_ADDON") == "None"
+        assert self._get(proj_path, "SEEN_BY_ADDON", "--spider", "sp") == "spider"
+
+    def test_unknown_spider(self, proj_path: Path) -> None:
+        returncode, _, err = proc(
+            "settings", "--get", "FROM_SPIDER", "--spider", "nope", cwd=proj_path
+        )
+        assert returncode == 2
+        assert "Unable to find spider: nope" in err
+
+
 class TestBenchCommand:
     @pytest.mark.parametrize("use_reactor", [True, False])
     def test_run(self, use_reactor: bool) -> None: