diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index ee2c3a3cd..60bdb0e2c 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -563,6 +563,17 @@ Example usage:: $ scrapy settings --get DOWNLOAD_DELAY 0 +The settings of enabled :ref:`add-ons ` are taken into account. +:ref:`Spider settings ` are only taken into account if you name +a spider with ``--spider``:: + + $ scrapy settings --get DOWNLOAD_DELAY --spider myspider + 2.0 + +.. versionchanged:: VERSION + Add-on settings are now taken into account, and the ``--spider`` option was + added. + .. command:: runspider runspider diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index 26a97ccbb..edd697965 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -1,9 +1,20 @@ -import argparse -import json -from typing import Any, ClassVar +from __future__ import annotations +import json +from typing import TYPE_CHECKING, Any, ClassVar + +from scrapy import Spider +from scrapy.addons import AddonManager from scrapy.commands import ScrapyCommand +from scrapy.crawler import Crawler +from scrapy.exceptions import UsageError from scrapy.settings import BaseSettings +from scrapy.spiderloader import get_spider_loader + +if TYPE_CHECKING: + import argparse + + from scrapy.settings import Settings class Command(ScrapyCommand): @@ -45,10 +56,33 @@ class Command(ScrapyCommand): metavar="SETTING", help="print setting value, interpreted as a list", ) + parser.add_argument( + "--spider", + dest="spider", + metavar="SPIDER", + help="also apply the settings of this spider", + ) + + def _build_settings(self, spider_name: str | None) -> Settings: + """Return the settings as they would be during a crawl.""" + assert self.settings is not None + # Must run before get_spider_loader(), because add-ons may change + # SPIDER_MODULES and other pre-crawler settings that it relies on. + AddonManager.load_pre_crawler_settings(self.settings) + spidercls: type[Spider] = Spider + if spider_name is not None: + try: + spidercls = get_spider_loader(self.settings).load(spider_name) + except KeyError: + raise UsageError(f"Unable to find spider: {spider_name}") from None + # Building a Crawler applies the spider settings and does not install a + # reactor or build any component. + crawler = Crawler(spidercls, self.settings) + crawler.addons.load_settings(crawler.settings) + return crawler.settings def run(self, args: list[str], opts: argparse.Namespace) -> None: - assert self.settings is not None - settings = self.settings + settings = self._build_settings(opts.spider) if opts.get: s = settings.get(opts.get) if isinstance(s, BaseSettings): diff --git a/tests/test_commands.py b/tests/test_commands.py index 3e687e811..40f476edc 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -558,6 +558,80 @@ class TestExecute: assert capsys.readouterr().out.strip() == "scrapybot" +class TestSettingsCommand(TestProjectBase): + @pytest.fixture(autouse=True) + def create_files(self, proj_path: Path) -> None: + proj_mod_path = proj_path / self.project_name + (proj_mod_path / "addons.py").write_text(""" +class MyAddon: + def update_settings(self, settings): + settings.set("FROM_ADDON", "addon", priority="addon") + settings.set("SPIDER_WINS", "addon", priority="addon") + settings.set("SEEN_BY_ADDON", settings.get("FROM_SPIDER"), priority="addon") + + @classmethod + def update_pre_crawler_settings(cls, settings): + settings.set("FROM_ADDON_PRE_CRAWLER", "addon", priority="addon") +""") + (proj_mod_path / "spiders" / "sp.py").write_text(""" +import scrapy + +class MySpider(scrapy.Spider): + name = "sp" + custom_settings = { + "FROM_SPIDER": "spider", + "SPIDER_WINS": "spider", + } +""") + self._append_settings( + proj_mod_path, + f'ADDONS = {{"{self.project_name}.addons.MyAddon": 100}}\n', + ) + + @staticmethod + def _get(proj_path: Path, setting: str, *args: str) -> str: + returncode, out, err = proc("settings", "--get", setting, *args, cwd=proj_path) + assert returncode == 0, err + return out.strip() + + def test_project_setting(self, proj_path: Path) -> None: + assert self._get(proj_path, "BOT_NAME") == self.project_name + + def test_addon(self, proj_path: Path) -> None: + assert self._get(proj_path, "FROM_ADDON") == "addon" + + def test_addon_pre_crawler(self, proj_path: Path) -> None: + assert self._get(proj_path, "FROM_ADDON_PRE_CRAWLER") == "addon" + + def test_cmdline_beats_addon(self, proj_path: Path) -> None: + assert ( + self._get(proj_path, "FROM_ADDON", "-s", "FROM_ADDON=cmdline") == "cmdline" + ) + + def test_no_spider(self, proj_path: Path) -> None: + assert self._get(proj_path, "FROM_SPIDER") == "None" + assert self._get(proj_path, "SPIDER_WINS") == "addon" + + def test_spider(self, proj_path: Path) -> None: + assert self._get(proj_path, "FROM_SPIDER", "--spider", "sp") == "spider" + assert self._get(proj_path, "FROM_ADDON", "--spider", "sp") == "addon" + + def test_spider_beats_addon(self, proj_path: Path) -> None: + assert self._get(proj_path, "SPIDER_WINS", "--spider", "sp") == "spider" + + def test_addons_see_spider_settings(self, proj_path: Path) -> None: + """Add-ons are loaded after spider settings are applied, as in a crawl.""" + assert self._get(proj_path, "SEEN_BY_ADDON") == "None" + assert self._get(proj_path, "SEEN_BY_ADDON", "--spider", "sp") == "spider" + + def test_unknown_spider(self, proj_path: Path) -> None: + returncode, _, err = proc( + "settings", "--get", "FROM_SPIDER", "--spider", "nope", cwd=proj_path + ) + assert returncode == 2 + assert "Unable to find spider: nope" in err + + class TestBenchCommand: @pytest.mark.parametrize("use_reactor", [True, False]) def test_run(self, use_reactor: bool) -> None: