diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 7e0f52a00..a3c5b9e1b 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -43,9 +43,15 @@ Global configuration -------------------- Project-independent configuration parameters may also be set for all your -projects in the :file:`.scrapy/config.toml` file of your home folder, i.e. -:file:`~/.scrapy/config.toml` on Linux and macOS or -:file:`%USERPROFILE%\\.scrapy\\config.toml` on Windows: +projects in the :file:`scrapy/config.toml` file of your user configuration +folder, as determined by platformdirs_: + +* Linux: :file:`$XDG_CONFIG_HOME/scrapy/config.toml`, i.e. + :file:`~/.config/scrapy/config.toml` by default +* macOS: :file:`~/Library/Application Support/scrapy/config.toml` +* Windows: :file:`%LOCALAPPDATA%\\scrapy\\config.toml` + +For example: .. code-block:: toml @@ -59,6 +65,8 @@ Only the following parameters are supported there: Any project may override them, using the same section and parameter name in its :file:`pyproject.toml` file. +.. _platformdirs: https://pypi.org/project/platformdirs/ + .. _topics-project-structure: Default structure of Scrapy projects diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 0b8acdea4..5e191b6b9 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -45,7 +45,7 @@ regardless of which are installed. This is done by setting the shell = "bpython" You can also define it for all your projects in the -:file:`.scrapy/config.toml` file of your home folder (see +:file:`scrapy/config.toml` file of your user configuration folder (see :ref:`global-config`): .. code-block:: toml diff --git a/pyproject.toml b/pyproject.toml index b7d141e8d..06cac6ad8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ dependencies = [ "lxml>=4.6.4", "packaging", "parsel>=1.5.0", + "platformdirs>=2.0.0", "protego>=0.1.15", "pyOpenSSL>=22.0.0", "queuelib>=1.4.2", diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 624571cca..33d11f04d 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -9,6 +9,8 @@ from operator import itemgetter from pathlib import Path from typing import TYPE_CHECKING, Any, cast +from platformdirs import user_config_dir + from scrapy.exceptions import ScrapyDeprecationWarning, UsageError from scrapy.settings import BaseSettings from scrapy.utils.deprecate import update_classpath @@ -22,13 +24,18 @@ if sys.version_info >= (3, 11): else: import tomli as tomllib -_GLOBAL_CONFIG_PATH = Path("~/.scrapy/config.toml") - # Project-independent options that the global configuration file supports, as # (section, option) pairs. _GLOBAL_OPTIONS = frozenset({("settings", "shell")}) +def _global_config_path() -> Path: + """Return the path to the global configuration file, which lives in the + user configuration folder of the running platform. + """ + return Path(user_config_dir("scrapy", appauthor=False), "config.toml") + + def build_component_list( compdict: MutableMapping[Any, Any], *, @@ -108,7 +115,7 @@ def _read_global_config(cfg: ConfigParser) -> None: """Read the supported options of the global configuration file into the specified :class:`~configparser.ConfigParser` object. """ - path = _GLOBAL_CONFIG_PATH.expanduser() + path = _global_config_path() if not path.is_file(): return ignored: list[str] = [] @@ -206,7 +213,7 @@ def get_config(use_closest: bool = True) -> ConfigParser: warnings.warn( f"Configuration read from {', '.join(global_files)}. Global " "scrapy.cfg files are deprecated, use " - f"{_GLOBAL_CONFIG_PATH.expanduser()} for " + f"{_global_config_path()} for " "project-independent options and the [tool.scrapy] table of the " "pyproject.toml file of your project for the rest. See " "https://docs.scrapy.org/en/latest/topics/commands.html#global-config", diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index 9a6f0f006..1c0a669fd 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -16,6 +16,7 @@ from scrapy.utils.conf import ( feed_complete_default_values_from_settings, feed_process_params_from_cli, get_config, + get_sources, ) @@ -77,7 +78,7 @@ SETTINGS_CFG = ( class TestConfig: @pytest.fixture(autouse=True) def home(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: - """Point the global config locations (see + """Point the deprecated global scrapy.cfg locations (see :func:`scrapy.utils.conf.get_sources`) at an initially empty folder.""" home = tmp_path / "home" home.mkdir() @@ -86,6 +87,20 @@ class TestConfig: monkeypatch.setenv("XDG_CONFIG_HOME", str(home)) return home + @pytest.fixture(autouse=True) + def config_dir(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Point the global configuration file at an initially empty folder. + + platformdirs does not determine the user configuration folder from + environment variables on every platform, hence the patching. + """ + config_dir = tmp_path / "config" + config_dir.mkdir() + monkeypatch.setattr( + "scrapy.utils.conf.user_config_dir", lambda *args, **kwargs: str(config_dir) + ) + return config_dir + def test_no_config(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.chdir(tmp_path) assert closest_config() == "" @@ -159,23 +174,21 @@ class TestConfig: assert Path(closest) == (tmp_path / "scrapy.cfg").resolve() @staticmethod - def _write_global_config(home: Path, content: str) -> None: - path = home / ".scrapy" / "config.toml" - path.parent.mkdir() - path.write_text(content, encoding="utf-8") + def _write_global_config(config_dir: Path, content: str) -> None: + (config_dir / "config.toml").write_text(content, encoding="utf-8") def test_global_config( - self, home: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + self, config_dir: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - self._write_global_config(home, '[settings]\nshell = "bpython"\n') + self._write_global_config(config_dir, '[settings]\nshell = "bpython"\n') monkeypatch.chdir(tmp_path) assert get_config().get("settings", "shell") == "bpython" def test_global_config_overridden_by_project( - self, home: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + self, config_dir: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - self._write_global_config(home, '[settings]\nshell = "bpython"\n') + self._write_global_config(config_dir, '[settings]\nshell = "bpython"\n') (tmp_path / "pyproject.toml").write_text( '[tool.scrapy.settings]\nshell = "python"\n', encoding="utf-8" ) @@ -184,10 +197,10 @@ class TestConfig: assert get_config().get("settings", "shell") == "python" def test_global_config_unsupported_options( - self, home: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + self, config_dir: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: self._write_global_config( - home, + config_dir, '[settings]\nshell = "bpython"\ndefault = "myproject.settings"\n' '[deploy]\nproject = "myproject"\nunsupported = 1\n', ) @@ -199,6 +212,50 @@ class TestConfig: assert not cfg.has_option("settings", "default") assert not cfg.has_section("deploy") + def test_global_config_non_table( + self, config_dir: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + self._write_global_config(config_dir, 'shell = "bpython"\n') + monkeypatch.chdir(tmp_path) + + with pytest.warns(UserWarning, match="Ignoring the following options"): + cfg = get_config() + assert not cfg.has_section("settings") + + def test_global_config_preferred_over_global_scrapy_cfg( + self, + config_dir: Path, + home: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + (home / "scrapy.cfg").write_text( + "[settings]\nshell = python\n", encoding="utf-8" + ) + self._write_global_config(config_dir, '[settings]\nshell = "bpython"\n') + monkeypatch.chdir(tmp_path) + + with pytest.warns( + ScrapyDeprecationWarning, match="Global scrapy.cfg files are deprecated" + ): + cfg = get_config() + assert cfg.get("settings", "shell") == "bpython" + + def test_get_sources(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + (tmp_path / "scrapy.cfg").write_text(SETTINGS_CFG, encoding="utf-8") + subdir = tmp_path / "a" + subdir.mkdir() + monkeypatch.chdir(subdir) + + assert Path(get_sources()[-1]) == (tmp_path / "scrapy.cfg").resolve() + + def test_get_sources_without_scrapy_cfg( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.chdir(tmp_path) + + assert get_sources()[-1] == "" + def test_global_scrapy_cfg_deprecated( self, home: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 74ce0587f..0960076bb 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -6,8 +6,9 @@ from typing import TYPE_CHECKING import pytest +from scrapy.exceptions import NotConfigured from scrapy.utils.misc import set_environ -from scrapy.utils.project import data_path, get_project_settings +from scrapy.utils.project import data_path, get_project_settings, project_data_dir if TYPE_CHECKING: from collections.abc import Generator @@ -40,6 +41,19 @@ def test_data_path_inside_project(proj_path: Path) -> None: assert abspath == data_path(abspath) +def test_project_data_dir_without_config_file( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A project defined only through the environment has no folder to infer + its data dir from.""" + monkeypatch.chdir(tmp_path) + with ( + set_environ(SCRAPY_SETTINGS_MODULE="tests.test_cmdline.settings"), + pytest.raises(NotConfigured, match=r"Unable to find a pyproject\.toml file"), + ): + project_data_dir() + + class TestGetProjectSettings: def test_valid_envvar(self): value = "tests.test_cmdline.settings" diff --git a/tox.ini b/tox.ini index d9780a89e..4b18a50f9 100644 --- a/tox.ini +++ b/tox.ini @@ -76,6 +76,7 @@ deps = h2==4.3.0 httpx2==2.7.0 itemadapter==0.13.1 + platformdirs==4.6.0 ptpython==3.0.32 # newer ones require newer Python ipython==8.39.0 @@ -140,6 +141,7 @@ deps = itemadapter==0.1.0 lxml==4.6.4 parsel==1.5.0 + platformdirs==2.0.0 pyOpenSSL==22.0.0 queuelib==1.4.2 service_identity==23.1.0