Use platformdirs

docs/topics/commands.rst

@@ -43,9 +43,15 @@ Global configuration
 --------------------
 
 Project-independent configuration parameters may also be set for all your
-projects in the :file:`.scrapy/config.toml` file of your home folder, i.e.
-:file:`~/.scrapy/config.toml` on Linux and macOS or
-:file:`%USERPROFILE%\\.scrapy\\config.toml` on Windows:
+projects in the :file:`scrapy/config.toml` file of your user configuration
+folder, as determined by platformdirs_:
+
+* Linux: :file:`$XDG_CONFIG_HOME/scrapy/config.toml`, i.e.
+  :file:`~/.config/scrapy/config.toml` by default
+* macOS: :file:`~/Library/Application Support/scrapy/config.toml`
+* Windows: :file:`%LOCALAPPDATA%\\scrapy\\config.toml`
+
+For example:
 
 .. code-block:: toml
 
@@ -59,6 +65,8 @@ Only the following parameters are supported there:
 Any project may override them, using the same section and parameter name in its
 :file:`pyproject.toml` file.
 
+.. _platformdirs: https://pypi.org/project/platformdirs/
+
 .. _topics-project-structure:
 
 Default structure of Scrapy projects

docs/topics/shell.rst

@@ -45,7 +45,7 @@ regardless of which are installed. This is done by setting the
     shell = "bpython"
 
 You can also define it for all your projects in the
-:file:`.scrapy/config.toml` file of your home folder (see
+:file:`scrapy/config.toml` file of your user configuration folder (see
 :ref:`global-config`):
 
 .. code-block:: toml

pyproject.toml

@@ -16,6 +16,7 @@ dependencies = [
     "lxml>=4.6.4",
     "packaging",
     "parsel>=1.5.0",
+    "platformdirs>=2.0.0",
     "protego>=0.1.15",
     "pyOpenSSL>=22.0.0",
     "queuelib>=1.4.2",

scrapy/utils/conf.py

@@ -9,6 +9,8 @@ from operator import itemgetter
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, cast
 
+from platformdirs import user_config_dir
+
 from scrapy.exceptions import ScrapyDeprecationWarning, UsageError
 from scrapy.settings import BaseSettings
 from scrapy.utils.deprecate import update_classpath
@@ -22,13 +24,18 @@ if sys.version_info >= (3, 11):
 else:
     import tomli as tomllib
 
-_GLOBAL_CONFIG_PATH = Path("~/.scrapy/config.toml")
-
 # Project-independent options that the global configuration file supports, as
 # (section, option) pairs.
 _GLOBAL_OPTIONS = frozenset({("settings", "shell")})
 
 
+def _global_config_path() -> Path:
+    """Return the path to the global configuration file, which lives in the
+    user configuration folder of the running platform.
+    """
+    return Path(user_config_dir("scrapy", appauthor=False), "config.toml")
+
+
 def build_component_list(
     compdict: MutableMapping[Any, Any],
     *,
@@ -108,7 +115,7 @@ def _read_global_config(cfg: ConfigParser) -> None:
     """Read the supported options of the global configuration file into the
     specified :class:`~configparser.ConfigParser` object.
     """
-    path = _GLOBAL_CONFIG_PATH.expanduser()
+    path = _global_config_path()
     if not path.is_file():
         return
     ignored: list[str] = []
@@ -206,7 +213,7 @@ def get_config(use_closest: bool = True) -> ConfigParser:
         warnings.warn(
             f"Configuration read from {', '.join(global_files)}. Global "
             "scrapy.cfg files are deprecated, use "
-            f"{_GLOBAL_CONFIG_PATH.expanduser()} for "
+            f"{_global_config_path()} for "
             "project-independent options and the [tool.scrapy] table of the "
             "pyproject.toml file of your project for the rest. See "
             "https://docs.scrapy.org/en/latest/topics/commands.html#global-config",

tests/test_utils_conf.py

@@ -16,6 +16,7 @@ from scrapy.utils.conf import (
     feed_complete_default_values_from_settings,
     feed_process_params_from_cli,
     get_config,
+    get_sources,
 )
 
 
@@ -77,7 +78,7 @@ SETTINGS_CFG = (
 class TestConfig:
     @pytest.fixture(autouse=True)
     def home(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
-        """Point the global config locations (see
+        """Point the deprecated global scrapy.cfg locations (see
         :func:`scrapy.utils.conf.get_sources`) at an initially empty folder."""
         home = tmp_path / "home"
         home.mkdir()
@@ -86,6 +87,20 @@ class TestConfig:
         monkeypatch.setenv("XDG_CONFIG_HOME", str(home))
         return home
 
+    @pytest.fixture(autouse=True)
+    def config_dir(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
+        """Point the global configuration file at an initially empty folder.
+
+        platformdirs does not determine the user configuration folder from
+        environment variables on every platform, hence the patching.
+        """
+        config_dir = tmp_path / "config"
+        config_dir.mkdir()
+        monkeypatch.setattr(
+            "scrapy.utils.conf.user_config_dir", lambda *args, **kwargs: str(config_dir)
+        )
+        return config_dir
+
     def test_no_config(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
         monkeypatch.chdir(tmp_path)
         assert closest_config() == ""
@@ -159,23 +174,21 @@ class TestConfig:
         assert Path(closest) == (tmp_path / "scrapy.cfg").resolve()
 
     @staticmethod
-    def _write_global_config(home: Path, content: str) -> None:
-        path = home / ".scrapy" / "config.toml"
-        path.parent.mkdir()
-        path.write_text(content, encoding="utf-8")
+    def _write_global_config(config_dir: Path, content: str) -> None:
+        (config_dir / "config.toml").write_text(content, encoding="utf-8")
 
     def test_global_config(
-        self, home: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+        self, config_dir: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
     ) -> None:
-        self._write_global_config(home, '[settings]\nshell = "bpython"\n')
+        self._write_global_config(config_dir, '[settings]\nshell = "bpython"\n')
         monkeypatch.chdir(tmp_path)
 
         assert get_config().get("settings", "shell") == "bpython"
 
     def test_global_config_overridden_by_project(
-        self, home: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+        self, config_dir: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
     ) -> None:
-        self._write_global_config(home, '[settings]\nshell = "bpython"\n')
+        self._write_global_config(config_dir, '[settings]\nshell = "bpython"\n')
         (tmp_path / "pyproject.toml").write_text(
             '[tool.scrapy.settings]\nshell = "python"\n', encoding="utf-8"
         )
@@ -184,10 +197,10 @@ class TestConfig:
         assert get_config().get("settings", "shell") == "python"
 
     def test_global_config_unsupported_options(
-        self, home: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+        self, config_dir: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
     ) -> None:
         self._write_global_config(
-            home,
+            config_dir,
             '[settings]\nshell = "bpython"\ndefault = "myproject.settings"\n'
             '[deploy]\nproject = "myproject"\nunsupported = 1\n',
         )
@@ -199,6 +212,50 @@ class TestConfig:
         assert not cfg.has_option("settings", "default")
         assert not cfg.has_section("deploy")
 
+    def test_global_config_non_table(
+        self, config_dir: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        self._write_global_config(config_dir, 'shell = "bpython"\n')
+        monkeypatch.chdir(tmp_path)
+
+        with pytest.warns(UserWarning, match="Ignoring the following options"):
+            cfg = get_config()
+        assert not cfg.has_section("settings")
+
+    def test_global_config_preferred_over_global_scrapy_cfg(
+        self,
+        config_dir: Path,
+        home: Path,
+        tmp_path: Path,
+        monkeypatch: pytest.MonkeyPatch,
+    ) -> None:
+        (home / "scrapy.cfg").write_text(
+            "[settings]\nshell = python\n", encoding="utf-8"
+        )
+        self._write_global_config(config_dir, '[settings]\nshell = "bpython"\n')
+        monkeypatch.chdir(tmp_path)
+
+        with pytest.warns(
+            ScrapyDeprecationWarning, match="Global scrapy.cfg files are deprecated"
+        ):
+            cfg = get_config()
+        assert cfg.get("settings", "shell") == "bpython"
+
+    def test_get_sources(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+        (tmp_path / "scrapy.cfg").write_text(SETTINGS_CFG, encoding="utf-8")
+        subdir = tmp_path / "a"
+        subdir.mkdir()
+        monkeypatch.chdir(subdir)
+
+        assert Path(get_sources()[-1]) == (tmp_path / "scrapy.cfg").resolve()
+
+    def test_get_sources_without_scrapy_cfg(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        monkeypatch.chdir(tmp_path)
+
+        assert get_sources()[-1] == ""
+
     def test_global_scrapy_cfg_deprecated(
         self, home: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
     ) -> None:

tests/test_utils_project.py

@@ -6,8 +6,9 @@ from typing import TYPE_CHECKING
 
 import pytest
 
+from scrapy.exceptions import NotConfigured
 from scrapy.utils.misc import set_environ
-from scrapy.utils.project import data_path, get_project_settings
+from scrapy.utils.project import data_path, get_project_settings, project_data_dir
 
 if TYPE_CHECKING:
     from collections.abc import Generator
@@ -40,6 +41,19 @@ def test_data_path_inside_project(proj_path: Path) -> None:
     assert abspath == data_path(abspath)
 
 
+def test_project_data_dir_without_config_file(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """A project defined only through the environment has no folder to infer
+    its data dir from."""
+    monkeypatch.chdir(tmp_path)
+    with (
+        set_environ(SCRAPY_SETTINGS_MODULE="tests.test_cmdline.settings"),
+        pytest.raises(NotConfigured, match=r"Unable to find a pyproject\.toml file"),
+    ):
+        project_data_dir()
+
+
 class TestGetProjectSettings:
     def test_valid_envvar(self):
         value = "tests.test_cmdline.settings"

tox.ini

@@ -76,6 +76,7 @@ deps =
     h2==4.3.0
     httpx2==2.7.0
     itemadapter==0.13.1
+    platformdirs==4.6.0
     ptpython==3.0.32
     # newer ones require newer Python
     ipython==8.39.0
@@ -140,6 +141,7 @@ deps =
     itemadapter==0.1.0
     lxml==4.6.4
     parsel==1.5.0
+    platformdirs==2.0.0
     pyOpenSSL==22.0.0
     queuelib==1.4.2
     service_identity==23.1.0