From 83f83ef82314ceee43ca12a2487fb5186d69d0ca Mon Sep 17 00:00:00 2001 From: Aiman Date: Sun, 22 Mar 2026 22:52:41 +0530 Subject: [PATCH] update comfig file searching logic and docs --- docs/topics/commands.rst | 36 +++++++++----- docs/topics/shell.rst | 11 ++++- pyproject.toml | 1 + scrapy/utils/conf.py | 103 ++++++++++++++++++++------------------- scrapy/utils/project.py | 16 +++--- tests/test_utils_conf.py | 24 +++++---- 6 files changed, 106 insertions(+), 85 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 8d1351eb9..4cde739d7 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -27,7 +27,17 @@ in standard locations: 1. ``/etc/scrapy.cfg`` or ``c:\scrapy\scrapy.cfg`` (system-wide), 2. ``~/.config/scrapy.cfg`` (``$XDG_CONFIG_HOME``) and ``~/.scrapy.cfg`` (``$HOME``) for global (user-wide) settings, and -3. ``scrapy.cfg`` inside a Scrapy project's root (see next section). +3. ``pyproject.toml`` (recommended) or ``scrapy.cfg`` inside a Scrapy project's root (see next section). + +For ``pyproject.toml``, the settings are read from the ``[tool.scrapy]`` section: + +.. code-block:: toml + + [tool.scrapy.settings] + default = "myproject.settings" + +Scrapy searches for the closest ``pyproject.toml`` or ``scrapy.cfg``, and if both +files exist, ``pyproject.toml`` takes precedence. Settings from these files are merged in the listed order of preference: user-defined values have higher priority than system-wide defaults @@ -51,7 +61,7 @@ understand the directory structure of a Scrapy project. Though it can be modified, all Scrapy projects have the same file structure by default, similar to this:: - scrapy.cfg + pyproject.toml myproject/ __init__.py items.py @@ -64,32 +74,32 @@ structure by default, similar to this:: spider2.py ... -The directory where the ``scrapy.cfg`` file resides is known as the *project +The directory where the ``pyproject.toml`` file resides is known as the *project root directory*. That file contains the name of the python module that defines the project settings. Here is an example: -.. code-block:: ini +.. code-block:: toml - [settings] - default = myproject.settings + [tool.scrapy.settings] + default = "myproject.settings" .. _topics-project-envvar: Sharing the root directory between projects =========================================== -A project root directory, the one that contains the ``scrapy.cfg``, may be +A project root directory, the one that contains the ``pyproject.toml``, may be shared by multiple Scrapy projects, each with its own settings module. In that case, you must define one or more aliases for those settings modules -under ``[settings]`` in your ``scrapy.cfg`` file: +under ``[tool.scrapy.settings]`` in your ``pyproject.toml`` file: -.. code-block:: ini +.. code-block:: toml - [settings] - default = myproject1.settings - project1 = myproject1.settings - project2 = myproject2.settings + [tool.scrapy.settings] + default = "myproject1.settings" + project1 = "myproject1.settings" + project2 = "myproject2.settings" By default, the ``scrapy`` command-line tool will use the ``default`` settings. Use the ``SCRAPY_PROJECT`` environment variable to specify a different project diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 8ae8ff512..a5a4ae64f 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -34,10 +34,17 @@ is unavailable. Through Scrapy's settings you can configure it to use any one of ``ipython``, ``bpython`` or the standard ``python`` shell, regardless of which are installed. This is done by setting the ``SCRAPY_PYTHON_SHELL`` environment -variable; or by defining it in your :ref:`scrapy.cfg `:: +variable; or by defining it in your :ref:`pyproject.toml `: + +.. code-block:: toml + + [tool.scrapy.settings] + shell = "bpython" + +or in ``scrapy.cfg``:: [settings] - shell = bpython + shell = "bpython" .. _IPython: https://ipython.org/ .. _IPython installation guide: https://ipython.org/install.html diff --git a/pyproject.toml b/pyproject.toml index 28e297401..61511673d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ dependencies = [ # Platform-specific dependencies 'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"', 'PyPyDispatcher>=2.1.0; platform_python_implementation == "PyPy"', + "tomli>=1.1.0; python_version < '3.11'", ] classifiers = [ "Development Status :: 5 - Production/Stable", diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 9bbcab544..50bda6d32 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -20,10 +20,7 @@ if TYPE_CHECKING: if sys.version_info >= (3, 11): import tomllib else: - try: - import tomli as tomllib - except ImportError: - tomllib = None # type: ignore[assignment] + import tomli as tomllib def build_component_list( @@ -79,6 +76,40 @@ def arglist_to_dict(arglist: list[str]) -> dict[str, str]: return dict(x.split("=", 1) for x in arglist) +def closest_config( + path: str | os.PathLike = ".", _resolved: Path | None = None +) -> tuple[str, str]: + """ + Returns the closest config file (pyproject.toml or scrapy.cfg) by + traversing the current directory and its parents + """ + current = Path(path).resolve() if _resolved is None else _resolved + + toml = current / "pyproject.toml" + if toml.exists(): + try: + with toml.open("rb") as f: + data = tomllib.load(f) + if data.get("tool", {}).get("scrapy"): + return "toml", str(toml) + except tomllib.TOMLDecodeError: + warnings.warn( + f"Scrapy could not parse {toml}: invalid TOML. " + "This file will be ignored.", + stacklevel=2, + ) + + cfgfile = current / "scrapy.cfg" + if cfgfile.exists(): + return "cfg", str(cfgfile) + + parent = current.parent + if parent == current: + return "", "" + + return closest_config(_resolved=parent) + + def closest_scrapy_cfg( path: str | os.PathLike = ".", prevpath: str | os.PathLike | None = None, @@ -95,31 +126,6 @@ def closest_scrapy_cfg( return closest_scrapy_cfg(path.parent, path) -def closest_pyproject_toml( - path: str | os.PathLike = ".", - prevpath: str | os.PathLike | None = None, -) -> str: - if tomllib is None: - return "" - if prevpath is not None and str(path) == str(prevpath): - return "" - path = Path(path).resolve() - toml_path = path / "pyproject.toml" - if toml_path.exists(): - try: - with toml_path.open("rb") as f: - data = tomllib.load(f) - if "scrapy" in data.get("tool", {}): - return str(toml_path) - except tomllib.TOMLDecodeError: - warnings.warn( - f"Scrapy could not parse {toml_path}: invalid TOML. " - "This file will be ignored.", - stacklevel=2, - ) - return closest_pyproject_toml(path.parent, path) - - def init_env(project: str = "default", set_syspath: bool = True) -> None: """Initialize environment to use command-line tool from inside a project dir. This sets the Scrapy settings module and modifies the Python path to @@ -128,49 +134,46 @@ def init_env(project: str = "default", set_syspath: bool = True) -> None: cfg = get_config() if cfg.has_option("settings", project): os.environ["SCRAPY_SETTINGS_MODULE"] = cfg.get("settings", project) - closest = closest_pyproject_toml() or closest_scrapy_cfg() + + _, closest = closest_config() if closest: projdir = str(Path(closest).parent) if set_syspath and projdir not in sys.path: sys.path.append(projdir) -def _load_toml_config(toml_path: str) -> ConfigParser: - """Load Scrapy config from a TOML file.""" - with Path(toml_path).open("rb") as f: - data = tomllib.load(f) - scrapy_data = data.get("tool", {}).get("scrapy", {}) - cfg = ConfigParser() - cfg.read_dict(scrapy_data) - return cfg - - def get_config(use_closest: bool = True) -> ConfigParser: """Get Scrapy config file as a ConfigParser""" - if use_closest and tomllib is not None: - toml_path = closest_pyproject_toml() - if toml_path: - return _load_toml_config(toml_path) + if use_closest: + config_type, config_path = closest_config() + if config_type == "toml": + with Path(config_path).open("rb") as f: + data = tomllib.load(f) + scrapy_data = data.get("tool", {}).get("scrapy", {}) + cfg = ConfigParser() + cfg.read_dict(scrapy_data) + return cfg + if config_type == "cfg": + cfg = ConfigParser() + cfg.read(config_path) + return cfg - sources = get_sources(use_closest) + sources = get_sources() cfg = ConfigParser() cfg.read(sources) return cfg -def get_sources(use_closest: bool = True) -> list[str]: +def get_sources() -> list[str]: xdg_config_home = ( os.environ.get("XDG_CONFIG_HOME") or Path("~/.config").expanduser() ) - sources = [ + return [ "/etc/scrapy.cfg", r"c:\scrapy\scrapy.cfg", str(Path(xdg_config_home) / "scrapy.cfg"), str(Path("~/.scrapy.cfg").expanduser()), ] - if use_closest: - sources.append(closest_scrapy_cfg()) - return sources def feed_complete_default_values_from_settings( diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index 0fe2c9389..2d92694eb 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -7,12 +7,7 @@ from pathlib import Path from scrapy.exceptions import NotConfigured from scrapy.settings import Settings -from scrapy.utils.conf import ( - closest_pyproject_toml, - closest_scrapy_cfg, - get_config, - init_env, -) +from scrapy.utils.conf import closest_config, get_config, init_env ENVVAR = "SCRAPY_SETTINGS_MODULE" DATADIR_CFG_SECTION = "datadir" @@ -29,23 +24,24 @@ def inside_project() -> bool: ) else: return True - return bool(closest_scrapy_cfg() or closest_pyproject_toml()) + _, closest = closest_config() + return bool(closest) def project_data_dir(project: str = "default") -> str: """Return the current project data dir, creating it if it doesn't exist""" if not inside_project(): raise NotConfigured("Not inside a project") + _, project_cfg = closest_config() cfg = get_config() if cfg.has_option(DATADIR_CFG_SECTION, project): d = Path(cfg.get(DATADIR_CFG_SECTION, project)) else: - scrapy_cfg = closest_scrapy_cfg() or closest_pyproject_toml() - if not scrapy_cfg: + if not project_cfg: raise NotConfigured( "Unable to find scrapy.cfg or pyproject.toml to infer project data dir" ) - d = (Path(scrapy_cfg).parent / ".scrapy").resolve() + d = (Path(project_cfg).parent / ".scrapy").resolve() if not d.exists(): d.mkdir(parents=True) return str(d) diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index db48f4b61..9acf48c11 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -1,4 +1,3 @@ -import os import warnings import pytest @@ -8,7 +7,7 @@ from scrapy.settings import BaseSettings, Settings from scrapy.utils.conf import ( arglist_to_dict, build_component_list, - closest_pyproject_toml, + closest_config, feed_complete_default_values_from_settings, feed_process_params_from_cli, get_config, @@ -60,16 +59,20 @@ def test_arglist_to_dict(): class TestPyprojectToml: - def test_pyproject_toml_takes_precedence_over_scrapy_cfg(self, tmp_path): + def test_pyproject_toml_takes_precedence_over_scrapy_cfg( + self, tmp_path, monkeypatch + ): (tmp_path / "scrapy.cfg").write_text("[settings]\ndefault = from_scrapy_cfg\n") (tmp_path / "pyproject.toml").write_text( "[tool.scrapy.settings]\ndefault = 'from_pyproject_toml'\n" ) - os.chdir(tmp_path) + monkeypatch.chdir(tmp_path) cfg = get_config() assert cfg.get("settings", "default") == "from_pyproject_toml" - def test_scrapy_cfg_not_read_when_pyproject_toml_present(self, tmp_path): + def test_scrapy_cfg_not_read_when_pyproject_toml_present( + self, tmp_path, monkeypatch + ): (tmp_path / "scrapy.cfg").write_text( "[settings]\ndefault = from_scrapy_cfg\n" "[deploy]\nproject = from_scrapy_cfg\n" @@ -77,17 +80,18 @@ class TestPyprojectToml: (tmp_path / "pyproject.toml").write_text( "[tool.scrapy.settings]\ndefault = 'from_pyproject_toml'\n" ) - os.chdir(tmp_path) + monkeypatch.chdir(tmp_path) cfg = get_config() assert not cfg.has_section("deploy") - def test_malformed_toml_warns_and_returns_empty(self, tmp_path): + def test_malformed_toml_warns_and_falls_back(self, tmp_path, monkeypatch): (tmp_path / "pyproject.toml").write_text("[tool.scrapy\nnot valid toml") - os.chdir(tmp_path) + (tmp_path / "scrapy.cfg").write_text("[settings]\ndefault = from_scrapy_cfg\n") + monkeypatch.chdir(tmp_path) with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") - result = closest_pyproject_toml() - assert result == "" + config_type, _ = closest_config() + assert config_type == "cfg" assert len(w) == 1 assert "invalid TOML" in str(w[0].message)