diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 8d1351eb9..e8d99fc55 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -21,17 +21,8 @@ standalone ``scrapyd-deploy``. See `Deploying your project`_.) Configuration settings ====================== -Scrapy will look for configuration parameters in ini-style ``scrapy.cfg`` files -in standard locations: - -1. ``/etc/scrapy.cfg`` or ``c:\scrapy\scrapy.cfg`` (system-wide), -2. ``~/.config/scrapy.cfg`` (``$XDG_CONFIG_HOME``) and ``~/.scrapy.cfg`` (``$HOME``) - for global (user-wide) settings, and -3. ``scrapy.cfg`` inside a Scrapy project's root (see next section). - -Settings from these files are merged in the listed order of preference: -user-defined values have higher priority than system-wide defaults -and project-wide settings will override all others, when defined. +Scrapy looks for project configuration in a ``pyproject.toml`` file located in +the project root directory (see next section). Scrapy also understands, and can be configured through, a number of environment variables. Currently these are: @@ -51,7 +42,7 @@ understand the directory structure of a Scrapy project. Though it can be modified, all Scrapy projects have the same file structure by default, similar to this:: - scrapy.cfg + pyproject.toml myproject/ __init__.py items.py @@ -64,32 +55,32 @@ structure by default, similar to this:: spider2.py ... -The directory where the ``scrapy.cfg`` file resides is known as the *project +The directory where the ``pyproject.toml`` file resides is known as the *project root directory*. That file contains the name of the python module that defines the project settings. Here is an example: -.. code-block:: ini +.. code-block:: toml - [settings] - default = myproject.settings + [tool.scrapy.settings] + default = "myproject.settings" .. _topics-project-envvar: Sharing the root directory between projects =========================================== -A project root directory, the one that contains the ``scrapy.cfg``, may be +A project root directory, the one that contains the ``pyproject.toml``, may be shared by multiple Scrapy projects, each with its own settings module. In that case, you must define one or more aliases for those settings modules -under ``[settings]`` in your ``scrapy.cfg`` file: +under ``[tool.scrapy.settings]`` in your ``pyproject.toml`` file: -.. code-block:: ini +.. code-block:: toml - [settings] - default = myproject1.settings - project1 = myproject1.settings - project2 = myproject2.settings + [tool.scrapy.settings] + default = "myproject1.settings" + project1 = "myproject1.settings" + project2 = "myproject2.settings" By default, the ``scrapy`` command-line tool will use the ``default`` settings. Use the ``SCRAPY_PROJECT`` environment variable to specify a different project diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 5b826ed18..5b77c38db 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -34,10 +34,12 @@ is unavailable. Through Scrapy's settings you can configure it to use any one of ``ipython``, ``bpython`` or the standard ``python`` shell, regardless of which are installed. This is done by setting the ``SCRAPY_PYTHON_SHELL`` environment -variable; or by defining it in your :ref:`scrapy.cfg `:: +variable; or by defining it in your :ref:`pyproject.toml `: - [settings] - shell = bpython +.. code-block:: toml + + [tool.scrapy.settings] + shell = "bpython" .. _IPython: https://ipython.org/ .. _IPython installation guide: https://ipython.org/install/ diff --git a/pyproject.toml b/pyproject.toml index b8d9067f9..52692ae30 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ dependencies = [ "queuelib>=1.4.2", "service_identity>=23.1.0", "tldextract", + "tomli>=1.1.0; python_version < '3.11'", "w3lib>=1.17.0", "zope.interface>=5.1.0", # Platform-specific dependencies diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 5661a0a48..c232516c2 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -2,6 +2,7 @@ from __future__ import annotations import re import string +import sys from importlib.util import find_spec from pathlib import Path from shutil import copy2, copystat, ignore_patterns, move @@ -16,15 +17,22 @@ from scrapy.utils.template import render_templatefile, string_camelcase if TYPE_CHECKING: import argparse + +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib + + TEMPLATES_TO_RENDER: tuple[tuple[str, ...], ...] = ( - ("scrapy.cfg",), + ("pyproject.toml",), ("${project_name}", "settings.py.tmpl"), ("${project_name}", "items.py.tmpl"), ("${project_name}", "pipelines.py.tmpl"), ("${project_name}", "middlewares.py.tmpl"), ) -IGNORE = ignore_patterns("*.pyc", "__pycache__", ".svn") +IGNORE = ignore_patterns("*.pyc", "__pycache__", ".svn", "scrapy.cfg") def _make_writable(path: Path) -> None: @@ -102,6 +110,17 @@ class Command(ScrapyCommand): print(f"Error: scrapy.cfg already exists in {project_dir.resolve()}") return + if (project_dir / "pyproject.toml").exists(): + with (project_dir / "pyproject.toml").open("rb") as f: + data = tomllib.load(f) + if data.get("tool", {}).get("scrapy"): + self.exitcode = 1 + print( + f"Error: pyproject.toml with [tool.scrapy] already exists " + f"in {project_dir.resolve()}" + ) + return + if not self._is_valid_name(project_name): self.exitcode = 1 return diff --git a/scrapy/templates/project/pyproject.toml b/scrapy/templates/project/pyproject.toml new file mode 100644 index 000000000..6d37ea3b4 --- /dev/null +++ b/scrapy/templates/project/pyproject.toml @@ -0,0 +1,11 @@ +# Automatically created by: scrapy startproject +# +# For more information about the [tool.scrapy.deploy] section see: +# https://scrapyd.readthedocs.io/en/latest/deploy.html + +[tool.scrapy.settings] +default = "${project_name}.settings" + +[tool.scrapy.deploy] +# url = "http://localhost:6800/" +project = "${project_name}" diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 4850b370b..5ccfb7c4e 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -3,6 +3,7 @@ from __future__ import annotations import numbers import os import sys +import warnings from configparser import ConfigParser from operator import itemgetter from pathlib import Path @@ -16,6 +17,11 @@ from scrapy.utils.python import without_none_values if TYPE_CHECKING: from collections.abc import Callable, Collection, Iterable, Mapping, MutableMapping +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib + def build_component_list( compdict: MutableMapping[Any, Any], @@ -70,6 +76,40 @@ def arglist_to_dict(arglist: list[str]) -> dict[str, str]: return dict(x.split("=", 1) for x in arglist) +def closest_config( + path: str | os.PathLike = ".", _resolved: Path | None = None +) -> tuple[str, str]: + """ + Returns the closest config file (pyproject.toml or scrapy.cfg) by + traversing the current directory and its parents + """ + current = Path(path).resolve() if _resolved is None else _resolved + + toml = current / "pyproject.toml" + if toml.exists(): + try: + with toml.open("rb") as f: + data = tomllib.load(f) + if data.get("tool", {}).get("scrapy"): + return "toml", str(toml) + except tomllib.TOMLDecodeError: + warnings.warn( + f"Scrapy could not parse {toml}: invalid TOML. " + "This file will be ignored.", + stacklevel=2, + ) + + cfgfile = current / "scrapy.cfg" + if cfgfile.exists(): + return "cfg", str(cfgfile) + + parent = current.parent + if parent == current: + return "", "" + + return closest_config(_resolved=parent) + + def closest_scrapy_cfg( path: str | os.PathLike[str] = ".", prevpath: str | os.PathLike[str] | None = None, @@ -94,7 +134,8 @@ def init_env(project: str = "default", set_syspath: bool = True) -> None: cfg = get_config() if cfg.has_option("settings", project): os.environ["SCRAPY_SETTINGS_MODULE"] = cfg.get("settings", project) - closest = closest_scrapy_cfg() + + _, closest = closest_config() if closest: projdir = str(Path(closest).parent) if set_syspath and projdir not in sys.path: @@ -103,25 +144,41 @@ def init_env(project: str = "default", set_syspath: bool = True) -> None: def get_config(use_closest: bool = True) -> ConfigParser: """Get Scrapy config file as a ConfigParser""" - sources = get_sources(use_closest) + if use_closest: + config_type, config_path = closest_config() + if config_type == "toml": + with Path(config_path).open("rb") as f: + data = tomllib.load(f) + scrapy_data = data.get("tool", {}).get("scrapy", {}) + cfg = ConfigParser() + cfg.read_dict(scrapy_data) + return cfg + if config_type == "cfg": + warnings.warn( + "scrapy.cfg is deprecated. Please use pyproject.toml instead.", + DeprecationWarning, + stacklevel=2, + ) + cfg = ConfigParser() + cfg.read(config_path) + return cfg + + sources = get_sources() cfg = ConfigParser() cfg.read(sources) return cfg -def get_sources(use_closest: bool = True) -> list[str]: +def get_sources() -> list[str]: xdg_config_home = ( os.environ.get("XDG_CONFIG_HOME") or Path("~/.config").expanduser() ) - sources = [ + return [ "/etc/scrapy.cfg", r"c:\scrapy\scrapy.cfg", str(Path(xdg_config_home) / "scrapy.cfg"), str(Path("~/.scrapy.cfg").expanduser()), ] - if use_closest: - sources.append(closest_scrapy_cfg()) - return sources def feed_complete_default_values_from_settings( diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index 3e75c7729..d72d2c847 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -7,7 +7,7 @@ from pathlib import Path from scrapy.exceptions import NotConfigured from scrapy.settings import Settings -from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env +from scrapy.utils.conf import closest_config, get_config, init_env ENVVAR = "SCRAPY_SETTINGS_MODULE" DATADIR_CFG_SECTION = "datadir" @@ -25,23 +25,24 @@ def inside_project() -> bool: ) else: return True - return bool(closest_scrapy_cfg()) + _, closest = closest_config() + return bool(closest) def project_data_dir(project: str = "default") -> str: """Return the current project data dir, creating it if it doesn't exist""" if not inside_project(): raise NotConfigured("Not inside a project") + _, project_cfg = closest_config() cfg = get_config() if cfg.has_option(DATADIR_CFG_SECTION, project): d = Path(cfg.get(DATADIR_CFG_SECTION, project)) else: - scrapy_cfg = closest_scrapy_cfg() - if not scrapy_cfg: + if not project_cfg: raise NotConfigured( - "Unable to find scrapy.cfg file to infer project data dir" + "Unable to find scrapy.cfg or pyproject.toml to infer project data dir" ) - d = (Path(scrapy_cfg).parent / ".scrapy").resolve() + d = (Path(project_cfg).parent / ".scrapy").resolve() if not d.exists(): d.mkdir(parents=True) return str(d) diff --git a/tests/test_command_startproject.py b/tests/test_command_startproject.py index 7ac6c4fb0..f0e1cece6 100644 --- a/tests/test_command_startproject.py +++ b/tests/test_command_startproject.py @@ -20,7 +20,8 @@ class TestStartprojectCommand: @staticmethod def _assert_files_exist(project_dir: Path, project_name: str) -> None: - assert (project_dir / "scrapy.cfg").exists() + assert (project_dir / "pyproject.toml").exists() + assert not (project_dir / "scrapy.cfg").exists() assert (project_dir / project_name).exists() assert (project_dir / project_name / "__init__.py").exists() assert (project_dir / project_name / "items.py").exists() @@ -77,6 +78,20 @@ class TestStartprojectCommand: assert call("startproject", project_name, cwd=tmp_path) == 0 self._assert_files_exist(project_path, project_name) + def test_startproject_with_existing_pyproject_toml(self, tmp_path: Path) -> None: + # startproject should fail if a pyproject.toml with [tool.scrapy] already exists + (tmp_path / "pyproject.toml").write_text( + "[tool.scrapy.settings]\ndefault = 'myproject.settings'\n" + ) + assert call("startproject", self.project_name, cwd=tmp_path) == 1 + + def test_startproject_with_unrelated_pyproject_toml(self, tmp_path: Path) -> None: + # a pyproject.toml without [tool.scrapy] should not block startproject + (tmp_path / "pyproject.toml").write_text("[tool.black]\nline-length = 88\n") + project_dir = tmp_path / self.project_name + assert call("startproject", self.project_name, cwd=tmp_path) == 0 + self._assert_files_exist(project_dir, self.project_name) + def get_permissions_dict( path: str | os.PathLike[str], renamings=None, ignore=None diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index b6a9d8b06..55a346600 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -1,3 +1,5 @@ +import warnings + import pytest from scrapy.exceptions import UsageError @@ -5,8 +7,10 @@ from scrapy.settings import BaseSettings, Settings from scrapy.utils.conf import ( arglist_to_dict, build_component_list, + closest_config, feed_complete_default_values_from_settings, feed_process_params_from_cli, + get_config, ) @@ -54,6 +58,57 @@ def test_arglist_to_dict(): } +class TestPyprojectToml: + def test_multiple_projects(self, tmp_path, monkeypatch): + (tmp_path / "pyproject.toml").write_text( + "[tool.scrapy.settings]\n" + 'default = "myproject1.settings"\n' + 'project1 = "myproject1.settings"\n' + 'project2 = "myproject2.settings"\n' + ) + monkeypatch.chdir(tmp_path) + cfg = get_config() + assert cfg.get("settings", "default") == "myproject1.settings" + assert cfg.get("settings", "project1") == "myproject1.settings" + assert cfg.get("settings", "project2") == "myproject2.settings" + + def test_pyproject_toml_takes_precedence_over_scrapy_cfg( + self, tmp_path, monkeypatch + ): + (tmp_path / "scrapy.cfg").write_text("[settings]\ndefault = from_scrapy_cfg\n") + (tmp_path / "pyproject.toml").write_text( + "[tool.scrapy.settings]\ndefault = 'from_pyproject_toml'\n" + ) + monkeypatch.chdir(tmp_path) + cfg = get_config() + assert cfg.get("settings", "default") == "from_pyproject_toml" + + def test_scrapy_cfg_not_read_when_pyproject_toml_present( + self, tmp_path, monkeypatch + ): + (tmp_path / "scrapy.cfg").write_text( + "[settings]\ndefault = from_scrapy_cfg\n" + "[deploy]\nproject = from_scrapy_cfg\n" + ) + (tmp_path / "pyproject.toml").write_text( + "[tool.scrapy.settings]\ndefault = 'from_pyproject_toml'\n" + ) + monkeypatch.chdir(tmp_path) + cfg = get_config() + assert not cfg.has_section("deploy") + + def test_malformed_toml_warns_and_falls_back(self, tmp_path, monkeypatch): + (tmp_path / "pyproject.toml").write_text("[tool.scrapy\nnot valid toml") + (tmp_path / "scrapy.cfg").write_text("[settings]\ndefault = from_scrapy_cfg\n") + monkeypatch.chdir(tmp_path) + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + config_type, _ = closest_config() + assert config_type == "cfg" + assert len(w) == 1 + assert "invalid TOML" in str(w[0].message) + + class TestFeedExportConfig: def test_feed_export_config_invalid_format(self): settings = Settings()