From fa25549f3f5ce56c7297977a2ef6e3de77edd8c5 Mon Sep 17 00:00:00 2001 From: Aiman Date: Sun, 22 Mar 2026 00:04:31 +0530 Subject: [PATCH 1/9] Add support for pyproject.toml as a preferred alternative to scrapy.cfg --- scrapy/utils/conf.py | 51 +++++++++++++++++++++++++++++++++++++++- tests/test_utils_conf.py | 38 ++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 5869cf52e..9bbcab544 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -3,6 +3,7 @@ from __future__ import annotations import numbers import os import sys +import warnings from configparser import ConfigParser from operator import itemgetter from pathlib import Path @@ -16,6 +17,14 @@ from scrapy.utils.python import without_none_values if TYPE_CHECKING: from collections.abc import Callable, Collection, Iterable, Mapping, MutableMapping +if sys.version_info >= (3, 11): + import tomllib +else: + try: + import tomli as tomllib + except ImportError: + tomllib = None # type: ignore[assignment] + def build_component_list( compdict: MutableMapping[Any, Any], @@ -86,6 +95,31 @@ def closest_scrapy_cfg( return closest_scrapy_cfg(path.parent, path) +def closest_pyproject_toml( + path: str | os.PathLike = ".", + prevpath: str | os.PathLike | None = None, +) -> str: + if tomllib is None: + return "" + if prevpath is not None and str(path) == str(prevpath): + return "" + path = Path(path).resolve() + toml_path = path / "pyproject.toml" + if toml_path.exists(): + try: + with toml_path.open("rb") as f: + data = tomllib.load(f) + if "scrapy" in data.get("tool", {}): + return str(toml_path) + except tomllib.TOMLDecodeError: + warnings.warn( + f"Scrapy could not parse {toml_path}: invalid TOML. " + "This file will be ignored.", + stacklevel=2, + ) + return closest_pyproject_toml(path.parent, path) + + def init_env(project: str = "default", set_syspath: bool = True) -> None: """Initialize environment to use command-line tool from inside a project dir. This sets the Scrapy settings module and modifies the Python path to @@ -94,15 +128,30 @@ def init_env(project: str = "default", set_syspath: bool = True) -> None: cfg = get_config() if cfg.has_option("settings", project): os.environ["SCRAPY_SETTINGS_MODULE"] = cfg.get("settings", project) - closest = closest_scrapy_cfg() + closest = closest_pyproject_toml() or closest_scrapy_cfg() if closest: projdir = str(Path(closest).parent) if set_syspath and projdir not in sys.path: sys.path.append(projdir) +def _load_toml_config(toml_path: str) -> ConfigParser: + """Load Scrapy config from a TOML file.""" + with Path(toml_path).open("rb") as f: + data = tomllib.load(f) + scrapy_data = data.get("tool", {}).get("scrapy", {}) + cfg = ConfigParser() + cfg.read_dict(scrapy_data) + return cfg + + def get_config(use_closest: bool = True) -> ConfigParser: """Get Scrapy config file as a ConfigParser""" + if use_closest and tomllib is not None: + toml_path = closest_pyproject_toml() + if toml_path: + return _load_toml_config(toml_path) + sources = get_sources(use_closest) cfg = ConfigParser() cfg.read(sources) diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index b6a9d8b06..db48f4b61 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -1,3 +1,6 @@ +import os +import warnings + import pytest from scrapy.exceptions import UsageError @@ -5,8 +8,10 @@ from scrapy.settings import BaseSettings, Settings from scrapy.utils.conf import ( arglist_to_dict, build_component_list, + closest_pyproject_toml, feed_complete_default_values_from_settings, feed_process_params_from_cli, + get_config, ) @@ -54,6 +59,39 @@ def test_arglist_to_dict(): } +class TestPyprojectToml: + def test_pyproject_toml_takes_precedence_over_scrapy_cfg(self, tmp_path): + (tmp_path / "scrapy.cfg").write_text("[settings]\ndefault = from_scrapy_cfg\n") + (tmp_path / "pyproject.toml").write_text( + "[tool.scrapy.settings]\ndefault = 'from_pyproject_toml'\n" + ) + os.chdir(tmp_path) + cfg = get_config() + assert cfg.get("settings", "default") == "from_pyproject_toml" + + def test_scrapy_cfg_not_read_when_pyproject_toml_present(self, tmp_path): + (tmp_path / "scrapy.cfg").write_text( + "[settings]\ndefault = from_scrapy_cfg\n" + "[deploy]\nproject = from_scrapy_cfg\n" + ) + (tmp_path / "pyproject.toml").write_text( + "[tool.scrapy.settings]\ndefault = 'from_pyproject_toml'\n" + ) + os.chdir(tmp_path) + cfg = get_config() + assert not cfg.has_section("deploy") + + def test_malformed_toml_warns_and_returns_empty(self, tmp_path): + (tmp_path / "pyproject.toml").write_text("[tool.scrapy\nnot valid toml") + os.chdir(tmp_path) + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + result = closest_pyproject_toml() + assert result == "" + assert len(w) == 1 + assert "invalid TOML" in str(w[0].message) + + class TestFeedExportConfig: def test_feed_export_config_invalid_format(self): settings = Settings() From 97862e765e633ce0370839a5c76e8605ff8c4886 Mon Sep 17 00:00:00 2001 From: Aiman Date: Sun, 22 Mar 2026 00:40:07 +0530 Subject: [PATCH 2/9] minor changes --- scrapy/utils/project.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index 0139720b7..0fe2c9389 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -7,7 +7,12 @@ from pathlib import Path from scrapy.exceptions import NotConfigured from scrapy.settings import Settings -from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env +from scrapy.utils.conf import ( + closest_pyproject_toml, + closest_scrapy_cfg, + get_config, + init_env, +) ENVVAR = "SCRAPY_SETTINGS_MODULE" DATADIR_CFG_SECTION = "datadir" @@ -24,7 +29,7 @@ def inside_project() -> bool: ) else: return True - return bool(closest_scrapy_cfg()) + return bool(closest_scrapy_cfg() or closest_pyproject_toml()) def project_data_dir(project: str = "default") -> str: @@ -35,10 +40,10 @@ def project_data_dir(project: str = "default") -> str: if cfg.has_option(DATADIR_CFG_SECTION, project): d = Path(cfg.get(DATADIR_CFG_SECTION, project)) else: - scrapy_cfg = closest_scrapy_cfg() + scrapy_cfg = closest_scrapy_cfg() or closest_pyproject_toml() if not scrapy_cfg: raise NotConfigured( - "Unable to find scrapy.cfg file to infer project data dir" + "Unable to find scrapy.cfg or pyproject.toml to infer project data dir" ) d = (Path(scrapy_cfg).parent / ".scrapy").resolve() if not d.exists(): From 83f83ef82314ceee43ca12a2487fb5186d69d0ca Mon Sep 17 00:00:00 2001 From: Aiman Date: Sun, 22 Mar 2026 22:52:41 +0530 Subject: [PATCH 3/9] update comfig file searching logic and docs --- docs/topics/commands.rst | 36 +++++++++----- docs/topics/shell.rst | 11 ++++- pyproject.toml | 1 + scrapy/utils/conf.py | 103 ++++++++++++++++++++------------------- scrapy/utils/project.py | 16 +++--- tests/test_utils_conf.py | 24 +++++---- 6 files changed, 106 insertions(+), 85 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 8d1351eb9..4cde739d7 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -27,7 +27,17 @@ in standard locations: 1. ``/etc/scrapy.cfg`` or ``c:\scrapy\scrapy.cfg`` (system-wide), 2. ``~/.config/scrapy.cfg`` (``$XDG_CONFIG_HOME``) and ``~/.scrapy.cfg`` (``$HOME``) for global (user-wide) settings, and -3. ``scrapy.cfg`` inside a Scrapy project's root (see next section). +3. ``pyproject.toml`` (recommended) or ``scrapy.cfg`` inside a Scrapy project's root (see next section). + +For ``pyproject.toml``, the settings are read from the ``[tool.scrapy]`` section: + +.. code-block:: toml + + [tool.scrapy.settings] + default = "myproject.settings" + +Scrapy searches for the closest ``pyproject.toml`` or ``scrapy.cfg``, and if both +files exist, ``pyproject.toml`` takes precedence. Settings from these files are merged in the listed order of preference: user-defined values have higher priority than system-wide defaults @@ -51,7 +61,7 @@ understand the directory structure of a Scrapy project. Though it can be modified, all Scrapy projects have the same file structure by default, similar to this:: - scrapy.cfg + pyproject.toml myproject/ __init__.py items.py @@ -64,32 +74,32 @@ structure by default, similar to this:: spider2.py ... -The directory where the ``scrapy.cfg`` file resides is known as the *project +The directory where the ``pyproject.toml`` file resides is known as the *project root directory*. That file contains the name of the python module that defines the project settings. Here is an example: -.. code-block:: ini +.. code-block:: toml - [settings] - default = myproject.settings + [tool.scrapy.settings] + default = "myproject.settings" .. _topics-project-envvar: Sharing the root directory between projects =========================================== -A project root directory, the one that contains the ``scrapy.cfg``, may be +A project root directory, the one that contains the ``pyproject.toml``, may be shared by multiple Scrapy projects, each with its own settings module. In that case, you must define one or more aliases for those settings modules -under ``[settings]`` in your ``scrapy.cfg`` file: +under ``[tool.scrapy.settings]`` in your ``pyproject.toml`` file: -.. code-block:: ini +.. code-block:: toml - [settings] - default = myproject1.settings - project1 = myproject1.settings - project2 = myproject2.settings + [tool.scrapy.settings] + default = "myproject1.settings" + project1 = "myproject1.settings" + project2 = "myproject2.settings" By default, the ``scrapy`` command-line tool will use the ``default`` settings. Use the ``SCRAPY_PROJECT`` environment variable to specify a different project diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 8ae8ff512..a5a4ae64f 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -34,10 +34,17 @@ is unavailable. Through Scrapy's settings you can configure it to use any one of ``ipython``, ``bpython`` or the standard ``python`` shell, regardless of which are installed. This is done by setting the ``SCRAPY_PYTHON_SHELL`` environment -variable; or by defining it in your :ref:`scrapy.cfg `:: +variable; or by defining it in your :ref:`pyproject.toml `: + +.. code-block:: toml + + [tool.scrapy.settings] + shell = "bpython" + +or in ``scrapy.cfg``:: [settings] - shell = bpython + shell = "bpython" .. _IPython: https://ipython.org/ .. _IPython installation guide: https://ipython.org/install.html diff --git a/pyproject.toml b/pyproject.toml index 28e297401..61511673d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ dependencies = [ # Platform-specific dependencies 'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"', 'PyPyDispatcher>=2.1.0; platform_python_implementation == "PyPy"', + "tomli>=1.1.0; python_version < '3.11'", ] classifiers = [ "Development Status :: 5 - Production/Stable", diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 9bbcab544..50bda6d32 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -20,10 +20,7 @@ if TYPE_CHECKING: if sys.version_info >= (3, 11): import tomllib else: - try: - import tomli as tomllib - except ImportError: - tomllib = None # type: ignore[assignment] + import tomli as tomllib def build_component_list( @@ -79,6 +76,40 @@ def arglist_to_dict(arglist: list[str]) -> dict[str, str]: return dict(x.split("=", 1) for x in arglist) +def closest_config( + path: str | os.PathLike = ".", _resolved: Path | None = None +) -> tuple[str, str]: + """ + Returns the closest config file (pyproject.toml or scrapy.cfg) by + traversing the current directory and its parents + """ + current = Path(path).resolve() if _resolved is None else _resolved + + toml = current / "pyproject.toml" + if toml.exists(): + try: + with toml.open("rb") as f: + data = tomllib.load(f) + if data.get("tool", {}).get("scrapy"): + return "toml", str(toml) + except tomllib.TOMLDecodeError: + warnings.warn( + f"Scrapy could not parse {toml}: invalid TOML. " + "This file will be ignored.", + stacklevel=2, + ) + + cfgfile = current / "scrapy.cfg" + if cfgfile.exists(): + return "cfg", str(cfgfile) + + parent = current.parent + if parent == current: + return "", "" + + return closest_config(_resolved=parent) + + def closest_scrapy_cfg( path: str | os.PathLike = ".", prevpath: str | os.PathLike | None = None, @@ -95,31 +126,6 @@ def closest_scrapy_cfg( return closest_scrapy_cfg(path.parent, path) -def closest_pyproject_toml( - path: str | os.PathLike = ".", - prevpath: str | os.PathLike | None = None, -) -> str: - if tomllib is None: - return "" - if prevpath is not None and str(path) == str(prevpath): - return "" - path = Path(path).resolve() - toml_path = path / "pyproject.toml" - if toml_path.exists(): - try: - with toml_path.open("rb") as f: - data = tomllib.load(f) - if "scrapy" in data.get("tool", {}): - return str(toml_path) - except tomllib.TOMLDecodeError: - warnings.warn( - f"Scrapy could not parse {toml_path}: invalid TOML. " - "This file will be ignored.", - stacklevel=2, - ) - return closest_pyproject_toml(path.parent, path) - - def init_env(project: str = "default", set_syspath: bool = True) -> None: """Initialize environment to use command-line tool from inside a project dir. This sets the Scrapy settings module and modifies the Python path to @@ -128,49 +134,46 @@ def init_env(project: str = "default", set_syspath: bool = True) -> None: cfg = get_config() if cfg.has_option("settings", project): os.environ["SCRAPY_SETTINGS_MODULE"] = cfg.get("settings", project) - closest = closest_pyproject_toml() or closest_scrapy_cfg() + + _, closest = closest_config() if closest: projdir = str(Path(closest).parent) if set_syspath and projdir not in sys.path: sys.path.append(projdir) -def _load_toml_config(toml_path: str) -> ConfigParser: - """Load Scrapy config from a TOML file.""" - with Path(toml_path).open("rb") as f: - data = tomllib.load(f) - scrapy_data = data.get("tool", {}).get("scrapy", {}) - cfg = ConfigParser() - cfg.read_dict(scrapy_data) - return cfg - - def get_config(use_closest: bool = True) -> ConfigParser: """Get Scrapy config file as a ConfigParser""" - if use_closest and tomllib is not None: - toml_path = closest_pyproject_toml() - if toml_path: - return _load_toml_config(toml_path) + if use_closest: + config_type, config_path = closest_config() + if config_type == "toml": + with Path(config_path).open("rb") as f: + data = tomllib.load(f) + scrapy_data = data.get("tool", {}).get("scrapy", {}) + cfg = ConfigParser() + cfg.read_dict(scrapy_data) + return cfg + if config_type == "cfg": + cfg = ConfigParser() + cfg.read(config_path) + return cfg - sources = get_sources(use_closest) + sources = get_sources() cfg = ConfigParser() cfg.read(sources) return cfg -def get_sources(use_closest: bool = True) -> list[str]: +def get_sources() -> list[str]: xdg_config_home = ( os.environ.get("XDG_CONFIG_HOME") or Path("~/.config").expanduser() ) - sources = [ + return [ "/etc/scrapy.cfg", r"c:\scrapy\scrapy.cfg", str(Path(xdg_config_home) / "scrapy.cfg"), str(Path("~/.scrapy.cfg").expanduser()), ] - if use_closest: - sources.append(closest_scrapy_cfg()) - return sources def feed_complete_default_values_from_settings( diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index 0fe2c9389..2d92694eb 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -7,12 +7,7 @@ from pathlib import Path from scrapy.exceptions import NotConfigured from scrapy.settings import Settings -from scrapy.utils.conf import ( - closest_pyproject_toml, - closest_scrapy_cfg, - get_config, - init_env, -) +from scrapy.utils.conf import closest_config, get_config, init_env ENVVAR = "SCRAPY_SETTINGS_MODULE" DATADIR_CFG_SECTION = "datadir" @@ -29,23 +24,24 @@ def inside_project() -> bool: ) else: return True - return bool(closest_scrapy_cfg() or closest_pyproject_toml()) + _, closest = closest_config() + return bool(closest) def project_data_dir(project: str = "default") -> str: """Return the current project data dir, creating it if it doesn't exist""" if not inside_project(): raise NotConfigured("Not inside a project") + _, project_cfg = closest_config() cfg = get_config() if cfg.has_option(DATADIR_CFG_SECTION, project): d = Path(cfg.get(DATADIR_CFG_SECTION, project)) else: - scrapy_cfg = closest_scrapy_cfg() or closest_pyproject_toml() - if not scrapy_cfg: + if not project_cfg: raise NotConfigured( "Unable to find scrapy.cfg or pyproject.toml to infer project data dir" ) - d = (Path(scrapy_cfg).parent / ".scrapy").resolve() + d = (Path(project_cfg).parent / ".scrapy").resolve() if not d.exists(): d.mkdir(parents=True) return str(d) diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index db48f4b61..9acf48c11 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -1,4 +1,3 @@ -import os import warnings import pytest @@ -8,7 +7,7 @@ from scrapy.settings import BaseSettings, Settings from scrapy.utils.conf import ( arglist_to_dict, build_component_list, - closest_pyproject_toml, + closest_config, feed_complete_default_values_from_settings, feed_process_params_from_cli, get_config, @@ -60,16 +59,20 @@ def test_arglist_to_dict(): class TestPyprojectToml: - def test_pyproject_toml_takes_precedence_over_scrapy_cfg(self, tmp_path): + def test_pyproject_toml_takes_precedence_over_scrapy_cfg( + self, tmp_path, monkeypatch + ): (tmp_path / "scrapy.cfg").write_text("[settings]\ndefault = from_scrapy_cfg\n") (tmp_path / "pyproject.toml").write_text( "[tool.scrapy.settings]\ndefault = 'from_pyproject_toml'\n" ) - os.chdir(tmp_path) + monkeypatch.chdir(tmp_path) cfg = get_config() assert cfg.get("settings", "default") == "from_pyproject_toml" - def test_scrapy_cfg_not_read_when_pyproject_toml_present(self, tmp_path): + def test_scrapy_cfg_not_read_when_pyproject_toml_present( + self, tmp_path, monkeypatch + ): (tmp_path / "scrapy.cfg").write_text( "[settings]\ndefault = from_scrapy_cfg\n" "[deploy]\nproject = from_scrapy_cfg\n" @@ -77,17 +80,18 @@ class TestPyprojectToml: (tmp_path / "pyproject.toml").write_text( "[tool.scrapy.settings]\ndefault = 'from_pyproject_toml'\n" ) - os.chdir(tmp_path) + monkeypatch.chdir(tmp_path) cfg = get_config() assert not cfg.has_section("deploy") - def test_malformed_toml_warns_and_returns_empty(self, tmp_path): + def test_malformed_toml_warns_and_falls_back(self, tmp_path, monkeypatch): (tmp_path / "pyproject.toml").write_text("[tool.scrapy\nnot valid toml") - os.chdir(tmp_path) + (tmp_path / "scrapy.cfg").write_text("[settings]\ndefault = from_scrapy_cfg\n") + monkeypatch.chdir(tmp_path) with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") - result = closest_pyproject_toml() - assert result == "" + config_type, _ = closest_config() + assert config_type == "cfg" assert len(w) == 1 assert "invalid TOML" in str(w[0].message) From 89a0c077382d51cf382c8767c28f347e74c717f9 Mon Sep 17 00:00:00 2001 From: Aiman Date: Mon, 23 Mar 2026 08:11:34 +0530 Subject: [PATCH 4/9] change default template to use pyproject.toml --- scrapy/commands/startproject.py | 23 +++++++++++++++++++++-- scrapy/templates/project/pyproject.toml | 11 +++++++++++ tests/test_command_startproject.py | 17 ++++++++++++++++- 3 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 scrapy/templates/project/pyproject.toml diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 8f4427580..76afe82e1 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -2,6 +2,7 @@ from __future__ import annotations import re import string +import sys from importlib.util import find_spec from pathlib import Path from shutil import copy2, copystat, ignore_patterns, move @@ -16,15 +17,22 @@ from scrapy.utils.template import render_templatefile, string_camelcase if TYPE_CHECKING: import argparse + +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib + + TEMPLATES_TO_RENDER: tuple[tuple[str, ...], ...] = ( - ("scrapy.cfg",), + ("pyproject.toml",), ("${project_name}", "settings.py.tmpl"), ("${project_name}", "items.py.tmpl"), ("${project_name}", "pipelines.py.tmpl"), ("${project_name}", "middlewares.py.tmpl"), ) -IGNORE = ignore_patterns("*.pyc", "__pycache__", ".svn") +IGNORE = ignore_patterns("*.pyc", "__pycache__", ".svn", "scrapy.cfg") def _make_writable(path: Path) -> None: @@ -102,6 +110,17 @@ class Command(ScrapyCommand): print(f"Error: scrapy.cfg already exists in {project_dir.resolve()}") return + if (project_dir / "pyproject.toml").exists(): + with (project_dir / "pyproject.toml").open("rb") as f: + data = tomllib.load(f) + if data.get("tool", {}).get("scrapy"): + self.exitcode = 1 + print( + f"Error: pyproject.toml with [tool.scrapy] already exists " + f"in {project_dir.resolve()}" + ) + return + if not self._is_valid_name(project_name): self.exitcode = 1 return diff --git a/scrapy/templates/project/pyproject.toml b/scrapy/templates/project/pyproject.toml new file mode 100644 index 000000000..504db0590 --- /dev/null +++ b/scrapy/templates/project/pyproject.toml @@ -0,0 +1,11 @@ +# Automatically created by: scrapy startproject +# +# For more information about the [tool.scrapy.deploy] section see: +# https://scrapyd.readthedocs.io/en/latest/deploy.html + +[tool.scrapy.settings] +default = "${project_name}.settings" + +[tool.scrapy.deploy] +# url = "http://localhost:6800/" +project = "${project_name}" \ No newline at end of file diff --git a/tests/test_command_startproject.py b/tests/test_command_startproject.py index 4eed09fcd..e2b00a4fe 100644 --- a/tests/test_command_startproject.py +++ b/tests/test_command_startproject.py @@ -20,7 +20,8 @@ class TestStartprojectCommand: @staticmethod def _assert_files_exist(project_dir: Path, project_name: str) -> None: - assert (project_dir / "scrapy.cfg").exists() + assert (project_dir / "pyproject.toml").exists() + assert not (project_dir / "scrapy.cfg").exists() assert (project_dir / project_name).exists() assert (project_dir / project_name / "__init__.py").exists() assert (project_dir / project_name / "items.py").exists() @@ -77,6 +78,20 @@ class TestStartprojectCommand: assert call("startproject", project_name, cwd=tmp_path) == 0 self._assert_files_exist(project_path, project_name) + def test_startproject_with_existing_pyproject_toml(self, tmp_path: Path) -> None: + # startproject should fail if a pyproject.toml with [tool.scrapy] already exists + (tmp_path / "pyproject.toml").write_text( + "[tool.scrapy.settings]\ndefault = 'myproject.settings'\n" + ) + assert call("startproject", self.project_name, cwd=tmp_path) == 1 + + def test_startproject_with_unrelated_pyproject_toml(self, tmp_path: Path) -> None: + # a pyproject.toml without [tool.scrapy] should not block startproject + (tmp_path / "pyproject.toml").write_text("[tool.black]\nline-length = 88\n") + project_dir = tmp_path / self.project_name + assert call("startproject", self.project_name, cwd=tmp_path) == 0 + self._assert_files_exist(project_dir, self.project_name) + def get_permissions_dict( path: str | os.PathLike, renamings=None, ignore=None From 24d50bc44c9ba469afe647069910cba3f84d9ddb Mon Sep 17 00:00:00 2001 From: Aiman Date: Mon, 23 Mar 2026 08:18:52 +0530 Subject: [PATCH 5/9] fix EOF error --- scrapy/templates/project/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/templates/project/pyproject.toml b/scrapy/templates/project/pyproject.toml index 504db0590..6d37ea3b4 100644 --- a/scrapy/templates/project/pyproject.toml +++ b/scrapy/templates/project/pyproject.toml @@ -8,4 +8,4 @@ default = "${project_name}.settings" [tool.scrapy.deploy] # url = "http://localhost:6800/" -project = "${project_name}" \ No newline at end of file +project = "${project_name}" From ff71b4660c3da6f8d4e8274e7123e2abe37dd48e Mon Sep 17 00:00:00 2001 From: Aiman Date: Tue, 24 Mar 2026 23:28:49 +0530 Subject: [PATCH 6/9] add scrapy.cfg deprecation warnings --- docs/topics/commands.rst | 20 +++++--------------- pyproject.toml | 3 ++- scrapy/utils/conf.py | 5 +++++ 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 4cde739d7..f49e4cf22 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -21,23 +21,13 @@ standalone ``scrapyd-deploy``. See `Deploying your project`_.) Configuration settings ====================== -Scrapy will look for configuration parameters in ini-style ``scrapy.cfg`` files -in standard locations: +Scrapy will look for configuration parameters in toml-style ``pyproject.toml`` +files in standard locations: -1. ``/etc/scrapy.cfg`` or ``c:\scrapy\scrapy.cfg`` (system-wide), -2. ``~/.config/scrapy.cfg`` (``$XDG_CONFIG_HOME``) and ``~/.scrapy.cfg`` (``$HOME``) +1. ``/etc/pyproject.toml`` or ``c:\scrapy\pyproject.toml`` (system-wide), +2. ``~/.config/pyproject.toml`` (``$XDG_CONFIG_HOME``) and ``~/.pyproject.toml`` (``$HOME``) for global (user-wide) settings, and -3. ``pyproject.toml`` (recommended) or ``scrapy.cfg`` inside a Scrapy project's root (see next section). - -For ``pyproject.toml``, the settings are read from the ``[tool.scrapy]`` section: - -.. code-block:: toml - - [tool.scrapy.settings] - default = "myproject.settings" - -Scrapy searches for the closest ``pyproject.toml`` or ``scrapy.cfg``, and if both -files exist, ``pyproject.toml`` takes precedence. +3. ``pyproject.toml`` inside a Scrapy project's root (see next section). Settings from these files are merged in the listed order of preference: user-defined values have higher priority than system-wide defaults diff --git a/pyproject.toml b/pyproject.toml index 61511673d..324168e9a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,12 +22,13 @@ dependencies = [ "queuelib>=1.4.2", "service_identity>=18.1.0", "tldextract", + "tomli>=1.1.0; python_version < '3.11'", "w3lib>=1.17.0", "zope.interface>=5.1.0", # Platform-specific dependencies 'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"', 'PyPyDispatcher>=2.1.0; platform_python_implementation == "PyPy"', - "tomli>=1.1.0; python_version < '3.11'", + "sphinx-autobuild>=2024.10.3", ] classifiers = [ "Development Status :: 5 - Production/Stable", diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 50bda6d32..3a31f2d76 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -154,6 +154,11 @@ def get_config(use_closest: bool = True) -> ConfigParser: cfg.read_dict(scrapy_data) return cfg if config_type == "cfg": + warnings.warn( + "scrapy.cfg is deprecated. Please use pyproject.toml instead.", + DeprecationWarning, + stacklevel=2, + ) cfg = ConfigParser() cfg.read(config_path) return cfg From 33be33cae65612c2182b51178316280302234403 Mon Sep 17 00:00:00 2001 From: Aiman Date: Tue, 24 Mar 2026 23:35:35 +0530 Subject: [PATCH 7/9] remove scrapy.cfg mention --- docs/topics/shell.rst | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index a5a4ae64f..bdb9ca5d2 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -41,11 +41,6 @@ variable; or by defining it in your :ref:`pyproject.toml Date: Wed, 25 Mar 2026 02:06:23 +0530 Subject: [PATCH 8/9] remove sphinx-autobuild dependency --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 324168e9a..608ea9466 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,6 @@ dependencies = [ # Platform-specific dependencies 'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"', 'PyPyDispatcher>=2.1.0; platform_python_implementation == "PyPy"', - "sphinx-autobuild>=2024.10.3", ] classifiers = [ "Development Status :: 5 - Production/Stable", From 1e844796c0974f5fbe0a9c06b36b184ddb3f506c Mon Sep 17 00:00:00 2001 From: Aiman Date: Thu, 26 Mar 2026 22:21:57 +0530 Subject: [PATCH 9/9] add test and update docs --- docs/topics/commands.rst | 13 ++----------- tests/test_utils_conf.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index f49e4cf22..e8d99fc55 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -21,17 +21,8 @@ standalone ``scrapyd-deploy``. See `Deploying your project`_.) Configuration settings ====================== -Scrapy will look for configuration parameters in toml-style ``pyproject.toml`` -files in standard locations: - -1. ``/etc/pyproject.toml`` or ``c:\scrapy\pyproject.toml`` (system-wide), -2. ``~/.config/pyproject.toml`` (``$XDG_CONFIG_HOME``) and ``~/.pyproject.toml`` (``$HOME``) - for global (user-wide) settings, and -3. ``pyproject.toml`` inside a Scrapy project's root (see next section). - -Settings from these files are merged in the listed order of preference: -user-defined values have higher priority than system-wide defaults -and project-wide settings will override all others, when defined. +Scrapy looks for project configuration in a ``pyproject.toml`` file located in +the project root directory (see next section). Scrapy also understands, and can be configured through, a number of environment variables. Currently these are: diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index 9acf48c11..55a346600 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -59,6 +59,19 @@ def test_arglist_to_dict(): class TestPyprojectToml: + def test_multiple_projects(self, tmp_path, monkeypatch): + (tmp_path / "pyproject.toml").write_text( + "[tool.scrapy.settings]\n" + 'default = "myproject1.settings"\n' + 'project1 = "myproject1.settings"\n' + 'project2 = "myproject2.settings"\n' + ) + monkeypatch.chdir(tmp_path) + cfg = get_config() + assert cfg.get("settings", "default") == "myproject1.settings" + assert cfg.get("settings", "project1") == "myproject1.settings" + assert cfg.get("settings", "project2") == "myproject2.settings" + def test_pyproject_toml_takes_precedence_over_scrapy_cfg( self, tmp_path, monkeypatch ):