From af7d9f41b163e381a23e69113c043c3807c3e718 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Thu, 30 Jul 2026 14:38:41 +0200 Subject: [PATCH] Switch Scrapy projects to pyproject.toml and ~/.scrapy/config.toml --- docs/intro/tutorial.rst | 2 +- docs/news.rst | 13 ++ docs/topics/commands.rst | 67 ++++++---- docs/topics/deploy.rst | 2 +- docs/topics/shell.rst | 15 ++- pyproject.toml | 1 + scrapy/commands/startproject.py | 27 +++- scrapy/shell.py | 7 +- scrapy/templates/project/pyproject.toml | 2 + scrapy/templates/project/scrapy.cfg | 11 -- scrapy/utils/conf.py | 132 ++++++++++++++++++- scrapy/utils/project.py | 12 +- tests/test_command_startproject.py | 30 ++++- tests/test_utils_conf.py | 167 +++++++++++++++++++++++- tests/test_utils_project.py | 2 +- tox.ini | 2 + 16 files changed, 432 insertions(+), 60 deletions(-) create mode 100644 scrapy/templates/project/pyproject.toml delete mode 100644 scrapy/templates/project/scrapy.cfg diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index c4e04364b..cb3a2d178 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -56,7 +56,7 @@ directory where you'd like to store your code and run:: This will create a ``tutorial`` directory with the following contents:: tutorial/ - scrapy.cfg # deploy configuration file + pyproject.toml # project configuration file tutorial/ # project's Python module, you'll import your code from here __init__.py diff --git a/docs/news.rst b/docs/news.rst index 8f8477eaa..b7d64d35f 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -25,6 +25,19 @@ Backward-incompatible changes (:issue:`6585`, :issue:`7731`) +- New projects created with the :command:`startproject` command define their + configuration in a :file:`pyproject.toml` file, instead of the now + deprecated :file:`scrapy.cfg` file. + + To deploy such a project you need ``scrapyd`` TODO or higher, + ``scrapyd-client`` TODO or higher, or ``shub`` TODO or higher. Earlier + versions of those tools only read :file:`scrapy.cfg` files. + (:issue:`7030`) + +.. TODO: Fill in the ``scrapyd``, ``scrapyd-client`` and ``shub`` versions + above. Those releases, with support for the ``[tool.scrapy]`` table of + :file:`pyproject.toml`, must happen before this Scrapy release. + .. _release-2.17.0: Scrapy 2.17.0 (2026-07-07) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index ee2c3a3cd..7e0f52a00 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -16,22 +16,19 @@ accepts a different set of arguments and options. (The ``scrapy deploy`` command has been removed in 1.0 in favor of the standalone ``scrapyd-deploy``. See `Deploying your project`_.) +.. _config: .. _topics-config-settings: Configuration settings ====================== -Scrapy will look for configuration parameters in ini-style ``scrapy.cfg`` files -in standard locations: +Scrapy reads configuration parameters from the ``[tool.scrapy]`` table of the +:file:`pyproject.toml` file at the root of your project (see next section): -1. ``/etc/scrapy.cfg`` or ``c:\scrapy\scrapy.cfg`` (system-wide), -2. ``~/.config/scrapy.cfg`` (``$XDG_CONFIG_HOME``) and ``~/.scrapy.cfg`` (``$HOME``) - for global (user-wide) settings, and -3. ``scrapy.cfg`` inside a Scrapy project's root (see next section). +.. code-block:: toml -Settings from these files are merged in the listed order of preference: -user-defined values have higher priority than system-wide defaults -and project-wide settings will override all others, when defined. + [tool.scrapy.settings] + default = "myproject.settings" Scrapy also understands, and can be configured through, a number of environment variables. Currently these are: @@ -40,6 +37,28 @@ variables. Currently these are: * ``SCRAPY_PROJECT`` (see :ref:`topics-project-envvar`) * ``SCRAPY_PYTHON_SHELL`` (see :ref:`topics-shell`) +.. _global-config: + +Global configuration +-------------------- + +Project-independent configuration parameters may also be set for all your +projects in the :file:`.scrapy/config.toml` file of your home folder, i.e. +:file:`~/.scrapy/config.toml` on Linux and macOS or +:file:`%USERPROFILE%\\.scrapy\\config.toml` on Windows: + +.. code-block:: toml + + [settings] + shell = "bpython" + +Only the following parameters are supported there: + +* ``settings.shell`` (see :ref:`topics-shell`) + +Any project may override them, using the same section and parameter name in its +:file:`pyproject.toml` file. + .. _topics-project-structure: Default structure of Scrapy projects @@ -51,7 +70,7 @@ understand the directory structure of a Scrapy project. Though it can be modified, all Scrapy projects have the same file structure by default, similar to this:: - scrapy.cfg + pyproject.toml myproject/ __init__.py items.py @@ -64,32 +83,32 @@ structure by default, similar to this:: spider2.py ... -The directory where the ``scrapy.cfg`` file resides is known as the *project -root directory*. That file contains the name of the python module that defines -the project settings. Here is an example: +The directory where the :file:`pyproject.toml` file resides is known as the +*project root directory*. That file contains the name of the python module that +defines the project settings. Here is an example: -.. code-block:: ini +.. code-block:: toml - [settings] - default = myproject.settings + [tool.scrapy.settings] + default = "myproject.settings" .. _topics-project-envvar: Sharing the root directory between projects =========================================== -A project root directory, the one that contains the ``scrapy.cfg``, may be -shared by multiple Scrapy projects, each with its own settings module. +A project root directory, the one that contains the :file:`pyproject.toml`, may +be shared by multiple Scrapy projects, each with its own settings module. In that case, you must define one or more aliases for those settings modules -under ``[settings]`` in your ``scrapy.cfg`` file: +under ``[tool.scrapy.settings]`` in your :file:`pyproject.toml` file: -.. code-block:: ini +.. code-block:: toml - [settings] - default = myproject1.settings - project1 = myproject1.settings - project2 = myproject2.settings + [tool.scrapy.settings] + default = "myproject1.settings" + project1 = "myproject1.settings" + project2 = "myproject2.settings" By default, the ``scrapy`` command-line tool will use the ``default`` settings. Use the ``SCRAPY_PROJECT`` environment variable to specify a different project diff --git a/docs/topics/deploy.rst b/docs/topics/deploy.rst index f3515b4be..8046313f4 100644 --- a/docs/topics/deploy.rst +++ b/docs/topics/deploy.rst @@ -46,7 +46,7 @@ tool. Please refer to the `Zyte Scrapy Cloud documentation`_ for more information. Zyte Scrapy Cloud is compatible with Scrapyd and one can switch between -them as needed - the configuration is read from the ``scrapy.cfg`` file +them as needed - the configuration is read from the :file:`pyproject.toml` file just like ``scrapyd-deploy``. .. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 6f7e67cf9..0b8acdea4 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -37,12 +37,21 @@ Through Scrapy's settings you can configure it to use any one of ``ptpython``, ``ipython``, ``bpython`` or the standard ``python`` shell, regardless of which are installed. This is done by setting the ``SCRAPY_PYTHON_SHELL`` environment variable; or by defining it in your -:ref:`scrapy.cfg `: +:file:`pyproject.toml` file (see :ref:`config`): -.. code-block:: ini +.. code-block:: toml + + [tool.scrapy.settings] + shell = "bpython" + +You can also define it for all your projects in the +:file:`.scrapy/config.toml` file of your home folder (see +:ref:`global-config`): + +.. code-block:: toml [settings] - shell = bpython + shell = "bpython" .. _ptpython: https://github.com/prompt-toolkit/ptpython .. _IPython: https://ipython.org/ diff --git a/pyproject.toml b/pyproject.toml index 576a42e5c..b7d141e8d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ dependencies = [ "queuelib>=1.4.2", "service_identity>=23.1.0", "tldextract", + "tomli>=2.0.1; python_version < '3.11'", "w3lib>=1.17.0", "zope.interface>=5.1.0", # Platform-specific dependencies diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 5661a0a48..3f5af58c0 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -11,13 +11,14 @@ from typing import TYPE_CHECKING, Any, ClassVar import scrapy from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError +from scrapy.utils.conf import _scrapy_table from scrapy.utils.template import render_templatefile, string_camelcase if TYPE_CHECKING: import argparse TEMPLATES_TO_RENDER: tuple[tuple[str, ...], ...] = ( - ("scrapy.cfg",), + ("pyproject.toml",), ("${project_name}", "settings.py.tmpl"), ("${project_name}", "items.py.tmpl"), ("${project_name}", "pipelines.py.tmpl"), @@ -102,6 +103,21 @@ class Command(ScrapyCommand): print(f"Error: scrapy.cfg already exists in {project_dir.resolve()}") return + pyproject_path = project_dir / "pyproject.toml" + # Read before the template overwrites it, to restore it below. + pyproject_content = ( + pyproject_path.read_text(encoding="utf-8") + if pyproject_path.is_file() + else None + ) + if pyproject_content is not None and _scrapy_table(pyproject_path) is not None: + self.exitcode = 1 + print( + "Error: pyproject.toml already has a [tool.scrapy] table in " + f"{project_dir.resolve()}" + ) + return + if not self._is_valid_name(project_name): self.exitcode = 1 return @@ -121,6 +137,15 @@ class Command(ScrapyCommand): project_name=project_name, ProjectName=string_camelcase(project_name), ) + if pyproject_content: + # Append the generated tables to the pre-existing pyproject.toml + # instead of replacing it. + pyproject_path.write_text( + pyproject_content.rstrip("\n") + + "\n\n" + + pyproject_path.read_text(encoding="utf-8"), + encoding="utf-8", + ) print( f"New Scrapy project '{project_name}', using template directory " f"'{self.templates_dir}', created in:\n", diff --git a/scrapy/shell.py b/scrapy/shell.py index dfea00c46..56ba8e0e7 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -162,13 +162,12 @@ class Shell: if self.code: print(eval(self.code, globals(), self.vars)) # noqa: S307 else: - # Detect interactive shell setting in scrapy.cfg - # e.g.: ~/.config/scrapy.cfg or ~/.scrapy.cfg - # [settings] + # Detect interactive shell setting in pyproject.toml + # [tool.scrapy.settings] # # shell can be one of ipython, bpython or python; # # to be used as the interactive python console, if available. # # (default is ipython, fallbacks in the order listed above) - # shell = python + # shell = "python" cfg = get_config() section, option = "settings", "shell" env = os.environ.get("SCRAPY_PYTHON_SHELL") diff --git a/scrapy/templates/project/pyproject.toml b/scrapy/templates/project/pyproject.toml new file mode 100644 index 000000000..44f1159b3 --- /dev/null +++ b/scrapy/templates/project/pyproject.toml @@ -0,0 +1,2 @@ +[tool.scrapy.settings] +default = "${project_name}.settings" diff --git a/scrapy/templates/project/scrapy.cfg b/scrapy/templates/project/scrapy.cfg deleted file mode 100644 index 1daeaa541..000000000 --- a/scrapy/templates/project/scrapy.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Automatically created by: scrapy startproject -# -# For more information about the [deploy] section see: -# https://scrapyd.readthedocs.io/en/latest/deploy.html - -[settings] -default = ${project_name}.settings - -[deploy] -#url = http://localhost:6800/ -project = ${project_name} diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 4850b370b..624571cca 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -3,12 +3,13 @@ from __future__ import annotations import numbers import os import sys +import warnings from configparser import ConfigParser from operator import itemgetter from pathlib import Path from typing import TYPE_CHECKING, Any, cast -from scrapy.exceptions import UsageError +from scrapy.exceptions import ScrapyDeprecationWarning, UsageError from scrapy.settings import BaseSettings from scrapy.utils.deprecate import update_classpath from scrapy.utils.python import without_none_values @@ -16,6 +17,17 @@ from scrapy.utils.python import without_none_values if TYPE_CHECKING: from collections.abc import Callable, Collection, Iterable, Mapping, MutableMapping +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib + +_GLOBAL_CONFIG_PATH = Path("~/.scrapy/config.toml") + +# Project-independent options that the global configuration file supports, as +# (section, option) pairs. +_GLOBAL_OPTIONS = frozenset({("settings", "shell")}) + def build_component_list( compdict: MutableMapping[Any, Any], @@ -70,20 +82,105 @@ def arglist_to_dict(arglist: list[str]) -> dict[str, str]: return dict(x.split("=", 1) for x in arglist) +def _load_toml(path: Path) -> dict[str, Any] | None: + """Return the content of the specified TOML file, or ``None`` if it cannot + be parsed. + """ + try: + with path.open("rb") as f: + return tomllib.load(f) + except tomllib.TOMLDecodeError as exc: + warnings.warn(f"Ignoring invalid TOML file {path}: {exc}", stacklevel=3) + return None + + +def _scrapy_table(path: Path) -> dict[str, Any] | None: + """Return the ``tool.scrapy`` table of the specified TOML file, or ``None`` + if the file has no such table or cannot be parsed. + """ + data = _load_toml(path) + tool = data.get("tool") if data else None + table = tool.get("scrapy") if isinstance(tool, dict) else None + return table if isinstance(table, dict) else None + + +def _read_global_config(cfg: ConfigParser) -> None: + """Read the supported options of the global configuration file into the + specified :class:`~configparser.ConfigParser` object. + """ + path = _GLOBAL_CONFIG_PATH.expanduser() + if not path.is_file(): + return + ignored: list[str] = [] + for section, options in (_load_toml(path) or {}).items(): + if not isinstance(options, dict): + ignored.append(section) + continue + for option, value in options.items(): + if (section, option) not in _GLOBAL_OPTIONS: + ignored.append(f"{section}.{option}") + continue + if not cfg.has_section(section): + cfg.add_section(section) + cfg.set(section, option, str(value)) + if ignored: + supported = ", ".join(f"{s}.{o}" for s, o in sorted(_GLOBAL_OPTIONS)) + warnings.warn( + f"Ignoring the following options of {path}: {', '.join(ignored)}. " + f"Only project-independent options are supported there: {supported}.", + stacklevel=3, + ) + + +def closest_config(path: str | os.PathLike[str] = ".") -> str: + """Return the path to the closest configuration file, found by traversing + the specified folder and its parents. + + A folder holds a configuration file if it contains a + :file:`pyproject.toml` file with a ``tool.scrapy`` table or a (deprecated) + :file:`scrapy.cfg` file. If it contains both, :file:`pyproject.toml` wins. + """ + start = Path(path).resolve() + for folder in (start, *start.parents): + toml_path = folder / "pyproject.toml" + if toml_path.is_file() and _scrapy_table(toml_path) is not None: + return str(toml_path) + cfg_path = folder / "scrapy.cfg" + if cfg_path.is_file(): + return str(cfg_path) + return "" + + def closest_scrapy_cfg( path: str | os.PathLike[str] = ".", prevpath: str | os.PathLike[str] | None = None, ) -> str: """Return the path to the closest scrapy.cfg file by traversing the current directory and its parents + + .. deprecated:: VERSION + Use :func:`closest_config` instead. """ + warnings.warn( + "scrapy.utils.conf.closest_scrapy_cfg() is deprecated, use " + "scrapy.utils.conf.closest_config() instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return _closest_scrapy_cfg(path, prevpath) + + +def _closest_scrapy_cfg( + path: str | os.PathLike[str] = ".", + prevpath: str | os.PathLike[str] | None = None, +) -> str: if prevpath is not None and str(path) == str(prevpath): return "" path = Path(path).resolve() cfgfile = path / "scrapy.cfg" if cfgfile.exists(): return str(cfgfile) - return closest_scrapy_cfg(path.parent, path) + return _closest_scrapy_cfg(path.parent, path) def init_env(project: str = "default", set_syspath: bool = True) -> None: @@ -94,7 +191,7 @@ def init_env(project: str = "default", set_syspath: bool = True) -> None: cfg = get_config() if cfg.has_option("settings", project): os.environ["SCRAPY_SETTINGS_MODULE"] = cfg.get("settings", project) - closest = closest_scrapy_cfg() + closest = closest_config() if closest: projdir = str(Path(closest).parent) if set_syspath and projdir not in sys.path: @@ -103,9 +200,32 @@ def init_env(project: str = "default", set_syspath: bool = True) -> None: def get_config(use_closest: bool = True) -> ConfigParser: """Get Scrapy config file as a ConfigParser""" - sources = get_sources(use_closest) cfg = ConfigParser() - cfg.read(sources) + global_files = cfg.read(get_sources(use_closest=False)) + if global_files: + warnings.warn( + f"Configuration read from {', '.join(global_files)}. Global " + "scrapy.cfg files are deprecated, use " + f"{_GLOBAL_CONFIG_PATH.expanduser()} for " + "project-independent options and the [tool.scrapy] table of the " + "pyproject.toml file of your project for the rest. See " + "https://docs.scrapy.org/en/latest/topics/commands.html#global-config", + ScrapyDeprecationWarning, + stacklevel=2, + ) + _read_global_config(cfg) + closest = closest_config() if use_closest else "" + if Path(closest).name == "pyproject.toml": + cfg.read_dict(_scrapy_table(Path(closest)) or {}) + elif closest: + warnings.warn( + f"{closest} is deprecated, define your project configuration in " + "the [tool.scrapy] table of a pyproject.toml file instead. See " + "https://docs.scrapy.org/en/latest/topics/commands.html#config", + ScrapyDeprecationWarning, + stacklevel=2, + ) + cfg.read(closest) return cfg @@ -120,7 +240,7 @@ def get_sources(use_closest: bool = True) -> list[str]: str(Path("~/.scrapy.cfg").expanduser()), ] if use_closest: - sources.append(closest_scrapy_cfg()) + sources.append(_closest_scrapy_cfg()) return sources diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index 3e75c7729..2742b5940 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -7,7 +7,7 @@ from pathlib import Path from scrapy.exceptions import NotConfigured from scrapy.settings import Settings -from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env +from scrapy.utils.conf import closest_config, get_config, init_env ENVVAR = "SCRAPY_SETTINGS_MODULE" DATADIR_CFG_SECTION = "datadir" @@ -25,7 +25,7 @@ def inside_project() -> bool: ) else: return True - return bool(closest_scrapy_cfg()) + return bool(closest_config()) def project_data_dir(project: str = "default") -> str: @@ -36,12 +36,12 @@ def project_data_dir(project: str = "default") -> str: if cfg.has_option(DATADIR_CFG_SECTION, project): d = Path(cfg.get(DATADIR_CFG_SECTION, project)) else: - scrapy_cfg = closest_scrapy_cfg() - if not scrapy_cfg: + config = closest_config() + if not config: raise NotConfigured( - "Unable to find scrapy.cfg file to infer project data dir" + "Unable to find a pyproject.toml file to infer project data dir" ) - d = (Path(scrapy_cfg).parent / ".scrapy").resolve() + d = (Path(config).parent / ".scrapy").resolve() if not d.exists(): d.mkdir(parents=True) return str(d) diff --git a/tests/test_command_startproject.py b/tests/test_command_startproject.py index 7ac6c4fb0..6496714a9 100644 --- a/tests/test_command_startproject.py +++ b/tests/test_command_startproject.py @@ -20,7 +20,7 @@ class TestStartprojectCommand: @staticmethod def _assert_files_exist(project_dir: Path, project_name: str) -> None: - assert (project_dir / "scrapy.cfg").exists() + assert (project_dir / "pyproject.toml").exists() assert (project_dir / project_name).exists() assert (project_dir / project_name / "__init__.py").exists() assert (project_dir / project_name / "items.py").exists() @@ -77,6 +77,34 @@ class TestStartprojectCommand: assert call("startproject", project_name, cwd=tmp_path) == 0 self._assert_files_exist(project_path, project_name) + def test_existing_pyproject_toml(self, tmp_path: Path) -> None: + pyproject_path = tmp_path / "pyproject.toml" + pyproject_path.write_text('[project]\nname = "existing"\n', encoding="utf-8") + + assert call("startproject", self.project_name, ".", cwd=tmp_path) == 0 + self._assert_files_exist(tmp_path, self.project_name) + content = pyproject_path.read_text(encoding="utf-8") + assert '[project]\nname = "existing"\n' in content + assert f'default = "{self.project_name}.settings"' in content + + def test_existing_scrapy_table(self, tmp_path: Path) -> None: + (tmp_path / "pyproject.toml").write_text( + '[tool.scrapy.deploy]\nproject = "existing"\n', encoding="utf-8" + ) + + returncode, out, _ = proc("startproject", self.project_name, ".", cwd=tmp_path) + assert returncode == 1 + assert "already has a [tool.scrapy] table" in out + + def test_existing_scrapy_cfg(self, tmp_path: Path) -> None: + (tmp_path / "scrapy.cfg").write_text( + "[deploy]\nproject = existing\n", encoding="utf-8" + ) + + returncode, out, _ = proc("startproject", self.project_name, ".", cwd=tmp_path) + assert returncode == 1 + assert "scrapy.cfg already exists" in out + def get_permissions_dict( path: str | os.PathLike[str], renamings=None, ignore=None diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index 518cc3518..9a6f0f006 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -1,16 +1,21 @@ from __future__ import annotations +import warnings +from pathlib import Path from typing import Any import pytest -from scrapy.exceptions import UsageError +from scrapy.exceptions import ScrapyDeprecationWarning, UsageError from scrapy.settings import BaseSettings, Settings from scrapy.utils.conf import ( arglist_to_dict, build_component_list, + closest_config, + closest_scrapy_cfg, feed_complete_default_values_from_settings, feed_process_params_from_cli, + get_config, ) @@ -58,6 +63,166 @@ def test_arglist_to_dict(): } +SETTINGS_TOML = ( + "[tool.scrapy.settings]\n" + 'default = "myproject1.settings"\n' + 'project1 = "myproject1.settings"\n' + 'project2 = "myproject2.settings"\n' +) +SETTINGS_CFG = ( + "[settings]\ndefault = myproject.settings\n[deploy]\nproject = myproject\n" +) + + +class TestConfig: + @pytest.fixture(autouse=True) + def home(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Point the global config locations (see + :func:`scrapy.utils.conf.get_sources`) at an initially empty folder.""" + home = tmp_path / "home" + home.mkdir() + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + monkeypatch.setenv("XDG_CONFIG_HOME", str(home)) + return home + + def test_no_config(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.chdir(tmp_path) + assert closest_config() == "" + + def test_pyproject_toml( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + (tmp_path / "pyproject.toml").write_text(SETTINGS_TOML, encoding="utf-8") + subdir = tmp_path / "a" / "b" + subdir.mkdir(parents=True) + monkeypatch.chdir(subdir) + + assert Path(closest_config()) == (tmp_path / "pyproject.toml").resolve() + with warnings.catch_warnings(): + warnings.simplefilter("error", ScrapyDeprecationWarning) + cfg = get_config() + assert cfg.get("settings", "default") == "myproject1.settings" + assert cfg.get("settings", "project1") == "myproject1.settings" + assert cfg.get("settings", "project2") == "myproject2.settings" + + def test_pyproject_toml_preferred( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + (tmp_path / "pyproject.toml").write_text(SETTINGS_TOML, encoding="utf-8") + (tmp_path / "scrapy.cfg").write_text(SETTINGS_CFG, encoding="utf-8") + monkeypatch.chdir(tmp_path) + + assert Path(closest_config()) == (tmp_path / "pyproject.toml").resolve() + cfg = get_config() + assert cfg.get("settings", "default") == "myproject1.settings" + assert not cfg.has_section("deploy") + + def test_closest_wins( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + (tmp_path / "pyproject.toml").write_text(SETTINGS_TOML, encoding="utf-8") + subdir = tmp_path / "a" + subdir.mkdir() + (subdir / "scrapy.cfg").write_text(SETTINGS_CFG, encoding="utf-8") + monkeypatch.chdir(subdir) + + assert Path(closest_config()) == (subdir / "scrapy.cfg").resolve() + with pytest.warns(ScrapyDeprecationWarning, match="scrapy.cfg is deprecated"): + cfg = get_config() + assert cfg.get("settings", "default") == "myproject.settings" + + def test_pyproject_toml_without_scrapy_table( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + (tmp_path / "pyproject.toml").write_text(SETTINGS_TOML, encoding="utf-8") + subdir = tmp_path / "a" + subdir.mkdir() + (subdir / "pyproject.toml").write_text( + '[project]\nname = "unrelated"\n', encoding="utf-8" + ) + monkeypatch.chdir(subdir) + + assert Path(closest_config()) == (tmp_path / "pyproject.toml").resolve() + + def test_invalid_pyproject_toml( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + (tmp_path / "pyproject.toml").write_text( + "[tool.scrapy\nnot valid toml", encoding="utf-8" + ) + (tmp_path / "scrapy.cfg").write_text(SETTINGS_CFG, encoding="utf-8") + monkeypatch.chdir(tmp_path) + + with pytest.warns(UserWarning, match="Ignoring invalid TOML file"): + closest = closest_config() + assert Path(closest) == (tmp_path / "scrapy.cfg").resolve() + + @staticmethod + def _write_global_config(home: Path, content: str) -> None: + path = home / ".scrapy" / "config.toml" + path.parent.mkdir() + path.write_text(content, encoding="utf-8") + + def test_global_config( + self, home: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + self._write_global_config(home, '[settings]\nshell = "bpython"\n') + monkeypatch.chdir(tmp_path) + + assert get_config().get("settings", "shell") == "bpython" + + def test_global_config_overridden_by_project( + self, home: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + self._write_global_config(home, '[settings]\nshell = "bpython"\n') + (tmp_path / "pyproject.toml").write_text( + '[tool.scrapy.settings]\nshell = "python"\n', encoding="utf-8" + ) + monkeypatch.chdir(tmp_path) + + assert get_config().get("settings", "shell") == "python" + + def test_global_config_unsupported_options( + self, home: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + self._write_global_config( + home, + '[settings]\nshell = "bpython"\ndefault = "myproject.settings"\n' + '[deploy]\nproject = "myproject"\nunsupported = 1\n', + ) + monkeypatch.chdir(tmp_path) + + with pytest.warns(UserWarning, match="settings.default, deploy.project"): + cfg = get_config() + assert cfg.get("settings", "shell") == "bpython" + assert not cfg.has_option("settings", "default") + assert not cfg.has_section("deploy") + + def test_global_scrapy_cfg_deprecated( + self, home: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + (home / "scrapy.cfg").write_text( + "[settings]\nshell = python\n", encoding="utf-8" + ) + monkeypatch.chdir(tmp_path) + + with pytest.warns( + ScrapyDeprecationWarning, match="Global scrapy.cfg files are deprecated" + ): + cfg = get_config() + assert cfg.get("settings", "shell") == "python" + + def test_closest_scrapy_cfg_deprecated( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + (tmp_path / "scrapy.cfg").write_text(SETTINGS_CFG, encoding="utf-8") + monkeypatch.chdir(tmp_path) + + with pytest.warns(ScrapyDeprecationWarning, match="closest_scrapy_cfg"): + assert Path(closest_scrapy_cfg()) == (tmp_path / "scrapy.cfg").resolve() + + class TestFeedExportConfig: def test_feed_export_config_invalid_format(self): settings = Settings() diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 7eade6463..74ce0587f 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -20,7 +20,7 @@ def proj_path(tmp_path: Path) -> Generator[Path]: try: os.chdir(project_dir) - Path("scrapy.cfg").touch() + Path("pyproject.toml").write_text("[tool.scrapy]\n", encoding="utf-8") yield project_dir finally: diff --git a/tox.ini b/tox.ini index e10a7cc0c..d9780a89e 100644 --- a/tox.ini +++ b/tox.ini @@ -82,6 +82,7 @@ deps = pyOpenSSL==26.3.0 pytest==9.1.1 socksio==1.0.0 + tomli==2.4.1 types-Pygments==2.20.0.20260518 types-defusedxml==0.7.0.20260504 types-lxml==2026.2.16 @@ -142,6 +143,7 @@ deps = pyOpenSSL==22.0.0 queuelib==1.4.2 service_identity==23.1.0 + tomli==2.0.1 w3lib==1.17.0 zope.interface==5.1.0 {[test-requirements]deps}