Merge 1e844796c0 into dd10cb8e9a

docs/topics/commands.rst

@@ -21,17 +21,8 @@ standalone ``scrapyd-deploy``. See `Deploying your project`_.)
 Configuration settings
 ======================
 
-Scrapy will look for configuration parameters in ini-style ``scrapy.cfg`` files
-in standard locations:
-
-1. ``/etc/scrapy.cfg`` or ``c:\scrapy\scrapy.cfg`` (system-wide),
-2. ``~/.config/scrapy.cfg`` (``$XDG_CONFIG_HOME``) and ``~/.scrapy.cfg`` (``$HOME``)
-   for global (user-wide) settings, and
-3. ``scrapy.cfg`` inside a Scrapy project's root (see next section).
-
-Settings from these files are merged in the listed order of preference:
-user-defined values have higher priority than system-wide defaults
-and project-wide settings will override all others, when defined.
+Scrapy looks for project configuration in a ``pyproject.toml`` file located in 
+the project root directory (see next section).
 
 Scrapy also understands, and can be configured through, a number of environment
 variables. Currently these are:
@@ -51,7 +42,7 @@ understand the directory structure of a Scrapy project.
 Though it can be modified, all Scrapy projects have the same file
 structure by default, similar to this::
 
-   scrapy.cfg
+   pyproject.toml
    myproject/
        __init__.py
        items.py
@@ -64,32 +55,32 @@ structure by default, similar to this::
            spider2.py
            ...
 
-The directory where the ``scrapy.cfg`` file resides is known as the *project
+The directory where the ``pyproject.toml`` file resides is known as the *project
 root directory*. That file contains the name of the python module that defines
 the project settings. Here is an example:
 
-.. code-block:: ini
+.. code-block:: toml
 
-    [settings]
-    default = myproject.settings
+    [tool.scrapy.settings]
+    default = "myproject.settings"
 
 .. _topics-project-envvar:
 
 Sharing the root directory between projects
 ===========================================
 
-A project root directory, the one that contains the ``scrapy.cfg``, may be
+A project root directory, the one that contains the ``pyproject.toml``, may be
 shared by multiple Scrapy projects, each with its own settings module.
 
 In that case, you must define one or more aliases for those settings modules
-under ``[settings]`` in your ``scrapy.cfg`` file:
+under ``[tool.scrapy.settings]`` in your ``pyproject.toml`` file:
 
-.. code-block:: ini
+.. code-block:: toml
 
-    [settings]
-    default = myproject1.settings
-    project1 = myproject1.settings
-    project2 = myproject2.settings
+    [tool.scrapy.settings]
+    default = "myproject1.settings"
+    project1 = "myproject1.settings"
+    project2 = "myproject2.settings"
 
 By default, the ``scrapy`` command-line tool will use the ``default`` settings.
 Use the ``SCRAPY_PROJECT`` environment variable to specify a different project

docs/topics/shell.rst

@@ -34,10 +34,12 @@ is unavailable.
 Through Scrapy's settings you can configure it to use any one of
 ``ipython``, ``bpython`` or the standard ``python`` shell, regardless of which
 are installed. This is done by setting the ``SCRAPY_PYTHON_SHELL`` environment
-variable; or by defining it in your :ref:`scrapy.cfg <topics-config-settings>`::
+variable; or by defining it in your :ref:`pyproject.toml <topics-config-settings>`:
 
-    [settings]
-    shell = bpython
+.. code-block:: toml
+
+    [tool.scrapy.settings]
+    shell = "bpython"
 
 .. _IPython: https://ipython.org/
 .. _IPython installation guide: https://ipython.org/install/

pyproject.toml

@@ -21,6 +21,7 @@ dependencies = [
     "queuelib>=1.4.2",
     "service_identity>=23.1.0",
     "tldextract",
+    "tomli>=1.1.0; python_version < '3.11'",
     "w3lib>=1.17.0",
     "zope.interface>=5.1.0",
     # Platform-specific dependencies

scrapy/commands/startproject.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import re
 import string
+import sys
 from importlib.util import find_spec
 from pathlib import Path
 from shutil import copy2, copystat, ignore_patterns, move
@@ -16,15 +17,22 @@ from scrapy.utils.template import render_templatefile, string_camelcase
 if TYPE_CHECKING:
     import argparse
 
+
+if sys.version_info >= (3, 11):
+    import tomllib
+else:
+    import tomli as tomllib
+
+
 TEMPLATES_TO_RENDER: tuple[tuple[str, ...], ...] = (
-    ("scrapy.cfg",),
+    ("pyproject.toml",),
     ("${project_name}", "settings.py.tmpl"),
     ("${project_name}", "items.py.tmpl"),
     ("${project_name}", "pipelines.py.tmpl"),
     ("${project_name}", "middlewares.py.tmpl"),
 )
 
-IGNORE = ignore_patterns("*.pyc", "__pycache__", ".svn")
+IGNORE = ignore_patterns("*.pyc", "__pycache__", ".svn", "scrapy.cfg")
 
 
 def _make_writable(path: Path) -> None:
@@ -102,6 +110,17 @@ class Command(ScrapyCommand):
             print(f"Error: scrapy.cfg already exists in {project_dir.resolve()}")
             return
 
+        if (project_dir / "pyproject.toml").exists():
+            with (project_dir / "pyproject.toml").open("rb") as f:
+                data = tomllib.load(f)
+            if data.get("tool", {}).get("scrapy"):
+                self.exitcode = 1
+                print(
+                    f"Error: pyproject.toml with [tool.scrapy] already exists "
+                    f"in {project_dir.resolve()}"
+                )
+                return
+
         if not self._is_valid_name(project_name):
             self.exitcode = 1
             return

scrapy/templates/project/pyproject.toml

@@ -0,0 +1,11 @@
+# Automatically created by: scrapy startproject
+#
+# For more information about the [tool.scrapy.deploy] section see:
+# https://scrapyd.readthedocs.io/en/latest/deploy.html
+
+[tool.scrapy.settings]
+default = "${project_name}.settings"
+
+[tool.scrapy.deploy]
+# url = "http://localhost:6800/"
+project = "${project_name}"

scrapy/utils/conf.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import numbers
 import os
 import sys
+import warnings
 from configparser import ConfigParser
 from operator import itemgetter
 from pathlib import Path
@@ -16,6 +17,11 @@ from scrapy.utils.python import without_none_values
 if TYPE_CHECKING:
     from collections.abc import Callable, Collection, Iterable, Mapping, MutableMapping
 
+if sys.version_info >= (3, 11):
+    import tomllib
+else:
+    import tomli as tomllib
+
 
 def build_component_list(
     compdict: MutableMapping[Any, Any],
@@ -70,6 +76,40 @@ def arglist_to_dict(arglist: list[str]) -> dict[str, str]:
     return dict(x.split("=", 1) for x in arglist)
 
 
+def closest_config(
+    path: str | os.PathLike = ".", _resolved: Path | None = None
+) -> tuple[str, str]:
+    """
+    Returns the closest config file (pyproject.toml or scrapy.cfg) by
+    traversing the current directory and its parents
+    """
+    current = Path(path).resolve() if _resolved is None else _resolved
+
+    toml = current / "pyproject.toml"
+    if toml.exists():
+        try:
+            with toml.open("rb") as f:
+                data = tomllib.load(f)
+            if data.get("tool", {}).get("scrapy"):
+                return "toml", str(toml)
+        except tomllib.TOMLDecodeError:
+            warnings.warn(
+                f"Scrapy could not parse {toml}: invalid TOML. "
+                "This file will be ignored.",
+                stacklevel=2,
+            )
+
+    cfgfile = current / "scrapy.cfg"
+    if cfgfile.exists():
+        return "cfg", str(cfgfile)
+
+    parent = current.parent
+    if parent == current:
+        return "", ""
+
+    return closest_config(_resolved=parent)
+
+
 def closest_scrapy_cfg(
     path: str | os.PathLike[str] = ".",
     prevpath: str | os.PathLike[str] | None = None,
@@ -94,7 +134,8 @@ def init_env(project: str = "default", set_syspath: bool = True) -> None:
     cfg = get_config()
     if cfg.has_option("settings", project):
         os.environ["SCRAPY_SETTINGS_MODULE"] = cfg.get("settings", project)
-    closest = closest_scrapy_cfg()
+
+    _, closest = closest_config()
     if closest:
         projdir = str(Path(closest).parent)
         if set_syspath and projdir not in sys.path:
@@ -103,25 +144,41 @@ def init_env(project: str = "default", set_syspath: bool = True) -> None:
 
 def get_config(use_closest: bool = True) -> ConfigParser:
     """Get Scrapy config file as a ConfigParser"""
-    sources = get_sources(use_closest)
+    if use_closest:
+        config_type, config_path = closest_config()
+        if config_type == "toml":
+            with Path(config_path).open("rb") as f:
+                data = tomllib.load(f)
+            scrapy_data = data.get("tool", {}).get("scrapy", {})
+            cfg = ConfigParser()
+            cfg.read_dict(scrapy_data)
+            return cfg
+        if config_type == "cfg":
+            warnings.warn(
+                "scrapy.cfg is deprecated. Please use pyproject.toml instead.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+            cfg = ConfigParser()
+            cfg.read(config_path)
+            return cfg
+
+    sources = get_sources()
     cfg = ConfigParser()
     cfg.read(sources)
     return cfg
 
 
-def get_sources(use_closest: bool = True) -> list[str]:
+def get_sources() -> list[str]:
     xdg_config_home = (
         os.environ.get("XDG_CONFIG_HOME") or Path("~/.config").expanduser()
     )
-    sources = [
+    return [
         "/etc/scrapy.cfg",
         r"c:\scrapy\scrapy.cfg",
         str(Path(xdg_config_home) / "scrapy.cfg"),
         str(Path("~/.scrapy.cfg").expanduser()),
     ]
-    if use_closest:
-        sources.append(closest_scrapy_cfg())
-    return sources
 
 
 def feed_complete_default_values_from_settings(

scrapy/utils/project.py

@@ -7,7 +7,7 @@ from pathlib import Path
 
 from scrapy.exceptions import NotConfigured
 from scrapy.settings import Settings
-from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env
+from scrapy.utils.conf import closest_config, get_config, init_env
 
 ENVVAR = "SCRAPY_SETTINGS_MODULE"
 DATADIR_CFG_SECTION = "datadir"
@@ -25,23 +25,24 @@ def inside_project() -> bool:
             )
         else:
             return True
-    return bool(closest_scrapy_cfg())
+    _, closest = closest_config()
+    return bool(closest)
 
 
 def project_data_dir(project: str = "default") -> str:
     """Return the current project data dir, creating it if it doesn't exist"""
     if not inside_project():
         raise NotConfigured("Not inside a project")
+    _, project_cfg = closest_config()
     cfg = get_config()
     if cfg.has_option(DATADIR_CFG_SECTION, project):
         d = Path(cfg.get(DATADIR_CFG_SECTION, project))
     else:
-        scrapy_cfg = closest_scrapy_cfg()
-        if not scrapy_cfg:
+        if not project_cfg:
             raise NotConfigured(
-                "Unable to find scrapy.cfg file to infer project data dir"
+                "Unable to find scrapy.cfg or pyproject.toml to infer project data dir"
             )
-        d = (Path(scrapy_cfg).parent / ".scrapy").resolve()
+        d = (Path(project_cfg).parent / ".scrapy").resolve()
     if not d.exists():
         d.mkdir(parents=True)
     return str(d)

tests/test_command_startproject.py

@@ -20,7 +20,8 @@ class TestStartprojectCommand:
 
     @staticmethod
     def _assert_files_exist(project_dir: Path, project_name: str) -> None:
-        assert (project_dir / "scrapy.cfg").exists()
+        assert (project_dir / "pyproject.toml").exists()
+        assert not (project_dir / "scrapy.cfg").exists()
         assert (project_dir / project_name).exists()
         assert (project_dir / project_name / "__init__.py").exists()
         assert (project_dir / project_name / "items.py").exists()
@@ -77,6 +78,20 @@ class TestStartprojectCommand:
         assert call("startproject", project_name, cwd=tmp_path) == 0
         self._assert_files_exist(project_path, project_name)
 
+    def test_startproject_with_existing_pyproject_toml(self, tmp_path: Path) -> None:
+        # startproject should fail if a pyproject.toml with [tool.scrapy] already exists
+        (tmp_path / "pyproject.toml").write_text(
+            "[tool.scrapy.settings]\ndefault = 'myproject.settings'\n"
+        )
+        assert call("startproject", self.project_name, cwd=tmp_path) == 1
+
+    def test_startproject_with_unrelated_pyproject_toml(self, tmp_path: Path) -> None:
+        # a pyproject.toml without [tool.scrapy] should not block startproject
+        (tmp_path / "pyproject.toml").write_text("[tool.black]\nline-length = 88\n")
+        project_dir = tmp_path / self.project_name
+        assert call("startproject", self.project_name, cwd=tmp_path) == 0
+        self._assert_files_exist(project_dir, self.project_name)
+
 
 def get_permissions_dict(
     path: str | os.PathLike[str], renamings=None, ignore=None

tests/test_utils_conf.py

@@ -1,3 +1,5 @@
+import warnings
+
 import pytest
 
 from scrapy.exceptions import UsageError
@@ -5,8 +7,10 @@ from scrapy.settings import BaseSettings, Settings
 from scrapy.utils.conf import (
     arglist_to_dict,
     build_component_list,
+    closest_config,
     feed_complete_default_values_from_settings,
     feed_process_params_from_cli,
+    get_config,
 )
 
 
@@ -54,6 +58,57 @@ def test_arglist_to_dict():
     }
 
 
+class TestPyprojectToml:
+    def test_multiple_projects(self, tmp_path, monkeypatch):
+        (tmp_path / "pyproject.toml").write_text(
+            "[tool.scrapy.settings]\n"
+            'default = "myproject1.settings"\n'
+            'project1 = "myproject1.settings"\n'
+            'project2 = "myproject2.settings"\n'
+        )
+        monkeypatch.chdir(tmp_path)
+        cfg = get_config()
+        assert cfg.get("settings", "default") == "myproject1.settings"
+        assert cfg.get("settings", "project1") == "myproject1.settings"
+        assert cfg.get("settings", "project2") == "myproject2.settings"
+
+    def test_pyproject_toml_takes_precedence_over_scrapy_cfg(
+        self, tmp_path, monkeypatch
+    ):
+        (tmp_path / "scrapy.cfg").write_text("[settings]\ndefault = from_scrapy_cfg\n")
+        (tmp_path / "pyproject.toml").write_text(
+            "[tool.scrapy.settings]\ndefault = 'from_pyproject_toml'\n"
+        )
+        monkeypatch.chdir(tmp_path)
+        cfg = get_config()
+        assert cfg.get("settings", "default") == "from_pyproject_toml"
+
+    def test_scrapy_cfg_not_read_when_pyproject_toml_present(
+        self, tmp_path, monkeypatch
+    ):
+        (tmp_path / "scrapy.cfg").write_text(
+            "[settings]\ndefault = from_scrapy_cfg\n"
+            "[deploy]\nproject = from_scrapy_cfg\n"
+        )
+        (tmp_path / "pyproject.toml").write_text(
+            "[tool.scrapy.settings]\ndefault = 'from_pyproject_toml'\n"
+        )
+        monkeypatch.chdir(tmp_path)
+        cfg = get_config()
+        assert not cfg.has_section("deploy")
+
+    def test_malformed_toml_warns_and_falls_back(self, tmp_path, monkeypatch):
+        (tmp_path / "pyproject.toml").write_text("[tool.scrapy\nnot valid toml")
+        (tmp_path / "scrapy.cfg").write_text("[settings]\ndefault = from_scrapy_cfg\n")
+        monkeypatch.chdir(tmp_path)
+        with warnings.catch_warnings(record=True) as w:
+            warnings.simplefilter("always")
+            config_type, _ = closest_config()
+        assert config_type == "cfg"
+        assert len(w) == 1
+        assert "invalid TOML" in str(w[0].message)
+
+
 class TestFeedExportConfig:
     def test_feed_export_config_invalid_format(self):
         settings = Settings()