RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update comfig file searching logic and docs

This commit is contained in:
Aiman 2026-03-22 22:52:41 +05:30
parent 97862e765e
commit 83f83ef823
6 changed files with 106 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
docs/topics/commands.rst
View File

@ -27,7 +27,17 @@ in standard locations:
1. ``/etc/scrapy.cfg`` or ``c:\scrapy\scrapy.cfg`` (system-wide),
2. ``~/.config/scrapy.cfg`` (``$XDG_CONFIG_HOME``) and ``~/.scrapy.cfg`` (``$HOME``)
for global (user-wide) settings, and
3. ``scrapy.cfg`` inside a Scrapy project's root (see next section).
3. ``pyproject.toml`` (recommended) or ``scrapy.cfg`` inside a Scrapy project's root (see next section).
For ``pyproject.toml``, the settings are read from the ``[tool.scrapy]`` section:
.. code-block:: toml
[tool.scrapy.settings]
default = "myproject.settings"
Scrapy searches for the closest ``pyproject.toml`` or ``scrapy.cfg``, and if both
files exist, ``pyproject.toml`` takes precedence.
Settings from these files are merged in the listed order of preference:
user-defined values have higher priority than system-wide defaults
@ -51,7 +61,7 @@ understand the directory structure of a Scrapy project.
Though it can be modified, all Scrapy projects have the same file
structure by default, similar to this::
scrapy.cfg
pyproject.toml
myproject/
__init__.py
items.py
@ -64,32 +74,32 @@ structure by default, similar to this::
spider2.py
...
The directory where the ``scrapy.cfg`` file resides is known as the *project
The directory where the ``pyproject.toml`` file resides is known as the *project
root directory*. That file contains the name of the python module that defines
the project settings. Here is an example:
.. code-block:: ini
.. code-block:: toml
[settings]
default = myproject.settings
[tool.scrapy.settings]
default = "myproject.settings"
.. _topics-project-envvar:
Sharing the root directory between projects
===========================================
A project root directory, the one that contains the ``scrapy.cfg``, may be
A project root directory, the one that contains the ``pyproject.toml``, may be
shared by multiple Scrapy projects, each with its own settings module.
In that case, you must define one or more aliases for those settings modules
under ``[settings]`` in your ``scrapy.cfg`` file:
under ``[tool.scrapy.settings]`` in your ``pyproject.toml`` file:
.. code-block:: ini
.. code-block:: toml
[settings]
default = myproject1.settings
project1 = myproject1.settings
project2 = myproject2.settings
[tool.scrapy.settings]
default = "myproject1.settings"
project1 = "myproject1.settings"
project2 = "myproject2.settings"
By default, the ``scrapy`` command-line tool will use the ``default`` settings.
Use the ``SCRAPY_PROJECT`` environment variable to specify a different project

11
docs/topics/shell.rst
View File

@ -34,10 +34,17 @@ is unavailable.
Through Scrapy's settings you can configure it to use any one of
``ipython``, ``bpython`` or the standard ``python`` shell, regardless of which
are installed. This is done by setting the ``SCRAPY_PYTHON_SHELL`` environment
variable; or by defining it in your :ref:`scrapy.cfg <topics-config-settings>`::
variable; or by defining it in your :ref:`pyproject.toml <topics-config-settings>`:
.. code-block:: toml
[tool.scrapy.settings]
shell = "bpython"
or in ``scrapy.cfg``::
[settings]
shell = bpython
shell = "bpython"
.. _IPython: https://ipython.org/
.. _IPython installation guide: https://ipython.org/install.html

1
pyproject.toml
View File

@ -27,6 +27,7 @@ dependencies = [
# Platform-specific dependencies
'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"',
'PyPyDispatcher>=2.1.0; platform_python_implementation == "PyPy"',
"tomli>=1.1.0; python_version < '3.11'",
]
classifiers = [
"Development Status :: 5 - Production/Stable",

103
scrapy/utils/conf.py
View File

@ -20,10 +20,7 @@ if TYPE_CHECKING:
if sys.version_info >= (3, 11):
import tomllib
else:
try:
import tomli as tomllib
except ImportError:
tomllib = None # type: ignore[assignment]
import tomli as tomllib
def build_component_list(
@ -79,6 +76,40 @@ def arglist_to_dict(arglist: list[str]) -> dict[str, str]:
return dict(x.split("=", 1) for x in arglist)
def closest_config(
path: str | os.PathLike = ".", _resolved: Path | None = None
) -> tuple[str, str]:
"""
Returns the closest config file (pyproject.toml or scrapy.cfg) by
traversing the current directory and its parents
"""
current = Path(path).resolve() if _resolved is None else _resolved
toml = current / "pyproject.toml"
if toml.exists():
try:
with toml.open("rb") as f:
data = tomllib.load(f)
if data.get("tool", {}).get("scrapy"):
return "toml", str(toml)
except tomllib.TOMLDecodeError:
warnings.warn(
f"Scrapy could not parse {toml}: invalid TOML. "
"This file will be ignored.",
stacklevel=2,
)
cfgfile = current / "scrapy.cfg"
if cfgfile.exists():
return "cfg", str(cfgfile)
parent = current.parent
if parent == current:
return "", ""
return closest_config(_resolved=parent)
def closest_scrapy_cfg(
path: str | os.PathLike = ".",
prevpath: str | os.PathLike | None = None,
@ -95,31 +126,6 @@ def closest_scrapy_cfg(
return closest_scrapy_cfg(path.parent, path)
def closest_pyproject_toml(
path: str | os.PathLike = ".",
prevpath: str | os.PathLike | None = None,
) -> str:
if tomllib is None:
return ""
if prevpath is not None and str(path) == str(prevpath):
return ""
path = Path(path).resolve()
toml_path = path / "pyproject.toml"
if toml_path.exists():
try:
with toml_path.open("rb") as f:
data = tomllib.load(f)
if "scrapy" in data.get("tool", {}):
return str(toml_path)
except tomllib.TOMLDecodeError:
warnings.warn(
f"Scrapy could not parse {toml_path}: invalid TOML. "
"This file will be ignored.",
stacklevel=2,
)
return closest_pyproject_toml(path.parent, path)
def init_env(project: str = "default", set_syspath: bool = True) -> None:
"""Initialize environment to use command-line tool from inside a project
dir. This sets the Scrapy settings module and modifies the Python path to
@ -128,49 +134,46 @@ def init_env(project: str = "default", set_syspath: bool = True) -> None:
cfg = get_config()
if cfg.has_option("settings", project):
os.environ["SCRAPY_SETTINGS_MODULE"] = cfg.get("settings", project)
closest = closest_pyproject_toml() or closest_scrapy_cfg()
_, closest = closest_config()
if closest:
projdir = str(Path(closest).parent)
if set_syspath and projdir not in sys.path:
sys.path.append(projdir)
def _load_toml_config(toml_path: str) -> ConfigParser:
"""Load Scrapy config from a TOML file."""
with Path(toml_path).open("rb") as f:
data = tomllib.load(f)
scrapy_data = data.get("tool", {}).get("scrapy", {})
cfg = ConfigParser()
cfg.read_dict(scrapy_data)
return cfg
def get_config(use_closest: bool = True) -> ConfigParser:
"""Get Scrapy config file as a ConfigParser"""
if use_closest and tomllib is not None:
toml_path = closest_pyproject_toml()
if toml_path:
return _load_toml_config(toml_path)
if use_closest:
config_type, config_path = closest_config()
if config_type == "toml":
with Path(config_path).open("rb") as f:
data = tomllib.load(f)
scrapy_data = data.get("tool", {}).get("scrapy", {})
cfg = ConfigParser()
cfg.read_dict(scrapy_data)
return cfg
if config_type == "cfg":
cfg = ConfigParser()
cfg.read(config_path)
return cfg
sources = get_sources(use_closest)
sources = get_sources()
cfg = ConfigParser()
cfg.read(sources)
return cfg
def get_sources(use_closest: bool = True) -> list[str]:
def get_sources() -> list[str]:
xdg_config_home = (
os.environ.get("XDG_CONFIG_HOME") or Path("~/.config").expanduser()
)
sources = [
return [
"/etc/scrapy.cfg",
r"c:\scrapy\scrapy.cfg",
str(Path(xdg_config_home) / "scrapy.cfg"),
str(Path("~/.scrapy.cfg").expanduser()),
]
if use_closest:
sources.append(closest_scrapy_cfg())
return sources
def feed_complete_default_values_from_settings(

16
scrapy/utils/project.py
View File

@ -7,12 +7,7 @@ from pathlib import Path
from scrapy.exceptions import NotConfigured
from scrapy.settings import Settings
from scrapy.utils.conf import (
closest_pyproject_toml,
closest_scrapy_cfg,
get_config,
init_env,
)
from scrapy.utils.conf import closest_config, get_config, init_env
ENVVAR = "SCRAPY_SETTINGS_MODULE"
DATADIR_CFG_SECTION = "datadir"
@ -29,23 +24,24 @@ def inside_project() -> bool:
)
else:
return True
return bool(closest_scrapy_cfg() or closest_pyproject_toml())
_, closest = closest_config()
return bool(closest)
def project_data_dir(project: str = "default") -> str:
"""Return the current project data dir, creating it if it doesn't exist"""
if not inside_project():
raise NotConfigured("Not inside a project")
_, project_cfg = closest_config()
cfg = get_config()
if cfg.has_option(DATADIR_CFG_SECTION, project):
d = Path(cfg.get(DATADIR_CFG_SECTION, project))
else:
scrapy_cfg = closest_scrapy_cfg() or closest_pyproject_toml()
if not scrapy_cfg:
if not project_cfg:
raise NotConfigured(
"Unable to find scrapy.cfg or pyproject.toml to infer project data dir"
)
d = (Path(scrapy_cfg).parent / ".scrapy").resolve()
d = (Path(project_cfg).parent / ".scrapy").resolve()
if not d.exists():
d.mkdir(parents=True)
return str(d)

24
tests/test_utils_conf.py
View File

@ -1,4 +1,3 @@
import os
import warnings
import pytest
@ -8,7 +7,7 @@ from scrapy.settings import BaseSettings, Settings
from scrapy.utils.conf import (
arglist_to_dict,
build_component_list,
closest_pyproject_toml,
closest_config,
feed_complete_default_values_from_settings,
feed_process_params_from_cli,
get_config,
@ -60,16 +59,20 @@ def test_arglist_to_dict():
class TestPyprojectToml:
def test_pyproject_toml_takes_precedence_over_scrapy_cfg(self, tmp_path):
def test_pyproject_toml_takes_precedence_over_scrapy_cfg(
self, tmp_path, monkeypatch
):
(tmp_path / "scrapy.cfg").write_text("[settings]\ndefault = from_scrapy_cfg\n")
(tmp_path / "pyproject.toml").write_text(
"[tool.scrapy.settings]\ndefault = 'from_pyproject_toml'\n"
)
os.chdir(tmp_path)
monkeypatch.chdir(tmp_path)
cfg = get_config()
assert cfg.get("settings", "default") == "from_pyproject_toml"
def test_scrapy_cfg_not_read_when_pyproject_toml_present(self, tmp_path):
def test_scrapy_cfg_not_read_when_pyproject_toml_present(
self, tmp_path, monkeypatch
):
(tmp_path / "scrapy.cfg").write_text(
"[settings]\ndefault = from_scrapy_cfg\n"
"[deploy]\nproject = from_scrapy_cfg\n"
@ -77,17 +80,18 @@ class TestPyprojectToml:
(tmp_path / "pyproject.toml").write_text(
"[tool.scrapy.settings]\ndefault = 'from_pyproject_toml'\n"
)
os.chdir(tmp_path)
monkeypatch.chdir(tmp_path)
cfg = get_config()
assert not cfg.has_section("deploy")
def test_malformed_toml_warns_and_returns_empty(self, tmp_path):
def test_malformed_toml_warns_and_falls_back(self, tmp_path, monkeypatch):
(tmp_path / "pyproject.toml").write_text("[tool.scrapy\nnot valid toml")
os.chdir(tmp_path)
(tmp_path / "scrapy.cfg").write_text("[settings]\ndefault = from_scrapy_cfg\n")
monkeypatch.chdir(tmp_path)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
result = closest_pyproject_toml()
assert result == ""
config_type, _ = closest_config()
assert config_type == "cfg"
assert len(w) == 1
assert "invalid TOML" in str(w[0].message)