diff --git a/hermes_cli/doctor.py b/hermes_cli/doctor.py index 981b885bfea43..d02b600f672b2 100644 --- a/hermes_cli/doctor.py +++ b/hermes_cli/doctor.py @@ -213,18 +213,36 @@ def _check_managed_runtimes() -> None: them rather than re-deriving existence by probing paths (hermes-home lifetime split, phase 3.10). Tools the registry does not know about are still reported from PATH so a system copy is visible. + + Drift is a WARNING in a git checkout and an ERROR in a sealed tree. + A checkout heals itself on the next `hermes update`; a sealed tree + cannot provision anything, so a mismatch there means the artifact was + built against a different pin table than the code it ships, and only + its steward can fix it. """ try: from hermes_cli.runtime_registry import load_facts, load_pins - from hermes_constants import get_runtime_dir + from hermes_cli.runtime_tree import Sealed, runtime_tree + from hermes_constants import get_install_root, get_runtime_dir runtime_dir = get_runtime_dir() pins = load_pins() facts = load_facts(runtime_dir) + tree = runtime_tree(get_install_root()) except Exception as exc: check_warn("Managed runtimes unreadable", f"({exc})") return + sealed = isinstance(tree, Sealed) + # What a user can actually do about it differs per install kind: + # a checkout runs an update, a sealed tree rebuilds with its steward. + remedy = ( + f"(rebuild this {tree.steward} artifact — a sealed install cannot provision)" + if sealed + else "(reprovisioned on the next 'hermes update')" + ) + report_drift = check_fail if sealed else check_warn + for tool, pin in pins.items(): fact = facts.get(tool) if fact is None: @@ -232,23 +250,20 @@ def _check_managed_runtimes() -> None: if system: check_ok(f"{tool} (system)", f"at {system}") else: - check_warn( - f"{tool} not provisioned", - "(installed on the next 'hermes update')", - ) + report_drift(f"{tool} not provisioned", remedy) continue if not (runtime_dir / fact.path).is_file(): - check_warn( + report_drift( f"{tool} recorded but missing", - "(runtime dir was modified; 'hermes update' reinstalls it)", + "(runtime dir was modified) " + remedy if not sealed else remedy, ) continue # Pins are exact, so this is equality: anything else means the # pin moved and this install has not caught up yet. if fact.version != pin["version"]: - check_warn( + report_drift( f"{tool} {fact.version} does not match the pin {pin['version']}", - "(reprovisioned on the next 'hermes update')", + remedy, ) continue check_ok(f"{tool} {fact.version}", "(managed)") diff --git a/hermes_cli/runtime-pins.json b/hermes_cli/runtime-pins.json new file mode 120000 index 0000000000000..055329decfd7d --- /dev/null +++ b/hermes_cli/runtime-pins.json @@ -0,0 +1 @@ +../runtime-pins.json \ No newline at end of file diff --git a/hermes_cli/runtime_provisioner.py b/hermes_cli/runtime_provisioner.py index b67d6eeb7942b..8daa354a337dd 100644 --- a/hermes_cli/runtime_provisioner.py +++ b/hermes_cli/runtime_provisioner.py @@ -41,7 +41,8 @@ from dataclasses import dataclass from pathlib import Path from typing import Callable, Optional -from hermes_constants import get_runtime_dir +from hermes_constants import get_install_root, get_runtime_dir +from hermes_cli.runtime_tree import Sealed, runtime_tree from hermes_cli.runtime_registry import ( PinnedFile, RuntimeFact, @@ -530,6 +531,80 @@ def provision_runtimes( return results +def stale_tools( + runtime_dir: Path | None = None, + install_root: Path | None = None, + target: str | None = None, +) -> dict[str, tuple[str, Optional[str]]]: + """Pinned tools whose installed state does not match the pin table. + + Maps tool → (pinned version, installed version or None). Empty means + every pin is satisfied. This is the same equality check + ``_provision_one`` makes before deciding to re-download — exact pins + make it an equality check, not a range check. + """ + rt = runtime_dir if runtime_dir is not None else get_runtime_dir() + resolved_target = target or current_target() + facts = load_facts(rt) + drift: dict[str, tuple[str, Optional[str]]] = {} + + for tool, entry in load_pins(install_root).items(): + fact = facts.get(tool) + installed = fact.version if fact is not None else None + if fact is not None and not (rt / _binary_rel(tool, resolved_target)).is_file(): + # Recorded but vanished reads as unprovisioned everywhere + # else; say so here too rather than reporting it as current. + installed = None + if installed != entry["version"]: + drift[tool] = (entry["version"], installed) + return drift + + +class StaleManagedRuntimes(RuntimeError): + """A sealed install's runtime tools disagree with its pin table.""" + + +def require_current_runtimes( + project_root: Path | None = None, + runtime_dir: Path | None = None, + install_root: Path | None = None, +) -> None: + """Fail fast when a SEALED install ships out-of-date runtime tools. + + A git checkout provisions on demand: drift there is a normal state + that the next `hermes update` (or the self-heal path) resolves, and + raising would break the very run that fixes it. + + A sealed tree cannot self-heal. Its steward — Nix, Docker, the + desktop bundle — builds the runtime tools as part of the artifact, so + drift means the artifact was assembled against a different pin table + than the code it ships. Every consequence of that is worse and more + confusing than stopping here: tools silently missing from PATH, + or a version the code does not expect. The steward has to rebuild. + """ + root = project_root if project_root is not None else get_install_root() + tree = runtime_tree(root) + if not isinstance(tree, Sealed): + return + + drift = stale_tools(runtime_dir=runtime_dir, install_root=install_root) + if not drift: + return + + lines = [ + f" {tool}: pinned {pinned}, installed {installed or 'nothing'}" + for tool, (pinned, installed) in sorted(drift.items()) + ] + raise StaleManagedRuntimes( + f"This Hermes is a sealed install managed by {tree.steward!r}, and its " + "managed runtime tools do not match runtime-pins.json:\n" + + "\n".join(lines) + + "\n\nThe artifact was built against a different pin table than the code " + "it ships. Rebuild it with its steward — a sealed tree cannot provision " + "these itself." + ) + + def step_provision_runtimes() -> dict: """post_update MACHINE_STEPS entry.""" results = provision_runtimes() diff --git a/hermes_cli/runtime_registry.py b/hermes_cli/runtime_registry.py index f2590b0212fbf..dc67c19c1a784 100644 --- a/hermes_cli/runtime_registry.py +++ b/hermes_cli/runtime_registry.py @@ -124,10 +124,20 @@ def pins_path(install_root: Path | None = None) -> Path: repo root for a checkout, the payload's repo/ dir for the desktop bundle) rather than ``get_install_root()`` — the install root is where tools get INSTALLED, and callers may point it elsewhere. + + A wheel has no repo root: `pip install .` lays out site-packages with + no parent-level data files, so the table is also packaged INSIDE + hermes_cli (see the package-data entry in pyproject.toml, fed by a + symlink to the root file so there is still one table). Sealed venv + installs — uv2nix, Docker, the desktop payload's site-packages — read + that copy; nothing else changes. """ if install_root is not None: return install_root / PINS_FILENAME - return Path(__file__).resolve().parent.parent / PINS_FILENAME + repo_copy = Path(__file__).resolve().parent.parent / PINS_FILENAME + if repo_copy.is_file(): + return repo_copy + return Path(__file__).resolve().parent / PINS_FILENAME # Loopback http is allowed so tests can serve real archives from a local diff --git a/nix/checks.nix b/nix/checks.nix index 1cf34212597ab..c886d2f593d8e 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -36,6 +36,128 @@ json.dump(sorted(leaf_paths(DEFAULT_CONFIG)), sys.stdout, indent=2) packages.configKeys = configKeys; checks = { + # The Nix runtime dir IS the pin table, not a copy of it. Before + # this, nix/npm-12-0-2.nix and runtime-pins.json pinned the same + # npm independently: two files to bump, and a devShell that could + # silently ship a different npm than every user's install. + # + # Asserting the relationship rather than the values means a pin + # bump needs no edit here — and the artifact this builds is the + # same runtime dir Hermes provisions, read by the same registry + # code, so a drift would fail `hermes doctor` on a nix install. + runtime-pins-are-the-source = + let + pins = (builtins.fromJSON (builtins.readFile ../runtime-pins.json)).tools; + runtimeDir = pkgs.callPackage ./runtime-pins.nix { }; + mismatched = lib.filterAttrs ( + name: entry: runtimeDir.${name}.pinnedVersion != entry.version + ) pins; + in + pkgs.runCommand "hermes-runtime-pins-source" { } ( + if mismatched != { } then + throw "Nix runtime tools disagree with runtime-pins.json: ${ + toString (builtins.attrNames mismatched) + }" + else + '' + echo "PASS: ${toString (builtins.length (builtins.attrNames pins))} tools built from runtime-pins.json" + mkdir -p $out && echo ok > $out/result + '' + ); + + # The nix-built runtime dir satisfies the Python readers with no + # nix-specific code: `hermes_cli.runtime_env` assembles PATH from + # its facts, every pinned tool runs (patchelf'd), and each reports + # the version the table pinned. That is the whole contract — if + # this passes, a nix install needs no special handling anywhere. + runtime-dir-serves-python = + let + runtimeDir = pkgs.callPackage ./runtime-pins.nix { }; + pins = (builtins.fromJSON (builtins.readFile ../runtime-pins.json)).tools; + # "How do I ask your version" is genuinely per-tool; the TOOLS + # come from the table, so a new pin surfaces here as a missing + # probe rather than going silently unchecked. + # + # npm is resolved from PATH but launched through node on + # purpose: its shim is `#!/usr/bin/env node`, and the Nix + # build sandbox has no /usr/bin/env (exit 126). That is a + # sandbox artifact, not a property of the artifact under test + # — `command -v npm` still proves the assembled PATH resolves + # to the pinned npm rather than node's bundled copy. + probes = { + node = "node --version"; + npm = "node \"$(command -v npm)\" --version"; + uv = "uv --version"; + git = "git --version"; + gh = "gh --version"; + ripgrep = "rg --version"; + }; + probeFor = + name: + probes.${name} + or (throw "runtime-dir-serves-python: no version probe for pinned tool '${name}'"); + in + pkgs.runCommand "hermes-runtime-dir-serves-python" + { + nativeBuildInputs = [ hermesVenv ]; + } + '' + set -euo pipefail + export HOME=$TMPDIR + + # PATH comes from the facts file, via the same assembler + # every Hermes subprocess uses. No hand-built PATH here: + # that would test a PATH this check invented. It is seeded + # with the build PATH because the assembler PREPENDS (that + # is the behaviour under test) and the script still needs + # coreutils afterwards. + eval "$(${hermesVenv}/bin/python3 - <<'PY' + import os + from pathlib import Path + from hermes_cli.runtime_env import with_managed_runtimes + + env = with_managed_runtimes( + {"PATH": os.environ["PATH"]}, runtime_dir=Path("${runtimeDir}") + ) + for key in ("PATH", "GIT_EXEC_PATH", "GIT_SSL_CAINFO", "npm_config_cache"): + if env.get(key): + print(f"export {key}={env[key]!r}") + PY + )" + + ${lib.concatStringsSep "\n" ( + lib.mapAttrsToList (name: entry: '' + echo "== ${name} (pinned ${entry.version})" + got=$(${probeFor name} 2>&1 | head -1) + echo " $got" + case "$got" in + *${entry.version}*) ;; + *) echo "expected ${entry.version} from ${name}"; exit 1 ;; + esac + '') pins + )} + + # And the sealed-install gate agrees this dir is current. + ${hermesVenv}/bin/python3 - <<'PY' + import json, tempfile + from pathlib import Path + from hermes_cli.runtime_provisioner import require_current_runtimes, stale_tools + + runtime_dir = Path("${runtimeDir}") + drift = stale_tools(runtime_dir=runtime_dir) + assert not drift, f"nix runtime dir is stale against its own pins: {drift}" + + with tempfile.TemporaryDirectory() as td: + sealed = Path(td) + (sealed / "install-stamp.json").write_text(json.dumps({"distribution": "nix"})) + require_current_runtimes(project_root=sealed, runtime_dir=runtime_dir) + print("sealed-install gate: current") + PY + + mkdir -p $out && echo ok > $out/result + ''; + + # Cross-platform evaluation — catches "not supported for interpreter" # errors (e.g. sphinx dropping python311) without needing a darwin builder. # Evaluation is pure and instant; it doesn't build anything. diff --git a/nix/lib.nix b/nix/lib.nix index 25c34f4ce8cea..b66138eea0d82 100644 --- a/nix/lib.nix +++ b/nix/lib.nix @@ -32,10 +32,21 @@ let repoRoot = ./..; - npm12 = callPackage ./npm-12-0-2.nix { }; + # The managed runtime tools come from the repo's pin table, not from a + # version and digest restated here. runtime-pins.json is what a source + # install, `hermes update` and the desktop payload all provision from, + # so the devShell and every nix build get the same tools those users + # get. npm used to be pinned twice (nix/npm-12-0-2.nix vs the table), + # which is two places to bump and one silent skew when someone bumps + # only one. + runtimeDir = callPackage ./runtime-pins.nix { }; + npm12 = runtimeDir.npm; node_gyp_11_4_0 = callPackage ./node-gyp-11-4-0.nix { }; nodejs_26_npm_12 = symlinkJoin { name = "nodejs-26-npm-12"; + # npm FIRST: it supersedes the npm bundled inside node, and + # symlinkJoin resolves a collision in favour of the earlier path. + # The pin table states that relationship as npm's `extends: [node]`. paths = [ npm12 nodejs_26 diff --git a/nix/npm-12-0-2.nix b/nix/npm-12-0-2.nix deleted file mode 100644 index a583b117970a4..0000000000000 --- a/nix/npm-12-0-2.nix +++ /dev/null @@ -1,29 +0,0 @@ -{ - stdenv, - makeWrapper, - fetchurl, - nodejs_26, -}: -stdenv.mkDerivation rec { - pname = "npm"; - version = "12.0.2"; - - src = fetchurl { - url = "https://registry.npmjs.org/npm/-/npm-${version}.tgz"; - hash = "sha256-XbuGxx0HoZV/LpBzQJLdali9zZ68LY1ByhxuaiHTZOE="; - }; - - nativeBuildInputs = [ makeWrapper ]; - dontBuild = true; - - installPhase = '' - mkdir -p $out/lib/npm12 - cp -r . $out/lib/npm12/ - mkdir -p $out/bin - - makeWrapper ${nodejs_26}/bin/node $out/bin/npm \ - --add-flags "$out/lib/npm12/bin/npm-cli.js" - makeWrapper ${nodejs_26}/bin/node $out/bin/npx \ - --add-flags "$out/lib/npm12/bin/npx-cli.js" - ''; -} diff --git a/nix/runtime-pins.nix b/nix/runtime-pins.nix new file mode 100644 index 0000000000000..abda9cefaa20b --- /dev/null +++ b/nix/runtime-pins.nix @@ -0,0 +1,243 @@ +# nix/runtime-pins.nix — managed runtime tools, built from runtime-pins.json +# +# runtime-pins.json is the ONE table of managed tool versions and digests. +# It already feeds the Python provisioner (source installs, `hermes +# update`, desktop payload staging). This file makes Nix a fourth consumer +# of that table rather than a second table: every version, URL and digest +# here is read from the JSON, so a pin bump stays one edit. +# +# Shape: +# +# * one derivation per pinned tool, each holding that tool's own tree +# exactly as upstream ships it; +# * `extends` in the table becomes a real Nix dependency — npm's +# derivation takes node's, so Nix orders the builds and neither this +# file nor a reader restates "npm needs node"; +# * `bundle` symlinks those derivations into the directory layout +# `hermes_cli/runtime_registry.py` describes, and writes `runtimes.json` +# with the registry's own code. +# +# Nothing here wraps a program or exports an environment variable. The +# bundle is a runtime dir, and `hermes_cli/runtime_env.py` already knows +# how to turn one of those into PATH, GIT_EXEC_PATH, npm_config_cache and +# the rest — on every install kind. A Nix-specific version of any of that +# would be a second implementation of tested behaviour. +{ + lib, + stdenv, + fetchurl, + autoPatchelfHook, + unzip, + python3, + runCommand, + curl, + expat, + fontconfig, + zlib, +}: +let + repoRoot = ../.; + pins = (builtins.fromJSON (builtins.readFile ../runtime-pins.json)).tools; + + # Pin-table target keys use Node/Python spellings so one string works on + # both sides of the JS/Python boundary; Nix systems spell it the other + # way round. This is the only place the two vocabularies meet. + targetBySystem = { + "x86_64-linux" = "linux-x64"; + "aarch64-linux" = "linux-arm64"; + "x86_64-darwin" = "darwin-x64"; + "aarch64-darwin" = "darwin-arm64"; + }; + + target = + targetBySystem.${stdenv.hostPlatform.system} + or (throw "runtime-pins: no pin target for ${stdenv.hostPlatform.system}"); + + # A tool either pins one target-independent artifact ('any', a registry + # tarball whose bytes do not vary) or one per target. Same resolution + # the Python registry does — see `pinned_file`. + artifactOf = + name: entry: + entry.files.any + or entry.files.${target} + or (throw "runtime-pins: ${name} has no pinned download for ${target}"); + + # fetchurl's `sha256` takes the bare lowercase hex the table already + # stores — the same string the Python provisioner verifies, so there is + # no second encoding to keep in sync. Nix enforces it as a fixed-output + # derivation: a tampered pin fails the build, as it fails provisioning. + fetchPinned = name: entry: fetchurl { inherit (artifactOf name entry) url sha256; }; + + extendsOf = entry: entry.extends or [ ]; + + # Prebuilt upstream binaries link against a normal FHS glibc, which does + # not exist here. autoPatchelfHook rewrites the interpreter and RPATH + # onto the nixpkgs runtime; macOS binaries are already relocatable. + # + # One library set covers every tool: these are the shared objects the + # pinned artifacts actually ask for (zlib broadly; curl/expat for + # dugite's http helpers; fontconfig for the Skia lib dugite ships). + patchelfInputs = lib.optionals stdenv.hostPlatform.isLinux [ + stdenv.cc.cc.lib + zlib + curl + expat + fontconfig + ]; + + mkToolBase = + name: entry: extra: + stdenv.mkDerivation ( + { + pname = "hermes-runtime-${name}"; + version = entry.version; + src = fetchPinned name entry; + + nativeBuildInputs = + [ unzip ] ++ lib.optionals stdenv.hostPlatform.isLinux [ autoPatchelfHook ]; + buildInputs = patchelfInputs; + + dontUnpack = true; + dontBuild = true; + dontConfigure = true; + + passthru = { + pinnedVersion = entry.version; + pinnedUrl = (artifactOf name entry).url; + extends = map (dep: tools.${dep}) (extendsOf entry); + }; + + meta = { + description = "Hermes managed runtime ${name} ${entry.version} (pinned in runtime-pins.json)"; + platforms = lib.platforms.unix; + }; + } + // extra + ); + + # The common case: unpack the artifact and keep upstream's own layout. + # + # Un-nesting a lone versioned wrapper directory is decided by what the + # archive CONTAINS, not by a per-tool list — the same rule the Python + # provisioner uses, and for the same reason (uv nests on POSIX and not + # on Windows, so a hardcoded list gets it wrong). + mkUnpackedTool = + name: entry: + mkToolBase name entry { + installPhase = '' + runHook preInstall + mkdir -p unpacked + tar -xf "$src" -C unpacked 2>/dev/null || unzip -q "$src" -d unpacked + + inner=unpacked + entries=("$inner"/*) + if [ ''${#entries[@]} -eq 1 ] && [ -d "''${entries[0]}" ]; then + inner="''${entries[0]}" + fi + + mkdir -p "$out" + cp -R "$inner"/. "$out/" + runHook postInstall + ''; + }; + + # npm is the one tool that cannot simply be unpacked. Its own bin/npm + # resolves npm-cli.js from dirname(process.execPath), so unpacked onto a + # PATH it finds the npm BUNDLED inside node — the copy this pin exists + # to supersede — and dies with MODULE_NOT_FOUND when that copy is gone. + # Letting npm install itself produces the launchers the platform + # actually needs instead of hand-written shims, and is the same install + # the Python provisioner performs (`_stage_npm`) from the same + # digest-verified tarball, offline. + # + # `extends` is what makes this work without a special case: node is a + # real build input, so Nix has already built and patched it. + mkNpmTool = + name: entry: + let + node = tools.node; + in + mkToolBase name entry { + installPhase = '' + runHook preInstall + mkdir -p "$out" + # npm insists on a writable HOME and cache; the sandbox's HOME is + # deliberately unwritable. Neither belongs in $out — a real + # install keeps its cache in the runtime dir, which + # managed_tool_env points npm_config_cache at. + HOME="$TMPDIR" npm_config_cache="$TMPDIR/npm-cache" \ + ${node}/bin/node ${node}/lib/node_modules/npm/bin/npm-cli.js \ + install --global --prefix "$out" --offline --no-audit --no-fund \ + "$src" + runHook postInstall + ''; + }; + + # An `extends` edge means "staged by what it extends", which today is + # npm-shaped: run the extended tool's installer. A second extender with + # different mechanics would add a branch here; one edge, one meaning + # until then. + mkTool = + name: entry: if extendsOf entry == [ ] then mkUnpackedTool name entry else mkNpmTool name entry; + + tools = lib.mapAttrs mkTool pins; + + # The registry code that writes the facts, as a store path. Only the + # leaf modules the fact-writing imports — all pure-stdlib, so a bare + # python3 loads them with no venv, and the bundle does not depend on + # the whole repo (which would rebuild it on any source change). + registrySrc = runCommand "hermes-runtime-registry-src" { } '' + mkdir -p "$out/hermes_cli" + touch "$out/hermes_cli/__init__.py" + cp ${../hermes_cli/runtime_registry.py} "$out/hermes_cli/runtime_registry.py" + cp ${../hermes_cli/runtime_provisioner.py} "$out/hermes_cli/runtime_provisioner.py" + cp ${../hermes_cli/runtime_tree.py} "$out/hermes_cli/runtime_tree.py" + cp ${../hermes_constants.py} "$out/hermes_constants.py" + cp ${../runtime-pins.json} "$out/runtime-pins.json" + ''; + + # The bundle IS a runtime dir: `