From 84874c58a5f9fe8117ce4890629d44e7c11e3315 Mon Sep 17 00:00:00 2001 From: ethernet Date: Tue, 28 Jul 2026 13:17:59 -0400 Subject: [PATCH] feat(dev-sandbox): support fake installer / fake main / git clones allow you to simulate the whole official curl | bash installer, and subsequent hermes updates. Run development commands in a bubblewrap filesystem and network sandbox with a local HTTPS MITM fixture server and a fake github git-upload-pack transport. Package the sandbox command and expose it from the nix devShell. Stage the local installer at its canonical fake HTTPS URL and add a persistent installation/update test path. Route root installs through sandbox-owned filesystem locations and snapshot dirty source worktrees into temporary fake commits so update tests can fast-forward without changing the real checkout. Includes a --install-ref sandbox installer mode that fetches any commit (--from-main is a nice shorthand for local development) outside the sealed sandbox, installs from that snapshot, and then promotes the fake remote to the current worktree so update flows can be exercised with FF. Notes on non-root sandboxes: Giving a non-root sandbox a network is tricky. slirp4netns joins the target userns and setuids to root before configuring the netns, so the userns must map a uid 0; bwrap's --unshare-user maps exactly ONE uid, so --uid 1000 leaves no root to become and slirp diedswith `setns(CLONE_NEWNET): Operation not permitted`. Stage 1 builds the user+net namespaces with `unshare` and two one-id ranges: inner 0 -> a subuid, unused by the payload, present only so slirp can become root inner 1000 -> our real host uid Mapping the payload to the *host* uid (not a second subuid) keeps everything the sandbox writes owned by us, so `rm -rf` on a persistent sandbox still needs no privileges. Stage 2 execs bwrap WITHOUT --unshare-user -- it only adds mount/pid -- sidestepping bwrap's refusal to accept --uid outside a userns it created. Costs a /etc/subuid range for the invoking user (we error with the exact line to add) and util-linux `unshare`; `--root` needs neither. --- hermes_cli/main.py | 3 + nix/devShell.nix | 5 +- nix/packages.nix | 5 + nix/sandbox.nix | 124 +++++ scripts/dev-sandbox.sh | 677 +++++++++++++++++++++------ scripts/sandbox/openssl.cnf | 43 ++ scripts/sandbox/pick-release-tags.sh | 110 +++++ scripts/sandbox/proxy.py | 237 ++++++++++ scripts/sandbox/ssh-shim.sh | 13 + scripts/sandbox/stage2-run.sh | 251 ++++++++++ 10 files changed, 1322 insertions(+), 146 deletions(-) create mode 100644 nix/sandbox.nix create mode 100644 scripts/sandbox/openssl.cnf create mode 100755 scripts/sandbox/pick-release-tags.sh create mode 100644 scripts/sandbox/proxy.py create mode 100644 scripts/sandbox/ssh-shim.sh create mode 100755 scripts/sandbox/stage2-run.sh diff --git a/hermes_cli/main.py b/hermes_cli/main.py index 624d0f10109f7..3d58833496c89 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -6876,6 +6876,9 @@ def _desktop_linux_needs_no_sandbox() -> bool: unprivileged desktop user on an AppArmor-restricted host. The root case should remain an explicit user choice. """ + if os.environ.get("ELECTRON_DISABLE_SANDBOX", 0) == "1": + return True + if sys.platform != "linux": return False if hasattr(os, "geteuid") and os.geteuid() == 0: diff --git a/nix/devShell.nix b/nix/devShell.nix index ac50beb0e699e..d1c0875045506 100644 --- a/nix/devShell.nix +++ b/nix/devShell.nix @@ -27,10 +27,7 @@ mkdir -p $out/bin install -Dm755 ${../hermes} $out/bin/hermes '') - (pkgs.runCommand "dev-sandbox" { } '' - mkdir -p $out/bin - install -Dm755 ${../scripts/dev-sandbox.sh} $out/bin/sandbox - '') + self'.packages.sandbox uv # Headless Wayland compositor for E2E tests (test:e2e:visual). # cage renders a single client with no window management, so diff --git a/nix/packages.nix b/nix/packages.nix index 5e4d32404f1ff..a496f73a3eeb2 100644 --- a/nix/packages.nix +++ b/nix/packages.nix @@ -9,6 +9,9 @@ ... }: let + + sandbox = pkgs.callPackage ./sandbox.nix { }; + minimal = pkgs.callPackage ./hermes-agent.nix { inherit (inputs) uv2nix pyproject-nix pyproject-build-systems; npm-lockfile-fix = inputs'.npm-lockfile-fix.packages.default; @@ -51,6 +54,8 @@ }).node-gyp; default = full; + inherit sandbox; + inherit minimal; # Ships discord.py + python-telegram-bot + slack-sdk so a plain diff --git a/nix/sandbox.nix b/nix/sandbox.nix new file mode 100644 index 0000000000000..cec5dc9868079 --- /dev/null +++ b/nix/sandbox.nix @@ -0,0 +1,124 @@ +{ + # electron deps + alsa-lib, + at-spi2-atk, + atk, + cairo, + cups, + dbus, + expat, + fontconfig, + freetype, + glib, + gtk3, + libdrm, + libgbm, + libxkbcommon, + mesa, + nspr, + nss, + pango, + systemd, + libX11, + libXcomposite, + libXdamage, + libXext, + libXfixes, + libXrandr, + libXrender, + libXtst, + libxcb, + + # sandbox deps + bash, + bubblewrap, + cacert, + coreutils, + curl, + gawk, + git, + glibc, + gnumake, + gnugrep, + gnused, + gzip, + nodejs_22, + openssl, + python3, + slirp4netns, + stdenv, + gnutar, + util-linux, + + # etc + writeShellApplication, + lib, +}: +let + electronRuntime = [ + alsa-lib + at-spi2-atk + atk + cairo + cups + dbus + expat + fontconfig + freetype + glib + gtk3 + libdrm + libgbm + libxkbcommon + mesa + nspr + nss + pango + systemd + libX11 + libXcomposite + libXdamage + libXext + libXfixes + libXrandr + libXrender + libXtst + libxcb + ]; +in +writeShellApplication { + name = "sandbox"; + runtimeInputs = [ + bash + bubblewrap + cacert + coreutils + curl + gawk + git + glibc.bin + gnumake + gnugrep + gnused + gzip + nodejs_22 + openssl + python3 + slirp4netns + stdenv.cc + gnutar + util-linux + ] + ++ electronRuntime; + text = '' + export DEV_SANDBOX_REAL_CA_CERT=${cacert}/etc/ssl/certs/ca-bundle.crt + export DEV_SANDBOX_DYNAMIC_LINKER=${stdenv.cc.bintools.dynamicLinker} + export DEV_SANDBOX_NODE_DIR=${nodejs_22} + export DEV_SANDBOX_ELECTRON_LD_LIBRARY_PATH=${lib.makeLibraryPath electronRuntime} + # The script is imported into the store as a single file, so its own + # directory has no scripts/sandbox/ beside it. Point it at the assets + # (fake-internet proxy, ssh shim) explicitly. + export DEV_SANDBOX_ASSETS=${../scripts/sandbox} + exec ${../scripts/dev-sandbox.sh} "$@" + ''; +} diff --git a/scripts/dev-sandbox.sh b/scripts/dev-sandbox.sh index 5ce459c94e9ab..dca11a72f3aa0 100755 --- a/scripts/dev-sandbox.sh +++ b/scripts/dev-sandbox.sh @@ -1,198 +1,591 @@ #!/usr/bin/env bash -# Run a Hermes instance in an isolated sandbox — separate HERMES_HOME, -# separate Electron userData, and a distinct Desktop app name so it doesn't compete -# with your main desktop instance's single-instance lock. +# Run a command in a disposable, network-isolated fake Internet. # -# By default the sandbox is throwaway: a temp dir is created and removed on -# exit. Use --persistent to keep the sandbox across restarts (stored under -# .hermes-sandbox/ in the worktree git root). -# -# Usage: -# scripts/dev-sandbox.sh python -m hermes_cli.main -# scripts/dev-sandbox.sh hermes desktop -# scripts/dev-sandbox.sh electron . -# scripts/dev-sandbox.sh -- npm run dev # from apps/desktop/ -# scripts/dev-sandbox.sh --persistent hermes desktop -# scripts/dev-sandbox.sh --persistent -- npm run dev -# -# Seed the sandbox HERMES_HOME from an existing directory (e.g. your main -# ~/.hermes) so config, sessions, skills, etc. are pre-populated: -# scripts/dev-sandbox.sh --from ~/.hermes hermes desktop -# -# Override the app name (default: HermesSandbox): -# HERMES_DEV_SANDBOX_NAME=Staging scripts/dev-sandbox.sh hermes desktop -# -# Override the persistent sandbox dir name (default: .hermes-sandbox): -# HERMES_DEV_SANDBOX_DIR=.staging-sandbox scripts/dev-sandbox.sh --persistent hermes desktop +# The command runs in private user, mount, PID, and network namespaces. This +# script is stage 1: it builds the sandbox tree, mints the fake CA, and creates +# the user+network namespaces with `unshare` (see the namespace plan further +# down), then re-execs into scripts/sandbox/stage2-run.sh, which adds the +# mount/pid namespaces with bubblewrap and runs the payload. Its only writable +# filesystem is SANDBOX_ROOT. HTTP(S) goes to a local static MITM proxy; +# github.com SSH uses a sandbox-local git-upload-pack shim; neither transport +# can reach the host network. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Helper files the sandbox needs: the stage-2 script it re-execs into, plus the +# files it copies in (the fake-internet proxy, the ssh shim, the openssl config). +# They sit next to this script in the repo, but the Nix wrapper installs the +# script into the store on its own, so it exports DEV_SANDBOX_ASSETS to point +# here. +SANDBOX_ASSETS="${DEV_SANDBOX_ASSETS:-$SCRIPT_DIR/sandbox}" +for asset in proxy.py ssh-shim.sh openssl.cnf stage2-run.sh; do + [ -f "$SANDBOX_ASSETS/$asset" ] || { + echo "error: missing sandbox asset: $SANDBOX_ASSETS/$asset" >&2 + exit 1 + } +done + print_help() { cat <<'EOF' -Usage: dev-sandbox.sh [--persistent] [--from DIR] [--] +Usage: dev-sandbox.sh [options] [--] + dev-sandbox.sh install [options] [--] [installer arguments...] -Run a Hermes instance in an isolated sandbox. +Run COMMAND in a throwaway chroot-like bubblewrap sandbox. The sandbox has no +writable host mounts: only its own root, mounted at /work, is writable. Options: - --persistent Keep the sandbox dir across restarts (under the worktree - git root, in .hermes-sandbox/). Without this flag the - sandbox is a temp dir that is removed on exit. - --from DIR Copy DIR into the sandbox HERMES_HOME as the starting - point (config, sessions, skills, etc.). - Ignored if the sandbox HERMES_HOME already has content - (e.g. reusing a --persistent sandbox) to avoid clobbering. - --delete Delete the existing persistent sandbox in .hermes-sandbox. - -h, --help Show this help message. + --persistent Keep the whole sandbox under .hermes-sandbox/. + --delete Delete the persistent sandbox (asks first). + --root Install as uid 0 with the root FHS layout: code in + /usr/local/lib/hermes-agent, command in + /usr/local/bin. Default is the user-level layout. + --from DIR One-time copy of DIR into the sandbox's $HOME. + Existing persistent sandboxes are never overwritten. + --http-root DIR Copy DIR into the fake web server root for this run. + Requests map to DIR//; no URL is forwarded. + --installer PATH With `install`, serve PATH at the canonical install.sh + URL. Default: scripts/install.sh in this worktree. + --from-main With `install`, fetch the real upstream main installer + and repository, then advance fake main to this folder + after a successful install for update testing. + Shorthand for --install-ref refs/heads/main. + --install-ref REF Like --from-main, but installs REF instead of main: + a branch, a tag (v2026.7.7), or a SHA reachable from main. + Use it to test updating from an older release, not just + from the tip. + -h, --help Show this help. + +Option order matters: every option above is consumed by THIS script, and +parsing stops at the first argument it does not recognize. Everything from +that point on is passed through to the command (or, with `install`, to the +installer). Put sandbox options first and separate installer arguments with +`--`, otherwise they arrive here and fail: + + # WRONG — --from-main reaches install.sh, which rejects it + scripts/dev-sandbox.sh install --skip-setup --from-main + + # RIGHT + scripts/dev-sandbox.sh install --from-main -- --skip-setup + +Install layout: `install.sh` picks its layout from `id -u` alone, so uid is what +separates the two real-world Linux installs. By default the sandbox runs as an +unprivileged `hermes` user, giving the layout most people have — +$HERMES_HOME/hermes-agent plus a ~/.local/bin launcher. Pass --root for the FHS +one. Both are worth testing; they differ in more than paths (root also relocates +uv's Python to /usr/local/share for world-readability). + +The fake web server signs certificates with a CA trusted only inside this +sandbox. HTTP_PROXY/HTTPS_PROXY send fixture URLs there first; other HTTP(S) +requests pass through the sandbox's rootless outbound network. SSH to github.com +runs a sandbox-local upload-pack shim, never your SSH config, agent, +known-hosts file, or authorized keys. + +Fake github main always comes from this folder. If it has staged, unstaged, or +non-ignored untracked changes, the sandbox warns and creates a temporary local +commit containing them; it never stages or commits the real worktree. Environment: - HERMES_DEV_SANDBOX_NAME Override the app name (default: HermesSandbox) - HERMES_DEV_SANDBOX_DIR Override the persistent dir name (default: .hermes-sandbox) + HERMES_DEV_SANDBOX_DIR Sandbox directory name, relative to the repo root + (default: .hermes-sandbox). Examples: - dev-sandbox.sh hermes desktop - dev-sandbox.sh --persistent hermes desktop - dev-sandbox.sh --from ~/.hermes hermes desktop - dev-sandbox.sh -- npm run dev + # create a sandbox, install this branch as `main`, and then drop to a shell, + # skipping `hermes setup` & the browser tools for speed. + scripts/dev-sandbox.sh install --persistent -- --skip-setup --skip-browser + + # Install the official upstream main. You're dropped into a shell where + # you can run `hermes update`. + scripts/dev-sandbox.sh install --persistent --from-main + EOF } PERSISTENT=false DELETE=false +RUN_AS_USER=true SEED_DIR="" +HTTP_ROOT="" +INSTALL_SHORTCUT=false +INSTALLER_PATH="" +# Which upstream commit the sandbox installs before the update routes run. +# Empty means "install this worktree's own installer" (no upstream fetch); set, +# it is anything git can resolve -- a branch, a tag (v2026.7.7), or a SHA +# reachable from main -- so "can a user two releases back still update?" is +# expressible. --from-main is shorthand for refs/heads/main. +INSTALL_REF="" +UPSTREAM_URL="${HERMES_DEV_SANDBOX_UPSTREAM:-https://github.com/NousResearch/hermes-agent.git}" + +if [ "${1:-}" = install ]; then + INSTALL_SHORTCUT=true + shift +fi while [ "$#" -gt 0 ]; do case "$1" in - --persistent) - PERSISTENT=true - shift - ;; + --persistent) PERSISTENT=true; shift ;; + --delete) DELETE=true; shift ;; + --root) RUN_AS_USER=false; shift ;; + --user) RUN_AS_USER=true; shift ;; # the default; accepted for symmetry --from) - if [ "$#" -lt 2 ] || [[ "$2" == -* ]]; then - echo "error: --from requires a directory argument" >&2 - exit 1 - fi - SEED_DIR="$2" - shift 2 - ;; - --from=*) - SEED_DIR="${1#--from=}" - if [ -z "$SEED_DIR" ]; then - echo "error: --from requires a directory argument" >&2 - exit 1 - fi - shift - ;; - --delete) - DELETE=true - shift - ;; - -h|--help) - print_help - exit 0 - ;; - --) - shift - break - ;; - *) - break - ;; + [ "$#" -ge 2 ] || { echo 'error: --from needs a directory' >&2; exit 1; } + SEED_DIR="$2"; shift 2 ;; + --http-root) + [ "$#" -ge 2 ] || { echo 'error: --http-root needs a directory' >&2; exit 1; } + HTTP_ROOT="$2"; shift 2 ;; + --installer) + [ "$#" -ge 2 ] || { echo 'error: --installer needs a file' >&2; exit 1; } + INSTALLER_PATH="$2"; shift 2 ;; + --from-main) INSTALL_REF="refs/heads/main"; shift ;; + --install-ref) + [ "$#" -ge 2 ] || { echo 'error: --install-ref needs a value' >&2; exit 1; } + INSTALL_REF="$2" + shift 2 ;; + --from=*|--http-root=*|--installer=*|--install-ref=*) + key="${1%%=*}"; value="${1#*=}" + [ -n "$value" ] || { echo "error: $key needs a value" >&2; exit 1; } + case "$key" in + --from) SEED_DIR="$value" ;; + --http-root) HTTP_ROOT="$value" ;; + --installer) INSTALLER_PATH="$value" ;; + --install-ref) INSTALL_REF="$value" ;; + esac + shift ;; + -h|--help) print_help; exit 0 ;; + --) shift; break ;; + *) break ;; esac done -if [ -n "$SEED_DIR" ]; then - if [ ! -d "$SEED_DIR" ]; then - echo "error: --from dir '$SEED_DIR' does not exist" >&2 - exit 1 - fi - # Resolve to absolute path so it's valid after we cd later. - SEED_DIR="$(cd "$SEED_DIR" && pwd)" -fi - -if [ "$#" -eq 0 ]; then +if [ "$INSTALL_SHORTCUT" = false ] && [ "$#" -eq 0 ]; then print_help >&2 exit 1 fi +if [ -n "$INSTALLER_PATH" ] && [ "$INSTALL_SHORTCUT" = false ]; then + echo 'error: --installer is only valid with the install shortcut' >&2 + exit 1 +fi +if [ -n "$INSTALL_REF" ] && [ "$INSTALL_SHORTCUT" = false ]; then + echo 'error: --from-main / --install-ref are only valid with the install shortcut' >&2 + exit 1 +fi +if [ -n "$INSTALL_REF" ] && [ -n "$INSTALLER_PATH" ]; then + echo 'error: --from-main / --install-ref cannot be combined with --installer' >&2 + exit 1 +fi -SANDBOX_DIR_NAME="${HERMES_DEV_SANDBOX_DIR:-.hermes-sandbox}" -GIT_ROOT="$(git rev-parse --show-toplevel 2>/dev/null || echo "$SCRIPT_DIR/..")" +for dir in "$SEED_DIR" "$HTTP_ROOT"; do + [ -z "$dir" ] || [ -d "$dir" ] || { echo "error: directory '$dir' does not exist" >&2; exit 1; } +done + +GIT_ROOT="${HERMES_SANDBOX_SOURCE_ROOT:-$(git rev-parse --show-toplevel)}" GIT_ROOT="$(cd "$GIT_ROOT" && pwd)" -PERSISTENT_SANDBOX_ROOT="$GIT_ROOT/$SANDBOX_DIR_NAME" +if [ "$INSTALL_SHORTCUT" = true ] && [ -z "$INSTALL_REF" ] && [ -z "$INSTALLER_PATH" ]; then + INSTALLER_PATH="$GIT_ROOT/scripts/install.sh" +fi +if [ -n "$INSTALLER_PATH" ] && [ ! -f "$INSTALLER_PATH" ]; then + echo "error: installer '$INSTALLER_PATH' does not exist" >&2 + exit 1 +fi +COMMIT="$(git -C "$GIT_ROOT" rev-parse --verify 'HEAD^{commit}')" || { + echo "error: current folder has no HEAD commit" >&2 + exit 1 +} +SANDBOX_DIR_NAME="${HERMES_DEV_SANDBOX_DIR:-.hermes-sandbox}" +PERSISTENT_ROOT="$GIT_ROOT/$SANDBOX_DIR_NAME" if [ "$DELETE" = true ]; then - if [ -d "$PERSISTENT_SANDBOX_ROOT" ]; then - read -r -p "[sandbox] delete $PERSISTENT_SANDBOX_ROOT? [y/N] " REPLY - case "$REPLY" in - [yY]|[yY][eE][sS]) - echo "[sandbox] deleting $PERSISTENT_SANDBOX_ROOT" >&2 - rm -rf -- "$PERSISTENT_SANDBOX_ROOT" - ;; - *) - echo "[sandbox] aborted" >&2 - exit 1 - ;; - esac - else - echo "[sandbox] nothing to delete at $PERSISTENT_SANDBOX_ROOT" >&2 + if [ ! -d "$PERSISTENT_ROOT" ]; then + echo "[sandbox] nothing to delete at $PERSISTENT_ROOT" >&2 + exit 0 fi + read -r -p "[sandbox] delete $PERSISTENT_ROOT? [y/N] " reply + case "$reply" in + y|Y|yes|YES) rm -rf -- "$PERSISTENT_ROOT" ;; + *) echo '[sandbox] aborted' >&2; exit 1 ;; + esac exit 0 fi -# Derive a per-worktree app name so multiple checkouts don't collide. -# Each worktree has its own toplevel path even though they share one repo, -# so we hash that path into a short, stable suffix. -WORKTREE_ROOT="$(git rev-parse --show-toplevel 2>/dev/null || echo "$SCRIPT_DIR/..")" -WORKTREE_ROOT="$(cd "$WORKTREE_ROOT" && pwd)" -WORKTREE_HASH="$(printf '%s' "$WORKTREE_ROOT" | cksum | cut -d' ' -f1)" -WORKTREE_NAME="$(basename "$WORKTREE_ROOT")" -DEFAULT_SANDBOX_NAME="HermesSandbox-${WORKTREE_NAME}-${WORKTREE_HASH}" - -SANDBOX_NAME="${HERMES_DEV_SANDBOX_NAME:-$DEFAULT_SANDBOX_NAME}" - if [ "$PERSISTENT" = true ]; then - SANDBOX_ROOT="$PERSISTENT_SANDBOX_ROOT" + SANDBOX_ROOT="$PERSISTENT_ROOT" else SANDBOX_ROOT="$(mktemp -d -t hermes-sandbox.XXXXXX)" + cleanup() { chmod -R u+w "$SANDBOX_ROOT"; rm -rf -- "$SANDBOX_ROOT"; } + trap cleanup EXIT INT TERM fi -export HERMES_HOME="$SANDBOX_ROOT/hermes-home" -export HERMES_DESKTOP_USER_DATA_DIR="$SANDBOX_ROOT/user-data" -export HERMES_DESKTOP_APP_NAME="$SANDBOX_NAME" - -mkdir -p "$HERMES_HOME" "$HERMES_DESKTOP_USER_DATA_DIR" - -if [ -n "$SEED_DIR" ]; then - # Only seed when the sandbox HERMES_HOME is empty — avoids clobbering an - # existing persistent sandbox on re-run. - if [ -z "$(ls -A "$HERMES_HOME" 2>/dev/null)" ]; then - echo "[sandbox] seeding HERMES_HOME from $SEED_DIR" >&2 - cp -a "$SEED_DIR/." "$HERMES_HOME/" +mkdir -p "$SANDBOX_ROOT"/{root,home,etc} +UPSTREAM_REPO="" +UPSTREAM_COMMIT="" +if [ -n "$INSTALL_REF" ]; then + echo "[sandbox] fetching upstream $INSTALL_REF for installer/update test" >&2 + UPSTREAM_REPO="$(mktemp -d -t hermes-sandbox-upstream.XXXXXX)" + git -C "$UPSTREAM_REPO" init -q + # Fetch the ref as given. A branch or tag name resolves on its own; a raw SHA + # needs the remote to allow fetching it directly, so fall back to fetching + # main and resolving the SHA locally (which works for any commit that is an + # ancestor of main -- the interesting case for "update from N versions ago"). + # + # Peel to ^{commit} in both cases: an annotated tag fetches as a tag OBJECT, + # and using it directly fails later with "trying to write non-commit object + # ... to branch 'refs/heads/main'". + if git -C "$UPSTREAM_REPO" fetch -q "$UPSTREAM_URL" "$INSTALL_REF" 2>/dev/null; then + UPSTREAM_COMMIT="$(git -C "$UPSTREAM_REPO" rev-parse "FETCH_HEAD^{commit}")" + elif git -C "$UPSTREAM_REPO" fetch -q "$UPSTREAM_URL" refs/heads/main \ + && UPSTREAM_COMMIT="$(git -C "$UPSTREAM_REPO" rev-parse --verify -q "$INSTALL_REF^{commit}")"; then + : else - echo "[sandbox] --from ignored: $HERMES_HOME already has content" >&2 + rm -rf -- "$UPSTREAM_REPO" + echo "error: could not resolve upstream ref: $INSTALL_REF" >&2 + echo ' Use a branch (main), a tag (v2026.7.7), or a SHA reachable from main.' >&2 + exit 1 fi fi +if [ ! -e "$SANDBOX_ROOT/root/repo/.sandbox-source" ]; then + mkdir -p "$SANDBOX_ROOT/root/repo" + # Persistent roots live under the worktree, so copying with cp would recurse + # into the sandbox itself. tar also lets us exclude a worktree's .git file, + # which can point at the host's shared worktree metadata. + tar -C "$GIT_ROOT" --exclude='./.git' --exclude="./$SANDBOX_DIR_NAME" -cf - . \ + | tar -C "$SANDBOX_ROOT/root/repo" -xf - + : > "$SANDBOX_ROOT/root/repo/.sandbox-source" +fi -echo "[sandbox] HERMES_HOME=$HERMES_HOME" >&2 -echo "[sandbox] userData=$HERMES_DESKTOP_USER_DATA_DIR" >&2 -echo "[sandbox] appName=$HERMES_DESKTOP_APP_NAME" >&2 -if [ "$PERSISTENT" = true ]; then - echo "[sandbox] persistent: $SANDBOX_ROOT" >&2 +if [ -n "$SEED_DIR" ] && [ ! -e "$SANDBOX_ROOT/.seeded" ]; then + echo "[sandbox] seeding home from $SEED_DIR" >&2 + cp -a "$SEED_DIR/." "$SANDBOX_ROOT/home/" + : > "$SANDBOX_ROOT/.seeded" +fi + +rm -rf "$SANDBOX_ROOT/root/http" +mkdir -p "$SANDBOX_ROOT/root/http" +if [ -n "$HTTP_ROOT" ]; then + cp -a "$HTTP_ROOT/." "$SANDBOX_ROOT/root/http/" +fi +if [ "$INSTALL_SHORTCUT" = true ]; then + mkdir -p "$SANDBOX_ROOT/root/http/hermes-agent.nousresearch.com" + if [ -n "$INSTALL_REF" ]; then + git -C "$UPSTREAM_REPO" show "$UPSTREAM_COMMIT:scripts/install.sh" \ + > "$SANDBOX_ROOT/root/http/hermes-agent.nousresearch.com/install.sh" + else + cp -a "$INSTALLER_PATH" "$SANDBOX_ROOT/root/http/hermes-agent.nousresearch.com/install.sh" + fi + set -- bash -c ' + set +e + curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash -s -- "$@" + install_status=$? + if [ "$install_status" -eq 0 ] && [ -f /work/promote-main ]; then + next_main=$(cat /work/promote-main) + if git --git-dir=/work/repos/hermes-agent.git update-ref refs/heads/main "$next_main"; then + rm -f /work/promote-main + printf "[sandbox] fake main advanced to this folder for update testing\n" >&2 + else + printf "[sandbox] failed to advance fake main after install\n" >&2 + install_status=1 + fi + fi + if [ "$DEV_SANDBOX_INTERACTIVE" = true ]; then + printf "\n[sandbox] installer exited %s; entering sandbox shell\n" "$install_status" >&2 + exec /dev/tty 2>&1 + exec bash -i + fi + exit "$install_status" + ' sandbox-installer "$@" +fi + +mkdir -p "$SANDBOX_ROOT/root"/{bin,certs,lib64,logs,repos,ssh,usr/bin,usr/local} +REAL_CA_CERT="${DEV_SANDBOX_REAL_CA_CERT:-}" +if [ -z "$REAL_CA_CERT" ]; then + for candidate in /etc/ssl/certs/ca-certificates.crt /etc/ssl/cert.pem; do + if [ -f "$candidate" ]; then + REAL_CA_CERT="$candidate" + break + fi + done +fi +if [ ! -f "$REAL_CA_CERT" ]; then + echo 'error: no system CA bundle found for outbound sandbox HTTPS' >&2 + exit 1 +fi +if [ ! -f "$SANDBOX_ROOT/root/certs/real-ca.pem" ]; then + cp "$REAL_CA_CERT" "$SANDBOX_ROOT/root/certs/real-ca.pem" +fi +printf 'nameserver 10.0.2.3\n' > "$SANDBOX_ROOT/etc/resolv.conf" +SANDBOX_SHELL="$(command -v bash)" +DYNAMIC_LINKER="${DEV_SANDBOX_DYNAMIC_LINKER:-}" +if [ -z "$DYNAMIC_LINKER" ]; then + # Nix store first: NixOS also ships a /lib64/ld-linux-x86-64.so.2 compat stub, + # so probing FHS paths first would quietly switch which loader a bare script + # invocation uses on this host. Globs that match nothing expand to themselves, + # so every candidate is -f tested. The FHS paths cover Debian/Ubuntu (where + # the loader is under /lib64 or a multiarch /lib dir), which is what CI runs. + for candidate in \ + /nix/store/*-glibc-*/lib/ld-linux-*.so.* \ + /lib64/ld-linux-x86-64.so.2 \ + /lib/ld-linux-aarch64.so.1 \ + /lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 \ + /lib/aarch64-linux-gnu/ld-linux-aarch64.so.1 + do + if [ -f "$candidate" ]; then + DYNAMIC_LINKER="$candidate" + break + fi + done +fi +if [ ! -f "$DYNAMIC_LINKER" ]; then + echo 'error: no glibc dynamic linker found for sandboxed release binaries' >&2 + echo ' Set DEV_SANDBOX_DYNAMIC_LINKER to its path.' >&2 + exit 1 +fi +ln -sf "$SANDBOX_SHELL" "$SANDBOX_ROOT/root/bin/sh" +ln -sf "$(command -v ls)" "$SANDBOX_ROOT/root/bin/ls" +ln -sf "$(command -v env)" "$SANDBOX_ROOT/root/usr/bin/env" +ln -sf "$DYNAMIC_LINKER" "$SANDBOX_ROOT/root/lib64/$(basename "$DYNAMIC_LINKER")" +# Identity inside the sandbox. install.sh chooses its layout from `id -u` +# alone (see resolve_install_layout), so the uid here is what decides between +# the root FHS install and a user-level one. +if [ "$RUN_AS_USER" = true ]; then + SANDBOX_UID=1000 + SANDBOX_GID=1000 + SANDBOX_USER=hermes + SANDBOX_HOME=/home/hermes else - echo "[sandbox] ephemeral (will be cleaned up on exit)" >&2 + SANDBOX_UID=0 + SANDBOX_GID=0 + SANDBOX_USER=root + SANDBOX_HOME=/root +fi +{ + printf 'root:x:0:0:Sandbox Root:/root:%s\n' "$SANDBOX_SHELL" + if [ "$RUN_AS_USER" = true ]; then + printf '%s:x:%s:%s:Sandbox User:%s:%s\n' \ + "$SANDBOX_USER" "$SANDBOX_UID" "$SANDBOX_GID" "$SANDBOX_HOME" "$SANDBOX_SHELL" + fi +} > "$SANDBOX_ROOT/etc/passwd" +{ + printf 'root:x:0:\n' + if [ "$RUN_AS_USER" = true ]; then + printf '%s:x:%s:\n' "$SANDBOX_USER" "$SANDBOX_GID" + fi +} > "$SANDBOX_ROOT/etc/group" +# A user-level install writes the `hermes` launcher to ~/.local/bin and the +# checkout to $HERMES_HOME; both live under the sandbox HOME, which is bound +# from $SANDBOX_ROOT/home. bwrap maps our real uid to $SANDBOX_UID, so the +# host-side ownership of that directory is what the sandbox sees as its own. +printf 'hosts: files dns\n' > "$SANDBOX_ROOT/etc/nsswitch.conf" +printf '127.0.0.1 localhost\n' > "$SANDBOX_ROOT/etc/hosts" + +SOURCE_REPO="$GIT_ROOT" +SOURCE_REF="$COMMIT" +SNAPSHOT_REPO="" +FAKE_REPO="$SANDBOX_ROOT/root/repos/hermes-agent.git" +git -C "$SANDBOX_ROOT/root/repos" init --bare -q hermes-agent.git +if [ -n "$INSTALL_REF" ]; then + git --git-dir="$FAKE_REPO" fetch -q --force "$UPSTREAM_REPO" \ + "$UPSTREAM_COMMIT:refs/heads/main" +fi +if [ -n "$(git -C "$GIT_ROOT" status --porcelain)" ]; then + echo '[sandbox] warning: current folder is dirty; creating a temporary fake commit for main' >&2 + SNAPSHOT_REPO="$(mktemp -d -t hermes-sandbox-snapshot.XXXXXX)" + git -C "$SNAPSHOT_REPO" init -q + git -C "$SNAPSHOT_REPO" fetch -q "$GIT_ROOT" "$COMMIT" + git -C "$SNAPSHOT_REPO" config user.name 'Hermes sandbox' + git -C "$SNAPSHOT_REPO" config user.email 'sandbox@invalid' + GIT_DIR="$SNAPSHOT_REPO/.git" GIT_WORK_TREE="$GIT_ROOT" git read-tree "$COMMIT" + GIT_DIR="$SNAPSHOT_REPO/.git" GIT_WORK_TREE="$GIT_ROOT" \ + git add -A -- . + SNAPSHOT_TREE="$(GIT_DIR="$SNAPSHOT_REPO/.git" git write-tree)" + SNAPSHOT_PARENT="$COMMIT" + if EXISTING_MAIN="$(git --git-dir="$FAKE_REPO" rev-parse --verify refs/heads/main 2>/dev/null)"; then + git -C "$SNAPSHOT_REPO" fetch -q "$FAKE_REPO" "$EXISTING_MAIN" + SNAPSHOT_PARENT="$EXISTING_MAIN" + fi + SOURCE_REF="$(GIT_DIR="$SNAPSHOT_REPO/.git" git commit-tree "$SNAPSHOT_TREE" -p "$SNAPSHOT_PARENT" \ + -m 'sandbox snapshot of dirty worktree')" + SOURCE_REPO="$SNAPSHOT_REPO" fi -if [ "$PERSISTENT" = false ]; then - cleanup() { - chmod -R u+w "$SANDBOX_ROOT" - rm -rf -- "$SANDBOX_ROOT" +if [ -n "$INSTALL_REF" ]; then + git --git-dir="$FAKE_REPO" fetch -q --force "$SOURCE_REPO" \ + "$SOURCE_REF:refs/hermes-sandbox/next" + printf '%s\n' "$SOURCE_REF" > "$SANDBOX_ROOT/root/promote-main" +else + git --git-dir="$FAKE_REPO" fetch -q --force "$SOURCE_REPO" \ + "$SOURCE_REF:refs/heads/main" +fi +git --git-dir="$FAKE_REPO" symbolic-ref HEAD refs/heads/main +if [ -n "$SNAPSHOT_REPO" ]; then + # Best-effort: it is a mktemp directory the OS will reap, and failing the whole + # run over a leftover object file would be worse than leaking it. Concurrent + # git activity in the worktree can still be writing here as we delete. + rm -rf -- "$SNAPSHOT_REPO" 2>/dev/null || true +fi +if [ -n "$UPSTREAM_REPO" ]; then + rm -rf -- "$UPSTREAM_REPO" +fi + +# openssl reads a config even for `req -addext`, and its compiled-in path is a +# symlink into /etc/ssl on Debian/Ubuntu -- which the sandbox replaces. Ship our +# own and point OPENSSL_CONF at it, both here and inside the sandbox. +cp "$SANDBOX_ASSETS/openssl.cnf" "$SANDBOX_ROOT/root/certs/openssl.cnf" + +if [ ! -f "$SANDBOX_ROOT/root/certs/ca.pem" ]; then + if ! ca_error="$(OPENSSL_CONF="$SANDBOX_ROOT/root/certs/openssl.cnf" \ + openssl req -x509 -newkey rsa:2048 -nodes -days 2 \ + -subj '/CN=Hermes dev sandbox CA' \ + -extensions sandbox_ca_ext \ + -keyout "$SANDBOX_ROOT/root/certs/ca.key" \ + -out "$SANDBOX_ROOT/root/certs/ca.pem" 2>&1 >/dev/null)"; then + echo 'error: could not create the sandbox CA:' >&2 + printf '%s\n' "$ca_error" >&2 + exit 1 + fi +fi +GIT_UPLOAD_PACK="$(command -v git-upload-pack)" +sed "s|@GIT_UPLOAD_PACK@|$GIT_UPLOAD_PACK|" "$SANDBOX_ASSETS/ssh-shim.sh" \ + > "$SANDBOX_ROOT/root/usr/bin/ssh" +chmod 700 "$SANDBOX_ROOT/root/usr/bin/ssh" + +# The fake-internet proxy and the ssh shim are real files under +# scripts/sandbox/ rather than heredocs, so they can be linted, syntax-checked +# and diffed like any other source. Copy them into the sandbox tree. +cp "$SANDBOX_ASSETS/proxy.py" "$SANDBOX_ROOT/root/proxy.py" + +if [ -n "$INSTALL_REF" ]; then + echo "[sandbox] fake main: upstream $INSTALL_REF ($UPSTREAM_COMMIT)" >&2 + echo "[sandbox] prepared update: current folder ($SOURCE_REF)" >&2 +else + echo "[sandbox] fake main: current folder ($SOURCE_REF)" >&2 +fi +echo "[sandbox] root: $SANDBOX_ROOT" >&2 +echo "[sandbox] http root: $SANDBOX_ROOT/root/http" >&2 +if [ "$RUN_AS_USER" = true ]; then + echo "[sandbox] identity: $SANDBOX_USER (uid $SANDBOX_UID) — installs are user-level under $SANDBOX_HOME" >&2 +else + echo '[sandbox] identity: root (uid 0) — installs use the /usr/local FHS layout' >&2 +fi +[ "$PERSISTENT" = true ] && echo '[sandbox] persistent' >&2 || echo '[sandbox] ephemeral' >&2 + +for command in awk bash bwrap curl git openssl python3 slirp4netns tar unshare; do + command -v "$command" >/dev/null || { + echo "error: missing required command: $command" >&2 + exit 1 } - trap cleanup EXIT - trap 'cleanup; exit 130' INT TERM +done + +INTERACTIVE=false +if [ -t 0 ] && [ -t 1 ]; then + INTERACTIVE=true +fi +NODE_DIR="${DEV_SANDBOX_NODE_DIR:-}" +if [ -z "$NODE_DIR" ] && command -v node >/dev/null; then + NODE_DIR="$(dirname "$(dirname "$(command -v node)")")" +fi +WAYLAND_SOCKET="" +if [ -n "${XDG_RUNTIME_DIR:-}" ] && [ -n "${WAYLAND_DISPLAY:-}" ] \ + && [ -S "$XDG_RUNTIME_DIR/$WAYLAND_DISPLAY" ]; then + WAYLAND_SOCKET="$XDG_RUNTIME_DIR/$WAYLAND_DISPLAY" fi -"$@" -rc=$? -exit $rc +# Namespace plan (stage 1 -> stage 2). +# +# slirp4netns joins the target's userns and setuids to root before configuring +# the netns, so the userns MUST map a uid 0. bwrap's own --unshare-user maps +# exactly one uid, so it cannot both run the payload as uid 1000 and offer slirp +# a root to become: that combination fails with +# setns(CLONE_NEWNET): Operation not permitted. +# +# So stage 1 builds the namespaces here with two ranges: +# inner 0 <- a subuid, unused by the payload, present only so slirp can +# become root inside the namespace +# inner $SANDBOX_UID <- our real host uid, so everything the sandbox writes +# stays owned by us and `rm -rf` on a persistent sandbox needs +# no privileges or chown dance +# The payload then runs in stage 2, where bwrap adds the mount/pid namespaces +# without creating a userns at all. +# +# The root layout needs no subuid at all: inner 0 IS the host uid there. +netns_args=(--user --net) +if [ "$RUN_AS_USER" = true ]; then + host_user="$(id -un)" + subuid_base="$(awk -F: -v u="$host_user" '$1 == u {print $2; exit}' /etc/subuid)" + subgid_base="$(awk -F: -v u="$host_user" '$1 == u {print $2; exit}' /etc/subgid)" + if [ -z "$subuid_base" ] || [ -z "$subgid_base" ]; then + echo "error: no /etc/subuid or /etc/subgid range for $host_user" >&2 + echo ' A user-level sandbox needs one spare subordinate id to host' >&2 + echo " its internal root. Add e.g. '$host_user:100000:65536' to both," >&2 + echo ' or use --root.' >&2 + exit 1 + fi + netns_args+=( + --map-users="0:$subuid_base:1" --map-users="$SANDBOX_UID:$(id -u):1" + --map-groups="0:$subgid_base:1" --map-groups="$SANDBOX_GID:$(id -g):1" + ) +else + netns_args+=(--map-root-user) +fi + +sandbox_pid_file="$SANDBOX_ROOT/root/logs/sandbox.pid" +slirp_ready="$SANDBOX_ROOT/root/logs/slirp.ready" +slirp_log="$SANDBOX_ROOT/root/logs/slirp.log" +: > "$sandbox_pid_file" +: > "$slirp_ready" + +env \ + DEV_SANDBOX_ROOT="$SANDBOX_ROOT" \ + DEV_SANDBOX_BASH="$(command -v bash)" \ + DEV_SANDBOX_REAL_CA_CERT="$REAL_CA_CERT" \ + DEV_SANDBOX_INTERACTIVE="$INTERACTIVE" \ + DEV_SANDBOX_USER="$SANDBOX_USER" \ + DEV_SANDBOX_HOME="$SANDBOX_HOME" \ + DEV_SANDBOX_NODE_DIR="$NODE_DIR" \ + DEV_SANDBOX_ELECTRON_LD_LIBRARY_PATH="${DEV_SANDBOX_ELECTRON_LD_LIBRARY_PATH:-}" \ + DEV_SANDBOX_XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-}" \ + DEV_SANDBOX_WAYLAND_DISPLAY="${WAYLAND_DISPLAY:-}" \ + DEV_SANDBOX_WAYLAND_SOCKET="$WAYLAND_SOCKET" \ + unshare "${netns_args[@]}" \ + "$SANDBOX_ASSETS/stage2-run.sh" "$@" & +sandbox_launcher=$! + +for _ in $(seq 1 200); do + [ -s "$sandbox_pid_file" ] && break + if ! kill -0 "$sandbox_launcher" 2>/dev/null; then + wait "$sandbox_launcher" + exit $? + fi + sleep 0.05 +done +sandbox_pid="$(tr -dc '0-9' < "$sandbox_pid_file")" +if [ -z "$sandbox_pid" ]; then + echo 'error: sandbox did not report its PID' >&2 + exit 1 +fi + +slirp4netns --configure --disable-host-loopback --ready-fd=3 \ + --userns-path="/proc/$sandbox_pid/ns/user" "$sandbox_pid" tap0 \ + 3>"$slirp_ready" >"$slirp_log" 2>&1 & +slirp_pid=$! +cleanup_slirp() { + kill "$slirp_pid" 2>/dev/null || true + wait "$slirp_pid" 2>/dev/null || true +} +trap cleanup_slirp EXIT INT TERM + +for _ in $(seq 1 200); do + [ -s "$slirp_ready" ] && break + if ! kill -0 "$slirp_pid" 2>/dev/null; then + cat "$slirp_log" >&2 || true + exit 1 + fi + sleep 0.05 +done +if [ ! -s "$slirp_ready" ]; then + echo 'error: timed out waiting for sandbox network setup' >&2 + exit 1 +fi + +wait "$sandbox_launcher" +exit $? \ No newline at end of file diff --git a/scripts/sandbox/openssl.cnf b/scripts/sandbox/openssl.cnf new file mode 100644 index 0000000000000..04884355bd16e --- /dev/null +++ b/scripts/sandbox/openssl.cnf @@ -0,0 +1,43 @@ +# Minimal openssl config for the dev sandbox. +# +# The sandbox replaces /etc wholesale, and on Debian/Ubuntu +# /usr/lib/ssl/openssl.cnf (openssl's compiled-in OPENSSLDIR) is a symlink into +# /etc/ssl -- so the config openssl insists on reading disappears and every +# `openssl req` fails with: +# +# Can't open "/usr/lib/ssl/openssl.cnf" for reading +# +# which surfaces to the payload as a bare `curl: (35) Recv failure`. Rather than +# reconstruct each distro's /etc/ssl, point OPENSSL_CONF at this file: the proxy +# only needs enough config for `req -addext` and `x509 -copy_extensions`. + +[ req ] +distinguished_name = req_distinguished_name + +[ req_distinguished_name ] + +# Used by `req -x509` for the sandbox's own CA. Without an explicit +# basicConstraints the generated certificate is not a CA, and every leaf it +# signs is rejected by the client with "invalid CA certificate (79)". +[ sandbox_ca_ext ] +basicConstraints = critical,CA:true +keyUsage = critical,keyCertSign,cRLSign +subjectKeyIdentifier = hash + +[ ca ] +default_ca = sandbox_ca + +[ sandbox_ca ] +default_md = sha256 +policy = policy_anything +email_in_dn = no +preserve = no + +[ policy_anything ] +commonName = optional +countryName = optional +stateOrProvinceName = optional +localityName = optional +organizationName = optional +organizationalUnitName = optional +emailAddress = optional diff --git a/scripts/sandbox/pick-release-tags.sh b/scripts/sandbox/pick-release-tags.sh new file mode 100755 index 0000000000000..1363eb14c5178 --- /dev/null +++ b/scripts/sandbox/pick-release-tags.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# Pick the release tags the install/update E2E should update FROM. +# +# Emits a JSON array of tag names on stdout, suitable for a GitHub Actions +# matrix (`fromJSON`). Choosing at runtime rather than hardcoding keeps the +# matrix honest as releases land: a pinned list silently stops covering the +# newest release the day after it ships, and pins the "oldest" forever even +# after it stops being a version anyone still runs. +# +# Selection: the newest tag, the oldest tag, and evenly spaced tags in between. +# Newest catches "did the last release break updating?", oldest is the longest +# upgrade jump anyone can still make, and the spread samples the migrations in +# between (config-schema bumps, venv layout changes, dependency floors). +# +# Usage: +# scripts/sandbox/pick-release-tags.sh [--count N] [--repo DIR] +# +# --count how many tags to emit (default 5, minimum 1). Fewer tags than +# requested emits all of them. +# --repo repository to read tags from (default: this checkout). +# +# Reads tags from the local checkout, so it needs one fetched with tags +# (actions/checkout with fetch-depth: 0, or `fetch-tags: true`). A shallow +# checkout has no tags and this exits non-zero rather than silently emitting an +# empty matrix. +# +# Only vYYYY.M.D[.N] release tags are considered; the repo also carries +# backup/* and one-off tags that are not releases. + +set -euo pipefail + +COUNT=5 +# Default to the repository containing this script, resolved through its real +# path so a symlinked or copied script still reads the checkout it lives in +# rather than whatever repo the caller happens to be standing in. +REPO="" +while [ "$#" -gt 0 ]; do + case "$1" in + --count) + [ "$#" -ge 2 ] || { echo 'error: --count needs a value' >&2; exit 1; } + COUNT="$2"; shift 2 ;; + --repo) + [ "$#" -ge 2 ] || { echo 'error: --repo needs a value' >&2; exit 1; } + REPO="$2"; shift 2 ;; + -h|--help) sed -n '2,30p' "$0"; exit 0 ;; + *) echo "error: unknown argument: $1" >&2; exit 1 ;; + esac +done +case "$COUNT" in + ''|*[!0-9]*) echo "error: --count must be a positive integer: $COUNT" >&2; exit 1 ;; +esac +[ "$COUNT" -ge 1 ] || { echo 'error: --count must be at least 1' >&2; exit 1; } + +# Resolve the script's own location through symlinks, then ask git which +# worktree that path belongs to. Deriving the repo from the script rather than +# from $PWD means a copied script cannot silently report a different checkout's +# tags, and --show-toplevel keeps it correct when invoked from a subdirectory. +if [ -z "$REPO" ]; then + script_path="${BASH_SOURCE[0]}" + if command -v readlink >/dev/null 2>&1; then + script_path="$(readlink -f "$script_path" 2>/dev/null || printf '%s' "$script_path")" + fi + script_dir="$(cd "$(dirname "$script_path")" && pwd)" + REPO="$(git -C "$script_dir" rev-parse --show-toplevel 2>/dev/null || printf '%s' "$script_dir")" +fi + +# sort -V orders v2026.4.8 before v2026.4.13 (numeric), which a plain +# lexicographic sort gets wrong. +mapfile -t tags < <( + git -C "$REPO" tag --list 'v*' \ + | grep -E '^v[0-9]{4}\.[0-9]+\.[0-9]+(\.[0-9]+)?$' \ + | sort -V +) + +total="${#tags[@]}" +if [ "$total" -eq 0 ]; then + echo "error: no release tags found in $REPO" >&2 + echo ' A shallow clone has no tags: fetch with tags (actions/checkout' >&2 + echo ' with fetch-depth: 0, or fetch-tags: true).' >&2 + exit 1 +fi + +if [ "$total" -le "$COUNT" ]; then + picked=("${tags[@]}") +elif [ "$COUNT" -eq 1 ]; then + # One slot means the newest release; there is no span to spread across. + picked=("${tags[$((total - 1))]}") +else + # Evenly spaced indices across [0, total-1], endpoints included, so the + # oldest and newest are always present and the rest are spread between them. + picked=() + for slot in $(seq 0 $((COUNT - 1))); do + # Round to nearest rather than truncate, so the spacing does not bunch + # toward the oldest end. + index=$(( (slot * (total - 1) * 2 + (COUNT - 1)) / ((COUNT - 1) * 2) )) + candidate="${tags[$index]}" + # Guard against a duplicate if rounding lands twice on the same tag. + case " ${picked[*]-} " in + *" $candidate "*) continue ;; + esac + picked+=("$candidate") + done +fi + +printf '[' +for i in "${!picked[@]}"; do + [ "$i" -eq 0 ] || printf ',' + printf '"%s"' "${picked[$i]}" +done +printf ']\n' diff --git a/scripts/sandbox/proxy.py b/scripts/sandbox/proxy.py new file mode 100644 index 0000000000000..f34c9a843a9a2 --- /dev/null +++ b/scripts/sandbox/proxy.py @@ -0,0 +1,237 @@ +"""MITM proxy backing the dev sandbox's fake Internet. + +Listens on 127.0.0.1:8080 and is pointed at by http_proxy/https_proxy inside +the sandbox. For each request it either serves a fixture from the filesystem or +forwards to the real host: + +* ``//`` exists -> serve it. This is how the sandbox answers + the canonical install URL with the installer under test, so the payload can + run the true ``curl -fsSL https://…/install.sh | bash`` one-liner. +* otherwise -> forward upstream, verifying against the real CA bundle. The + sandbox is isolated from the *host*, not from the internet: a real install + still has to reach PyPI and npm. + +HTTPS is intercepted by minting a per-host certificate from the sandbox's own +throwaway CA, which the payload trusts via CURL_CA_BUNDLE / SSL_CERT_FILE. + +Usage: proxy.py +""" + +import os +import pathlib +import socket +import ssl +import subprocess +import sys +import threading +from urllib.parse import unquote, urlsplit + +ROOT, CERTS, REAL_CA = map(pathlib.Path, sys.argv[1:]) + +LISTEN_ADDRESS = ('127.0.0.1', 8080) +MAX_REQUEST_BYTES = 65536 +UPSTREAM_TIMEOUT_SECONDS = 30 +CERT_VALIDITY_DAYS = 2 + + +def read_request(conn): + data = b"" + while b"\r\n\r\n" not in data and len(data) < MAX_REQUEST_BYTES: + part = conn.recv(4096) + if not part: + return b"" + data += part + return data + + +def run_openssl(args): + """Run openssl, raising with its stderr when it fails. + + Discarding stderr here costs real debugging time: the caller sees only a + dropped connection (``curl: (35) Recv failure``) and the log holds nothing + but the argv, so an unwritable directory, a missing CA key, and an option + the host's openssl rejects all look identical. + """ + done = subprocess.run( + ['openssl', *args], stdout=subprocess.DEVNULL, stderr=subprocess.PIPE + ) + if done.returncode != 0: + detail = done.stderr.decode('utf-8', 'replace').strip() + raise RuntimeError( + f'openssl {args[0]} failed (exit {done.returncode}): {detail}' + ) + + +_CERT_LOCK = threading.Lock() + + +def cert_for(host): + """Return a (cert, key) pair for host, minting it from the sandbox CA. + + Minting is serialized and published atomically. The proxy is threaded, so + two concurrent requests for the same host would otherwise both run openssl + into the same paths, and a reader could pick up a finished certificate + beside a key from the other writer -- which TLS rejects as + ``[X509: KEY_VALUES_MISMATCH] key values mismatch``. + """ + safe = ''.join(char if char.isalnum() or char in '.-' else '_' for char in host) + cert, key = CERTS / f'{safe}.pem', CERTS / f'{safe}.key' + if cert.exists() and key.exists(): + return cert, key + with _CERT_LOCK: + # Re-check: another thread may have finished while we waited. + if cert.exists() and key.exists(): + return cert, key + # Build under unique temp names, then rename into place. os.replace is + # atomic, so a reader sees either the old pair or the new one, never a + # half-written mix. The key lands first: the certificate's existence is + # what everything else keys off. + stamp = f'{os.getpid()}.{threading.get_ident()}' + tmp_key = CERTS / f'{safe}.key.{stamp}' + tmp_cert = CERTS / f'{safe}.pem.{stamp}' + csr = CERTS / f'{safe}.csr.{stamp}' + run_openssl([ + 'req', '-newkey', 'rsa:2048', '-nodes', + '-subj', f'/CN={host}', + '-addext', f'subjectAltName=DNS:{host}', + '-keyout', str(tmp_key), '-out', str(csr), + ]) + run_openssl([ + 'x509', '-req', '-days', str(CERT_VALIDITY_DAYS), '-in', str(csr), + '-CA', str(CERTS / 'ca.pem'), '-CAkey', str(CERTS / 'ca.key'), + '-CAcreateserial', '-copy_extensions', 'copy', '-out', str(tmp_cert), + ]) + csr.unlink(missing_ok=True) + os.replace(tmp_key, key) + os.replace(tmp_cert, cert) + return cert, key + + +def file_for(host, target): + """Resolve a request to a fixture file, or None to forward upstream.""" + path = urlsplit(target).path or '/' + parts = pathlib.PurePosixPath(unquote(path)).parts + if '..' in parts: + return None + candidate = ROOT / host / pathlib.PurePosixPath(*[p for p in parts if p != '/']) + if candidate.is_dir(): + candidate /= 'index.html' + return candidate if candidate.is_file() else None + + +def respond_fixture(conn, found): + body = found.read_bytes() + headers = ( + f'Content-Length: {len(body)}\r\nConnection: close\r\n\r\n'.encode() + ) + conn.sendall(b'HTTP/1.1 200 OK\r\n' + headers + body) + + +def close_request(request, target=None): + """Rewrite a proxied request for a direct upstream connection.""" + headers, separator, body = request.partition(b'\r\n\r\n') + lines = headers.split(b'\r\n') + if target is not None: + method, _, version = lines[0].split(b' ', 2) + lines[0] = b' '.join((method, target.encode(), version)) + lines = [ + line for line in lines + if not line.lower().startswith(b'proxy-connection:') + ] + lines.append(b'Connection: close') + return b'\r\n'.join(lines) + separator + body + + +def relay(source, destination): + while True: + chunk = source.recv(MAX_REQUEST_BYTES) + if not chunk: + return + destination.sendall(chunk) + + +def forward_https(conn, host, port, request): + context = ssl.create_default_context(cafile=str(REAL_CA)) + with socket.create_connection((host, port), timeout=UPSTREAM_TIMEOUT_SECONDS) as raw: + with context.wrap_socket(raw, server_hostname=host) as upstream: + upstream.sendall(close_request(request)) + relay(upstream, conn) + + +def forward_http(conn, host, port, request, target): + parsed = urlsplit(target) + path = parsed.path or '/' + if parsed.query: + path += f'?{parsed.query}' + with socket.create_connection((host, port), timeout=UPSTREAM_TIMEOUT_SECONDS) as upstream: + upstream.sendall(close_request(request, path)) + relay(upstream, conn) + + +def handle_connect(conn, target): + """Intercept a CONNECT tunnel, terminating TLS with a minted cert.""" + host, _, port_text = target.rpartition(':') + port = int(port_text or '443') + conn.sendall(b'HTTP/1.1 200 Connection Established\r\n\r\n') + cert, key = cert_for(host) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(cert, key) + with context.wrap_socket(conn, server_side=True) as tls: + nested = read_request(tls) + if not nested: + return + line = nested.split(b'\r\n', 1)[0].decode('iso-8859-1') + nested_target = line.split(' ', 2)[1] + found = file_for(host, nested_target) + if found is not None: + respond_fixture(tls, found) + else: + forward_https(tls, host, port, nested) + + +def host_from_headers(request): + for header in request.split(b'\r\n')[1:]: + if header.lower().startswith(b'host:'): + value = header.split(b':', 1)[1].strip().decode() + return value.split(':', 1)[0] + return None + + +def handle_request(conn): + with conn: + request = read_request(conn) + if not request: + return + line = request.split(b'\r\n', 1)[0].decode('iso-8859-1') + method, target, _ = line.split(' ', 2) + if method.upper() == 'CONNECT': + handle_connect(conn, target) + return + parsed = urlsplit(target) + host = parsed.hostname or host_from_headers(request) or 'unknown' + found = file_for(host, target) + if found is not None: + respond_fixture(conn, found) + else: + forward_http(conn, host, parsed.port or 80, request, target) + + +def handle(conn): + try: + handle_request(conn) + except Exception as error: + print(f'proxy request failed: {error!r}', file=sys.stderr, flush=True) + + +def main(): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server: + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(LISTEN_ADDRESS) + server.listen() + while True: + conn, _ = server.accept() + threading.Thread(target=handle, args=(conn,), daemon=True).start() + + +if __name__ == '__main__': + main() diff --git a/scripts/sandbox/ssh-shim.sh b/scripts/sandbox/ssh-shim.sh new file mode 100644 index 0000000000000..1b90035eea4c5 --- /dev/null +++ b/scripts/sandbox/ssh-shim.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Stand-in for ssh inside the dev sandbox. +# +# install.sh and `hermes update` clone over ssh first (git@github.com:...), so +# the sandbox needs an `ssh` that answers. Rather than run a real sshd, this +# ignores the host, user, and command git asked for and speaks the +# upload-pack protocol directly against the sandbox's bare repo -- which is +# what makes the ssh-first code path exercisable with no keys, no known_hosts, +# and no network. +# +# GIT_UPLOAD_PACK is substituted by dev-sandbox.sh when it installs this shim, +# because the host's git-upload-pack is not necessarily on the sandbox PATH. +exec @GIT_UPLOAD_PACK@ /work/repos/hermes-agent.git diff --git a/scripts/sandbox/stage2-run.sh b/scripts/sandbox/stage2-run.sh new file mode 100755 index 0000000000000..42d2800ec7eb0 --- /dev/null +++ b/scripts/sandbox/stage2-run.sh @@ -0,0 +1,251 @@ +#!/usr/bin/env bash +# Stage 2 of the dev sandbox: build the mounts and run the payload. +# +# Not called directly. scripts/dev-sandbox.sh (stage 1) creates the user and +# network namespaces with `unshare` and re-execs into this script inside them, +# so by the time this runs we are already at the target uid with a private +# netns. bwrap therefore does NOT create a userns here -- it only adds the +# mount and pid namespaces. (`unshare --user` grants its creator full +# capabilities in the new userns regardless of which uid it maps, which is what +# lets bwrap mount as a non-root uid.) +# +# The whole interface with stage 1 is the DEV_SANDBOX_* environment, asserted +# below: there are no shared functions or variables between the two stages. +# Stage 1 locates this script alongside the other sandbox assets (see +# DEV_SANDBOX_ASSETS in dev-sandbox.sh), so the Nix wrapper's store copy and a +# plain repo checkout both work. + +set -euo pipefail + +: "${DEV_SANDBOX_ROOT:?missing DEV_SANDBOX_ROOT}" +: "${DEV_SANDBOX_BASH:?missing DEV_SANDBOX_BASH}" +: "${DEV_SANDBOX_INTERACTIVE:?missing DEV_SANDBOX_INTERACTIVE}" +: "${DEV_SANDBOX_USER:?missing DEV_SANDBOX_USER}" +: "${DEV_SANDBOX_HOME:?missing DEV_SANDBOX_HOME}" + +# Announce our pid so stage 1 can point slirp4netns at these namespaces, +# then hold until it reports the network is up. +slirp_ready="$DEV_SANDBOX_ROOT/root/logs/slirp.ready" +printf '%s\n' "$$" > "$DEV_SANDBOX_ROOT/root/logs/sandbox.pid" +for _ in $(seq 1 200); do + [ -s "$slirp_ready" ] && break + sleep 0.05 +done +if [ ! -s "$slirp_ready" ]; then + echo 'error: timed out waiting for sandbox network setup' >&2 + cat "$DEV_SANDBOX_ROOT/root/logs/slirp.log" >&2 || true + exit 1 +fi + +# The sandbox HOME is /root for a root install and /home/ for a +# user-level one. Only the latter needs its parent created first; --dir / +# is not a thing bwrap accepts. +home_mounts=() +home_parent="$(dirname "$DEV_SANDBOX_HOME")" +if [ "$home_parent" != / ]; then + home_mounts+=(--dir "$home_parent") +fi +home_mounts+=(--bind "$DEV_SANDBOX_ROOT/home" "$DEV_SANDBOX_HOME") + +node_env=() +if [ -n "${DEV_SANDBOX_NODE_DIR:-}" ]; then + node_env+=(--setenv npm_config_nodedir "$DEV_SANDBOX_NODE_DIR") +fi +electron_env=() +if [ -n "${DEV_SANDBOX_ELECTRON_LD_LIBRARY_PATH:-}" ]; then + electron_env+=( + --setenv LD_LIBRARY_PATH "$DEV_SANDBOX_ELECTRON_LD_LIBRARY_PATH" + --setenv HERMES_DESKTOP_DISABLE_GPU 1 + ) +fi +gui_mounts=() +if [ -n "${DEV_SANDBOX_WAYLAND_SOCKET:-}" ]; then + runtime_dir="${DEV_SANDBOX_XDG_RUNTIME_DIR:?missing DEV_SANDBOX_XDG_RUNTIME_DIR}" + runtime_parent="$(dirname "$runtime_dir")" + runtime_grandparent="$(dirname "$runtime_parent")" + gui_mounts+=( + --dir "$runtime_grandparent" + --dir "$runtime_parent" + --dir "$runtime_dir" + --bind "$DEV_SANDBOX_WAYLAND_SOCKET" "$DEV_SANDBOX_WAYLAND_SOCKET" + --setenv XDG_RUNTIME_DIR "$runtime_dir" + --setenv WAYLAND_DISPLAY "${DEV_SANDBOX_WAYLAND_DISPLAY:?missing DEV_SANDBOX_WAYLAND_DISPLAY}" + ) +fi + +# How the sandbox gets a usable runtime, and where its own shims go. +# +# On Nix, every binary lives under /nix/store, so the sandbox can own /bin, +# /lib64 and /usr/bin outright and fill them with symlinks into the store. +# +# Elsewhere the runtime IS /usr, /bin, /lib, /lib64 -- so binding the +# sandbox's near-empty versions over them hides the real thing, and bwrap +# dies with `execvp /usr/bin/bash: No such file or directory`. Keep the host +# directories read-only and override only the individual files we shim. +# +# The same answer decides how /etc is handled further down. +if [ -d /nix ] && [[ "$(readlink -f "$DEV_SANDBOX_BASH")" == /nix/* ]]; then + USE_HOST_RUNTIME=false +else + USE_HOST_RUNTIME=true +fi + +runtime_mounts=() +shim_mounts=() +if [ "$USE_HOST_RUNTIME" = false ]; then + runtime_mounts+=(--ro-bind /nix /nix) + shim_mounts+=( + --dir /usr + --dir /bin + --dir /lib64 + --bind "$DEV_SANDBOX_ROOT/root/bin" /bin + --bind "$DEV_SANDBOX_ROOT/root/lib64" /lib64 + --bind "$DEV_SANDBOX_ROOT/root/usr/bin" /usr/bin + ) +else + for path in /usr /bin /sbin /lib /lib64; do + [ -e "$path" ] && runtime_mounts+=(--ro-bind "$path" "$path") + done + # The git-upload-pack shim standing in for github.com is the only file that + # must beat the host's copy; sh/ls/env are already there for real. + shim_mounts+=(--bind "$DEV_SANDBOX_ROOT/root/usr/bin/ssh" /usr/bin/ssh) +fi + +# /etc: start from a copy of the host's and overwrite only the files we fake. +# +# Replacing the whole directory with a five-file one is the tempting shortcut +# and it is wrong: a distro puts things under /etc that binaries outside /etc +# depend on, so hiding all of it breaks tools that look fine on PATH. Two real +# examples, both Debian/Ubuntu: openssl's compiled-in openssl.cnf is a symlink +# into /etc/ssl, and /usr/bin/awk is a symlink to /etc/alternatives/awk -- with +# /etc replaced, openssl cannot mint a certificate and awk reports "not found". +# Those are two symptoms of one cause, and nothing says there are only two. +# +# Copying rather than mount-overlaying the individual files, because several of +# these are symlinks in the wild (resolv.conf -> ../run/systemd/... on Ubuntu, +# hosts and nsswitch.conf -> /etc/static/... on NixOS) and bwrap cannot bind a +# file onto a symlink whose target does not exist inside the sandbox. +# +# Symlinks are copied as symlinks, never dereferenced: on NixOS /etc/static +# points into the store and following it would copy gigabytes per sandbox. The +# store is already mounted at /nix on that path, and the host runtime dirs are +# mounted at their own paths, so absolute symlinks still resolve. +# +# The five we override, and why each must differ from the host's: +# passwd, group the sandbox identity, which does not exist on the host +# resolv.conf slirp4netns's DNS, not the host resolver +# nsswitch.conf files+dns only, so nothing consults host NSS modules +# hosts minimal, so no host entry leaks in +# +# os-release is removed rather than replaced. Installers branch on it to reach +# for a package manager -- `install.sh` reads ID from it and, on debian/ubuntu, +# offers to apt-get build tools, prompting on /dev/tty when sudo exists but is +# not passwordless. That prompt cannot be satisfied here (no terminal) and it is +# fatal under `set -e`. Inheriting the host's file would make the sandbox claim +# to be a distro whose package manager it cannot actually use; absent means +# DISTRO="unknown" and the apt path is skipped, which is the truth. +etc_mounts=() +if [ "$USE_HOST_RUNTIME" = true ] && [ -d /etc ]; then + sandbox_etc="$DEV_SANDBOX_ROOT/etc-merged" + rm -rf -- "$sandbox_etc" + mkdir -p "$sandbox_etc" + # -a keeps symlinks as symlinks; unreadable entries (shadow, sudoers) are + # skipped rather than failing the run. + cp -a /etc/. "$sandbox_etc/" 2>/dev/null || true + for etc_file in passwd group resolv.conf nsswitch.conf hosts; do + [ -f "$DEV_SANDBOX_ROOT/etc/$etc_file" ] || continue + rm -f "$sandbox_etc/$etc_file" + cp "$DEV_SANDBOX_ROOT/etc/$etc_file" "$sandbox_etc/$etc_file" + done + rm -f "$sandbox_etc/os-release" "$sandbox_etc/lsb-release" + etc_mounts+=(--ro-bind "$sandbox_etc" /etc) +else + etc_mounts+=(--bind "$DEV_SANDBOX_ROOT/etc" /etc) +fi + +# /dev without a tty, so a script guarding on `[ -e /dev/tty ]` takes its +# no-terminal path. +# +# bwrap's --dev creates a /dev/tty NODE, but nothing in here has a controlling +# terminal, so opening it fails with "No such device or address". That is the +# worst of both: the guard passes and the read then fails. Under `set -e` -- +# which install.sh uses -- a failed read inside a function aborts the whole +# installer, which is exactly how older releases died here while prompting for +# sudo to install ripgrep/ffmpeg. +# +# Making the tty real is not the fix: with an openable terminal that prompt +# blocks forever waiting for input nobody will type. Absent is what a headless +# machine looks like, and what every prompt in here should assume. +# +# --dev cannot be used with the node removed afterwards (bwrap refuses to mount +# a directory over a device node), so /dev is assembled explicitly. +dev_mounts=( + --tmpfs /dev + --dev-bind /dev/null /dev/null + --dev-bind /dev/zero /dev/zero + --dev-bind /dev/full /dev/full + --dev-bind /dev/random /dev/random + --dev-bind /dev/urandom /dev/urandom + --symlink /proc/self/fd /dev/fd + --symlink /proc/self/fd/0 /dev/stdin + --symlink /proc/self/fd/1 /dev/stdout + --symlink /proc/self/fd/2 /dev/stderr +) +if [ "$DEV_SANDBOX_INTERACTIVE" = true ]; then + # An interactive shell is deliberately given a terminal; keep bwrap's /dev. + dev_mounts=(--dev /dev) +fi + +exec bwrap \ + --unshare-pid \ + --die-with-parent --proc /proc --tmpfs /tmp \ + "${dev_mounts[@]}" \ + "${gui_mounts[@]}" \ + "${runtime_mounts[@]}" \ + --bind "$DEV_SANDBOX_ROOT/root" /work \ + "${shim_mounts[@]}" \ + --bind "$DEV_SANDBOX_ROOT/root/usr/local" /usr/local \ + "${home_mounts[@]}" \ + "${etc_mounts[@]}" \ + --chdir /work/repo \ + --clearenv \ + --setenv PATH "$DEV_SANDBOX_HOME/.local/bin:/usr/local/bin:/usr/bin:$PATH" \ + --setenv HOME "$DEV_SANDBOX_HOME" \ + --setenv USER "$DEV_SANDBOX_USER" \ + --setenv LOGNAME "$DEV_SANDBOX_USER" \ + --setenv CURL_CA_BUNDLE /work/certs/ca.pem \ + --setenv SSL_CERT_FILE /work/certs/ca.pem \ + --setenv GIT_SSL_CAINFO /work/certs/ca.pem \ + --setenv NODE_EXTRA_CA_CERTS /work/certs/real-ca.pem \ + --setenv OPENSSL_CONF /work/certs/openssl.cnf \ + --setenv HTTP_PROXY http://127.0.0.1:8080 \ + --setenv HTTPS_PROXY http://127.0.0.1:8080 \ + --setenv ALL_PROXY http://127.0.0.1:8080 \ + --setenv NO_PROXY '' \ + --setenv DEV_SANDBOX_INTERACTIVE "$DEV_SANDBOX_INTERACTIVE" \ + --setenv ELECTRON_DISABLE_SANDBOX 1 \ + "${node_env[@]}" \ + "${electron_env[@]}" \ + -- "$DEV_SANDBOX_BASH" -ceu ' + python3 /work/proxy.py /work/http /work/certs /work/certs/real-ca.pem >/work/logs/proxy.log 2>&1 & + proxy_pid=$! + cleanup() { + kill "$proxy_pid" 2>/dev/null || true + wait "$proxy_pid" 2>/dev/null || true + } + trap cleanup EXIT INT TERM + # Bash opens /dev/tcp itself, so the readiness probe needs no netcat -- + # one less binary the sandbox has to find on the host (GitHub runners + # ship no `nc`). + proxy_up() { (exec 3<>/dev/tcp/127.0.0.1/8080) 2>/dev/null; } + for _ in $(seq 1 100); do + proxy_up && break + sleep 0.05 + done + if ! proxy_up; then + echo "error: the sandbox fake-internet proxy never came up" >&2 + cat /work/logs/proxy.log >&2 || true + exit 1 + fi + "$@" + ' sandbox-command "$@"