diff --git a/hermes_cli/main.py b/hermes_cli/main.py index 624d0f10109f7..3d58833496c89 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -6876,6 +6876,9 @@ def _desktop_linux_needs_no_sandbox() -> bool: unprivileged desktop user on an AppArmor-restricted host. The root case should remain an explicit user choice. """ + if os.environ.get("ELECTRON_DISABLE_SANDBOX", 0) == "1": + return True + if sys.platform != "linux": return False if hasattr(os, "geteuid") and os.geteuid() == 0: diff --git a/nix/devShell.nix b/nix/devShell.nix index ac50beb0e699e..d1c0875045506 100644 --- a/nix/devShell.nix +++ b/nix/devShell.nix @@ -27,10 +27,7 @@ mkdir -p $out/bin install -Dm755 ${../hermes} $out/bin/hermes '') - (pkgs.runCommand "dev-sandbox" { } '' - mkdir -p $out/bin - install -Dm755 ${../scripts/dev-sandbox.sh} $out/bin/sandbox - '') + self'.packages.sandbox uv # Headless Wayland compositor for E2E tests (test:e2e:visual). # cage renders a single client with no window management, so diff --git a/nix/packages.nix b/nix/packages.nix index 5e4d32404f1ff..a496f73a3eeb2 100644 --- a/nix/packages.nix +++ b/nix/packages.nix @@ -9,6 +9,9 @@ ... }: let + + sandbox = pkgs.callPackage ./sandbox.nix { }; + minimal = pkgs.callPackage ./hermes-agent.nix { inherit (inputs) uv2nix pyproject-nix pyproject-build-systems; npm-lockfile-fix = inputs'.npm-lockfile-fix.packages.default; @@ -51,6 +54,8 @@ }).node-gyp; default = full; + inherit sandbox; + inherit minimal; # Ships discord.py + python-telegram-bot + slack-sdk so a plain diff --git a/nix/sandbox.nix b/nix/sandbox.nix new file mode 100644 index 0000000000000..cec5dc9868079 --- /dev/null +++ b/nix/sandbox.nix @@ -0,0 +1,124 @@ +{ + # electron deps + alsa-lib, + at-spi2-atk, + atk, + cairo, + cups, + dbus, + expat, + fontconfig, + freetype, + glib, + gtk3, + libdrm, + libgbm, + libxkbcommon, + mesa, + nspr, + nss, + pango, + systemd, + libX11, + libXcomposite, + libXdamage, + libXext, + libXfixes, + libXrandr, + libXrender, + libXtst, + libxcb, + + # sandbox deps + bash, + bubblewrap, + cacert, + coreutils, + curl, + gawk, + git, + glibc, + gnumake, + gnugrep, + gnused, + gzip, + nodejs_22, + openssl, + python3, + slirp4netns, + stdenv, + gnutar, + util-linux, + + # etc + writeShellApplication, + lib, +}: +let + electronRuntime = [ + alsa-lib + at-spi2-atk + atk + cairo + cups + dbus + expat + fontconfig + freetype + glib + gtk3 + libdrm + libgbm + libxkbcommon + mesa + nspr + nss + pango + systemd + libX11 + libXcomposite + libXdamage + libXext + libXfixes + libXrandr + libXrender + libXtst + libxcb + ]; +in +writeShellApplication { + name = "sandbox"; + runtimeInputs = [ + bash + bubblewrap + cacert + coreutils + curl + gawk + git + glibc.bin + gnumake + gnugrep + gnused + gzip + nodejs_22 + openssl + python3 + slirp4netns + stdenv.cc + gnutar + util-linux + ] + ++ electronRuntime; + text = '' + export DEV_SANDBOX_REAL_CA_CERT=${cacert}/etc/ssl/certs/ca-bundle.crt + export DEV_SANDBOX_DYNAMIC_LINKER=${stdenv.cc.bintools.dynamicLinker} + export DEV_SANDBOX_NODE_DIR=${nodejs_22} + export DEV_SANDBOX_ELECTRON_LD_LIBRARY_PATH=${lib.makeLibraryPath electronRuntime} + # The script is imported into the store as a single file, so its own + # directory has no scripts/sandbox/ beside it. Point it at the assets + # (fake-internet proxy, ssh shim) explicitly. + export DEV_SANDBOX_ASSETS=${../scripts/sandbox} + exec ${../scripts/dev-sandbox.sh} "$@" + ''; +} diff --git a/scripts/dev-sandbox.sh b/scripts/dev-sandbox.sh index 5ce459c94e9ab..dca11a72f3aa0 100755 --- a/scripts/dev-sandbox.sh +++ b/scripts/dev-sandbox.sh @@ -1,198 +1,591 @@ #!/usr/bin/env bash -# Run a Hermes instance in an isolated sandbox — separate HERMES_HOME, -# separate Electron userData, and a distinct Desktop app name so it doesn't compete -# with your main desktop instance's single-instance lock. +# Run a command in a disposable, network-isolated fake Internet. # -# By default the sandbox is throwaway: a temp dir is created and removed on -# exit. Use --persistent to keep the sandbox across restarts (stored under -# .hermes-sandbox/ in the worktree git root). -# -# Usage: -# scripts/dev-sandbox.sh python -m hermes_cli.main -# scripts/dev-sandbox.sh hermes desktop -# scripts/dev-sandbox.sh electron . -# scripts/dev-sandbox.sh -- npm run dev # from apps/desktop/ -# scripts/dev-sandbox.sh --persistent hermes desktop -# scripts/dev-sandbox.sh --persistent -- npm run dev -# -# Seed the sandbox HERMES_HOME from an existing directory (e.g. your main -# ~/.hermes) so config, sessions, skills, etc. are pre-populated: -# scripts/dev-sandbox.sh --from ~/.hermes hermes desktop -# -# Override the app name (default: HermesSandbox): -# HERMES_DEV_SANDBOX_NAME=Staging scripts/dev-sandbox.sh hermes desktop -# -# Override the persistent sandbox dir name (default: .hermes-sandbox): -# HERMES_DEV_SANDBOX_DIR=.staging-sandbox scripts/dev-sandbox.sh --persistent hermes desktop +# The command runs in private user, mount, PID, and network namespaces. This +# script is stage 1: it builds the sandbox tree, mints the fake CA, and creates +# the user+network namespaces with `unshare` (see the namespace plan further +# down), then re-execs into scripts/sandbox/stage2-run.sh, which adds the +# mount/pid namespaces with bubblewrap and runs the payload. Its only writable +# filesystem is SANDBOX_ROOT. HTTP(S) goes to a local static MITM proxy; +# github.com SSH uses a sandbox-local git-upload-pack shim; neither transport +# can reach the host network. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Helper files the sandbox needs: the stage-2 script it re-execs into, plus the +# files it copies in (the fake-internet proxy, the ssh shim, the openssl config). +# They sit next to this script in the repo, but the Nix wrapper installs the +# script into the store on its own, so it exports DEV_SANDBOX_ASSETS to point +# here. +SANDBOX_ASSETS="${DEV_SANDBOX_ASSETS:-$SCRIPT_DIR/sandbox}" +for asset in proxy.py ssh-shim.sh openssl.cnf stage2-run.sh; do + [ -f "$SANDBOX_ASSETS/$asset" ] || { + echo "error: missing sandbox asset: $SANDBOX_ASSETS/$asset" >&2 + exit 1 + } +done + print_help() { cat <<'EOF' -Usage: dev-sandbox.sh [--persistent] [--from DIR] [--] +Usage: dev-sandbox.sh [options] [--] + dev-sandbox.sh install [options] [--] [installer arguments...] -Run a Hermes instance in an isolated sandbox. +Run COMMAND in a throwaway chroot-like bubblewrap sandbox. The sandbox has no +writable host mounts: only its own root, mounted at /work, is writable. Options: - --persistent Keep the sandbox dir across restarts (under the worktree - git root, in .hermes-sandbox/). Without this flag the - sandbox is a temp dir that is removed on exit. - --from DIR Copy DIR into the sandbox HERMES_HOME as the starting - point (config, sessions, skills, etc.). - Ignored if the sandbox HERMES_HOME already has content - (e.g. reusing a --persistent sandbox) to avoid clobbering. - --delete Delete the existing persistent sandbox in .hermes-sandbox. - -h, --help Show this help message. + --persistent Keep the whole sandbox under .hermes-sandbox/. + --delete Delete the persistent sandbox (asks first). + --root Install as uid 0 with the root FHS layout: code in + /usr/local/lib/hermes-agent, command in + /usr/local/bin. Default is the user-level layout. + --from DIR One-time copy of DIR into the sandbox's $HOME. + Existing persistent sandboxes are never overwritten. + --http-root DIR Copy DIR into the fake web server root for this run. + Requests map to DIR//; no URL is forwarded. + --installer PATH With `install`, serve PATH at the canonical install.sh + URL. Default: scripts/install.sh in this worktree. + --from-main With `install`, fetch the real upstream main installer + and repository, then advance fake main to this folder + after a successful install for update testing. + Shorthand for --install-ref refs/heads/main. + --install-ref REF Like --from-main, but installs REF instead of main: + a branch, a tag (v2026.7.7), or a SHA reachable from main. + Use it to test updating from an older release, not just + from the tip. + -h, --help Show this help. + +Option order matters: every option above is consumed by THIS script, and +parsing stops at the first argument it does not recognize. Everything from +that point on is passed through to the command (or, with `install`, to the +installer). Put sandbox options first and separate installer arguments with +`--`, otherwise they arrive here and fail: + + # WRONG — --from-main reaches install.sh, which rejects it + scripts/dev-sandbox.sh install --skip-setup --from-main + + # RIGHT + scripts/dev-sandbox.sh install --from-main -- --skip-setup + +Install layout: `install.sh` picks its layout from `id -u` alone, so uid is what +separates the two real-world Linux installs. By default the sandbox runs as an +unprivileged `hermes` user, giving the layout most people have — +$HERMES_HOME/hermes-agent plus a ~/.local/bin launcher. Pass --root for the FHS +one. Both are worth testing; they differ in more than paths (root also relocates +uv's Python to /usr/local/share for world-readability). + +The fake web server signs certificates with a CA trusted only inside this +sandbox. HTTP_PROXY/HTTPS_PROXY send fixture URLs there first; other HTTP(S) +requests pass through the sandbox's rootless outbound network. SSH to github.com +runs a sandbox-local upload-pack shim, never your SSH config, agent, +known-hosts file, or authorized keys. + +Fake github main always comes from this folder. If it has staged, unstaged, or +non-ignored untracked changes, the sandbox warns and creates a temporary local +commit containing them; it never stages or commits the real worktree. Environment: - HERMES_DEV_SANDBOX_NAME Override the app name (default: HermesSandbox) - HERMES_DEV_SANDBOX_DIR Override the persistent dir name (default: .hermes-sandbox) + HERMES_DEV_SANDBOX_DIR Sandbox directory name, relative to the repo root + (default: .hermes-sandbox). Examples: - dev-sandbox.sh hermes desktop - dev-sandbox.sh --persistent hermes desktop - dev-sandbox.sh --from ~/.hermes hermes desktop - dev-sandbox.sh -- npm run dev + # create a sandbox, install this branch as `main`, and then drop to a shell, + # skipping `hermes setup` & the browser tools for speed. + scripts/dev-sandbox.sh install --persistent -- --skip-setup --skip-browser + + # Install the official upstream main. You're dropped into a shell where + # you can run `hermes update`. + scripts/dev-sandbox.sh install --persistent --from-main + EOF } PERSISTENT=false DELETE=false +RUN_AS_USER=true SEED_DIR="" +HTTP_ROOT="" +INSTALL_SHORTCUT=false +INSTALLER_PATH="" +# Which upstream commit the sandbox installs before the update routes run. +# Empty means "install this worktree's own installer" (no upstream fetch); set, +# it is anything git can resolve -- a branch, a tag (v2026.7.7), or a SHA +# reachable from main -- so "can a user two releases back still update?" is +# expressible. --from-main is shorthand for refs/heads/main. +INSTALL_REF="" +UPSTREAM_URL="${HERMES_DEV_SANDBOX_UPSTREAM:-https://github.com/NousResearch/hermes-agent.git}" + +if [ "${1:-}" = install ]; then + INSTALL_SHORTCUT=true + shift +fi while [ "$#" -gt 0 ]; do case "$1" in - --persistent) - PERSISTENT=true - shift - ;; + --persistent) PERSISTENT=true; shift ;; + --delete) DELETE=true; shift ;; + --root) RUN_AS_USER=false; shift ;; + --user) RUN_AS_USER=true; shift ;; # the default; accepted for symmetry --from) - if [ "$#" -lt 2 ] || [[ "$2" == -* ]]; then - echo "error: --from requires a directory argument" >&2 - exit 1 - fi - SEED_DIR="$2" - shift 2 - ;; - --from=*) - SEED_DIR="${1#--from=}" - if [ -z "$SEED_DIR" ]; then - echo "error: --from requires a directory argument" >&2 - exit 1 - fi - shift - ;; - --delete) - DELETE=true - shift - ;; - -h|--help) - print_help - exit 0 - ;; - --) - shift - break - ;; - *) - break - ;; + [ "$#" -ge 2 ] || { echo 'error: --from needs a directory' >&2; exit 1; } + SEED_DIR="$2"; shift 2 ;; + --http-root) + [ "$#" -ge 2 ] || { echo 'error: --http-root needs a directory' >&2; exit 1; } + HTTP_ROOT="$2"; shift 2 ;; + --installer) + [ "$#" -ge 2 ] || { echo 'error: --installer needs a file' >&2; exit 1; } + INSTALLER_PATH="$2"; shift 2 ;; + --from-main) INSTALL_REF="refs/heads/main"; shift ;; + --install-ref) + [ "$#" -ge 2 ] || { echo 'error: --install-ref needs a value' >&2; exit 1; } + INSTALL_REF="$2" + shift 2 ;; + --from=*|--http-root=*|--installer=*|--install-ref=*) + key="${1%%=*}"; value="${1#*=}" + [ -n "$value" ] || { echo "error: $key needs a value" >&2; exit 1; } + case "$key" in + --from) SEED_DIR="$value" ;; + --http-root) HTTP_ROOT="$value" ;; + --installer) INSTALLER_PATH="$value" ;; + --install-ref) INSTALL_REF="$value" ;; + esac + shift ;; + -h|--help) print_help; exit 0 ;; + --) shift; break ;; + *) break ;; esac done -if [ -n "$SEED_DIR" ]; then - if [ ! -d "$SEED_DIR" ]; then - echo "error: --from dir '$SEED_DIR' does not exist" >&2 - exit 1 - fi - # Resolve to absolute path so it's valid after we cd later. - SEED_DIR="$(cd "$SEED_DIR" && pwd)" -fi - -if [ "$#" -eq 0 ]; then +if [ "$INSTALL_SHORTCUT" = false ] && [ "$#" -eq 0 ]; then print_help >&2 exit 1 fi +if [ -n "$INSTALLER_PATH" ] && [ "$INSTALL_SHORTCUT" = false ]; then + echo 'error: --installer is only valid with the install shortcut' >&2 + exit 1 +fi +if [ -n "$INSTALL_REF" ] && [ "$INSTALL_SHORTCUT" = false ]; then + echo 'error: --from-main / --install-ref are only valid with the install shortcut' >&2 + exit 1 +fi +if [ -n "$INSTALL_REF" ] && [ -n "$INSTALLER_PATH" ]; then + echo 'error: --from-main / --install-ref cannot be combined with --installer' >&2 + exit 1 +fi -SANDBOX_DIR_NAME="${HERMES_DEV_SANDBOX_DIR:-.hermes-sandbox}" -GIT_ROOT="$(git rev-parse --show-toplevel 2>/dev/null || echo "$SCRIPT_DIR/..")" +for dir in "$SEED_DIR" "$HTTP_ROOT"; do + [ -z "$dir" ] || [ -d "$dir" ] || { echo "error: directory '$dir' does not exist" >&2; exit 1; } +done + +GIT_ROOT="${HERMES_SANDBOX_SOURCE_ROOT:-$(git rev-parse --show-toplevel)}" GIT_ROOT="$(cd "$GIT_ROOT" && pwd)" -PERSISTENT_SANDBOX_ROOT="$GIT_ROOT/$SANDBOX_DIR_NAME" +if [ "$INSTALL_SHORTCUT" = true ] && [ -z "$INSTALL_REF" ] && [ -z "$INSTALLER_PATH" ]; then + INSTALLER_PATH="$GIT_ROOT/scripts/install.sh" +fi +if [ -n "$INSTALLER_PATH" ] && [ ! -f "$INSTALLER_PATH" ]; then + echo "error: installer '$INSTALLER_PATH' does not exist" >&2 + exit 1 +fi +COMMIT="$(git -C "$GIT_ROOT" rev-parse --verify 'HEAD^{commit}')" || { + echo "error: current folder has no HEAD commit" >&2 + exit 1 +} +SANDBOX_DIR_NAME="${HERMES_DEV_SANDBOX_DIR:-.hermes-sandbox}" +PERSISTENT_ROOT="$GIT_ROOT/$SANDBOX_DIR_NAME" if [ "$DELETE" = true ]; then - if [ -d "$PERSISTENT_SANDBOX_ROOT" ]; then - read -r -p "[sandbox] delete $PERSISTENT_SANDBOX_ROOT? [y/N] " REPLY - case "$REPLY" in - [yY]|[yY][eE][sS]) - echo "[sandbox] deleting $PERSISTENT_SANDBOX_ROOT" >&2 - rm -rf -- "$PERSISTENT_SANDBOX_ROOT" - ;; - *) - echo "[sandbox] aborted" >&2 - exit 1 - ;; - esac - else - echo "[sandbox] nothing to delete at $PERSISTENT_SANDBOX_ROOT" >&2 + if [ ! -d "$PERSISTENT_ROOT" ]; then + echo "[sandbox] nothing to delete at $PERSISTENT_ROOT" >&2 + exit 0 fi + read -r -p "[sandbox] delete $PERSISTENT_ROOT? [y/N] " reply + case "$reply" in + y|Y|yes|YES) rm -rf -- "$PERSISTENT_ROOT" ;; + *) echo '[sandbox] aborted' >&2; exit 1 ;; + esac exit 0 fi -# Derive a per-worktree app name so multiple checkouts don't collide. -# Each worktree has its own toplevel path even though they share one repo, -# so we hash that path into a short, stable suffix. -WORKTREE_ROOT="$(git rev-parse --show-toplevel 2>/dev/null || echo "$SCRIPT_DIR/..")" -WORKTREE_ROOT="$(cd "$WORKTREE_ROOT" && pwd)" -WORKTREE_HASH="$(printf '%s' "$WORKTREE_ROOT" | cksum | cut -d' ' -f1)" -WORKTREE_NAME="$(basename "$WORKTREE_ROOT")" -DEFAULT_SANDBOX_NAME="HermesSandbox-${WORKTREE_NAME}-${WORKTREE_HASH}" - -SANDBOX_NAME="${HERMES_DEV_SANDBOX_NAME:-$DEFAULT_SANDBOX_NAME}" - if [ "$PERSISTENT" = true ]; then - SANDBOX_ROOT="$PERSISTENT_SANDBOX_ROOT" + SANDBOX_ROOT="$PERSISTENT_ROOT" else SANDBOX_ROOT="$(mktemp -d -t hermes-sandbox.XXXXXX)" + cleanup() { chmod -R u+w "$SANDBOX_ROOT"; rm -rf -- "$SANDBOX_ROOT"; } + trap cleanup EXIT INT TERM fi -export HERMES_HOME="$SANDBOX_ROOT/hermes-home" -export HERMES_DESKTOP_USER_DATA_DIR="$SANDBOX_ROOT/user-data" -export HERMES_DESKTOP_APP_NAME="$SANDBOX_NAME" - -mkdir -p "$HERMES_HOME" "$HERMES_DESKTOP_USER_DATA_DIR" - -if [ -n "$SEED_DIR" ]; then - # Only seed when the sandbox HERMES_HOME is empty — avoids clobbering an - # existing persistent sandbox on re-run. - if [ -z "$(ls -A "$HERMES_HOME" 2>/dev/null)" ]; then - echo "[sandbox] seeding HERMES_HOME from $SEED_DIR" >&2 - cp -a "$SEED_DIR/." "$HERMES_HOME/" +mkdir -p "$SANDBOX_ROOT"/{root,home,etc} +UPSTREAM_REPO="" +UPSTREAM_COMMIT="" +if [ -n "$INSTALL_REF" ]; then + echo "[sandbox] fetching upstream $INSTALL_REF for installer/update test" >&2 + UPSTREAM_REPO="$(mktemp -d -t hermes-sandbox-upstream.XXXXXX)" + git -C "$UPSTREAM_REPO" init -q + # Fetch the ref as given. A branch or tag name resolves on its own; a raw SHA + # needs the remote to allow fetching it directly, so fall back to fetching + # main and resolving the SHA locally (which works for any commit that is an + # ancestor of main -- the interesting case for "update from N versions ago"). + # + # Peel to ^{commit} in both cases: an annotated tag fetches as a tag OBJECT, + # and using it directly fails later with "trying to write non-commit object + # ... to branch 'refs/heads/main'". + if git -C "$UPSTREAM_REPO" fetch -q "$UPSTREAM_URL" "$INSTALL_REF" 2>/dev/null; then + UPSTREAM_COMMIT="$(git -C "$UPSTREAM_REPO" rev-parse "FETCH_HEAD^{commit}")" + elif git -C "$UPSTREAM_REPO" fetch -q "$UPSTREAM_URL" refs/heads/main \ + && UPSTREAM_COMMIT="$(git -C "$UPSTREAM_REPO" rev-parse --verify -q "$INSTALL_REF^{commit}")"; then + : else - echo "[sandbox] --from ignored: $HERMES_HOME already has content" >&2 + rm -rf -- "$UPSTREAM_REPO" + echo "error: could not resolve upstream ref: $INSTALL_REF" >&2 + echo ' Use a branch (main), a tag (v2026.7.7), or a SHA reachable from main.' >&2 + exit 1 fi fi +if [ ! -e "$SANDBOX_ROOT/root/repo/.sandbox-source" ]; then + mkdir -p "$SANDBOX_ROOT/root/repo" + # Persistent roots live under the worktree, so copying with cp would recurse + # into the sandbox itself. tar also lets us exclude a worktree's .git file, + # which can point at the host's shared worktree metadata. + tar -C "$GIT_ROOT" --exclude='./.git' --exclude="./$SANDBOX_DIR_NAME" -cf - . \ + | tar -C "$SANDBOX_ROOT/root/repo" -xf - + : > "$SANDBOX_ROOT/root/repo/.sandbox-source" +fi -echo "[sandbox] HERMES_HOME=$HERMES_HOME" >&2 -echo "[sandbox] userData=$HERMES_DESKTOP_USER_DATA_DIR" >&2 -echo "[sandbox] appName=$HERMES_DESKTOP_APP_NAME" >&2 -if [ "$PERSISTENT" = true ]; then - echo "[sandbox] persistent: $SANDBOX_ROOT" >&2 +if [ -n "$SEED_DIR" ] && [ ! -e "$SANDBOX_ROOT/.seeded" ]; then + echo "[sandbox] seeding home from $SEED_DIR" >&2 + cp -a "$SEED_DIR/." "$SANDBOX_ROOT/home/" + : > "$SANDBOX_ROOT/.seeded" +fi + +rm -rf "$SANDBOX_ROOT/root/http" +mkdir -p "$SANDBOX_ROOT/root/http" +if [ -n "$HTTP_ROOT" ]; then + cp -a "$HTTP_ROOT/." "$SANDBOX_ROOT/root/http/" +fi +if [ "$INSTALL_SHORTCUT" = true ]; then + mkdir -p "$SANDBOX_ROOT/root/http/hermes-agent.nousresearch.com" + if [ -n "$INSTALL_REF" ]; then + git -C "$UPSTREAM_REPO" show "$UPSTREAM_COMMIT:scripts/install.sh" \ + > "$SANDBOX_ROOT/root/http/hermes-agent.nousresearch.com/install.sh" + else + cp -a "$INSTALLER_PATH" "$SANDBOX_ROOT/root/http/hermes-agent.nousresearch.com/install.sh" + fi + set -- bash -c ' + set +e + curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash -s -- "$@" + install_status=$? + if [ "$install_status" -eq 0 ] && [ -f /work/promote-main ]; then + next_main=$(cat /work/promote-main) + if git --git-dir=/work/repos/hermes-agent.git update-ref refs/heads/main "$next_main"; then + rm -f /work/promote-main + printf "[sandbox] fake main advanced to this folder for update testing\n" >&2 + else + printf "[sandbox] failed to advance fake main after install\n" >&2 + install_status=1 + fi + fi + if [ "$DEV_SANDBOX_INTERACTIVE" = true ]; then + printf "\n[sandbox] installer exited %s; entering sandbox shell\n" "$install_status" >&2 + exec /dev/tty 2>&1 + exec bash -i + fi + exit "$install_status" + ' sandbox-installer "$@" +fi + +mkdir -p "$SANDBOX_ROOT/root"/{bin,certs,lib64,logs,repos,ssh,usr/bin,usr/local} +REAL_CA_CERT="${DEV_SANDBOX_REAL_CA_CERT:-}" +if [ -z "$REAL_CA_CERT" ]; then + for candidate in /etc/ssl/certs/ca-certificates.crt /etc/ssl/cert.pem; do + if [ -f "$candidate" ]; then + REAL_CA_CERT="$candidate" + break + fi + done +fi +if [ ! -f "$REAL_CA_CERT" ]; then + echo 'error: no system CA bundle found for outbound sandbox HTTPS' >&2 + exit 1 +fi +if [ ! -f "$SANDBOX_ROOT/root/certs/real-ca.pem" ]; then + cp "$REAL_CA_CERT" "$SANDBOX_ROOT/root/certs/real-ca.pem" +fi +printf 'nameserver 10.0.2.3\n' > "$SANDBOX_ROOT/etc/resolv.conf" +SANDBOX_SHELL="$(command -v bash)" +DYNAMIC_LINKER="${DEV_SANDBOX_DYNAMIC_LINKER:-}" +if [ -z "$DYNAMIC_LINKER" ]; then + # Nix store first: NixOS also ships a /lib64/ld-linux-x86-64.so.2 compat stub, + # so probing FHS paths first would quietly switch which loader a bare script + # invocation uses on this host. Globs that match nothing expand to themselves, + # so every candidate is -f tested. The FHS paths cover Debian/Ubuntu (where + # the loader is under /lib64 or a multiarch /lib dir), which is what CI runs. + for candidate in \ + /nix/store/*-glibc-*/lib/ld-linux-*.so.* \ + /lib64/ld-linux-x86-64.so.2 \ + /lib/ld-linux-aarch64.so.1 \ + /lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 \ + /lib/aarch64-linux-gnu/ld-linux-aarch64.so.1 + do + if [ -f "$candidate" ]; then + DYNAMIC_LINKER="$candidate" + break + fi + done +fi +if [ ! -f "$DYNAMIC_LINKER" ]; then + echo 'error: no glibc dynamic linker found for sandboxed release binaries' >&2 + echo ' Set DEV_SANDBOX_DYNAMIC_LINKER to its path.' >&2 + exit 1 +fi +ln -sf "$SANDBOX_SHELL" "$SANDBOX_ROOT/root/bin/sh" +ln -sf "$(command -v ls)" "$SANDBOX_ROOT/root/bin/ls" +ln -sf "$(command -v env)" "$SANDBOX_ROOT/root/usr/bin/env" +ln -sf "$DYNAMIC_LINKER" "$SANDBOX_ROOT/root/lib64/$(basename "$DYNAMIC_LINKER")" +# Identity inside the sandbox. install.sh chooses its layout from `id -u` +# alone (see resolve_install_layout), so the uid here is what decides between +# the root FHS install and a user-level one. +if [ "$RUN_AS_USER" = true ]; then + SANDBOX_UID=1000 + SANDBOX_GID=1000 + SANDBOX_USER=hermes + SANDBOX_HOME=/home/hermes else - echo "[sandbox] ephemeral (will be cleaned up on exit)" >&2 + SANDBOX_UID=0 + SANDBOX_GID=0 + SANDBOX_USER=root + SANDBOX_HOME=/root +fi +{ + printf 'root:x:0:0:Sandbox Root:/root:%s\n' "$SANDBOX_SHELL" + if [ "$RUN_AS_USER" = true ]; then + printf '%s:x:%s:%s:Sandbox User:%s:%s\n' \ + "$SANDBOX_USER" "$SANDBOX_UID" "$SANDBOX_GID" "$SANDBOX_HOME" "$SANDBOX_SHELL" + fi +} > "$SANDBOX_ROOT/etc/passwd" +{ + printf 'root:x:0:\n' + if [ "$RUN_AS_USER" = true ]; then + printf '%s:x:%s:\n' "$SANDBOX_USER" "$SANDBOX_GID" + fi +} > "$SANDBOX_ROOT/etc/group" +# A user-level install writes the `hermes` launcher to ~/.local/bin and the +# checkout to $HERMES_HOME; both live under the sandbox HOME, which is bound +# from $SANDBOX_ROOT/home. bwrap maps our real uid to $SANDBOX_UID, so the +# host-side ownership of that directory is what the sandbox sees as its own. +printf 'hosts: files dns\n' > "$SANDBOX_ROOT/etc/nsswitch.conf" +printf '127.0.0.1 localhost\n' > "$SANDBOX_ROOT/etc/hosts" + +SOURCE_REPO="$GIT_ROOT" +SOURCE_REF="$COMMIT" +SNAPSHOT_REPO="" +FAKE_REPO="$SANDBOX_ROOT/root/repos/hermes-agent.git" +git -C "$SANDBOX_ROOT/root/repos" init --bare -q hermes-agent.git +if [ -n "$INSTALL_REF" ]; then + git --git-dir="$FAKE_REPO" fetch -q --force "$UPSTREAM_REPO" \ + "$UPSTREAM_COMMIT:refs/heads/main" +fi +if [ -n "$(git -C "$GIT_ROOT" status --porcelain)" ]; then + echo '[sandbox] warning: current folder is dirty; creating a temporary fake commit for main' >&2 + SNAPSHOT_REPO="$(mktemp -d -t hermes-sandbox-snapshot.XXXXXX)" + git -C "$SNAPSHOT_REPO" init -q + git -C "$SNAPSHOT_REPO" fetch -q "$GIT_ROOT" "$COMMIT" + git -C "$SNAPSHOT_REPO" config user.name 'Hermes sandbox' + git -C "$SNAPSHOT_REPO" config user.email 'sandbox@invalid' + GIT_DIR="$SNAPSHOT_REPO/.git" GIT_WORK_TREE="$GIT_ROOT" git read-tree "$COMMIT" + GIT_DIR="$SNAPSHOT_REPO/.git" GIT_WORK_TREE="$GIT_ROOT" \ + git add -A -- . + SNAPSHOT_TREE="$(GIT_DIR="$SNAPSHOT_REPO/.git" git write-tree)" + SNAPSHOT_PARENT="$COMMIT" + if EXISTING_MAIN="$(git --git-dir="$FAKE_REPO" rev-parse --verify refs/heads/main 2>/dev/null)"; then + git -C "$SNAPSHOT_REPO" fetch -q "$FAKE_REPO" "$EXISTING_MAIN" + SNAPSHOT_PARENT="$EXISTING_MAIN" + fi + SOURCE_REF="$(GIT_DIR="$SNAPSHOT_REPO/.git" git commit-tree "$SNAPSHOT_TREE" -p "$SNAPSHOT_PARENT" \ + -m 'sandbox snapshot of dirty worktree')" + SOURCE_REPO="$SNAPSHOT_REPO" fi -if [ "$PERSISTENT" = false ]; then - cleanup() { - chmod -R u+w "$SANDBOX_ROOT" - rm -rf -- "$SANDBOX_ROOT" +if [ -n "$INSTALL_REF" ]; then + git --git-dir="$FAKE_REPO" fetch -q --force "$SOURCE_REPO" \ + "$SOURCE_REF:refs/hermes-sandbox/next" + printf '%s\n' "$SOURCE_REF" > "$SANDBOX_ROOT/root/promote-main" +else + git --git-dir="$FAKE_REPO" fetch -q --force "$SOURCE_REPO" \ + "$SOURCE_REF:refs/heads/main" +fi +git --git-dir="$FAKE_REPO" symbolic-ref HEAD refs/heads/main +if [ -n "$SNAPSHOT_REPO" ]; then + # Best-effort: it is a mktemp directory the OS will reap, and failing the whole + # run over a leftover object file would be worse than leaking it. Concurrent + # git activity in the worktree can still be writing here as we delete. + rm -rf -- "$SNAPSHOT_REPO" 2>/dev/null || true +fi +if [ -n "$UPSTREAM_REPO" ]; then + rm -rf -- "$UPSTREAM_REPO" +fi + +# openssl reads a config even for `req -addext`, and its compiled-in path is a +# symlink into /etc/ssl on Debian/Ubuntu -- which the sandbox replaces. Ship our +# own and point OPENSSL_CONF at it, both here and inside the sandbox. +cp "$SANDBOX_ASSETS/openssl.cnf" "$SANDBOX_ROOT/root/certs/openssl.cnf" + +if [ ! -f "$SANDBOX_ROOT/root/certs/ca.pem" ]; then + if ! ca_error="$(OPENSSL_CONF="$SANDBOX_ROOT/root/certs/openssl.cnf" \ + openssl req -x509 -newkey rsa:2048 -nodes -days 2 \ + -subj '/CN=Hermes dev sandbox CA' \ + -extensions sandbox_ca_ext \ + -keyout "$SANDBOX_ROOT/root/certs/ca.key" \ + -out "$SANDBOX_ROOT/root/certs/ca.pem" 2>&1 >/dev/null)"; then + echo 'error: could not create the sandbox CA:' >&2 + printf '%s\n' "$ca_error" >&2 + exit 1 + fi +fi +GIT_UPLOAD_PACK="$(command -v git-upload-pack)" +sed "s|@GIT_UPLOAD_PACK@|$GIT_UPLOAD_PACK|" "$SANDBOX_ASSETS/ssh-shim.sh" \ + > "$SANDBOX_ROOT/root/usr/bin/ssh" +chmod 700 "$SANDBOX_ROOT/root/usr/bin/ssh" + +# The fake-internet proxy and the ssh shim are real files under +# scripts/sandbox/ rather than heredocs, so they can be linted, syntax-checked +# and diffed like any other source. Copy them into the sandbox tree. +cp "$SANDBOX_ASSETS/proxy.py" "$SANDBOX_ROOT/root/proxy.py" + +if [ -n "$INSTALL_REF" ]; then + echo "[sandbox] fake main: upstream $INSTALL_REF ($UPSTREAM_COMMIT)" >&2 + echo "[sandbox] prepared update: current folder ($SOURCE_REF)" >&2 +else + echo "[sandbox] fake main: current folder ($SOURCE_REF)" >&2 +fi +echo "[sandbox] root: $SANDBOX_ROOT" >&2 +echo "[sandbox] http root: $SANDBOX_ROOT/root/http" >&2 +if [ "$RUN_AS_USER" = true ]; then + echo "[sandbox] identity: $SANDBOX_USER (uid $SANDBOX_UID) — installs are user-level under $SANDBOX_HOME" >&2 +else + echo '[sandbox] identity: root (uid 0) — installs use the /usr/local FHS layout' >&2 +fi +[ "$PERSISTENT" = true ] && echo '[sandbox] persistent' >&2 || echo '[sandbox] ephemeral' >&2 + +for command in awk bash bwrap curl git openssl python3 slirp4netns tar unshare; do + command -v "$command" >/dev/null || { + echo "error: missing required command: $command" >&2 + exit 1 } - trap cleanup EXIT - trap 'cleanup; exit 130' INT TERM +done + +INTERACTIVE=false +if [ -t 0 ] && [ -t 1 ]; then + INTERACTIVE=true +fi +NODE_DIR="${DEV_SANDBOX_NODE_DIR:-}" +if [ -z "$NODE_DIR" ] && command -v node >/dev/null; then + NODE_DIR="$(dirname "$(dirname "$(command -v node)")")" +fi +WAYLAND_SOCKET="" +if [ -n "${XDG_RUNTIME_DIR:-}" ] && [ -n "${WAYLAND_DISPLAY:-}" ] \ + && [ -S "$XDG_RUNTIME_DIR/$WAYLAND_DISPLAY" ]; then + WAYLAND_SOCKET="$XDG_RUNTIME_DIR/$WAYLAND_DISPLAY" fi -"$@" -rc=$? -exit $rc +# Namespace plan (stage 1 -> stage 2). +# +# slirp4netns joins the target's userns and setuids to root before configuring +# the netns, so the userns MUST map a uid 0. bwrap's own --unshare-user maps +# exactly one uid, so it cannot both run the payload as uid 1000 and offer slirp +# a root to become: that combination fails with +# setns(CLONE_NEWNET): Operation not permitted. +# +# So stage 1 builds the namespaces here with two ranges: +# inner 0 <- a subuid, unused by the payload, present only so slirp can +# become root inside the namespace +# inner $SANDBOX_UID <- our real host uid, so everything the sandbox writes +# stays owned by us and `rm -rf` on a persistent sandbox needs +# no privileges or chown dance +# The payload then runs in stage 2, where bwrap adds the mount/pid namespaces +# without creating a userns at all. +# +# The root layout needs no subuid at all: inner 0 IS the host uid there. +netns_args=(--user --net) +if [ "$RUN_AS_USER" = true ]; then + host_user="$(id -un)" + subuid_base="$(awk -F: -v u="$host_user" '$1 == u {print $2; exit}' /etc/subuid)" + subgid_base="$(awk -F: -v u="$host_user" '$1 == u {print $2; exit}' /etc/subgid)" + if [ -z "$subuid_base" ] || [ -z "$subgid_base" ]; then + echo "error: no /etc/subuid or /etc/subgid range for $host_user" >&2 + echo ' A user-level sandbox needs one spare subordinate id to host' >&2 + echo " its internal root. Add e.g. '$host_user:100000:65536' to both," >&2 + echo ' or use --root.' >&2 + exit 1 + fi + netns_args+=( + --map-users="0:$subuid_base:1" --map-users="$SANDBOX_UID:$(id -u):1" + --map-groups="0:$subgid_base:1" --map-groups="$SANDBOX_GID:$(id -g):1" + ) +else + netns_args+=(--map-root-user) +fi + +sandbox_pid_file="$SANDBOX_ROOT/root/logs/sandbox.pid" +slirp_ready="$SANDBOX_ROOT/root/logs/slirp.ready" +slirp_log="$SANDBOX_ROOT/root/logs/slirp.log" +: > "$sandbox_pid_file" +: > "$slirp_ready" + +env \ + DEV_SANDBOX_ROOT="$SANDBOX_ROOT" \ + DEV_SANDBOX_BASH="$(command -v bash)" \ + DEV_SANDBOX_REAL_CA_CERT="$REAL_CA_CERT" \ + DEV_SANDBOX_INTERACTIVE="$INTERACTIVE" \ + DEV_SANDBOX_USER="$SANDBOX_USER" \ + DEV_SANDBOX_HOME="$SANDBOX_HOME" \ + DEV_SANDBOX_NODE_DIR="$NODE_DIR" \ + DEV_SANDBOX_ELECTRON_LD_LIBRARY_PATH="${DEV_SANDBOX_ELECTRON_LD_LIBRARY_PATH:-}" \ + DEV_SANDBOX_XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-}" \ + DEV_SANDBOX_WAYLAND_DISPLAY="${WAYLAND_DISPLAY:-}" \ + DEV_SANDBOX_WAYLAND_SOCKET="$WAYLAND_SOCKET" \ + unshare "${netns_args[@]}" \ + "$SANDBOX_ASSETS/stage2-run.sh" "$@" & +sandbox_launcher=$! + +for _ in $(seq 1 200); do + [ -s "$sandbox_pid_file" ] && break + if ! kill -0 "$sandbox_launcher" 2>/dev/null; then + wait "$sandbox_launcher" + exit $? + fi + sleep 0.05 +done +sandbox_pid="$(tr -dc '0-9' < "$sandbox_pid_file")" +if [ -z "$sandbox_pid" ]; then + echo 'error: sandbox did not report its PID' >&2 + exit 1 +fi + +slirp4netns --configure --disable-host-loopback --ready-fd=3 \ + --userns-path="/proc/$sandbox_pid/ns/user" "$sandbox_pid" tap0 \ + 3>"$slirp_ready" >"$slirp_log" 2>&1 & +slirp_pid=$! +cleanup_slirp() { + kill "$slirp_pid" 2>/dev/null || true + wait "$slirp_pid" 2>/dev/null || true +} +trap cleanup_slirp EXIT INT TERM + +for _ in $(seq 1 200); do + [ -s "$slirp_ready" ] && break + if ! kill -0 "$slirp_pid" 2>/dev/null; then + cat "$slirp_log" >&2 || true + exit 1 + fi + sleep 0.05 +done +if [ ! -s "$slirp_ready" ]; then + echo 'error: timed out waiting for sandbox network setup' >&2 + exit 1 +fi + +wait "$sandbox_launcher" +exit $? \ No newline at end of file diff --git a/scripts/sandbox/openssl.cnf b/scripts/sandbox/openssl.cnf new file mode 100644 index 0000000000000..04884355bd16e --- /dev/null +++ b/scripts/sandbox/openssl.cnf @@ -0,0 +1,43 @@ +# Minimal openssl config for the dev sandbox. +# +# The sandbox replaces /etc wholesale, and on Debian/Ubuntu +# /usr/lib/ssl/openssl.cnf (openssl's compiled-in OPENSSLDIR) is a symlink into +# /etc/ssl -- so the config openssl insists on reading disappears and every +# `openssl req` fails with: +# +# Can't open "/usr/lib/ssl/openssl.cnf" for reading +# +# which surfaces to the payload as a bare `curl: (35) Recv failure`. Rather than +# reconstruct each distro's /etc/ssl, point OPENSSL_CONF at this file: the proxy +# only needs enough config for `req -addext` and `x509 -copy_extensions`. + +[ req ] +distinguished_name = req_distinguished_name + +[ req_distinguished_name ] + +# Used by `req -x509` for the sandbox's own CA. Without an explicit +# basicConstraints the generated certificate is not a CA, and every leaf it +# signs is rejected by the client with "invalid CA certificate (79)". +[ sandbox_ca_ext ] +basicConstraints = critical,CA:true +keyUsage = critical,keyCertSign,cRLSign +subjectKeyIdentifier = hash + +[ ca ] +default_ca = sandbox_ca + +[ sandbox_ca ] +default_md = sha256 +policy = policy_anything +email_in_dn = no +preserve = no + +[ policy_anything ] +commonName = optional +countryName = optional +stateOrProvinceName = optional +localityName = optional +organizationName = optional +organizationalUnitName = optional +emailAddress = optional diff --git a/scripts/sandbox/pick-release-tags.sh b/scripts/sandbox/pick-release-tags.sh new file mode 100755 index 0000000000000..1363eb14c5178 --- /dev/null +++ b/scripts/sandbox/pick-release-tags.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# Pick the release tags the install/update E2E should update FROM. +# +# Emits a JSON array of tag names on stdout, suitable for a GitHub Actions +# matrix (`fromJSON`). Choosing at runtime rather than hardcoding keeps the +# matrix honest as releases land: a pinned list silently stops covering the +# newest release the day after it ships, and pins the "oldest" forever even +# after it stops being a version anyone still runs. +# +# Selection: the newest tag, the oldest tag, and evenly spaced tags in between. +# Newest catches "did the last release break updating?", oldest is the longest +# upgrade jump anyone can still make, and the spread samples the migrations in +# between (config-schema bumps, venv layout changes, dependency floors). +# +# Usage: +# scripts/sandbox/pick-release-tags.sh [--count N] [--repo DIR] +# +# --count how many tags to emit (default 5, minimum 1). Fewer tags than +# requested emits all of them. +# --repo repository to read tags from (default: this checkout). +# +# Reads tags from the local checkout, so it needs one fetched with tags +# (actions/checkout with fetch-depth: 0, or `fetch-tags: true`). A shallow +# checkout has no tags and this exits non-zero rather than silently emitting an +# empty matrix. +# +# Only vYYYY.M.D[.N] release tags are considered; the repo also carries +# backup/* and one-off tags that are not releases. + +set -euo pipefail + +COUNT=5 +# Default to the repository containing this script, resolved through its real +# path so a symlinked or copied script still reads the checkout it lives in +# rather than whatever repo the caller happens to be standing in. +REPO="" +while [ "$#" -gt 0 ]; do + case "$1" in + --count) + [ "$#" -ge 2 ] || { echo 'error: --count needs a value' >&2; exit 1; } + COUNT="$2"; shift 2 ;; + --repo) + [ "$#" -ge 2 ] || { echo 'error: --repo needs a value' >&2; exit 1; } + REPO="$2"; shift 2 ;; + -h|--help) sed -n '2,30p' "$0"; exit 0 ;; + *) echo "error: unknown argument: $1" >&2; exit 1 ;; + esac +done +case "$COUNT" in + ''|*[!0-9]*) echo "error: --count must be a positive integer: $COUNT" >&2; exit 1 ;; +esac +[ "$COUNT" -ge 1 ] || { echo 'error: --count must be at least 1' >&2; exit 1; } + +# Resolve the script's own location through symlinks, then ask git which +# worktree that path belongs to. Deriving the repo from the script rather than +# from $PWD means a copied script cannot silently report a different checkout's +# tags, and --show-toplevel keeps it correct when invoked from a subdirectory. +if [ -z "$REPO" ]; then + script_path="${BASH_SOURCE[0]}" + if command -v readlink >/dev/null 2>&1; then + script_path="$(readlink -f "$script_path" 2>/dev/null || printf '%s' "$script_path")" + fi + script_dir="$(cd "$(dirname "$script_path")" && pwd)" + REPO="$(git -C "$script_dir" rev-parse --show-toplevel 2>/dev/null || printf '%s' "$script_dir")" +fi + +# sort -V orders v2026.4.8 before v2026.4.13 (numeric), which a plain +# lexicographic sort gets wrong. +mapfile -t tags < <( + git -C "$REPO" tag --list 'v*' \ + | grep -E '^v[0-9]{4}\.[0-9]+\.[0-9]+(\.[0-9]+)?$' \ + | sort -V +) + +total="${#tags[@]}" +if [ "$total" -eq 0 ]; then + echo "error: no release tags found in $REPO" >&2 + echo ' A shallow clone has no tags: fetch with tags (actions/checkout' >&2 + echo ' with fetch-depth: 0, or fetch-tags: true).' >&2 + exit 1 +fi + +if [ "$total" -le "$COUNT" ]; then + picked=("${tags[@]}") +elif [ "$COUNT" -eq 1 ]; then + # One slot means the newest release; there is no span to spread across. + picked=("${tags[$((total - 1))]}") +else + # Evenly spaced indices across [0, total-1], endpoints included, so the + # oldest and newest are always present and the rest are spread between them. + picked=() + for slot in $(seq 0 $((COUNT - 1))); do + # Round to nearest rather than truncate, so the spacing does not bunch + # toward the oldest end. + index=$(( (slot * (total - 1) * 2 + (COUNT - 1)) / ((COUNT - 1) * 2) )) + candidate="${tags[$index]}" + # Guard against a duplicate if rounding lands twice on the same tag. + case " ${picked[*]-} " in + *" $candidate "*) continue ;; + esac + picked+=("$candidate") + done +fi + +printf '[' +for i in "${!picked[@]}"; do + [ "$i" -eq 0 ] || printf ',' + printf '"%s"' "${picked[$i]}" +done +printf ']\n' diff --git a/scripts/sandbox/proxy.py b/scripts/sandbox/proxy.py new file mode 100644 index 0000000000000..f34c9a843a9a2 --- /dev/null +++ b/scripts/sandbox/proxy.py @@ -0,0 +1,237 @@ +"""MITM proxy backing the dev sandbox's fake Internet. + +Listens on 127.0.0.1:8080 and is pointed at by http_proxy/https_proxy inside +the sandbox. For each request it either serves a fixture from the filesystem or +forwards to the real host: + +* ``//`` exists -> serve it. This is how the sandbox answers + the canonical install URL with the installer under test, so the payload can + run the true ``curl -fsSL https://…/install.sh | bash`` one-liner. +* otherwise -> forward upstream, verifying against the real CA bundle. The + sandbox is isolated from the *host*, not from the internet: a real install + still has to reach PyPI and npm. + +HTTPS is intercepted by minting a per-host certificate from the sandbox's own +throwaway CA, which the payload trusts via CURL_CA_BUNDLE / SSL_CERT_FILE. + +Usage: proxy.py +""" + +import os +import pathlib +import socket +import ssl +import subprocess +import sys +import threading +from urllib.parse import unquote, urlsplit + +ROOT, CERTS, REAL_CA = map(pathlib.Path, sys.argv[1:]) + +LISTEN_ADDRESS = ('127.0.0.1', 8080) +MAX_REQUEST_BYTES = 65536 +UPSTREAM_TIMEOUT_SECONDS = 30 +CERT_VALIDITY_DAYS = 2 + + +def read_request(conn): + data = b"" + while b"\r\n\r\n" not in data and len(data) < MAX_REQUEST_BYTES: + part = conn.recv(4096) + if not part: + return b"" + data += part + return data + + +def run_openssl(args): + """Run openssl, raising with its stderr when it fails. + + Discarding stderr here costs real debugging time: the caller sees only a + dropped connection (``curl: (35) Recv failure``) and the log holds nothing + but the argv, so an unwritable directory, a missing CA key, and an option + the host's openssl rejects all look identical. + """ + done = subprocess.run( + ['openssl', *args], stdout=subprocess.DEVNULL, stderr=subprocess.PIPE + ) + if done.returncode != 0: + detail = done.stderr.decode('utf-8', 'replace').strip() + raise RuntimeError( + f'openssl {args[0]} failed (exit {done.returncode}): {detail}' + ) + + +_CERT_LOCK = threading.Lock() + + +def cert_for(host): + """Return a (cert, key) pair for host, minting it from the sandbox CA. + + Minting is serialized and published atomically. The proxy is threaded, so + two concurrent requests for the same host would otherwise both run openssl + into the same paths, and a reader could pick up a finished certificate + beside a key from the other writer -- which TLS rejects as + ``[X509: KEY_VALUES_MISMATCH] key values mismatch``. + """ + safe = ''.join(char if char.isalnum() or char in '.-' else '_' for char in host) + cert, key = CERTS / f'{safe}.pem', CERTS / f'{safe}.key' + if cert.exists() and key.exists(): + return cert, key + with _CERT_LOCK: + # Re-check: another thread may have finished while we waited. + if cert.exists() and key.exists(): + return cert, key + # Build under unique temp names, then rename into place. os.replace is + # atomic, so a reader sees either the old pair or the new one, never a + # half-written mix. The key lands first: the certificate's existence is + # what everything else keys off. + stamp = f'{os.getpid()}.{threading.get_ident()}' + tmp_key = CERTS / f'{safe}.key.{stamp}' + tmp_cert = CERTS / f'{safe}.pem.{stamp}' + csr = CERTS / f'{safe}.csr.{stamp}' + run_openssl([ + 'req', '-newkey', 'rsa:2048', '-nodes', + '-subj', f'/CN={host}', + '-addext', f'subjectAltName=DNS:{host}', + '-keyout', str(tmp_key), '-out', str(csr), + ]) + run_openssl([ + 'x509', '-req', '-days', str(CERT_VALIDITY_DAYS), '-in', str(csr), + '-CA', str(CERTS / 'ca.pem'), '-CAkey', str(CERTS / 'ca.key'), + '-CAcreateserial', '-copy_extensions', 'copy', '-out', str(tmp_cert), + ]) + csr.unlink(missing_ok=True) + os.replace(tmp_key, key) + os.replace(tmp_cert, cert) + return cert, key + + +def file_for(host, target): + """Resolve a request to a fixture file, or None to forward upstream.""" + path = urlsplit(target).path or '/' + parts = pathlib.PurePosixPath(unquote(path)).parts + if '..' in parts: + return None + candidate = ROOT / host / pathlib.PurePosixPath(*[p for p in parts if p != '/']) + if candidate.is_dir(): + candidate /= 'index.html' + return candidate if candidate.is_file() else None + + +def respond_fixture(conn, found): + body = found.read_bytes() + headers = ( + f'Content-Length: {len(body)}\r\nConnection: close\r\n\r\n'.encode() + ) + conn.sendall(b'HTTP/1.1 200 OK\r\n' + headers + body) + + +def close_request(request, target=None): + """Rewrite a proxied request for a direct upstream connection.""" + headers, separator, body = request.partition(b'\r\n\r\n') + lines = headers.split(b'\r\n') + if target is not None: + method, _, version = lines[0].split(b' ', 2) + lines[0] = b' '.join((method, target.encode(), version)) + lines = [ + line for line in lines + if not line.lower().startswith(b'proxy-connection:') + ] + lines.append(b'Connection: close') + return b'\r\n'.join(lines) + separator + body + + +def relay(source, destination): + while True: + chunk = source.recv(MAX_REQUEST_BYTES) + if not chunk: + return + destination.sendall(chunk) + + +def forward_https(conn, host, port, request): + context = ssl.create_default_context(cafile=str(REAL_CA)) + with socket.create_connection((host, port), timeout=UPSTREAM_TIMEOUT_SECONDS) as raw: + with context.wrap_socket(raw, server_hostname=host) as upstream: + upstream.sendall(close_request(request)) + relay(upstream, conn) + + +def forward_http(conn, host, port, request, target): + parsed = urlsplit(target) + path = parsed.path or '/' + if parsed.query: + path += f'?{parsed.query}' + with socket.create_connection((host, port), timeout=UPSTREAM_TIMEOUT_SECONDS) as upstream: + upstream.sendall(close_request(request, path)) + relay(upstream, conn) + + +def handle_connect(conn, target): + """Intercept a CONNECT tunnel, terminating TLS with a minted cert.""" + host, _, port_text = target.rpartition(':') + port = int(port_text or '443') + conn.sendall(b'HTTP/1.1 200 Connection Established\r\n\r\n') + cert, key = cert_for(host) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(cert, key) + with context.wrap_socket(conn, server_side=True) as tls: + nested = read_request(tls) + if not nested: + return + line = nested.split(b'\r\n', 1)[0].decode('iso-8859-1') + nested_target = line.split(' ', 2)[1] + found = file_for(host, nested_target) + if found is not None: + respond_fixture(tls, found) + else: + forward_https(tls, host, port, nested) + + +def host_from_headers(request): + for header in request.split(b'\r\n')[1:]: + if header.lower().startswith(b'host:'): + value = header.split(b':', 1)[1].strip().decode() + return value.split(':', 1)[0] + return None + + +def handle_request(conn): + with conn: + request = read_request(conn) + if not request: + return + line = request.split(b'\r\n', 1)[0].decode('iso-8859-1') + method, target, _ = line.split(' ', 2) + if method.upper() == 'CONNECT': + handle_connect(conn, target) + return + parsed = urlsplit(target) + host = parsed.hostname or host_from_headers(request) or 'unknown' + found = file_for(host, target) + if found is not None: + respond_fixture(conn, found) + else: + forward_http(conn, host, parsed.port or 80, request, target) + + +def handle(conn): + try: + handle_request(conn) + except Exception as error: + print(f'proxy request failed: {error!r}', file=sys.stderr, flush=True) + + +def main(): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server: + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(LISTEN_ADDRESS) + server.listen() + while True: + conn, _ = server.accept() + threading.Thread(target=handle, args=(conn,), daemon=True).start() + + +if __name__ == '__main__': + main() diff --git a/scripts/sandbox/ssh-shim.sh b/scripts/sandbox/ssh-shim.sh new file mode 100644 index 0000000000000..1b90035eea4c5 --- /dev/null +++ b/scripts/sandbox/ssh-shim.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Stand-in for ssh inside the dev sandbox. +# +# install.sh and `hermes update` clone over ssh first (git@github.com:...), so +# the sandbox needs an `ssh` that answers. Rather than run a real sshd, this +# ignores the host, user, and command git asked for and speaks the +# upload-pack protocol directly against the sandbox's bare repo -- which is +# what makes the ssh-first code path exercisable with no keys, no known_hosts, +# and no network. +# +# GIT_UPLOAD_PACK is substituted by dev-sandbox.sh when it installs this shim, +# because the host's git-upload-pack is not necessarily on the sandbox PATH. +exec @GIT_UPLOAD_PACK@ /work/repos/hermes-agent.git diff --git a/scripts/sandbox/stage2-run.sh b/scripts/sandbox/stage2-run.sh new file mode 100755 index 0000000000000..42d2800ec7eb0 --- /dev/null +++ b/scripts/sandbox/stage2-run.sh @@ -0,0 +1,251 @@ +#!/usr/bin/env bash +# Stage 2 of the dev sandbox: build the mounts and run the payload. +# +# Not called directly. scripts/dev-sandbox.sh (stage 1) creates the user and +# network namespaces with `unshare` and re-execs into this script inside them, +# so by the time this runs we are already at the target uid with a private +# netns. bwrap therefore does NOT create a userns here -- it only adds the +# mount and pid namespaces. (`unshare --user` grants its creator full +# capabilities in the new userns regardless of which uid it maps, which is what +# lets bwrap mount as a non-root uid.) +# +# The whole interface with stage 1 is the DEV_SANDBOX_* environment, asserted +# below: there are no shared functions or variables between the two stages. +# Stage 1 locates this script alongside the other sandbox assets (see +# DEV_SANDBOX_ASSETS in dev-sandbox.sh), so the Nix wrapper's store copy and a +# plain repo checkout both work. + +set -euo pipefail + +: "${DEV_SANDBOX_ROOT:?missing DEV_SANDBOX_ROOT}" +: "${DEV_SANDBOX_BASH:?missing DEV_SANDBOX_BASH}" +: "${DEV_SANDBOX_INTERACTIVE:?missing DEV_SANDBOX_INTERACTIVE}" +: "${DEV_SANDBOX_USER:?missing DEV_SANDBOX_USER}" +: "${DEV_SANDBOX_HOME:?missing DEV_SANDBOX_HOME}" + +# Announce our pid so stage 1 can point slirp4netns at these namespaces, +# then hold until it reports the network is up. +slirp_ready="$DEV_SANDBOX_ROOT/root/logs/slirp.ready" +printf '%s\n' "$$" > "$DEV_SANDBOX_ROOT/root/logs/sandbox.pid" +for _ in $(seq 1 200); do + [ -s "$slirp_ready" ] && break + sleep 0.05 +done +if [ ! -s "$slirp_ready" ]; then + echo 'error: timed out waiting for sandbox network setup' >&2 + cat "$DEV_SANDBOX_ROOT/root/logs/slirp.log" >&2 || true + exit 1 +fi + +# The sandbox HOME is /root for a root install and /home/ for a +# user-level one. Only the latter needs its parent created first; --dir / +# is not a thing bwrap accepts. +home_mounts=() +home_parent="$(dirname "$DEV_SANDBOX_HOME")" +if [ "$home_parent" != / ]; then + home_mounts+=(--dir "$home_parent") +fi +home_mounts+=(--bind "$DEV_SANDBOX_ROOT/home" "$DEV_SANDBOX_HOME") + +node_env=() +if [ -n "${DEV_SANDBOX_NODE_DIR:-}" ]; then + node_env+=(--setenv npm_config_nodedir "$DEV_SANDBOX_NODE_DIR") +fi +electron_env=() +if [ -n "${DEV_SANDBOX_ELECTRON_LD_LIBRARY_PATH:-}" ]; then + electron_env+=( + --setenv LD_LIBRARY_PATH "$DEV_SANDBOX_ELECTRON_LD_LIBRARY_PATH" + --setenv HERMES_DESKTOP_DISABLE_GPU 1 + ) +fi +gui_mounts=() +if [ -n "${DEV_SANDBOX_WAYLAND_SOCKET:-}" ]; then + runtime_dir="${DEV_SANDBOX_XDG_RUNTIME_DIR:?missing DEV_SANDBOX_XDG_RUNTIME_DIR}" + runtime_parent="$(dirname "$runtime_dir")" + runtime_grandparent="$(dirname "$runtime_parent")" + gui_mounts+=( + --dir "$runtime_grandparent" + --dir "$runtime_parent" + --dir "$runtime_dir" + --bind "$DEV_SANDBOX_WAYLAND_SOCKET" "$DEV_SANDBOX_WAYLAND_SOCKET" + --setenv XDG_RUNTIME_DIR "$runtime_dir" + --setenv WAYLAND_DISPLAY "${DEV_SANDBOX_WAYLAND_DISPLAY:?missing DEV_SANDBOX_WAYLAND_DISPLAY}" + ) +fi + +# How the sandbox gets a usable runtime, and where its own shims go. +# +# On Nix, every binary lives under /nix/store, so the sandbox can own /bin, +# /lib64 and /usr/bin outright and fill them with symlinks into the store. +# +# Elsewhere the runtime IS /usr, /bin, /lib, /lib64 -- so binding the +# sandbox's near-empty versions over them hides the real thing, and bwrap +# dies with `execvp /usr/bin/bash: No such file or directory`. Keep the host +# directories read-only and override only the individual files we shim. +# +# The same answer decides how /etc is handled further down. +if [ -d /nix ] && [[ "$(readlink -f "$DEV_SANDBOX_BASH")" == /nix/* ]]; then + USE_HOST_RUNTIME=false +else + USE_HOST_RUNTIME=true +fi + +runtime_mounts=() +shim_mounts=() +if [ "$USE_HOST_RUNTIME" = false ]; then + runtime_mounts+=(--ro-bind /nix /nix) + shim_mounts+=( + --dir /usr + --dir /bin + --dir /lib64 + --bind "$DEV_SANDBOX_ROOT/root/bin" /bin + --bind "$DEV_SANDBOX_ROOT/root/lib64" /lib64 + --bind "$DEV_SANDBOX_ROOT/root/usr/bin" /usr/bin + ) +else + for path in /usr /bin /sbin /lib /lib64; do + [ -e "$path" ] && runtime_mounts+=(--ro-bind "$path" "$path") + done + # The git-upload-pack shim standing in for github.com is the only file that + # must beat the host's copy; sh/ls/env are already there for real. + shim_mounts+=(--bind "$DEV_SANDBOX_ROOT/root/usr/bin/ssh" /usr/bin/ssh) +fi + +# /etc: start from a copy of the host's and overwrite only the files we fake. +# +# Replacing the whole directory with a five-file one is the tempting shortcut +# and it is wrong: a distro puts things under /etc that binaries outside /etc +# depend on, so hiding all of it breaks tools that look fine on PATH. Two real +# examples, both Debian/Ubuntu: openssl's compiled-in openssl.cnf is a symlink +# into /etc/ssl, and /usr/bin/awk is a symlink to /etc/alternatives/awk -- with +# /etc replaced, openssl cannot mint a certificate and awk reports "not found". +# Those are two symptoms of one cause, and nothing says there are only two. +# +# Copying rather than mount-overlaying the individual files, because several of +# these are symlinks in the wild (resolv.conf -> ../run/systemd/... on Ubuntu, +# hosts and nsswitch.conf -> /etc/static/... on NixOS) and bwrap cannot bind a +# file onto a symlink whose target does not exist inside the sandbox. +# +# Symlinks are copied as symlinks, never dereferenced: on NixOS /etc/static +# points into the store and following it would copy gigabytes per sandbox. The +# store is already mounted at /nix on that path, and the host runtime dirs are +# mounted at their own paths, so absolute symlinks still resolve. +# +# The five we override, and why each must differ from the host's: +# passwd, group the sandbox identity, which does not exist on the host +# resolv.conf slirp4netns's DNS, not the host resolver +# nsswitch.conf files+dns only, so nothing consults host NSS modules +# hosts minimal, so no host entry leaks in +# +# os-release is removed rather than replaced. Installers branch on it to reach +# for a package manager -- `install.sh` reads ID from it and, on debian/ubuntu, +# offers to apt-get build tools, prompting on /dev/tty when sudo exists but is +# not passwordless. That prompt cannot be satisfied here (no terminal) and it is +# fatal under `set -e`. Inheriting the host's file would make the sandbox claim +# to be a distro whose package manager it cannot actually use; absent means +# DISTRO="unknown" and the apt path is skipped, which is the truth. +etc_mounts=() +if [ "$USE_HOST_RUNTIME" = true ] && [ -d /etc ]; then + sandbox_etc="$DEV_SANDBOX_ROOT/etc-merged" + rm -rf -- "$sandbox_etc" + mkdir -p "$sandbox_etc" + # -a keeps symlinks as symlinks; unreadable entries (shadow, sudoers) are + # skipped rather than failing the run. + cp -a /etc/. "$sandbox_etc/" 2>/dev/null || true + for etc_file in passwd group resolv.conf nsswitch.conf hosts; do + [ -f "$DEV_SANDBOX_ROOT/etc/$etc_file" ] || continue + rm -f "$sandbox_etc/$etc_file" + cp "$DEV_SANDBOX_ROOT/etc/$etc_file" "$sandbox_etc/$etc_file" + done + rm -f "$sandbox_etc/os-release" "$sandbox_etc/lsb-release" + etc_mounts+=(--ro-bind "$sandbox_etc" /etc) +else + etc_mounts+=(--bind "$DEV_SANDBOX_ROOT/etc" /etc) +fi + +# /dev without a tty, so a script guarding on `[ -e /dev/tty ]` takes its +# no-terminal path. +# +# bwrap's --dev creates a /dev/tty NODE, but nothing in here has a controlling +# terminal, so opening it fails with "No such device or address". That is the +# worst of both: the guard passes and the read then fails. Under `set -e` -- +# which install.sh uses -- a failed read inside a function aborts the whole +# installer, which is exactly how older releases died here while prompting for +# sudo to install ripgrep/ffmpeg. +# +# Making the tty real is not the fix: with an openable terminal that prompt +# blocks forever waiting for input nobody will type. Absent is what a headless +# machine looks like, and what every prompt in here should assume. +# +# --dev cannot be used with the node removed afterwards (bwrap refuses to mount +# a directory over a device node), so /dev is assembled explicitly. +dev_mounts=( + --tmpfs /dev + --dev-bind /dev/null /dev/null + --dev-bind /dev/zero /dev/zero + --dev-bind /dev/full /dev/full + --dev-bind /dev/random /dev/random + --dev-bind /dev/urandom /dev/urandom + --symlink /proc/self/fd /dev/fd + --symlink /proc/self/fd/0 /dev/stdin + --symlink /proc/self/fd/1 /dev/stdout + --symlink /proc/self/fd/2 /dev/stderr +) +if [ "$DEV_SANDBOX_INTERACTIVE" = true ]; then + # An interactive shell is deliberately given a terminal; keep bwrap's /dev. + dev_mounts=(--dev /dev) +fi + +exec bwrap \ + --unshare-pid \ + --die-with-parent --proc /proc --tmpfs /tmp \ + "${dev_mounts[@]}" \ + "${gui_mounts[@]}" \ + "${runtime_mounts[@]}" \ + --bind "$DEV_SANDBOX_ROOT/root" /work \ + "${shim_mounts[@]}" \ + --bind "$DEV_SANDBOX_ROOT/root/usr/local" /usr/local \ + "${home_mounts[@]}" \ + "${etc_mounts[@]}" \ + --chdir /work/repo \ + --clearenv \ + --setenv PATH "$DEV_SANDBOX_HOME/.local/bin:/usr/local/bin:/usr/bin:$PATH" \ + --setenv HOME "$DEV_SANDBOX_HOME" \ + --setenv USER "$DEV_SANDBOX_USER" \ + --setenv LOGNAME "$DEV_SANDBOX_USER" \ + --setenv CURL_CA_BUNDLE /work/certs/ca.pem \ + --setenv SSL_CERT_FILE /work/certs/ca.pem \ + --setenv GIT_SSL_CAINFO /work/certs/ca.pem \ + --setenv NODE_EXTRA_CA_CERTS /work/certs/real-ca.pem \ + --setenv OPENSSL_CONF /work/certs/openssl.cnf \ + --setenv HTTP_PROXY http://127.0.0.1:8080 \ + --setenv HTTPS_PROXY http://127.0.0.1:8080 \ + --setenv ALL_PROXY http://127.0.0.1:8080 \ + --setenv NO_PROXY '' \ + --setenv DEV_SANDBOX_INTERACTIVE "$DEV_SANDBOX_INTERACTIVE" \ + --setenv ELECTRON_DISABLE_SANDBOX 1 \ + "${node_env[@]}" \ + "${electron_env[@]}" \ + -- "$DEV_SANDBOX_BASH" -ceu ' + python3 /work/proxy.py /work/http /work/certs /work/certs/real-ca.pem >/work/logs/proxy.log 2>&1 & + proxy_pid=$! + cleanup() { + kill "$proxy_pid" 2>/dev/null || true + wait "$proxy_pid" 2>/dev/null || true + } + trap cleanup EXIT INT TERM + # Bash opens /dev/tcp itself, so the readiness probe needs no netcat -- + # one less binary the sandbox has to find on the host (GitHub runners + # ship no `nc`). + proxy_up() { (exec 3<>/dev/tcp/127.0.0.1/8080) 2>/dev/null; } + for _ in $(seq 1 100); do + proxy_up && break + sleep 0.05 + done + if ! proxy_up; then + echo "error: the sandbox fake-internet proxy never came up" >&2 + cat /work/logs/proxy.log >&2 || true + exit 1 + fi + "$@" + ' sandbox-command "$@"