diff --git a/.github/workflows/newci-install-e2e-run.yml b/.github/workflows/newci-install-e2e-run.yml new file mode 100644 index 0000000000000..c72931996486a --- /dev/null +++ b/.github/workflows/newci-install-e2e-run.yml @@ -0,0 +1,246 @@ +name: '[newci] Install & Update E2E (reusable)' + +# ⚠️ SHADOW WORKFLOW — temporary, for the ARC runner migration. +# +# A duplicate of the production install-e2e-run.yml, running on the GKE +# self-hosted (ARC) runners so the migration can be observed without touching +# the workflows that gate merges. Production CI in this branch is +# byte-identical to main. +# +# Safety properties (keep these when editing): +# - concurrency groups are newci-prefixed, so a shadow run can never +# cancel the production run it shadows +# - artifact names are newci-prefixed, so production artifacts stay clean +# - reusable-workflow calls point only at other newci-* workflows +# - this workflow is never called by production install-e2e.yml, so its +# result cannot fail the production job +# +# To retire: delete .github/workflows/newci-*.yml. +# +# ───────────────────────────────────────────────────────────────────────── +# Why this shadow is expected to SKIP its legs today +# ───────────────────────────────────────────────────────────────────────── +# +# tests/install/install-update-e2e.sh runs inside scripts/dev-sandbox.sh, +# which is built on bubblewrap. bwrap needs to remount / as slave and mount +# a fresh /proc, and in a stock ARC pod both are denied. Measured in-cluster +# on nousresearch/nous-gke-runner (2026-08-04), installing the same deps the +# production job apt-installs: +# +# pod securityContext bwrap result +# ---------------------------------------- ---------------------------- +# default (what arc-runner-set gives you) Failed to make / slave: EPERM +# capabilities.add: [SYS_ADMIN] Can't mount proc: EPERM +# SYS_ADMIN + apparmor/seccomp Unconfined Can't mount proc: EPERM +# privileged: true OK (all probes pass) +# +# So this needs a scale set whose *runner* container is privileged. +# arc-runner-docker does NOT qualify — only its dind sidecar is privileged, +# the runner container is not. Until such a set exists, the preflight job +# below detects the missing capability and skips the legs with a warning +# rather than burning ~11 minutes per leg to fail at the same place. +# +# When infra adds the set: pass its label as `runner` from the caller and +# the legs start running with no other change here. + +on: + workflow_call: + inputs: + route: + description: 'Update path to exercise: update (hermes update) or installer (re-run install.sh).' + required: true + type: string + install-ref: + description: 'What to install before updating: a branch, a tag (v2026.7.7), or a SHA reachable from main.' + required: false + type: string + default: refs/heads/main + runner: + description: 'Runner label. Needs a privileged runner container for bubblewrap — see the header.' + required: false + type: string + default: arc-runner-set + timeout-minutes: + description: 'Job timeout. A cold run installs real toolchains twice.' + required: false + type: number + default: 45 + +permissions: + contents: read + +jobs: + # Probe bwrap before paying for a full install. This is the whole point of + # the shadow: it answers "can ARC run the install E2E yet?" in ~30 seconds + # instead of ~11 minutes, and says exactly what is missing when it cannot. + preflight: + name: Probe sandbox capability + runs-on: ${{ inputs.runner }} + timeout-minutes: 10 + outputs: + supported: ${{ steps.probe.outputs.supported }} + steps: + - name: Probe bubblewrap + id: probe + run: | + set -uo pipefail + + # The production job apt-installs these on ubuntu-latest; do the same + # here so the probe measures the sandbox, not a missing package. + sudo apt-get update -qq + sudo apt-get install -y -qq bubblewrap slirp4netns uidmap util-linux + + supported=true + note="" + + if ! bwrap --ro-bind / / --dev /dev --proc /proc --unshare-all true 2>/tmp/bwrap-err; then + supported=false + note="$(cat /tmp/bwrap-err)" + fi + + echo "supported=$supported" >> "$GITHUB_OUTPUT" + + if [ "$supported" = true ]; then + echo "✅ bubblewrap works on this runner — running the real legs." + { + echo "## Sandbox preflight: supported ✅" + echo "" + echo "\`bwrap\` can create its namespaces on \`${{ inputs.runner }}\`." + } >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + echo "::warning::bubblewrap cannot run on ${{ inputs.runner }}; skipping the install E2E legs. ${note}" + { + echo "## Sandbox preflight: unsupported ⚠️" + echo "" + echo "\`bwrap\` failed on \`${{ inputs.runner }}\`:" + echo "" + echo '```' + echo "${note}" + echo '```' + echo "" + echo "The install E2E runs inside \`scripts/dev-sandbox.sh\` (bubblewrap)," + echo "which needs to remount \`/\` as slave and mount a fresh \`/proc\`." + echo "A stock ARC pod denies both." + echo "" + echo "| pod securityContext | bwrap |" + echo "|---|---|" + echo "| default | \`Failed to make / slave: EPERM\` |" + echo "| \`capabilities.add: [SYS_ADMIN]\` | \`Can't mount proc: EPERM\` |" + echo "| SYS_ADMIN + apparmor/seccomp Unconfined | \`Can't mount proc: EPERM\` |" + echo "| \`privileged: true\` | works |" + echo "" + echo "This needs a scale set whose **runner** container is privileged." + echo "\`arc-runner-docker\` does not qualify — only its dind sidecar is." + echo "" + echo "**This is a skip, not a regression.** Production \`install-e2e.yml\`" + echo "on GitHub-hosted runners is unaffected." + } >> "$GITHUB_STEP_SUMMARY" + + e2e: + name: ${{ inputs.route }} from ${{ inputs.install-ref }} + needs: preflight + if: needs.preflight.outputs.supported == 'true' + runs-on: ${{ inputs.runner }} + timeout-minutes: ${{ inputs.timeout-minutes }} + + steps: + # Full history: the sandbox fetches the starting commit and the test + # compares against this commit, so a shallow clone is not enough. + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + + # bubblewrap + slirp4netns are what the sandbox is built on; util-linux + # supplies the `unshare` that builds the multi-uid userns for the + # user-level (non-root) install. + - name: Install sandbox dependencies + run: | + set -euo pipefail + sudo apt-get update -qq + sudo apt-get install -y -qq bubblewrap slirp4netns uidmap util-linux + + # The production job flips kernel.apparmor_restrict_unprivileged_userns + # on its Ubuntu 24.04 VM. In an ARC pod that sysctl is not present in + # /proc at all (verified in-cluster), and a pod cannot set node-level + # sysctls anyway — so report the state and move on rather than failing + # on a `sysctl -w` that was never going to apply here. + - name: Report user-namespace state + run: | + set -euo pipefail + echo "--- kernel userns settings" + sysctl kernel.unprivileged_userns_clone 2>/dev/null || echo " (sysctl absent — expected in a pod)" + sysctl kernel.apparmor_restrict_unprivileged_userns 2>/dev/null || echo " (sysctl absent — expected in a pod)" + echo "--- subuid/subgid for $(id -un)" + grep "^$(id -un):" /etc/subuid /etc/subgid || echo " (none — sandbox will say so)" + + - name: Run install + update E2E + id: run + # Step-level continue-on-error IS supported (job-level is not, for jobs + # that call a reusable workflow). This is what makes the shadow + # non-blocking: the script's exit code is captured as an outcome and + # reported below, but never fails the job — so the shadow workflow's + # own check stays green and nobody has to triage a red X that only + # means "the migration isn't ready yet". + continue-on-error: true + run: | + set -euo pipefail + tests/install/install-update-e2e.sh \ + --route '${{ inputs.route }}' \ + --install-ref '${{ inputs.install-ref }}' + env: + # Outside the workspace on purpose: the script creates this directory + # up front, and an untracked dir inside the repo makes the worktree + # dirty -- which dev-sandbox reacts to by snapshotting the working + # copy into a fresh fake-main commit on every invocation, moving the + # update target mid-run. + HERMES_E2E_LOG_DIR: ${{ runner.temp }}/e2e-logs + + - name: Report leg outcome + if: always() + run: | + set -uo pipefail + outcome='${{ steps.run.outcome }}' + if [ "$outcome" = "success" ]; then + icon="✅" + else + icon="❌" + echo "::warning::[newci shadow] ${{ inputs.route }} from ${{ inputs.install-ref }} failed ($outcome). Not blocking — production install-e2e.yml is unaffected." + fi + { + echo "## ${icon} ${{ inputs.route }} from ${{ inputs.install-ref }} — ${outcome}" + echo "" + echo "Runner: \`${{ inputs.runner }}\`" + echo "" + echo "Shadow leg. A failure here is migration data, not a broken build;" + echo "production \`Install & Update E2E\` on GitHub-hosted runners is" + echo "unaffected by this result." + } >> "$GITHUB_STEP_SUMMARY" + + + # Artifact names cannot contain '/', and install-ref may be a full ref + # like refs/heads/main. GitHub Actions expressions have no string-replace + # function, so build the safe name here. Runs even on failure -- that is + # exactly when the logs are wanted. + - name: Build artifact name + if: always() + id: artifact + run: | + set -euo pipefail + safe_ref='${{ inputs.install-ref }}' + safe_ref="${safe_ref//\//-}" + echo "name=newci-install-e2e-${{ inputs.route }}-${safe_ref}" >> "$GITHUB_OUTPUT" + + # The installer's own transcripts say far more than the assertion that + # tripped when a real install breaks. + - name: Upload installer logs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + # Unique per leg: a matrix over releases runs this workflow several + # times per route, and same-named artifacts collide. + name: ${{ steps.artifact.outputs.name }}-${{ github.sha }} + path: ${{ runner.temp }}/e2e-logs + retention-days: 14 + if-no-files-found: ignore diff --git a/.github/workflows/newci-install-e2e.yml b/.github/workflows/newci-install-e2e.yml new file mode 100644 index 0000000000000..670543b83066d --- /dev/null +++ b/.github/workflows/newci-install-e2e.yml @@ -0,0 +1,199 @@ +name: '[newci] Install & Update E2E' + +# ⚠️ SHADOW WORKFLOW — temporary, for the ARC runner migration. +# +# A duplicate of the production install-e2e.yml, running on the GKE self-hosted +# (ARC) runners so the migration can be observed without touching the workflows +# that gate merges. Production CI in this branch is byte-identical to main. +# +# Safety properties (keep these when editing): +# - concurrency groups are newci-prefixed, so a shadow run can never +# cancel the production run it shadows +# - artifact names are newci-prefixed, so production artifacts stay clean +# - reusable-workflow calls point only at other newci-* workflows +# - the E2E step is continue-on-error, so a failing leg never reddens a check +# +# To retire: delete .github/workflows/newci-*.yml. +# +# ───────────────────────────────────────────────────────────────────────── +# This workflow CANNOT fail the production Install & Update E2E +# ───────────────────────────────────────────────────────────────────────── +# +# Two independent reasons, both deliberate: +# +# 1. It is a separate workflow with its own triggers. GitHub reports it as +# its own check run; production install-e2e.yml never calls it and never +# reads its result. Nothing about this file can turn that one red. +# 2. Belt and braces: inside newci-install-e2e-run.yml the E2E step carries +# `continue-on-error: true`, so a failing leg is recorded as an outcome +# and surfaced in the summary rather than failing its job. Even this +# shadow's own check stays green — a migration probe should produce +# information, not a red X someone has to triage. +# +# NOTE on (2): job-level `continue-on-error` is NOT valid on a job that calls +# a reusable workflow (GitHub allows only name/uses/with/secrets/strategy/ +# needs/if/concurrency/permissions there), which is why the tolerance lives on +# the step inside the reusable workflow instead. Do not "fix" this by adding +# continue-on-error to the matrix jobs below — it will not parse. +# +# Read the results in the run summary, not in the check status. +# +# Triggers mirror production install-e2e.yml so the shadow fires whenever the +# real one does: every 12 hours, on release tags, and manually. The cron is +# offset by 5 minutes so the two runs do not contend for the same runner pool +# at the same instant. +# +# NOTE: `schedule` only fires from the default branch. Until this branch is +# merged, the scheduled leg will not run on its own — use workflow_dispatch +# (pick this branch in the Run workflow dropdown) to exercise it meanwhile. + +on: + workflow_dispatch: + inputs: + route: + description: 'Which update route to exercise.' + required: false + type: choice + default: both + options: [both, update, installer] + tag-count: + description: 'How many release tags to sample (newest, oldest, and a spread between).' + required: false + type: string + default: '5' + runner: + description: 'Runner label to shadow on. Needs a privileged runner container for bubblewrap.' + required: false + type: string + default: arc-runner-set + schedule: + # Production runs at :20; offset so the shadow does not contend with it. + - cron: '25 7,19 * * *' + push: + tags: + # Release tags only: the repo also carries backup/* and one-off tags. + - 'v[0-9]+.[0-9]+.[0-9]+' + - 'v[0-9]+.[0-9]+.[0-9]+.[0-9]+' + +permissions: + contents: read + +concurrency: + group: newci-install-e2e-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Which released versions do we test updating FROM? Resolved once and shared + # by both route matrices, so the two routes cover the same set. + pick-releases: + name: Pick release tags + # Small runner: reads tag names and runs one script. + runs-on: arc-runner-small + timeout-minutes: 5 + outputs: + tags: ${{ steps.pick.outputs.tags }} + steps: + # This job only reads tag names and runs one script, so take the cheap + # checkout: no blobs (filter), no other files (sparse), but DO fetch tags + # -- they are the whole input, and the default shallow checkout has none. + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + filter: blob:none + fetch-tags: true + sparse-checkout: scripts/sandbox/pick-release-tags.sh + sparse-checkout-cone-mode: false + - id: pick + run: | + set -euo pipefail + tags="$(scripts/sandbox/pick-release-tags.sh --count '${{ inputs.tag-count || 5 }}')" + echo "Testing updates from: $tags" + echo "tags=$tags" >> "$GITHUB_OUTPUT" + + # `hermes update` -- the route most users take. + update: + if: github.event_name != 'workflow_dispatch' || inputs.route != 'installer' + needs: pick-releases + strategy: + # One release breaking is worth knowing about even if another already + # failed, so let every leg report. + fail-fast: false + matrix: + install-ref: ${{ fromJSON(needs.pick-releases.outputs.tags) }} + uses: ./.github/workflows/newci-install-e2e-run.yml + with: + route: update + install-ref: ${{ matrix.install-ref }} + runner: ${{ inputs.runner || 'arc-runner-set' }} + + # Re-running the curl one-liner over an existing checkout: autostash + pull + # rather than the updater's own git handling. + installer: + if: github.event_name != 'workflow_dispatch' || inputs.route != 'update' + needs: pick-releases + strategy: + fail-fast: false + max-parallel: 3 + matrix: + install-ref: ${{ fromJSON(needs.pick-releases.outputs.tags) }} + uses: ./.github/workflows/newci-install-e2e-run.yml + with: + route: installer + install-ref: ${{ matrix.install-ref }} + runner: ${{ inputs.runner || 'arc-runner-set' }} + + # Report what the shadow saw WITHOUT ever failing. This is the job to read; + # it converts leg results into a summary table instead of a red X. It has no + # `exit 1` path on purpose — see the header. + shadow-summary: + name: '[newci] Shadow result (informational — never fails)' + needs: [pick-releases, update, installer] + if: always() + # Small runner: one script step, no checkout. + runs-on: arc-runner-small + timeout-minutes: 10 + steps: + - name: Summarize + env: + NEEDS: ${{ toJSON(needs) }} + shell: python + run: | + import json, os + + needs = json.loads(os.environ["NEEDS"]) + icons = { + "success": "✅", + "skipped": "⏭️", + "cancelled": "⚪", + "failure": "❌", + } + + lines = [ + "## [newci] Install & Update E2E — shadow result", + "", + "Observation only. This never fails, and it cannot affect the", + "production `Install & Update E2E` check.", + "", + "| Job | Result |", + "|---|---|", + ] + for name, info in sorted(needs.items()): + result = info.get("result", "unknown") + lines.append(f"| {name} | {icons.get(result, '❓')} {result} |") + + failed = sorted(n for n, i in needs.items() if i.get("result") == "failure") + lines.append("") + if failed: + lines += [ + f"⚠️ {len(failed)} shadow job(s) failed: {', '.join(failed)}", + "", + "If the legs were skipped instead, the runner lacks bubblewrap", + "support — see the preflight summary in", + "`newci-install-e2e-run.yml` for the capability matrix.", + ] + else: + lines.append("No shadow failures.") + + body = "\n".join(lines) + print(body) + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as fh: + fh.write(body + "\n")