diff --git a/.github/workflows/release-smoke.yml b/.github/workflows/release-smoke.yml index 5ec66da8a6..1656d4f16f 100644 --- a/.github/workflows/release-smoke.yml +++ b/.github/workflows/release-smoke.yml @@ -38,6 +38,65 @@ on: type: string jobs: + # The Docker smoke below can never exercise the background-service leg of + # onboarding: containers have no service manager, so v2026.824.0 shipped a + # service install that crash-looped on a missing shim while every + # golden-path check stayed green. Run the same published artifact directly + # on the runner VM's systemd and require the installed service to end up + # serving. + smoke_service: + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Setup Node.js + uses: actions/setup-node@v7 + with: + node-version: 24 + + - name: Start a user systemd session + # The hosted runner has no login session for the runner user, so + # `systemctl --user` cannot reach a user manager until lingering + # starts one. Export the session address for the steps below. + run: | + sudo loginctl enable-linger "$(id -un)" + uid="$(id -u)" + for _ in $(seq 1 30); do + [[ -S "/run/user/$uid/bus" ]] && break + sleep 1 + done + [[ -S "/run/user/$uid/bus" ]] + { + echo "XDG_RUNTIME_DIR=/run/user/$uid" + echo "DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$uid/bus" + } >> "$GITHUB_ENV" + + - name: Onboard with the background service + env: + PAPERCLIPAI_VERSION: ${{ inputs.paperclip_version }} + DATA_DIR: ${{ runner.temp }}/service-smoke-data + SMOKE_CLEANUP: "false" + run: ./scripts/service-onboard-smoke.sh + + - name: Capture service diagnostics + if: always() + run: | + { + systemctl --user --no-pager status paperclipai.service || true + journalctl --user -u paperclipai.service --no-pager || true + } > "$RUNNER_TEMP/paperclipai-service.log" 2>&1 + + - name: Upload service diagnostics + if: always() + uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.artifact_name }}-service + path: ${{ runner.temp }}/paperclipai-service.log + retention-days: 14 + smoke: runs-on: ubuntu-latest timeout-minutes: 45 diff --git a/package.json b/package.json index 16132aeaae..2966139626 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "smoke:mcp-fixtures": "node scripts/smoke/mcp-fixture-harness.mjs", "smoke:pipelines-tutorial": "./scripts/smoke/pipelines-tutorial-smoke.sh", "smoke:terminal-bench-loop-skill": "node scripts/smoke/terminal-bench-loop-skill-smoke.mjs", - "test:release-registry": "node --test scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs", + "test:release-registry": "node --test scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs", "storybook-visual:baseline": "node scripts/storybook-visual-baseline.mjs", "test:storybook-visual": "node scripts/storybook-visual-baseline.mjs download && node scripts/storybook-visual-baseline.mjs verify && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts", "test:storybook-visual:update": "node scripts/storybook-visual-baseline.mjs download && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts --update-snapshots && node scripts/storybook-visual-baseline.mjs pack", diff --git a/scripts/service-onboard-smoke.sh b/scripts/service-onboard-smoke.sh new file mode 100755 index 0000000000..4b2d5ec9e4 --- /dev/null +++ b/scripts/service-onboard-smoke.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Prove that `onboard --install-service` on a released artifact leaves a +# working background service. The Docker onboard smoke can never cover this +# leg: containers have no service manager, so a release whose service install +# crash-loops on a missing shim (v2026.824.0) still passes every golden-path +# check. This script runs the published npm artifact on a real systemd user +# session and fails unless the installed service itself ends up serving +# /api/health. +# +# Requirements: a Linux host with a user systemd session. In CI that means +# `loginctl enable-linger` plus XDG_RUNTIME_DIR / DBUS_SESSION_BUS_ADDRESS +# pointing at /run/user/; see the smoke_service job in +# .github/workflows/release-smoke.yml. + +PAPERCLIPAI_VERSION="${PAPERCLIPAI_VERSION:-latest}" +DATA_DIR="${DATA_DIR:-$(mktemp -d "${TMPDIR:-/tmp}/paperclip-service-smoke.XXXXXX")}" +ONBOARD_TIMEOUT_SECONDS="${ONBOARD_TIMEOUT_SECONDS:-600}" +SMOKE_READY_TIMEOUT_SECONDS="${SMOKE_READY_TIMEOUT_SECONDS:-420}" +HEALTH_URL="${HEALTH_URL:-http://127.0.0.1:3100/api/health}" +SERVICE_NAME="paperclipai.service" +SHIM_PATH="${PAPERCLIP_SHIM_PATH:-$HOME/.local/bin/paperclipai}" +# Cleanup defaults to on so a local run does not leave a service behind; CI +# disables it so the diagnostics step can still inspect the unit. +SMOKE_CLEANUP="${SMOKE_CLEANUP:-true}" +SMOKE_FORCE="${SMOKE_FORCE:-false}" + +fail() { + echo "Service smoke failed: $*" >&2 + exit 1 +} + +diagnostics() { + echo "--- systemctl --user status $SERVICE_NAME ---" >&2 + systemctl --user --no-pager status "$SERVICE_NAME" >&2 || true + echo "--- journalctl --user -u $SERVICE_NAME (last 100 lines) ---" >&2 + journalctl --user -u "$SERVICE_NAME" --no-pager -n 100 >&2 || true +} + +cleanup() { + if [[ "$SMOKE_CLEANUP" == "true" ]]; then + if [[ -x "$SHIM_PATH" ]]; then + "$SHIM_PATH" service uninstall >/dev/null 2>&1 || true + fi + systemctl --user stop "$SERVICE_NAME" >/dev/null 2>&1 || true + fi +} +trap cleanup EXIT INT TERM + +command -v systemctl >/dev/null 2>&1 || fail "systemctl is not available on this host" +systemctl --user show-environment >/dev/null 2>&1 \ + || fail "no user systemd session; enable lingering and export XDG_RUNTIME_DIR first" + +# Refuse to smoke over a host that already has a managed install: the +# assertions below would prove nothing, and cleanup would tear down a real +# service. +if [[ "$SMOKE_FORCE" != "true" ]]; then + if [[ -e "$SHIM_PATH" ]]; then + fail "$SHIM_PATH already exists; set SMOKE_FORCE=true to smoke over it" + fi + if systemctl --user cat "$SERVICE_NAME" >/dev/null 2>&1; then + fail "$SERVICE_NAME is already installed; set SMOKE_FORCE=true to smoke over it" + fi +fi + +echo "==> Onboarding paperclipai@$PAPERCLIPAI_VERSION with --install-service" +echo " Data dir: $DATA_DIR" +if ! timeout "$ONBOARD_TIMEOUT_SECONDS" \ + npx --yes "paperclipai@${PAPERCLIPAI_VERSION}" onboard --yes --install-service --data-dir "$DATA_DIR"; then + diagnostics + fail "onboard exited non-zero" +fi + +echo "==> Verifying the managed shim" +if [[ ! -x "$SHIM_PATH" ]]; then + diagnostics + fail "no executable shim at $SHIM_PATH after onboarding" +fi + +echo "==> Waiting for $SERVICE_NAME to serve $HEALTH_URL" +for ((i = 1; i <= SMOKE_READY_TIMEOUT_SECONDS; i += 1)); do + state="$(systemctl --user is-active "$SERVICE_NAME" 2>/dev/null || true)" + if [[ "$state" == "failed" ]]; then + diagnostics + fail "$SERVICE_NAME entered the failed state" + fi + if curl -fsS "$HEALTH_URL" >/dev/null 2>&1; then + if [[ "$state" != "active" ]]; then + diagnostics + fail "$HEALTH_URL answers but $SERVICE_NAME is '$state' - something other than the service is serving" + fi + echo "==> Service smoke passed: $SERVICE_NAME is active and serving $HEALTH_URL" + exit 0 + fi + sleep 1 +done + +diagnostics +fail "$HEALTH_URL not ready after ${SMOKE_READY_TIMEOUT_SECONDS}s (unit state: $(systemctl --user is-active "$SERVICE_NAME" 2>/dev/null || echo unknown))" diff --git a/scripts/service-onboard-smoke.test.mjs b/scripts/service-onboard-smoke.test.mjs new file mode 100644 index 0000000000..b859cd3f1b --- /dev/null +++ b/scripts/service-onboard-smoke.test.mjs @@ -0,0 +1,52 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { accessSync, constants, readFileSync } from "node:fs"; +import { join } from "node:path"; +import test from "node:test"; + +// Pins the wiring that makes the background-service smoke an effective gate. +// The service leg exists because v2026.824.0 shipped a service install that +// crash-looped on a missing shim while the Docker smoke stayed green; these +// assertions keep the job from being silently disconnected or weakened. + +const repoRoot = new URL("..", import.meta.url).pathname.replace(/\/$/, ""); +const scriptPath = join(repoRoot, "scripts", "service-onboard-smoke.sh"); +const script = readFileSync(scriptPath, "utf8"); +const smokeWorkflow = readFileSync(join(repoRoot, ".github", "workflows", "release-smoke.yml"), "utf8"); +const releaseWorkflow = readFileSync(join(repoRoot, ".github", "workflows", "release.yml"), "utf8"); + +test("smoke script is executable and parses", () => { + accessSync(scriptPath, constants.X_OK); + execFileSync("bash", ["-n", scriptPath]); +}); + +test("smoke script keeps its load-bearing assertions", () => { + assert.match(script, /^set -euo pipefail$/m); + // Onboards the published artifact with the service leg forced on. + assert.match(script, /onboard --yes --install-service/); + // Fails when the shim never materialized. + assert.match(script, /no executable shim at .*after onboarding/); + // Fails when the unit dies instead of serving. + assert.match(script, /entered the failed state/); + // Fails when health answers but the service is not what is serving -- + // the exact signature of the v2026.824.0 defect. + assert.match(script, /something other than the service is serving/); + // Refuses to smoke over a real install unless forced. + assert.match(script, /SMOKE_FORCE/); +}); + +test("release-smoke workflow runs the service leg against the input version", () => { + assert.match(smokeWorkflow, /^ smoke_service:$/m); + assert.match(smokeWorkflow, /scripts\/service-onboard-smoke\.sh/); + const serviceJob = smokeWorkflow.split(/^ smoke:$/m)[0]; + assert.match(serviceJob, /PAPERCLIPAI_VERSION: \$\{\{ inputs\.paperclip_version \}\}/); + // Diagnostics must survive the run: cleanup stays off in CI and the + // artifact name cannot collide with the Docker job's upload. + assert.match(serviceJob, /SMOKE_CLEANUP: "false"/); + assert.match(serviceJob, /\$\{\{ inputs\.artifact_name \}\}-service/); +}); + +test("nightly and beta smokes still route through the reusable workflow", () => { + const calls = releaseWorkflow.match(/uses: \.\/\.github\/workflows\/release-smoke\.yml/g) ?? []; + assert.ok(calls.length >= 2, "smoke_nightly and smoke_beta must call release-smoke.yml so smoke_service gates them"); +});