diff --git a/scripts/__tests__/provision-worktree-self-heal.test.mjs b/scripts/__tests__/provision-worktree-self-heal.test.mjs new file mode 100644 index 0000000000..9f44379501 --- /dev/null +++ b/scripts/__tests__/provision-worktree-self-heal.test.mjs @@ -0,0 +1,204 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import test from "node:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const script = new URL("../provision-worktree.sh", import.meta.url).pathname; + +// Keep the PATH minimal so the fallback ladder is deterministic: node must be +// reachable, but a globally installed `paperclipai` must not shadow the paths +// under test. +const testPath = [path.dirname(process.execPath), "/usr/bin", "/bin"].join(":"); + +const cleanupDirs = []; + +function makeTempDir(prefix) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + cleanupDirs.push(dir); + return dir; +} + +test.after(() => { + for (const dir of cleanupDirs) { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +/** + * Writes a fake base workspace whose "tsx runner" is a plain node script, so + * the provision script's health check and init call can be steered per test. + * + * helpExit: exit code for `... index.ts --help` (the health check boot). + * initExit: exit code for `... index.ts worktree init ...`; on 0 the fake CLI + * writes a marker config so tests can tell CLI init from fallback. + */ +function makeBaseWorkspace({ helpExit, initExit }) { + const baseCwd = makeTempDir("paperclip-provision-base-"); + const runnerPath = path.join(baseCwd, "cli", "node_modules", "tsx", "dist", "cli.mjs"); + const entryPath = path.join(baseCwd, "cli", "src", "index.ts"); + fs.mkdirSync(path.dirname(runnerPath), { recursive: true }); + fs.mkdirSync(path.dirname(entryPath), { recursive: true }); + fs.writeFileSync(entryPath, "// fake CLI entry\n"); + fs.writeFileSync( + runnerPath, + ` +import fs from "node:fs"; +const cliArgs = process.argv.slice(3); +if (cliArgs.includes("--help")) { + if (${helpExit} !== 0) console.error("ERR_MODULE_NOT_FOUND: drizzle-orm"); + process.exit(${helpExit}); +} +if (cliArgs[0] === "worktree" && cliArgs[1] === "init") { + if (${initExit} !== 0) { + console.error("fake worktree init failure"); + process.exit(${initExit}); + } + fs.mkdirSync(".paperclip", { recursive: true }); + fs.writeFileSync(".paperclip/config.json", JSON.stringify({ $meta: { source: "fake-cli" } })); + fs.writeFileSync(".paperclip/.env", "PAPERCLIP_IN_WORKTREE=true\\n"); + process.exit(0); +} +process.exit(0); +`, + ); + return baseCwd; +} + +function runProvision(baseCwd, { pathPrefix } = {}) { + const worktreeCwd = makeTempDir("paperclip-provision-worktree-"); + const worktreesHome = makeTempDir("paperclip-provision-home-"); + const result = spawnSync("bash", [script], { + cwd: worktreeCwd, + encoding: "utf8", + env: { + PATH: pathPrefix ? `${pathPrefix}:${testPath}` : testPath, + HOME: os.homedir(), + PAPERCLIP_WORKSPACE_BASE_CWD: baseCwd, + PAPERCLIP_WORKSPACE_CWD: worktreeCwd, + PAPERCLIP_WORKSPACE_BRANCH: "feature/provision-test", + PAPERCLIP_WORKTREES_DIR: worktreesHome, + PAPERCLIP_HOME: path.join(worktreesHome, "no-such-instance-home"), + }, + }); + return { result, worktreeCwd, worktreesHome }; +} + +function readWorktreeConfig(worktreeCwd) { + const configPath = path.join(worktreeCwd, ".paperclip", "config.json"); + assert.ok(fs.existsSync(configPath), `expected ${configPath} to exist`); + return JSON.parse(fs.readFileSync(configPath, "utf8")); +} + +test("uses the base CLI when its import graph boots", () => { + const baseCwd = makeBaseWorkspace({ helpExit: 0, initExit: 0 }); + const { result, worktreeCwd } = runProvision(baseCwd); + + assert.equal(result.status, 0, result.stderr); + const config = readWorktreeConfig(worktreeCwd); + assert.equal(config.$meta.source, "fake-cli"); +}); + +test("falls back to an isolated config when the base CLI cannot boot", () => { + // Simulates the dangling pnpm symlink incident: the runner and entry files + // exist, but booting the CLI fails ESM resolution. The base has no + // package.json/pnpm-lock.yaml, so the repair install is not possible and the + // script must degrade to the no-CLI fallback config instead of failing. + const baseCwd = makeBaseWorkspace({ helpExit: 1, initExit: 0 }); + const { result, worktreeCwd, worktreesHome } = runProvision(baseCwd); + + assert.equal(result.status, 0, result.stderr); + assert.match(result.stderr, /writing isolated fallback config/); + const config = readWorktreeConfig(worktreeCwd); + assert.equal(config.$meta.source, "configure"); + const dataDir = config.database.embeddedPostgresDataDir; + assert.ok( + !path.relative(worktreesHome, dataDir).startsWith(".."), + `expected ${dataDir} to live under ${worktreesHome}`, + ); + const env = fs.readFileSync(path.join(worktreeCwd, ".paperclip", ".env"), "utf8"); + assert.match(env, /PAPERCLIP_IN_WORKTREE=true/); +}); + +test("repairs an unhealthy base install under the lock and then uses the CLI", (t) => { + const hasTools = ["flock", "git"].every( + (tool) => spawnSync("bash", ["-lc", `command -v ${tool}`], { env: { PATH: testPath } }).status === 0, + ); + if (!hasTools) { + t.skip("flock or git not available on this host"); + return; + } + + // The CLI's health is controlled by a flag file, and a fake `pnpm install` + // creates that flag — modeling a forced reinstall that relinks the store. + const baseCwd = makeTempDir("paperclip-provision-repair-base-"); + const healthFlag = path.join(baseCwd, "cli-healthy.flag"); + const runnerPath = path.join(baseCwd, "cli", "node_modules", "tsx", "dist", "cli.mjs"); + const entryPath = path.join(baseCwd, "cli", "src", "index.ts"); + fs.mkdirSync(path.dirname(runnerPath), { recursive: true }); + fs.mkdirSync(path.dirname(entryPath), { recursive: true }); + fs.writeFileSync(entryPath, "// fake CLI entry\n"); + fs.writeFileSync( + runnerPath, + ` +import fs from "node:fs"; +const cliArgs = process.argv.slice(3); +if (cliArgs.includes("--help")) { + process.exit(fs.existsSync(${JSON.stringify(healthFlag)}) ? 0 : 1); +} +if (cliArgs[0] === "worktree" && cliArgs[1] === "init") { + fs.mkdirSync(".paperclip", { recursive: true }); + fs.writeFileSync(".paperclip/config.json", JSON.stringify({ $meta: { source: "fake-cli" } })); + fs.writeFileSync(".paperclip/.env", "PAPERCLIP_IN_WORKTREE=true\\n"); + process.exit(0); +} +process.exit(0); +`, + ); + fs.writeFileSync(path.join(baseCwd, "package.json"), "{}\n"); + fs.writeFileSync(path.join(baseCwd, "pnpm-lock.yaml"), "lockfileVersion: '9.0'\n"); + spawnSync("git", ["init", "-q", baseCwd], { env: { PATH: testPath } }); + + const fakeBin = makeTempDir("paperclip-provision-fakebin-"); + const installLog = path.join(baseCwd, "pnpm-invocations.log"); + fs.writeFileSync( + path.join(fakeBin, "pnpm"), + `#!/usr/bin/env bash +if [[ "$1" == "install" ]]; then + echo "$@" >> ${JSON.stringify(installLog)} + touch ${JSON.stringify(healthFlag)} + exit 0 +fi +exit 1 +`, + { mode: 0o755 }, + ); + + const { result, worktreeCwd } = runProvision(baseCwd, { pathPrefix: fakeBin }); + + assert.equal(result.status, 0, result.stderr); + const config = readWorktreeConfig(worktreeCwd); + assert.equal(config.$meta.source, "fake-cli"); + const installs = fs.readFileSync(installLog, "utf8").trim().split("\n"); + assert.equal(installs.length, 1, `expected exactly one repair install, got: ${installs.join(" | ")}`); + assert.match(installs[0], /--force/); + assert.match(installs[0], /--frozen-lockfile/); + assert.ok( + fs.existsSync(path.join(baseCwd, ".git", "paperclip-provision-repair.lock")), + "expected the repair lock file inside the resolved git dir", + ); +}); + +test("a failed CLI init fails provisioning instead of being masked as success", () => { + // Regression test for the masked `return 0` after the init subshell: a CLI + // that passes the health check but fails `worktree init` signals a real + // problem, so the script must propagate the failure rather than report + // success or write an unseeded fallback config over it. + const baseCwd = makeBaseWorkspace({ helpExit: 0, initExit: 3 }); + const { result, worktreeCwd } = runProvision(baseCwd); + + assert.equal(result.status, 3, result.stderr); + assert.match(result.stderr, /fake worktree init failure/); + assert.ok(!fs.existsSync(path.join(worktreeCwd, ".paperclip", "config.json"))); +}); diff --git a/scripts/provision-worktree.sh b/scripts/provision-worktree.sh index 2435ba4aa6..fbfdda97b8 100644 --- a/scripts/provision-worktree.sh +++ b/scripts/provision-worktree.sh @@ -31,32 +31,89 @@ source_env_path="$(dirname "$source_config_path")/.env" mkdir -p "$paperclip_dir" -run_isolated_worktree_init() { - local base_cli_runner="$base_cwd/cli/node_modules/tsx/dist/cli.mjs" - local base_cli_entry="$base_cwd/cli/src/index.ts" +base_cli_runner_path="$base_cwd/cli/node_modules/tsx/dist/cli.mjs" +base_cli_entry_path="$base_cwd/cli/src/index.ts" - if [[ -f "$base_cli_runner" && -f "$base_cli_entry" ]]; then +base_cli_files_present() { + [[ -f "$base_cli_runner_path" && -f "$base_cli_entry_path" ]] +} + +# File existence is not enough: pnpm links package node_modules into the +# versioned virtual store, so a lockfile change plus a partial/filtered install +# in the base workspace leaves dangling symlinks that fail ESM resolution at +# runtime. Actually boot the CLI to prove its import graph resolves. +base_cli_healthy() { + base_cli_files_present || return 1 + (cd "$base_cwd" && node "$base_cli_runner_path" "$base_cli_entry_path" --help >/dev/null 2>&1) +} + +repair_base_workspace_install() { + command -v pnpm >/dev/null 2>&1 || return 1 + [[ -f "$base_cwd/package.json" && -f "$base_cwd/pnpm-lock.yaml" ]] || return 1 + echo "Base workspace CLI at $base_cli_entry_path failed its health check (typically dangling pnpm symlinks after a partial install); repairing with pnpm install in $base_cwd." >&2 + # --force guarantees relinking even when pnpm's up-to-date heuristics would + # otherwise skip the dangling symlinks; --frozen-lockfile keeps the repair + # from mutating the shared base workspace's lockfile. + local repair_cmd=(pnpm install --prod=false --force --frozen-lockfile --config.confirmModulesPurge=false) + # Resolve the real git dir so locking also covers base workspaces that are + # linked worktrees, where "$base_cwd/.git" is a file rather than a directory. + local repair_lock_dir="" + if command -v git >/dev/null 2>&1; then + repair_lock_dir="$(git -C "$base_cwd" rev-parse --absolute-git-dir 2>/dev/null || true)" + fi + if [[ ! -d "$repair_lock_dir" && -d "$base_cwd/.git" ]]; then + repair_lock_dir="$base_cwd/.git" + fi + if command -v flock >/dev/null 2>&1 && [[ -d "$repair_lock_dir" ]]; then + # The post-repair verification must run under the same lock: a concurrent + # provision's forced install could be mid-relink during an unlocked check + # and fail a repair that actually succeeded. Holding the lock also means a + # process that queued behind a peer's repair can skip its own reinstall. ( - cd "$worktree_cwd" - node "$base_cli_runner" "$base_cli_entry" worktree init --force --seed-mode minimal --name "$worktree_name" --from-config "$source_config_path" + cd "$base_cwd" || exit 1 + exec 9>"$repair_lock_dir/paperclip-provision-repair.lock" + flock 9 + if base_cli_healthy; then + echo "Base workspace CLI became healthy while waiting for the repair lock; skipping reinstall." >&2 + exit 0 + fi + env -u NODE_ENV CI=true "${repair_cmd[@]}" >&2 || exit 1 + base_cli_healthy ) - return 0 + else + (cd "$base_cwd" && env -u NODE_ENV CI=true "${repair_cmd[@]}" >&2 && base_cli_healthy) + fi +} + +ensure_base_cli_healthy() { + base_cli_files_present || return 1 + base_cli_healthy && return 0 + repair_base_workspace_install +} + +run_isolated_worktree_init() { + if ensure_base_cli_healthy; then + ( + cd "$worktree_cwd" && + node "$base_cli_runner_path" "$base_cli_entry_path" worktree init --force --seed-mode minimal --name "$worktree_name" --from-config "$source_config_path" + ) + return fi if command -v pnpm >/dev/null 2>&1 && pnpm paperclipai --help >/dev/null 2>&1; then ( - cd "$worktree_cwd" - pnpm paperclipai worktree init --force --seed-mode minimal --name "$worktree_name" --from-config "$source_config_path" + cd "$worktree_cwd" && + pnpm paperclipai worktree init --force --seed-mode minimal --name "$worktree_name" --from-config "$source_config_path" ) - return 0 + return fi if command -v paperclipai >/dev/null 2>&1; then ( - cd "$worktree_cwd" - paperclipai worktree init --force --seed-mode minimal --name "$worktree_name" --from-config "$source_config_path" + cd "$worktree_cwd" && + paperclipai worktree init --force --seed-mode minimal --name "$worktree_name" --from-config "$source_config_path" ) - return 0 + return fi return 127 @@ -67,9 +124,7 @@ paperclipai_command_available() { return 0 fi - local base_cli_tsx_path="$base_cwd/cli/node_modules/tsx/dist/cli.mjs" - local base_cli_entry_path="$base_cwd/cli/src/index.ts" - if command -v node >/dev/null 2>&1 && [[ -f "$base_cli_tsx_path" ]] && [[ -f "$base_cli_entry_path" ]]; then + if command -v node >/dev/null 2>&1 && base_cli_files_present; then return 0 fi @@ -429,9 +484,24 @@ else echo "Existing isolated Paperclip worktree config is stale for this host; regenerating." >&2 fi if paperclipai_command_available; then - run_isolated_worktree_init + if run_isolated_worktree_init; then + : + else + init_exit_code=$? + if [[ "$init_exit_code" -eq 127 ]]; then + # Every CLI candidate was unusable (e.g. an unhealthy base install that + # the repair could not fix); degrade instead of stranding the run. + echo "No usable paperclipai CLI found; writing isolated fallback config without DB seeding." >&2 + write_fallback_worktree_config + else + # A CLI that ran and failed signals a real problem; do not paper over + # it with an unseeded fallback config. + echo "paperclipai worktree init failed (exit $init_exit_code); failing provisioning instead of writing an unseeded fallback config." >&2 + exit "$init_exit_code" + fi + fi else - echo "paperclipai CLI not available in this workspace; writing isolated fallback config without DB seeding." >&2 + echo "paperclipai worktree init unavailable; writing isolated fallback config without DB seeding." >&2 write_fallback_worktree_config fi fi