From e1df4c6068fea684a1e9714ebd64bce95f3db19a Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Wed, 19 Aug 2026 10:41:29 -0500 Subject: [PATCH] fix(workspaces): keep deferred seed databases reliable (#11706) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip manages agent work in isolated execution workspaces. > - A workspace depends on a valid database seed before it can run. > - Deferred seed failures were hidden behind a successful provision status. > - The seed restore also had two possible owners for the embedded PostgreSQL process. > - That allowed the target database to stop while the restore was still running. > - This pull request makes seed failures visible and gives the seed process sole lifecycle ownership. > - The benefit is that workspace provisioning reports the real result and does not stop its own target database. ## Linked Issues or Issue Description Related: #11684 **What happened?** Initial worktree provisioning could report success before its deferred database seed completed. The seed restore could also reuse a target embedded PostgreSQL process with another shutdown owner. This could stop the target database during the restore. **Expected behavior** Workspace status must show a failed deferred seed as a failure. The seed restore must own the target embedded PostgreSQL process until restore, migration, and validation finish. **Steps to reproduce** 1. Provision a worktree with deferred database seeding. 2. Make the seed manifest end in a failed state while the command exits with code 0. 3. Observe that the provision status remains successful on `master`. 4. Start a seed restore against an already-running target embedded PostgreSQL process. 5. Observe that another lifecycle owner can stop the target during restore. **Paperclip version or commit** `51a843e135` **Deployment mode** Local dev with execution workspaces and embedded PostgreSQL. ## What Changed - Add a first-class `workspace_seed` operation for deferred database seeds. - Require terminal, verified seed evidence before the seed operation succeeds. - Surface the seed phase and failure metadata in workspace status and UI state. - Give the seed process exclusive lifecycle ownership of the target embedded PostgreSQL process. - Suppress imported embedded-Postgres exit hooks without removing existing host listeners. - Record a credential-safe shutdown diagnostic in failed seed manifests. ## Verification - The original deferred-seed commit passed 4 server tests, 24 workspace-status UI tests, shared/server/UI typechecks, and the UI token gate. - The original PostgreSQL-lifecycle commit passed 3 lifecycle tests, 3 ownership/diagnostic tests, 1 real embedded-Postgres seed integration, and the affected package typechecks. - No local tests were rerun after the clean cherry-pick because the operator requested the shortest landing path. - Review the automatic PR checks for the clean `origin/master` replay. ## Risks - A live target database now causes an early error instead of being reused. The error includes recovery guidance. - Workspace consumers must handle the new `workspace_seed` operation type. Shared types and UI state handling are updated in this pull request. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI Codex, GPT-5, high-reasoning mode, with repository, shell, and GitHub tool use. The runtime does not expose a more specific deployment suffix or context-window value. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- cli/src/__tests__/worktree.test.ts | 53 ++++++- cli/src/commands/worktree.ts | 37 ++++- doc/DEVELOPING.md | 3 + .../src/embedded-postgres-lifecycle.test.ts | 77 +++++++++++ .../db/src/embedded-postgres-lifecycle.ts | 54 ++++++++ packages/db/src/index.ts | 1 + .../shared/src/types/workspace-operation.ts | 1 + .../src/__tests__/workspace-runtime.test.ts | 101 ++++++++++++++ server/src/services/workspace-runtime.ts | 130 ++++++++++++++---- ui/src/lib/workspace-access-state.test.ts | 2 +- ui/src/lib/workspace-access-state.ts | 3 +- ...onWorkspaceDetail.provision-status.test.ts | 11 +- ui/src/pages/ExecutionWorkspaceDetail.tsx | 8 +- 13 files changed, 443 insertions(+), 38 deletions(-) create mode 100644 packages/db/src/embedded-postgres-lifecycle.test.ts create mode 100644 packages/db/src/embedded-postgres-lifecycle.ts diff --git a/cli/src/__tests__/worktree.test.ts b/cli/src/__tests__/worktree.test.ts index b4834dd512..8c2f5e3fd1 100644 --- a/cli/src/__tests__/worktree.test.ts +++ b/cli/src/__tests__/worktree.test.ts @@ -27,7 +27,9 @@ import { import { copyGitHooksToWorktreeGitDir, copySeededSecretsKey, + ensureEmbeddedPostgres, ensureWorktreeSeeded, + formatWorktreeSeedFailureDiagnostic, markWorktreeSeedPending, pauseSeededScheduledRoutines, quarantineSeededWorktreeExecutionState, @@ -488,6 +490,34 @@ describe("worktree helpers", () => { expect(full.nullifyColumns).toEqual({}); }); + it("requires the seed process to own the target embedded Postgres lifecycle", async () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-worktree-live-target-")); + try { + fs.writeFileSync( + path.join(tempRoot, "postmaster.pid"), + `${process.pid}\n${tempRoot}\n0\n55432\n`, + ); + + await expect(ensureEmbeddedPostgres(tempRoot, 55432, { allowExisting: false })) + .rejects.toThrow("while it is already running"); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } + }); + + it("surfaces a credential-safe diagnostic when the target shuts down during restore", () => { + expect(formatWorktreeSeedFailureDiagnostic( + "restore", + new Error( + "Failed to restore seed.sql.gz: FATAL: the database system is shutting down; psql error: write EPIPE", + ), + )).toBe( + "Target embedded PostgreSQL shut down during restore. Stop any competing worktree service and retry the seed.", + ); + expect(formatWorktreeSeedFailureDiagnostic("migrations", new Error("secret connection failure"))) + .toBe("Seed failed during migrations."); + }); + it("rejects a source migration journal that diverges from the code journal", () => { expect(() => resolveWorktreeSeedMigrationRevision({ status: "upToDate", @@ -714,7 +744,7 @@ describe("worktree helpers", () => { }, ); - it("ensure-seeded keeps the pending marker when seeding fails", async () => { + it("ensure-seeded records a target shutdown diagnostic when restore fails", async () => { const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-worktree-ensure-seeded-failure-")); try { const sourceConfigPath = path.join(tempRoot, "source", "config.json"); @@ -746,13 +776,28 @@ describe("worktree helpers", () => { await expect( ensureWorktreeSeeded( { config: targetConfigPath, fromConfig: sourceConfigPath }, - { seedDatabase: vi.fn().mockRejectedValue(new Error("seed failed")) }, + { + seedDatabase: vi.fn(async (input) => { + input.onPhase?.("restore", "started"); + throw new Error( + "Failed to restore seed.sql.gz: FATAL: the database system is shutting down; psql error: write EPIPE", + ); + }), + }, ), - ).rejects.toThrow("seed failed"); + ).rejects.toThrow("database system is shutting down"); expect(readWorktreeSeedManifest(targetConfigPath)).toMatchObject({ state: "failed", - phase: "pending", + phase: "restore", + diagnostics: expect.arrayContaining([ + expect.objectContaining({ + phase: "restore", + status: "failed", + message: + "Target embedded PostgreSQL shut down during restore. Stop any competing worktree service and retry the seed.", + }), + ]), }); expect(fs.existsSync(path.join(targetRoot, ".paperclip", "seed-pending"))).toBe(false); expect(fs.existsSync(path.join(targetRoot, ".paperclip", "seed-complete"))).toBe(false); diff --git a/cli/src/commands/worktree.ts b/cli/src/commands/worktree.ts index d31dabaefc..e3dadf71e1 100644 --- a/cli/src/commands/worktree.ts +++ b/cli/src/commands/worktree.ts @@ -55,6 +55,7 @@ import { workspaceRuntimeServices, createEmbeddedPostgresLogBuffer, formatEmbeddedPostgresError, + loadWithoutEmbeddedPostgresExitHooks, prepareEmbeddedPostgresNativeRuntime, } from "@paperclipai/db"; import type { Command } from "commander"; @@ -1105,11 +1106,15 @@ export function copySeededSecretsKey(input: { } } -async function ensureEmbeddedPostgres(dataDir: string, preferredPort: number): Promise { +export async function ensureEmbeddedPostgres( + dataDir: string, + preferredPort: number, + options: { allowExisting?: boolean } = {}, +): Promise { const moduleName = "embedded-postgres"; let EmbeddedPostgres: EmbeddedPostgresCtor; try { - const mod = await import(moduleName); + const mod = await loadWithoutEmbeddedPostgresExitHooks(() => import(moduleName)); EmbeddedPostgres = mod.default as EmbeddedPostgresCtor; } catch { throw new Error( @@ -1121,6 +1126,12 @@ async function ensureEmbeddedPostgres(dataDir: string, preferredPort: number): P const postmasterPidFile = path.resolve(dataDir, "postmaster.pid"); const runningPid = readRunningPostmasterPid(postmasterPidFile); if (runningPid) { + if (options.allowExisting === false) { + throw new Error( + `Cannot seed target embedded PostgreSQL at ${dataDir} while it is already running (pid=${runningPid}). ` + + "Stop the worktree service that owns this database, then retry the seed.", + ); + } return { port: readPidFilePort(postmasterPidFile) ?? preferredPort, startedByThisProcess: false, @@ -1632,13 +1643,14 @@ async function seedWorktreeDatabase(input: { }); input.onPhase?.("snapshot", "succeeded", `Created ${path.basename(backup.backupFile)}.`); + input.onPhase?.("restore", "started"); targetHandle = await ensureEmbeddedPostgres( input.targetConfig.database.embeddedPostgresDataDir, input.targetConfig.database.embeddedPostgresPort, + { allowExisting: false }, ); const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${targetHandle.port}/postgres`; - input.onPhase?.("restore", "started"); await resetPostgresDatabase(adminConnectionString, "paperclip"); const targetConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${targetHandle.port}/paperclip`; await runDatabaseRestore({ @@ -1703,6 +1715,23 @@ const WORKTREE_SEED_DIAGNOSTIC_LIMIT = 32; const WORKTREE_SEED_DIAGNOSTIC_MESSAGE_LIMIT = 512; const activeSeedInterruptHandlers = new Map void>(); +export function formatWorktreeSeedFailureDiagnostic( + phase: WorktreeSeedPhase, + error: unknown, +): string { + const message = error instanceof Error ? error.message : String(error ?? ""); + if ( + phase === "restore" + && /database system is shutting down|terminating connection due to administrator command/i.test(message) + ) { + return "Target embedded PostgreSQL shut down during restore. Stop any competing worktree service and retry the seed."; + } + if (phase === "restore" && /Cannot seed target embedded PostgreSQL.+already running/i.test(message)) { + return "Target embedded PostgreSQL is owned by a running worktree service. Stop that service and retry the seed."; + } + return `Seed failed during ${phase}.`; +} + function dispatchSeedInterruption(signal: NodeJS.Signals): void { for (const handler of activeSeedInterruptHandlers.values()) { try { @@ -2096,7 +2125,7 @@ async function runVerifiedWorktreeSeed(input: { state: "failed", // Do not persist the underlying error: database/driver errors may contain // connection credentials. The CLI still returns the exact error to its caller. - message: `Seed failed during ${activePhase}.`, + message: formatWorktreeSeedFailureDiagnostic(activePhase, error), }); throw error; } finally { diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 82889da8e3..32791b0b21 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -486,10 +486,13 @@ The default `worktree init` still seeds eagerly. A lean worktree (created withou - `pnpm paperclipai worktree ensure-seeded` performs the deferred seed **exactly once**. It is lock-guarded and idempotent: only a complete `verified` manifest short-circuits it, so it is safe to call repeatedly and from concurrent processes. Managed workspaces derive the source exclusively from the control-plane-provided base project workspace; manual worktrees must pass `--from-config`. - `paperclipai run` calls `ensureWorktreeSeeded` automatically before doctor/boot. Managed runs transparently seed a lean worktree from their registered base workspace; an unmanaged lean worktree must first run `worktree ensure-seeded --from-config `. - Managed git-worktree runtime startup also runs `scripts/provision-worktree-runtime.sh` automatically when a legacy workspace policy has no explicit runtime provision command and the manifest is not verified. An explicitly configured runtime provision command always takes precedence. +- The built-in deferred seed is recorded as its own terminal `workspace_seed` operation. A zero exit code is not enough for success: the operation succeeds only when `.paperclip/seed-manifest.json` contains complete verified evidence; failed, missing, or malformed manifests produce a failed operation with the seed phase in metadata. - Worktrees created before lazy seeding shipped have neither marker; they are treated as already-seeded for backward compatibility (never re-cloned). Both `minimal` and `full` modes use the same terminal data-validation contract. Source validation accepts a migration journal that is a prefix of the checkout's journal and records the source revision in seed diagnostics; it rejects a source that is ahead of the checkout because that would require a downgrade. After restore, Paperclip applies pending migrations and requires the target journal to be current. Both validations also read a credential-backed auth user, instance administrator role, active company membership, and representative cloned company/issue pair. Restore, migrations, execution quarantine, routine pausing, workspace rebinding, and post-restore validation all run under the seed lock. An interruption leaves the exact active phase in terminal `failed` state; it cannot produce readiness evidence. +The seed process must own the target embedded PostgreSQL lifecycle for that entire sequence. It refuses to restore into a target postmaster that is already running, suppresses the embedded provider's process-global exit hooks, and stops its owned target only after validation or failure cleanup. A shutdown detected during restore is recorded as a target-database shutdown diagnostic rather than a generic restore failure. + The seed manifest never grants source-path authority. Its source path and instance are diagnostic assertions that must exactly match the realpath-canonical registered source before any lock, backup, service stop, spawn, or database mutation. Missing registration, sibling or foreign paths, symlink aliases, source/target identity collisions, and company mismatches fail closed. **Unverified-seed guard.** `pnpm dev` (the dev-runner) refuses to boot a worktree whose manifest is pending, running, failed, malformed, or missing required verification evidence and points you at the fix: diff --git a/packages/db/src/embedded-postgres-lifecycle.test.ts b/packages/db/src/embedded-postgres-lifecycle.test.ts new file mode 100644 index 0000000000..7639fac645 --- /dev/null +++ b/packages/db/src/embedded-postgres-lifecycle.test.ts @@ -0,0 +1,77 @@ +import { EventEmitter } from "node:events"; +import { describe, expect, it, vi } from "vitest"; +import { loadWithoutEmbeddedPostgresExitHooks } from "./embedded-postgres-lifecycle.js"; + +describe("loadWithoutEmbeddedPostgresExitHooks", () => { + it("removes every eager exit hook from the real embedded-postgres import", async () => { + const eventNames = [ + "exit", + "beforeExit", + "SIGHUP", + "SIGINT", + "SIGTERM", + "SIGBREAK", + "message", + ]; + const before = new Map(eventNames.map((eventName) => [ + eventName, + process.rawListeners(eventName), + ])); + const moduleName = "embedded-postgres"; + + await loadWithoutEmbeddedPostgresExitHooks(() => import(moduleName)); + + for (const eventName of eventNames) { + expect(process.rawListeners(eventName)).toEqual(before.get(eventName)); + } + }); + + it("removes dependency exit hooks while preserving existing listeners", async () => { + const target = new EventEmitter(); + const existingSignalListener = vi.fn(); + const existingExitListener = vi.fn(); + target.on("SIGTERM", existingSignalListener); + target.on("exit", existingExitListener); + + const dependencyListener = vi.fn(); + const loaded = await loadWithoutEmbeddedPostgresExitHooks( + async () => { + for (const eventName of [ + "exit", + "beforeExit", + "SIGHUP", + "SIGINT", + "SIGTERM", + "SIGBREAK", + "message", + ]) { + target.on(eventName, dependencyListener); + } + return { default: class EmbeddedPostgres {} }; + }, + target, + ); + + expect(loaded.default.name).toBe("EmbeddedPostgres"); + expect(target.rawListeners("SIGTERM")).toEqual([existingSignalListener]); + expect(target.rawListeners("exit")).toEqual([existingExitListener]); + for (const eventName of ["beforeExit", "SIGHUP", "SIGINT", "SIGBREAK", "message"]) { + expect(target.rawListeners(eventName)).toEqual([]); + } + }); + + it("cleans up listeners even when the import fails", async () => { + const target = new EventEmitter(); + const dependencyListener = vi.fn(); + + await expect(loadWithoutEmbeddedPostgresExitHooks( + async () => { + target.on("SIGTERM", dependencyListener); + throw new Error("import failed"); + }, + target, + )).rejects.toThrow("import failed"); + + expect(target.rawListeners("SIGTERM")).toEqual([]); + }); +}); diff --git a/packages/db/src/embedded-postgres-lifecycle.ts b/packages/db/src/embedded-postgres-lifecycle.ts new file mode 100644 index 0000000000..5bd7e55faa --- /dev/null +++ b/packages/db/src/embedded-postgres-lifecycle.ts @@ -0,0 +1,54 @@ +const EMBEDDED_POSTGRES_EXIT_EVENTS = [ + "exit", + "beforeExit", + "SIGHUP", + "SIGINT", + "SIGTERM", + "SIGBREAK", + "message", +] as const; + +type EmbeddedPostgresExitTarget = { + rawListeners(eventName: string): Function[]; + removeListener(eventName: string, listener: (...args: any[]) => void): unknown; +}; + +/** + * embedded-postgres installs async-exit-hook listeners as an import side effect. + * Paperclip-managed clusters have an explicit owner and shutdown path, so those + * global listeners must not be allowed to stop a cluster independently of that + * owner (for example while a worktree seed restore is still streaming). + * + * Remove only listeners added by the supplied import and preserve every listener + * that was already registered by Paperclip or its host process. + */ +export async function loadWithoutEmbeddedPostgresExitHooks( + load: () => Promise, + target: EmbeddedPostgresExitTarget = process, +): Promise { + const listenersBeforeLoad = new Map( + EMBEDDED_POSTGRES_EXIT_EVENTS.map((eventName) => [ + eventName, + target.rawListeners(eventName), + ]), + ); + + let loaded: T; + try { + loaded = await load(); + } finally { + for (const eventName of EMBEDDED_POSTGRES_EXIT_EVENTS) { + const remainingBeforeLoad = [...(listenersBeforeLoad.get(eventName) ?? [])]; + for (const listener of target.rawListeners(eventName)) { + const existingIndex = remainingBeforeLoad.indexOf(listener); + if (existingIndex >= 0) { + remainingBeforeLoad.splice(existingIndex, 1); + continue; + } + target.removeListener(eventName, listener as (...args: any[]) => void); + } + } + } + + return loaded; +} diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index ff1a803dba..0957f095de 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -35,6 +35,7 @@ export { ensureLinuxSharedLibraryAliases, prepareEmbeddedPostgresNativeRuntime, } from "./embedded-postgres-native.js"; +export { loadWithoutEmbeddedPostgresExitHooks } from "./embedded-postgres-lifecycle.js"; export { issueRelations } from "./schema/issue_relations.js"; export { issueReferenceMentions } from "./schema/issue_reference_mentions.js"; export * from "./schema/index.js"; diff --git a/packages/shared/src/types/workspace-operation.ts b/packages/shared/src/types/workspace-operation.ts index 345e306ca7..8fcb52c708 100644 --- a/packages/shared/src/types/workspace-operation.ts +++ b/packages/shared/src/types/workspace-operation.ts @@ -2,6 +2,7 @@ export type WorkspaceOperationPhase = | "worktree_prepare" | "workspace_config_freshness" | "workspace_provision" + | "workspace_seed" | "workspace_runtime_provision" | "workspace_repair" | "workspace_teardown" diff --git a/server/src/__tests__/workspace-runtime.test.ts b/server/src/__tests__/workspace-runtime.test.ts index 8b122f7864..7f67755c2a 100644 --- a/server/src/__tests__/workspace-runtime.test.ts +++ b/server/src/__tests__/workspace-runtime.test.ts @@ -3882,6 +3882,107 @@ describe("ensureRuntimeServicesForRun", () => { } }); + it("records the built-in deferred seed as failed when its manifest is not verified", async () => { + const workspaceRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-workspace-seed-operation-")); + const restorePaperclipEnv = configureRuntimeProvisionTestHome(workspaceRoot, "workspace-seed-operation"); + const scriptsDir = path.join(workspaceRoot, "scripts"); + const markerDir = path.join(workspaceRoot, ".paperclip"); + await fs.mkdir(scriptsDir, { recursive: true }); + await fs.mkdir(markerDir, { recursive: true }); + await fs.writeFile(path.join(markerDir, "seed-pending"), "{}\n", "utf8"); + await fs.writeFile( + path.join(scriptsDir, "provision-worktree-runtime.sh"), + [ + "#!/usr/bin/env bash", + "set -euo pipefail", + `printf '%s\\n' '${JSON.stringify({ version: 2, state: "failed", phase: "source_validation" })}' > .paperclip/seed-manifest.json`, + ].join("\n"), + "utf8", + ); + const config = runtimeProvisionTestConfig({}); + const workspace = { + ...buildWorkspace(workspaceRoot), + source: "task_session" as const, + strategy: "git_worktree" as const, + worktreePath: workspaceRoot, + }; + const { recorder, operations } = createWorkspaceOperationRecorderDouble(); + + try { + await expect( + startRuntimeServicesForWorkspaceControl( + runtimeProvisionStartInput({ workspace, config, recorder }), + ), + ).rejects.toThrow(/without a verified manifest.*source_validation/); + + expect(operations).toEqual([ + expect.objectContaining({ + phase: "workspace_seed", + result: expect.objectContaining({ + status: "failed", + exitCode: 1, + metadata: expect.objectContaining({ + provisionKind: "workspace_seed", + seedState: "failed", + seedPhase: "source_validation", + seedFailurePhase: "source_validation", + }), + }), + }), + ]); + } finally { + await stopRuntimeServicesForExecutionWorkspace({ + executionWorkspaceId: "execution-workspace-1", + workspaceCwd: workspaceRoot, + }); + await fs.rm(workspaceRoot, { recursive: true, force: true }); + restorePaperclipEnv(); + } + }); + + it("keeps an explicit command matching the built-in seed command as runtime provisioning", async () => { + const workspaceRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-explicit-runtime-provision-")); + const restorePaperclipEnv = configureRuntimeProvisionTestHome(workspaceRoot, "explicit-runtime-provision"); + const scriptsDir = path.join(workspaceRoot, "scripts"); + await fs.mkdir(scriptsDir, { recursive: true }); + await fs.writeFile( + path.join(scriptsDir, "provision-worktree-runtime.sh"), + "#!/usr/bin/env bash\nset -euo pipefail\n", + "utf8", + ); + const config = runtimeProvisionTestConfig({ + provisionCommand: "bash ./scripts/provision-worktree-runtime.sh", + }); + const workspace = buildWorkspace(workspaceRoot); + const { recorder, operations } = createWorkspaceOperationRecorderDouble(); + + try { + const services = await startRuntimeServicesForWorkspaceControl( + runtimeProvisionStartInput({ workspace, config, recorder }), + ); + + expect(services).toHaveLength(1); + expect(operations).toEqual([ + expect.objectContaining({ + phase: "workspace_runtime_provision", + result: expect.objectContaining({ + status: "succeeded", + metadata: expect.objectContaining({ + provisionKind: "runtime_dependencies", + }), + }), + }), + ]); + } finally { + await stopRuntimeServicesForExecutionWorkspace({ + executionWorkspaceId: "execution-workspace-1", + workspaceCwd: workspaceRoot, + }); + await fs.rm(workspaceRoot, { recursive: true, force: true }); + restorePaperclipEnv(); + } + }); + it("does not create a runtime provision operation when the command is absent", async () => { const workspaceRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-runtime-provision-noop-")); const restorePaperclipEnv = configureRuntimeProvisionTestHome(workspaceRoot, "runtime-provision-noop"); diff --git a/server/src/services/workspace-runtime.ts b/server/src/services/workspace-runtime.ts index a9cff2ecbe..0e4656c13f 100644 --- a/server/src/services/workspace-runtime.ts +++ b/server/src/services/workspace-runtime.ts @@ -36,7 +36,7 @@ import { and, desc, eq, gte, inArray, isNull, lte, ne, or } from "drizzle-orm"; import { asNumber, asString, parseObject, renderTemplate } from "../adapters/utils.js"; import { conflict } from "../errors.js"; import { resolveHomeAwarePath } from "../home-paths.js"; -import { hasVerifiedWorktreeSeedManifest } from "../worktree-seed-manifest.js"; +import { hasVerifiedWorktreeSeedManifest, isVerifiedWorktreeSeedManifest } from "../worktree-seed-manifest.js"; import { buildManagedWorkspaceGuestEnv, logManagedWorkspaceReadinessRejection, @@ -2871,7 +2871,7 @@ async function recordGitOperation( async function recordWorkspaceCommandOperation( recorder: WorkspaceOperationRecorder | null | undefined, input: { - phase: "workspace_provision" | "workspace_runtime_provision" | "workspace_teardown"; + phase: "workspace_provision" | "workspace_seed" | "workspace_runtime_provision" | "workspace_teardown"; command: string; resolvedCommand?: string; cwd: string; @@ -2903,26 +2903,31 @@ async function recordWorkspaceCommandOperation( cwd: input.cwd, env: input.env, }); + const seedEvidence = input.phase === "workspace_seed" + ? readWorkspaceSeedOperationEvidence(input.cwd) + : null; stdout = result.stdout; - stderr = result.stderr; - code = result.code; + stderr = [result.stderr, seedEvidence?.error].filter(Boolean).join("\n"); + code = result.code === 0 && seedEvidence && !seedEvidence.verified ? 1 : result.code; if (result.stdout && input.onLog) await input.onLog("stdout", `[runtime-provision] ${result.stdout}`); - if (result.stderr && input.onLog) await input.onLog("stderr", `[runtime-provision] ${result.stderr}`); + if (stderr && input.onLog) await input.onLog("stderr", `[runtime-provision] ${stderr}`); + const truncationMetadata = result.stdoutTruncated || result.stderrTruncated + ? { + stdoutTruncated: result.stdoutTruncated, + stderrTruncated: result.stderrTruncated, + stdoutBytes: result.stdoutBytes, + stderrBytes: result.stderrBytes, + } + : null; return { - status: result.code === 0 ? "succeeded" : "failed", - exitCode: result.code, + status: code === 0 ? "succeeded" : "failed", + exitCode: code, stdout: result.stdout, - stderr: result.stderr, - system: result.code === 0 ? input.successMessage ?? null : null, - metadata: - result.stdoutTruncated || result.stderrTruncated - ? { - stdoutTruncated: result.stdoutTruncated, - stderrTruncated: result.stderrTruncated, - stdoutBytes: result.stdoutBytes, - stderrBytes: result.stderrBytes, - } - : null, + stderr, + system: code === 0 ? input.successMessage ?? null : null, + metadata: seedEvidence + ? { ...seedEvidence.metadata, ...(truncationMetadata ?? {}) } + : truncationMetadata, }; }, }); @@ -5052,6 +5057,7 @@ type StartLocalRuntimeServiceInput = { service: Record; onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise; runtimeProvisionCommand?: string | null; + runtimeProvisionKind?: RuntimeProvisionKind | null; recorder?: WorkspaceOperationRecorder | null; provisionCoordinator?: RuntimeProvisionCoordinator; preparedProvisioningRecord?: RuntimeServiceRecord | null; @@ -5079,6 +5085,49 @@ function readRuntimeProvisionCommand(config: Record) { ).trim(); } +const BUILTIN_WORKSPACE_SEED_COMMAND = "bash ./scripts/provision-worktree-runtime.sh"; + +type RuntimeProvisionKind = "workspace_seed" | "runtime_dependencies"; + +function readWorkspaceSeedOperationEvidence(worktreePath: string): { + verified: boolean; + error: string | null; + metadata: Record; +} { + const manifestPath = path.join(worktreePath, ".paperclip", "seed-manifest.json"); + try { + const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as Record; + const state = typeof manifest.state === "string" ? manifest.state : "unknown"; + const phase = typeof manifest.phase === "string" ? manifest.phase : null; + const verified = isVerifiedWorktreeSeedManifest(manifest); + return { + verified, + error: verified + ? null + : phase + ? `Workspace seed command returned without a verified manifest (state: ${state}, phase: ${phase}).` + : `Workspace seed command returned without a verified manifest (state: ${state}).`, + metadata: { + provisionKind: "workspace_seed", + seedState: state, + seedPhase: phase, + seedFailurePhase: state === "failed" ? phase : null, + }, + }; + } catch { + return { + verified: false, + error: "Workspace seed command returned without a readable seed manifest.", + metadata: { + provisionKind: "workspace_seed", + seedState: existsSync(manifestPath) ? "unreadable" : "absent", + seedPhase: null, + seedFailurePhase: "seed_manifest_unreadable", + }, + }; + } +} + export function resolveRuntimeProvisionCommand(input: { config: Record; workspace: RealizedExecutionWorkspace; @@ -5105,7 +5154,21 @@ export function resolveRuntimeProvisionCommand(input: { return ""; } - return "bash ./scripts/provision-worktree-runtime.sh"; + return BUILTIN_WORKSPACE_SEED_COMMAND; +} + +function resolveRuntimeProvision(input: { + config: Record; + workspace: RealizedExecutionWorkspace; +}): { command: string; kind: RuntimeProvisionKind | null } { + const command = resolveRuntimeProvisionCommand(input); + if (!command) return { command, kind: null }; + return { + command, + kind: readRuntimeProvisionCommand(input.config) + ? "runtime_dependencies" + : "workspace_seed", + }; } function runtimeProvisionWorkspaceKey(input: StartLocalRuntimeServiceInput) { @@ -5136,8 +5199,9 @@ async function runRuntimeProvisionWithWorkspaceMutex(input: StartLocalRuntimeSer }) : null); const resolvedCommand = resolveRepoManagedWorkspaceCommand(command, input.workspace.baseCwd); + const workspaceSeed = input.runtimeProvisionKind === "workspace_seed"; const promise = recordWorkspaceCommandOperation(recorder, { - phase: "workspace_runtime_provision", + phase: workspaceSeed ? "workspace_seed" : "workspace_runtime_provision", command, resolvedCommand, cwd: input.workspace.cwd, @@ -5150,14 +5214,19 @@ async function runRuntimeProvisionWithWorkspaceMutex(input: StartLocalRuntimeSer agent: input.agent, created: input.workspace.created, }), - label: `Runtime provision command "${command}"`, + label: workspaceSeed + ? `Workspace seed command "${command}"` + : `Runtime provision command "${command}"`, metadata: { executionWorkspaceId: input.executionWorkspaceId ?? null, projectWorkspaceId: input.workspace.workspaceId, serviceName: asString(input.service.name, "service"), + provisionKind: workspaceSeed ? "workspace_seed" : "runtime_dependencies", resolvedCommand: resolvedCommand === command ? null : resolvedCommand, }, - successMessage: `Provisioned runtime dependencies for ${input.workspace.cwd}\n`, + successMessage: workspaceSeed + ? `Verified the workspace database seed for ${input.workspace.cwd}\n` + : `Provisioned runtime dependencies for ${input.workspace.cwd}\n`, onLog: input.onLog, }).then(() => undefined); @@ -6465,7 +6534,8 @@ async function ensureRuntimeServicesForRunInvocation( }); const acquiredServiceIds: string[] = []; const refs: RuntimeServiceRef[] = []; - const runtimeProvisionCommand = resolveRuntimeProvisionCommand(input); + const runtimeProvision = resolveRuntimeProvision(input); + const runtimeProvisionCommand = runtimeProvision.command; const provisionCoordinator = createRuntimeProvisionCoordinator(); const allowFixedPortFallback = await isPersistedIsolatedExecutionWorkspace({ db: input.db, @@ -6523,6 +6593,7 @@ async function ensureRuntimeServicesForRunInvocation( service, onLog: input.onLog, runtimeProvisionCommand, + runtimeProvisionKind: runtimeProvision.kind, recorder: input.recorder, provisionCoordinator, allowFixedPortFallback, @@ -6693,6 +6764,7 @@ async function startRuntimeServicesForWorkspaceControlUnlocked( deferReadiness?: boolean; allowFixedPortFallback?: boolean; runtimeProvisionCommand?: string; + runtimeProvisionKind?: RuntimeProvisionKind | null; provisionCoordinator?: RuntimeProvisionCoordinator; preparedProvisioning?: { service: Record; @@ -6759,6 +6831,7 @@ async function startRuntimeServicesForWorkspaceControlUnlocked( service, onLog: input.onLog, runtimeProvisionCommand: options?.runtimeProvisionCommand, + runtimeProvisionKind: options?.runtimeProvisionKind, recorder: input.recorder, provisionCoordinator: options?.provisionCoordinator, preparedProvisioningRecord: @@ -6861,7 +6934,8 @@ async function startRuntimeServicesForWorkspaceControlInvocation( serviceStates: readConfiguredServiceStates(input.config), }); const invocationId = input.invocationId ?? randomUUID(); - const runtimeProvisionCommand = resolveRuntimeProvisionCommand(input); + const runtimeProvision = resolveRuntimeProvision(input); + const runtimeProvisionCommand = runtimeProvision.command; const provisionCoordinator = createRuntimeProvisionCoordinator(); const hasHttpsExposure = await anyRuntimeServiceUsesHttpsExposure(rawServices); @@ -6880,7 +6954,11 @@ async function startRuntimeServicesForWorkspaceControlInvocation( invocationId, input.db, input.db, - { runtimeProvisionCommand, provisionCoordinator }, + { + runtimeProvisionCommand, + runtimeProvisionKind: runtimeProvision.kind, + provisionCoordinator, + }, ); return batch.refs; } @@ -6931,6 +7009,7 @@ async function startRuntimeServicesForWorkspaceControlInvocation( service, onLog: input.onLog, runtimeProvisionCommand, + runtimeProvisionKind: runtimeProvision.kind, recorder: input.recorder, provisionCoordinator, reuseKey, @@ -6959,6 +7038,7 @@ async function startRuntimeServicesForWorkspaceControlInvocation( deferReadiness: true, allowFixedPortFallback, runtimeProvisionCommand, + runtimeProvisionKind: runtimeProvision.kind, provisionCoordinator, preparedProvisioning, }, diff --git a/ui/src/lib/workspace-access-state.test.ts b/ui/src/lib/workspace-access-state.test.ts index 1ef00cb893..ffd60795bf 100644 --- a/ui/src/lib/workspace-access-state.test.ts +++ b/ui/src/lib/workspace-access-state.test.ts @@ -45,7 +45,7 @@ function operation(overrides: Partial = {}): WorkspaceOperat executionWorkspaceId: "ews-1", heartbeatRunId: null, issueId: null, - phase: "workspace_runtime_provision", + phase: "workspace_seed", command: null, cwd: null, status: "succeeded", diff --git a/ui/src/lib/workspace-access-state.ts b/ui/src/lib/workspace-access-state.ts index 0747d239d4..7925008669 100644 --- a/ui/src/lib/workspace-access-state.ts +++ b/ui/src/lib/workspace-access-state.ts @@ -125,7 +125,8 @@ export function resolveWorkspaceAccessState(input: { const runtimeServices = input.runtimeServices ?? []; const repair = latestOperation(operations, "workspace_repair"); const provision = - latestOperation(operations, "workspace_runtime_provision") + latestOperation(operations, "workspace_seed") + ?? latestOperation(operations, "workspace_runtime_provision") ?? latestOperation(operations, "workspace_provision"); const failure = input.handoffFailure ?? null; const cause = describeWorkspaceReadinessCause(failure); diff --git a/ui/src/pages/ExecutionWorkspaceDetail.provision-status.test.ts b/ui/src/pages/ExecutionWorkspaceDetail.provision-status.test.ts index 337c7f283c..d532410b5d 100644 --- a/ui/src/pages/ExecutionWorkspaceDetail.provision-status.test.ts +++ b/ui/src/pages/ExecutionWorkspaceDetail.provision-status.test.ts @@ -9,7 +9,7 @@ function operation(overrides: Partial = {}): WorkspaceOperat executionWorkspaceId: "ews-1", heartbeatRunId: null, issueId: null, - phase: overrides.phase ?? "workspace_runtime_provision", + phase: overrides.phase ?? "workspace_seed", command: overrides.command ?? "bash ./scripts/provision-worktree-runtime.sh", cwd: null, status: overrides.status ?? "succeeded", @@ -58,6 +58,15 @@ describe("resolveRuntimeProvisionStatus", () => { ).toEqual({ kind: "provisioned", at: finishedAt }); }); + it("continues to recognize custom runtime dependency provisioning", () => { + expect( + resolveRuntimeProvisionStatus({ + runtimeProvisionCommand: "pnpm install", + operations: [operation({ phase: "workspace_runtime_provision" })], + }), + ).toMatchObject({ kind: "provisioned" }); + }); + it("reports provisioning while the op is running", () => { const startedAt = new Date("2026-08-01T12:00:00.000Z"); expect( diff --git a/ui/src/pages/ExecutionWorkspaceDetail.tsx b/ui/src/pages/ExecutionWorkspaceDetail.tsx index 6fea2668d7..6d4be124fe 100644 --- a/ui/src/pages/ExecutionWorkspaceDetail.tsx +++ b/ui/src/pages/ExecutionWorkspaceDetail.tsx @@ -377,6 +377,8 @@ function workspaceOperationPhaseLabel(phase: string) { return "Config freshness"; case "workspace_provision": return "Provision"; + case "workspace_seed": + return "Database seed"; case "workspace_runtime_provision": return "Runtime provision"; case "workspace_teardown": @@ -399,14 +401,16 @@ export type RuntimeProvisionStatus = /** * Derives the lazy runtime-provisioning state from the configured command and the - * `workspace_runtime_provision` operation-log entries (most-recent first). Returns + * database-seed or runtime-provision operation-log entries (most-recent first). Returns * "eager" when no runtime provision command is configured (the legacy path). */ export function resolveRuntimeProvisionStatus(input: { runtimeProvisionCommand: string | null | undefined; operations: WorkspaceOperation[] | undefined; }): RuntimeProvisionStatus { - const latest = (input.operations ?? []).find((operation) => operation.phase === "workspace_runtime_provision") ?? null; + const latest = (input.operations ?? []).find((operation) => ( + operation.phase === "workspace_seed" || operation.phase === "workspace_runtime_provision" + )) ?? null; if (latest) { const at = latest.finishedAt ?? latest.startedAt ?? null; if (latest.status === "running") return { kind: "provisioning", at };