fix(workspaces): keep deferred seed databases reliable (#11706)
## Thinking Path > - Paperclip manages agent work in isolated execution workspaces. > - A workspace depends on a valid database seed before it can run. > - Deferred seed failures were hidden behind a successful provision status. > - The seed restore also had two possible owners for the embedded PostgreSQL process. > - That allowed the target database to stop while the restore was still running. > - This pull request makes seed failures visible and gives the seed process sole lifecycle ownership. > - The benefit is that workspace provisioning reports the real result and does not stop its own target database. ## Linked Issues or Issue Description Related: #11684 **What happened?** Initial worktree provisioning could report success before its deferred database seed completed. The seed restore could also reuse a target embedded PostgreSQL process with another shutdown owner. This could stop the target database during the restore. **Expected behavior** Workspace status must show a failed deferred seed as a failure. The seed restore must own the target embedded PostgreSQL process until restore, migration, and validation finish. **Steps to reproduce** 1. Provision a worktree with deferred database seeding. 2. Make the seed manifest end in a failed state while the command exits with code 0. 3. Observe that the provision status remains successful on `master`. 4. Start a seed restore against an already-running target embedded PostgreSQL process. 5. Observe that another lifecycle owner can stop the target during restore. **Paperclip version or commit** `51a843e135` **Deployment mode** Local dev with execution workspaces and embedded PostgreSQL. ## What Changed - Add a first-class `workspace_seed` operation for deferred database seeds. - Require terminal, verified seed evidence before the seed operation succeeds. - Surface the seed phase and failure metadata in workspace status and UI state. - Give the seed process exclusive lifecycle ownership of the target embedded PostgreSQL process. - Suppress imported embedded-Postgres exit hooks without removing existing host listeners. - Record a credential-safe shutdown diagnostic in failed seed manifests. ## Verification - The original deferred-seed commit passed 4 server tests, 24 workspace-status UI tests, shared/server/UI typechecks, and the UI token gate. - The original PostgreSQL-lifecycle commit passed 3 lifecycle tests, 3 ownership/diagnostic tests, 1 real embedded-Postgres seed integration, and the affected package typechecks. - No local tests were rerun after the clean cherry-pick because the operator requested the shortest landing path. - Review the automatic PR checks for the clean `origin/master` replay. ## Risks - A live target database now causes an early error instead of being reused. The error includes recovery guidance. - Workspace consumers must handle the new `workspace_seed` operation type. Shared types and UI state handling are updated in this pull request. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI Codex, GPT-5, high-reasoning mode, with repository, shell, and GitHub tool use. The runtime does not expose a more specific deployment suffix or context-window value. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta
GitHub
parent
51a843e135
commit
e1df4c6068
|
|
@ -27,7 +27,9 @@ import {
|
|||
import {
|
||||
copyGitHooksToWorktreeGitDir,
|
||||
copySeededSecretsKey,
|
||||
ensureEmbeddedPostgres,
|
||||
ensureWorktreeSeeded,
|
||||
formatWorktreeSeedFailureDiagnostic,
|
||||
markWorktreeSeedPending,
|
||||
pauseSeededScheduledRoutines,
|
||||
quarantineSeededWorktreeExecutionState,
|
||||
|
|
@ -488,6 +490,34 @@ describe("worktree helpers", () => {
|
|||
expect(full.nullifyColumns).toEqual({});
|
||||
});
|
||||
|
||||
it("requires the seed process to own the target embedded Postgres lifecycle", async () => {
|
||||
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-worktree-live-target-"));
|
||||
try {
|
||||
fs.writeFileSync(
|
||||
path.join(tempRoot, "postmaster.pid"),
|
||||
`${process.pid}\n${tempRoot}\n0\n55432\n`,
|
||||
);
|
||||
|
||||
await expect(ensureEmbeddedPostgres(tempRoot, 55432, { allowExisting: false }))
|
||||
.rejects.toThrow("while it is already running");
|
||||
} finally {
|
||||
fs.rmSync(tempRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("surfaces a credential-safe diagnostic when the target shuts down during restore", () => {
|
||||
expect(formatWorktreeSeedFailureDiagnostic(
|
||||
"restore",
|
||||
new Error(
|
||||
"Failed to restore seed.sql.gz: FATAL: the database system is shutting down; psql error: write EPIPE",
|
||||
),
|
||||
)).toBe(
|
||||
"Target embedded PostgreSQL shut down during restore. Stop any competing worktree service and retry the seed.",
|
||||
);
|
||||
expect(formatWorktreeSeedFailureDiagnostic("migrations", new Error("secret connection failure")))
|
||||
.toBe("Seed failed during migrations.");
|
||||
});
|
||||
|
||||
it("rejects a source migration journal that diverges from the code journal", () => {
|
||||
expect(() => resolveWorktreeSeedMigrationRevision({
|
||||
status: "upToDate",
|
||||
|
|
@ -714,7 +744,7 @@ describe("worktree helpers", () => {
|
|||
},
|
||||
);
|
||||
|
||||
it("ensure-seeded keeps the pending marker when seeding fails", async () => {
|
||||
it("ensure-seeded records a target shutdown diagnostic when restore fails", async () => {
|
||||
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-worktree-ensure-seeded-failure-"));
|
||||
try {
|
||||
const sourceConfigPath = path.join(tempRoot, "source", "config.json");
|
||||
|
|
@ -746,13 +776,28 @@ describe("worktree helpers", () => {
|
|||
await expect(
|
||||
ensureWorktreeSeeded(
|
||||
{ config: targetConfigPath, fromConfig: sourceConfigPath },
|
||||
{ seedDatabase: vi.fn().mockRejectedValue(new Error("seed failed")) },
|
||||
{
|
||||
seedDatabase: vi.fn(async (input) => {
|
||||
input.onPhase?.("restore", "started");
|
||||
throw new Error(
|
||||
"Failed to restore seed.sql.gz: FATAL: the database system is shutting down; psql error: write EPIPE",
|
||||
);
|
||||
}),
|
||||
},
|
||||
),
|
||||
).rejects.toThrow("seed failed");
|
||||
).rejects.toThrow("database system is shutting down");
|
||||
|
||||
expect(readWorktreeSeedManifest(targetConfigPath)).toMatchObject({
|
||||
state: "failed",
|
||||
phase: "pending",
|
||||
phase: "restore",
|
||||
diagnostics: expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
phase: "restore",
|
||||
status: "failed",
|
||||
message:
|
||||
"Target embedded PostgreSQL shut down during restore. Stop any competing worktree service and retry the seed.",
|
||||
}),
|
||||
]),
|
||||
});
|
||||
expect(fs.existsSync(path.join(targetRoot, ".paperclip", "seed-pending"))).toBe(false);
|
||||
expect(fs.existsSync(path.join(targetRoot, ".paperclip", "seed-complete"))).toBe(false);
|
||||
|
|
|
|||
|
|
@ -55,6 +55,7 @@ import {
|
|||
workspaceRuntimeServices,
|
||||
createEmbeddedPostgresLogBuffer,
|
||||
formatEmbeddedPostgresError,
|
||||
loadWithoutEmbeddedPostgresExitHooks,
|
||||
prepareEmbeddedPostgresNativeRuntime,
|
||||
} from "@paperclipai/db";
|
||||
import type { Command } from "commander";
|
||||
|
|
@ -1105,11 +1106,15 @@ export function copySeededSecretsKey(input: {
|
|||
}
|
||||
}
|
||||
|
||||
async function ensureEmbeddedPostgres(dataDir: string, preferredPort: number): Promise<EmbeddedPostgresHandle> {
|
||||
export async function ensureEmbeddedPostgres(
|
||||
dataDir: string,
|
||||
preferredPort: number,
|
||||
options: { allowExisting?: boolean } = {},
|
||||
): Promise<EmbeddedPostgresHandle> {
|
||||
const moduleName = "embedded-postgres";
|
||||
let EmbeddedPostgres: EmbeddedPostgresCtor;
|
||||
try {
|
||||
const mod = await import(moduleName);
|
||||
const mod = await loadWithoutEmbeddedPostgresExitHooks(() => import(moduleName));
|
||||
EmbeddedPostgres = mod.default as EmbeddedPostgresCtor;
|
||||
} catch {
|
||||
throw new Error(
|
||||
|
|
@ -1121,6 +1126,12 @@ async function ensureEmbeddedPostgres(dataDir: string, preferredPort: number): P
|
|||
const postmasterPidFile = path.resolve(dataDir, "postmaster.pid");
|
||||
const runningPid = readRunningPostmasterPid(postmasterPidFile);
|
||||
if (runningPid) {
|
||||
if (options.allowExisting === false) {
|
||||
throw new Error(
|
||||
`Cannot seed target embedded PostgreSQL at ${dataDir} while it is already running (pid=${runningPid}). `
|
||||
+ "Stop the worktree service that owns this database, then retry the seed.",
|
||||
);
|
||||
}
|
||||
return {
|
||||
port: readPidFilePort(postmasterPidFile) ?? preferredPort,
|
||||
startedByThisProcess: false,
|
||||
|
|
@ -1632,13 +1643,14 @@ async function seedWorktreeDatabase(input: {
|
|||
});
|
||||
input.onPhase?.("snapshot", "succeeded", `Created ${path.basename(backup.backupFile)}.`);
|
||||
|
||||
input.onPhase?.("restore", "started");
|
||||
targetHandle = await ensureEmbeddedPostgres(
|
||||
input.targetConfig.database.embeddedPostgresDataDir,
|
||||
input.targetConfig.database.embeddedPostgresPort,
|
||||
{ allowExisting: false },
|
||||
);
|
||||
|
||||
const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${targetHandle.port}/postgres`;
|
||||
input.onPhase?.("restore", "started");
|
||||
await resetPostgresDatabase(adminConnectionString, "paperclip");
|
||||
const targetConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${targetHandle.port}/paperclip`;
|
||||
await runDatabaseRestore({
|
||||
|
|
@ -1703,6 +1715,23 @@ const WORKTREE_SEED_DIAGNOSTIC_LIMIT = 32;
|
|||
const WORKTREE_SEED_DIAGNOSTIC_MESSAGE_LIMIT = 512;
|
||||
const activeSeedInterruptHandlers = new Map<string, (signal: NodeJS.Signals) => void>();
|
||||
|
||||
export function formatWorktreeSeedFailureDiagnostic(
|
||||
phase: WorktreeSeedPhase,
|
||||
error: unknown,
|
||||
): string {
|
||||
const message = error instanceof Error ? error.message : String(error ?? "");
|
||||
if (
|
||||
phase === "restore"
|
||||
&& /database system is shutting down|terminating connection due to administrator command/i.test(message)
|
||||
) {
|
||||
return "Target embedded PostgreSQL shut down during restore. Stop any competing worktree service and retry the seed.";
|
||||
}
|
||||
if (phase === "restore" && /Cannot seed target embedded PostgreSQL.+already running/i.test(message)) {
|
||||
return "Target embedded PostgreSQL is owned by a running worktree service. Stop that service and retry the seed.";
|
||||
}
|
||||
return `Seed failed during ${phase}.`;
|
||||
}
|
||||
|
||||
function dispatchSeedInterruption(signal: NodeJS.Signals): void {
|
||||
for (const handler of activeSeedInterruptHandlers.values()) {
|
||||
try {
|
||||
|
|
@ -2096,7 +2125,7 @@ async function runVerifiedWorktreeSeed(input: {
|
|||
state: "failed",
|
||||
// Do not persist the underlying error: database/driver errors may contain
|
||||
// connection credentials. The CLI still returns the exact error to its caller.
|
||||
message: `Seed failed during ${activePhase}.`,
|
||||
message: formatWorktreeSeedFailureDiagnostic(activePhase, error),
|
||||
});
|
||||
throw error;
|
||||
} finally {
|
||||
|
|
|
|||
|
|
@ -486,10 +486,13 @@ The default `worktree init` still seeds eagerly. A lean worktree (created withou
|
|||
- `pnpm paperclipai worktree ensure-seeded` performs the deferred seed **exactly once**. It is lock-guarded and idempotent: only a complete `verified` manifest short-circuits it, so it is safe to call repeatedly and from concurrent processes. Managed workspaces derive the source exclusively from the control-plane-provided base project workspace; manual worktrees must pass `--from-config`.
|
||||
- `paperclipai run` calls `ensureWorktreeSeeded` automatically before doctor/boot. Managed runs transparently seed a lean worktree from their registered base workspace; an unmanaged lean worktree must first run `worktree ensure-seeded --from-config <source-config>`.
|
||||
- Managed git-worktree runtime startup also runs `scripts/provision-worktree-runtime.sh` automatically when a legacy workspace policy has no explicit runtime provision command and the manifest is not verified. An explicitly configured runtime provision command always takes precedence.
|
||||
- The built-in deferred seed is recorded as its own terminal `workspace_seed` operation. A zero exit code is not enough for success: the operation succeeds only when `.paperclip/seed-manifest.json` contains complete verified evidence; failed, missing, or malformed manifests produce a failed operation with the seed phase in metadata.
|
||||
- Worktrees created before lazy seeding shipped have neither marker; they are treated as already-seeded for backward compatibility (never re-cloned).
|
||||
|
||||
Both `minimal` and `full` modes use the same terminal data-validation contract. Source validation accepts a migration journal that is a prefix of the checkout's journal and records the source revision in seed diagnostics; it rejects a source that is ahead of the checkout because that would require a downgrade. After restore, Paperclip applies pending migrations and requires the target journal to be current. Both validations also read a credential-backed auth user, instance administrator role, active company membership, and representative cloned company/issue pair. Restore, migrations, execution quarantine, routine pausing, workspace rebinding, and post-restore validation all run under the seed lock. An interruption leaves the exact active phase in terminal `failed` state; it cannot produce readiness evidence.
|
||||
|
||||
The seed process must own the target embedded PostgreSQL lifecycle for that entire sequence. It refuses to restore into a target postmaster that is already running, suppresses the embedded provider's process-global exit hooks, and stops its owned target only after validation or failure cleanup. A shutdown detected during restore is recorded as a target-database shutdown diagnostic rather than a generic restore failure.
|
||||
|
||||
The seed manifest never grants source-path authority. Its source path and instance are diagnostic assertions that must exactly match the realpath-canonical registered source before any lock, backup, service stop, spawn, or database mutation. Missing registration, sibling or foreign paths, symlink aliases, source/target identity collisions, and company mismatches fail closed.
|
||||
|
||||
**Unverified-seed guard.** `pnpm dev` (the dev-runner) refuses to boot a worktree whose manifest is pending, running, failed, malformed, or missing required verification evidence and points you at the fix:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,77 @@
|
|||
import { EventEmitter } from "node:events";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { loadWithoutEmbeddedPostgresExitHooks } from "./embedded-postgres-lifecycle.js";
|
||||
|
||||
describe("loadWithoutEmbeddedPostgresExitHooks", () => {
|
||||
it("removes every eager exit hook from the real embedded-postgres import", async () => {
|
||||
const eventNames = [
|
||||
"exit",
|
||||
"beforeExit",
|
||||
"SIGHUP",
|
||||
"SIGINT",
|
||||
"SIGTERM",
|
||||
"SIGBREAK",
|
||||
"message",
|
||||
];
|
||||
const before = new Map(eventNames.map((eventName) => [
|
||||
eventName,
|
||||
process.rawListeners(eventName),
|
||||
]));
|
||||
const moduleName = "embedded-postgres";
|
||||
|
||||
await loadWithoutEmbeddedPostgresExitHooks(() => import(moduleName));
|
||||
|
||||
for (const eventName of eventNames) {
|
||||
expect(process.rawListeners(eventName)).toEqual(before.get(eventName));
|
||||
}
|
||||
});
|
||||
|
||||
it("removes dependency exit hooks while preserving existing listeners", async () => {
|
||||
const target = new EventEmitter();
|
||||
const existingSignalListener = vi.fn();
|
||||
const existingExitListener = vi.fn();
|
||||
target.on("SIGTERM", existingSignalListener);
|
||||
target.on("exit", existingExitListener);
|
||||
|
||||
const dependencyListener = vi.fn();
|
||||
const loaded = await loadWithoutEmbeddedPostgresExitHooks(
|
||||
async () => {
|
||||
for (const eventName of [
|
||||
"exit",
|
||||
"beforeExit",
|
||||
"SIGHUP",
|
||||
"SIGINT",
|
||||
"SIGTERM",
|
||||
"SIGBREAK",
|
||||
"message",
|
||||
]) {
|
||||
target.on(eventName, dependencyListener);
|
||||
}
|
||||
return { default: class EmbeddedPostgres {} };
|
||||
},
|
||||
target,
|
||||
);
|
||||
|
||||
expect(loaded.default.name).toBe("EmbeddedPostgres");
|
||||
expect(target.rawListeners("SIGTERM")).toEqual([existingSignalListener]);
|
||||
expect(target.rawListeners("exit")).toEqual([existingExitListener]);
|
||||
for (const eventName of ["beforeExit", "SIGHUP", "SIGINT", "SIGBREAK", "message"]) {
|
||||
expect(target.rawListeners(eventName)).toEqual([]);
|
||||
}
|
||||
});
|
||||
|
||||
it("cleans up listeners even when the import fails", async () => {
|
||||
const target = new EventEmitter();
|
||||
const dependencyListener = vi.fn();
|
||||
|
||||
await expect(loadWithoutEmbeddedPostgresExitHooks(
|
||||
async () => {
|
||||
target.on("SIGTERM", dependencyListener);
|
||||
throw new Error("import failed");
|
||||
},
|
||||
target,
|
||||
)).rejects.toThrow("import failed");
|
||||
|
||||
expect(target.rawListeners("SIGTERM")).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
const EMBEDDED_POSTGRES_EXIT_EVENTS = [
|
||||
"exit",
|
||||
"beforeExit",
|
||||
"SIGHUP",
|
||||
"SIGINT",
|
||||
"SIGTERM",
|
||||
"SIGBREAK",
|
||||
"message",
|
||||
] as const;
|
||||
|
||||
type EmbeddedPostgresExitTarget = {
|
||||
rawListeners(eventName: string): Function[];
|
||||
removeListener(eventName: string, listener: (...args: any[]) => void): unknown;
|
||||
};
|
||||
|
||||
/**
|
||||
* embedded-postgres installs async-exit-hook listeners as an import side effect.
|
||||
* Paperclip-managed clusters have an explicit owner and shutdown path, so those
|
||||
* global listeners must not be allowed to stop a cluster independently of that
|
||||
* owner (for example while a worktree seed restore is still streaming).
|
||||
*
|
||||
* Remove only listeners added by the supplied import and preserve every listener
|
||||
* that was already registered by Paperclip or its host process.
|
||||
*/
|
||||
export async function loadWithoutEmbeddedPostgresExitHooks<T>(
|
||||
load: () => Promise<T>,
|
||||
target: EmbeddedPostgresExitTarget = process,
|
||||
): Promise<T> {
|
||||
const listenersBeforeLoad = new Map(
|
||||
EMBEDDED_POSTGRES_EXIT_EVENTS.map((eventName) => [
|
||||
eventName,
|
||||
target.rawListeners(eventName),
|
||||
]),
|
||||
);
|
||||
|
||||
let loaded: T;
|
||||
try {
|
||||
loaded = await load();
|
||||
} finally {
|
||||
for (const eventName of EMBEDDED_POSTGRES_EXIT_EVENTS) {
|
||||
const remainingBeforeLoad = [...(listenersBeforeLoad.get(eventName) ?? [])];
|
||||
for (const listener of target.rawListeners(eventName)) {
|
||||
const existingIndex = remainingBeforeLoad.indexOf(listener);
|
||||
if (existingIndex >= 0) {
|
||||
remainingBeforeLoad.splice(existingIndex, 1);
|
||||
continue;
|
||||
}
|
||||
target.removeListener(eventName, listener as (...args: any[]) => void);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return loaded;
|
||||
}
|
||||
|
|
@ -35,6 +35,7 @@ export {
|
|||
ensureLinuxSharedLibraryAliases,
|
||||
prepareEmbeddedPostgresNativeRuntime,
|
||||
} from "./embedded-postgres-native.js";
|
||||
export { loadWithoutEmbeddedPostgresExitHooks } from "./embedded-postgres-lifecycle.js";
|
||||
export { issueRelations } from "./schema/issue_relations.js";
|
||||
export { issueReferenceMentions } from "./schema/issue_reference_mentions.js";
|
||||
export * from "./schema/index.js";
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ export type WorkspaceOperationPhase =
|
|||
| "worktree_prepare"
|
||||
| "workspace_config_freshness"
|
||||
| "workspace_provision"
|
||||
| "workspace_seed"
|
||||
| "workspace_runtime_provision"
|
||||
| "workspace_repair"
|
||||
| "workspace_teardown"
|
||||
|
|
|
|||
|
|
@ -3882,6 +3882,107 @@ describe("ensureRuntimeServicesForRun", () => {
|
|||
}
|
||||
});
|
||||
|
||||
it("records the built-in deferred seed as failed when its manifest is not verified", async () => {
|
||||
const workspaceRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-workspace-seed-operation-"));
|
||||
const restorePaperclipEnv = configureRuntimeProvisionTestHome(workspaceRoot, "workspace-seed-operation");
|
||||
const scriptsDir = path.join(workspaceRoot, "scripts");
|
||||
const markerDir = path.join(workspaceRoot, ".paperclip");
|
||||
await fs.mkdir(scriptsDir, { recursive: true });
|
||||
await fs.mkdir(markerDir, { recursive: true });
|
||||
await fs.writeFile(path.join(markerDir, "seed-pending"), "{}\n", "utf8");
|
||||
await fs.writeFile(
|
||||
path.join(scriptsDir, "provision-worktree-runtime.sh"),
|
||||
[
|
||||
"#!/usr/bin/env bash",
|
||||
"set -euo pipefail",
|
||||
`printf '%s\\n' '${JSON.stringify({ version: 2, state: "failed", phase: "source_validation" })}' > .paperclip/seed-manifest.json`,
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
const config = runtimeProvisionTestConfig({});
|
||||
const workspace = {
|
||||
...buildWorkspace(workspaceRoot),
|
||||
source: "task_session" as const,
|
||||
strategy: "git_worktree" as const,
|
||||
worktreePath: workspaceRoot,
|
||||
};
|
||||
const { recorder, operations } = createWorkspaceOperationRecorderDouble();
|
||||
|
||||
try {
|
||||
await expect(
|
||||
startRuntimeServicesForWorkspaceControl(
|
||||
runtimeProvisionStartInput({ workspace, config, recorder }),
|
||||
),
|
||||
).rejects.toThrow(/without a verified manifest.*source_validation/);
|
||||
|
||||
expect(operations).toEqual([
|
||||
expect.objectContaining({
|
||||
phase: "workspace_seed",
|
||||
result: expect.objectContaining({
|
||||
status: "failed",
|
||||
exitCode: 1,
|
||||
metadata: expect.objectContaining({
|
||||
provisionKind: "workspace_seed",
|
||||
seedState: "failed",
|
||||
seedPhase: "source_validation",
|
||||
seedFailurePhase: "source_validation",
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
await stopRuntimeServicesForExecutionWorkspace({
|
||||
executionWorkspaceId: "execution-workspace-1",
|
||||
workspaceCwd: workspaceRoot,
|
||||
});
|
||||
await fs.rm(workspaceRoot, { recursive: true, force: true });
|
||||
restorePaperclipEnv();
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps an explicit command matching the built-in seed command as runtime provisioning", async () => {
|
||||
const workspaceRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-explicit-runtime-provision-"));
|
||||
const restorePaperclipEnv = configureRuntimeProvisionTestHome(workspaceRoot, "explicit-runtime-provision");
|
||||
const scriptsDir = path.join(workspaceRoot, "scripts");
|
||||
await fs.mkdir(scriptsDir, { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(scriptsDir, "provision-worktree-runtime.sh"),
|
||||
"#!/usr/bin/env bash\nset -euo pipefail\n",
|
||||
"utf8",
|
||||
);
|
||||
const config = runtimeProvisionTestConfig({
|
||||
provisionCommand: "bash ./scripts/provision-worktree-runtime.sh",
|
||||
});
|
||||
const workspace = buildWorkspace(workspaceRoot);
|
||||
const { recorder, operations } = createWorkspaceOperationRecorderDouble();
|
||||
|
||||
try {
|
||||
const services = await startRuntimeServicesForWorkspaceControl(
|
||||
runtimeProvisionStartInput({ workspace, config, recorder }),
|
||||
);
|
||||
|
||||
expect(services).toHaveLength(1);
|
||||
expect(operations).toEqual([
|
||||
expect.objectContaining({
|
||||
phase: "workspace_runtime_provision",
|
||||
result: expect.objectContaining({
|
||||
status: "succeeded",
|
||||
metadata: expect.objectContaining({
|
||||
provisionKind: "runtime_dependencies",
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
await stopRuntimeServicesForExecutionWorkspace({
|
||||
executionWorkspaceId: "execution-workspace-1",
|
||||
workspaceCwd: workspaceRoot,
|
||||
});
|
||||
await fs.rm(workspaceRoot, { recursive: true, force: true });
|
||||
restorePaperclipEnv();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not create a runtime provision operation when the command is absent", async () => {
|
||||
const workspaceRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-runtime-provision-noop-"));
|
||||
const restorePaperclipEnv = configureRuntimeProvisionTestHome(workspaceRoot, "runtime-provision-noop");
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ import { and, desc, eq, gte, inArray, isNull, lte, ne, or } from "drizzle-orm";
|
|||
import { asNumber, asString, parseObject, renderTemplate } from "../adapters/utils.js";
|
||||
import { conflict } from "../errors.js";
|
||||
import { resolveHomeAwarePath } from "../home-paths.js";
|
||||
import { hasVerifiedWorktreeSeedManifest } from "../worktree-seed-manifest.js";
|
||||
import { hasVerifiedWorktreeSeedManifest, isVerifiedWorktreeSeedManifest } from "../worktree-seed-manifest.js";
|
||||
import {
|
||||
buildManagedWorkspaceGuestEnv,
|
||||
logManagedWorkspaceReadinessRejection,
|
||||
|
|
@ -2871,7 +2871,7 @@ async function recordGitOperation(
|
|||
async function recordWorkspaceCommandOperation(
|
||||
recorder: WorkspaceOperationRecorder | null | undefined,
|
||||
input: {
|
||||
phase: "workspace_provision" | "workspace_runtime_provision" | "workspace_teardown";
|
||||
phase: "workspace_provision" | "workspace_seed" | "workspace_runtime_provision" | "workspace_teardown";
|
||||
command: string;
|
||||
resolvedCommand?: string;
|
||||
cwd: string;
|
||||
|
|
@ -2903,26 +2903,31 @@ async function recordWorkspaceCommandOperation(
|
|||
cwd: input.cwd,
|
||||
env: input.env,
|
||||
});
|
||||
const seedEvidence = input.phase === "workspace_seed"
|
||||
? readWorkspaceSeedOperationEvidence(input.cwd)
|
||||
: null;
|
||||
stdout = result.stdout;
|
||||
stderr = result.stderr;
|
||||
code = result.code;
|
||||
stderr = [result.stderr, seedEvidence?.error].filter(Boolean).join("\n");
|
||||
code = result.code === 0 && seedEvidence && !seedEvidence.verified ? 1 : result.code;
|
||||
if (result.stdout && input.onLog) await input.onLog("stdout", `[runtime-provision] ${result.stdout}`);
|
||||
if (result.stderr && input.onLog) await input.onLog("stderr", `[runtime-provision] ${result.stderr}`);
|
||||
if (stderr && input.onLog) await input.onLog("stderr", `[runtime-provision] ${stderr}`);
|
||||
const truncationMetadata = result.stdoutTruncated || result.stderrTruncated
|
||||
? {
|
||||
stdoutTruncated: result.stdoutTruncated,
|
||||
stderrTruncated: result.stderrTruncated,
|
||||
stdoutBytes: result.stdoutBytes,
|
||||
stderrBytes: result.stderrBytes,
|
||||
}
|
||||
: null;
|
||||
return {
|
||||
status: result.code === 0 ? "succeeded" : "failed",
|
||||
exitCode: result.code,
|
||||
status: code === 0 ? "succeeded" : "failed",
|
||||
exitCode: code,
|
||||
stdout: result.stdout,
|
||||
stderr: result.stderr,
|
||||
system: result.code === 0 ? input.successMessage ?? null : null,
|
||||
metadata:
|
||||
result.stdoutTruncated || result.stderrTruncated
|
||||
? {
|
||||
stdoutTruncated: result.stdoutTruncated,
|
||||
stderrTruncated: result.stderrTruncated,
|
||||
stdoutBytes: result.stdoutBytes,
|
||||
stderrBytes: result.stderrBytes,
|
||||
}
|
||||
: null,
|
||||
stderr,
|
||||
system: code === 0 ? input.successMessage ?? null : null,
|
||||
metadata: seedEvidence
|
||||
? { ...seedEvidence.metadata, ...(truncationMetadata ?? {}) }
|
||||
: truncationMetadata,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
|
@ -5052,6 +5057,7 @@ type StartLocalRuntimeServiceInput = {
|
|||
service: Record<string, unknown>;
|
||||
onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise<void>;
|
||||
runtimeProvisionCommand?: string | null;
|
||||
runtimeProvisionKind?: RuntimeProvisionKind | null;
|
||||
recorder?: WorkspaceOperationRecorder | null;
|
||||
provisionCoordinator?: RuntimeProvisionCoordinator;
|
||||
preparedProvisioningRecord?: RuntimeServiceRecord | null;
|
||||
|
|
@ -5079,6 +5085,49 @@ function readRuntimeProvisionCommand(config: Record<string, unknown>) {
|
|||
).trim();
|
||||
}
|
||||
|
||||
const BUILTIN_WORKSPACE_SEED_COMMAND = "bash ./scripts/provision-worktree-runtime.sh";
|
||||
|
||||
type RuntimeProvisionKind = "workspace_seed" | "runtime_dependencies";
|
||||
|
||||
function readWorkspaceSeedOperationEvidence(worktreePath: string): {
|
||||
verified: boolean;
|
||||
error: string | null;
|
||||
metadata: Record<string, unknown>;
|
||||
} {
|
||||
const manifestPath = path.join(worktreePath, ".paperclip", "seed-manifest.json");
|
||||
try {
|
||||
const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as Record<string, unknown>;
|
||||
const state = typeof manifest.state === "string" ? manifest.state : "unknown";
|
||||
const phase = typeof manifest.phase === "string" ? manifest.phase : null;
|
||||
const verified = isVerifiedWorktreeSeedManifest(manifest);
|
||||
return {
|
||||
verified,
|
||||
error: verified
|
||||
? null
|
||||
: phase
|
||||
? `Workspace seed command returned without a verified manifest (state: ${state}, phase: ${phase}).`
|
||||
: `Workspace seed command returned without a verified manifest (state: ${state}).`,
|
||||
metadata: {
|
||||
provisionKind: "workspace_seed",
|
||||
seedState: state,
|
||||
seedPhase: phase,
|
||||
seedFailurePhase: state === "failed" ? phase : null,
|
||||
},
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
verified: false,
|
||||
error: "Workspace seed command returned without a readable seed manifest.",
|
||||
metadata: {
|
||||
provisionKind: "workspace_seed",
|
||||
seedState: existsSync(manifestPath) ? "unreadable" : "absent",
|
||||
seedPhase: null,
|
||||
seedFailurePhase: "seed_manifest_unreadable",
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export function resolveRuntimeProvisionCommand(input: {
|
||||
config: Record<string, unknown>;
|
||||
workspace: RealizedExecutionWorkspace;
|
||||
|
|
@ -5105,7 +5154,21 @@ export function resolveRuntimeProvisionCommand(input: {
|
|||
return "";
|
||||
}
|
||||
|
||||
return "bash ./scripts/provision-worktree-runtime.sh";
|
||||
return BUILTIN_WORKSPACE_SEED_COMMAND;
|
||||
}
|
||||
|
||||
function resolveRuntimeProvision(input: {
|
||||
config: Record<string, unknown>;
|
||||
workspace: RealizedExecutionWorkspace;
|
||||
}): { command: string; kind: RuntimeProvisionKind | null } {
|
||||
const command = resolveRuntimeProvisionCommand(input);
|
||||
if (!command) return { command, kind: null };
|
||||
return {
|
||||
command,
|
||||
kind: readRuntimeProvisionCommand(input.config)
|
||||
? "runtime_dependencies"
|
||||
: "workspace_seed",
|
||||
};
|
||||
}
|
||||
|
||||
function runtimeProvisionWorkspaceKey(input: StartLocalRuntimeServiceInput) {
|
||||
|
|
@ -5136,8 +5199,9 @@ async function runRuntimeProvisionWithWorkspaceMutex(input: StartLocalRuntimeSer
|
|||
})
|
||||
: null);
|
||||
const resolvedCommand = resolveRepoManagedWorkspaceCommand(command, input.workspace.baseCwd);
|
||||
const workspaceSeed = input.runtimeProvisionKind === "workspace_seed";
|
||||
const promise = recordWorkspaceCommandOperation(recorder, {
|
||||
phase: "workspace_runtime_provision",
|
||||
phase: workspaceSeed ? "workspace_seed" : "workspace_runtime_provision",
|
||||
command,
|
||||
resolvedCommand,
|
||||
cwd: input.workspace.cwd,
|
||||
|
|
@ -5150,14 +5214,19 @@ async function runRuntimeProvisionWithWorkspaceMutex(input: StartLocalRuntimeSer
|
|||
agent: input.agent,
|
||||
created: input.workspace.created,
|
||||
}),
|
||||
label: `Runtime provision command "${command}"`,
|
||||
label: workspaceSeed
|
||||
? `Workspace seed command "${command}"`
|
||||
: `Runtime provision command "${command}"`,
|
||||
metadata: {
|
||||
executionWorkspaceId: input.executionWorkspaceId ?? null,
|
||||
projectWorkspaceId: input.workspace.workspaceId,
|
||||
serviceName: asString(input.service.name, "service"),
|
||||
provisionKind: workspaceSeed ? "workspace_seed" : "runtime_dependencies",
|
||||
resolvedCommand: resolvedCommand === command ? null : resolvedCommand,
|
||||
},
|
||||
successMessage: `Provisioned runtime dependencies for ${input.workspace.cwd}\n`,
|
||||
successMessage: workspaceSeed
|
||||
? `Verified the workspace database seed for ${input.workspace.cwd}\n`
|
||||
: `Provisioned runtime dependencies for ${input.workspace.cwd}\n`,
|
||||
onLog: input.onLog,
|
||||
}).then(() => undefined);
|
||||
|
||||
|
|
@ -6465,7 +6534,8 @@ async function ensureRuntimeServicesForRunInvocation(
|
|||
});
|
||||
const acquiredServiceIds: string[] = [];
|
||||
const refs: RuntimeServiceRef[] = [];
|
||||
const runtimeProvisionCommand = resolveRuntimeProvisionCommand(input);
|
||||
const runtimeProvision = resolveRuntimeProvision(input);
|
||||
const runtimeProvisionCommand = runtimeProvision.command;
|
||||
const provisionCoordinator = createRuntimeProvisionCoordinator();
|
||||
const allowFixedPortFallback = await isPersistedIsolatedExecutionWorkspace({
|
||||
db: input.db,
|
||||
|
|
@ -6523,6 +6593,7 @@ async function ensureRuntimeServicesForRunInvocation(
|
|||
service,
|
||||
onLog: input.onLog,
|
||||
runtimeProvisionCommand,
|
||||
runtimeProvisionKind: runtimeProvision.kind,
|
||||
recorder: input.recorder,
|
||||
provisionCoordinator,
|
||||
allowFixedPortFallback,
|
||||
|
|
@ -6693,6 +6764,7 @@ async function startRuntimeServicesForWorkspaceControlUnlocked(
|
|||
deferReadiness?: boolean;
|
||||
allowFixedPortFallback?: boolean;
|
||||
runtimeProvisionCommand?: string;
|
||||
runtimeProvisionKind?: RuntimeProvisionKind | null;
|
||||
provisionCoordinator?: RuntimeProvisionCoordinator;
|
||||
preparedProvisioning?: {
|
||||
service: Record<string, unknown>;
|
||||
|
|
@ -6759,6 +6831,7 @@ async function startRuntimeServicesForWorkspaceControlUnlocked(
|
|||
service,
|
||||
onLog: input.onLog,
|
||||
runtimeProvisionCommand: options?.runtimeProvisionCommand,
|
||||
runtimeProvisionKind: options?.runtimeProvisionKind,
|
||||
recorder: input.recorder,
|
||||
provisionCoordinator: options?.provisionCoordinator,
|
||||
preparedProvisioningRecord:
|
||||
|
|
@ -6861,7 +6934,8 @@ async function startRuntimeServicesForWorkspaceControlInvocation(
|
|||
serviceStates: readConfiguredServiceStates(input.config),
|
||||
});
|
||||
const invocationId = input.invocationId ?? randomUUID();
|
||||
const runtimeProvisionCommand = resolveRuntimeProvisionCommand(input);
|
||||
const runtimeProvision = resolveRuntimeProvision(input);
|
||||
const runtimeProvisionCommand = runtimeProvision.command;
|
||||
const provisionCoordinator = createRuntimeProvisionCoordinator();
|
||||
const hasHttpsExposure = await anyRuntimeServiceUsesHttpsExposure(rawServices);
|
||||
|
||||
|
|
@ -6880,7 +6954,11 @@ async function startRuntimeServicesForWorkspaceControlInvocation(
|
|||
invocationId,
|
||||
input.db,
|
||||
input.db,
|
||||
{ runtimeProvisionCommand, provisionCoordinator },
|
||||
{
|
||||
runtimeProvisionCommand,
|
||||
runtimeProvisionKind: runtimeProvision.kind,
|
||||
provisionCoordinator,
|
||||
},
|
||||
);
|
||||
return batch.refs;
|
||||
}
|
||||
|
|
@ -6931,6 +7009,7 @@ async function startRuntimeServicesForWorkspaceControlInvocation(
|
|||
service,
|
||||
onLog: input.onLog,
|
||||
runtimeProvisionCommand,
|
||||
runtimeProvisionKind: runtimeProvision.kind,
|
||||
recorder: input.recorder,
|
||||
provisionCoordinator,
|
||||
reuseKey,
|
||||
|
|
@ -6959,6 +7038,7 @@ async function startRuntimeServicesForWorkspaceControlInvocation(
|
|||
deferReadiness: true,
|
||||
allowFixedPortFallback,
|
||||
runtimeProvisionCommand,
|
||||
runtimeProvisionKind: runtimeProvision.kind,
|
||||
provisionCoordinator,
|
||||
preparedProvisioning,
|
||||
},
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ function operation(overrides: Partial<WorkspaceOperation> = {}): WorkspaceOperat
|
|||
executionWorkspaceId: "ews-1",
|
||||
heartbeatRunId: null,
|
||||
issueId: null,
|
||||
phase: "workspace_runtime_provision",
|
||||
phase: "workspace_seed",
|
||||
command: null,
|
||||
cwd: null,
|
||||
status: "succeeded",
|
||||
|
|
|
|||
|
|
@ -125,7 +125,8 @@ export function resolveWorkspaceAccessState(input: {
|
|||
const runtimeServices = input.runtimeServices ?? [];
|
||||
const repair = latestOperation(operations, "workspace_repair");
|
||||
const provision =
|
||||
latestOperation(operations, "workspace_runtime_provision")
|
||||
latestOperation(operations, "workspace_seed")
|
||||
?? latestOperation(operations, "workspace_runtime_provision")
|
||||
?? latestOperation(operations, "workspace_provision");
|
||||
const failure = input.handoffFailure ?? null;
|
||||
const cause = describeWorkspaceReadinessCause(failure);
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ function operation(overrides: Partial<WorkspaceOperation> = {}): WorkspaceOperat
|
|||
executionWorkspaceId: "ews-1",
|
||||
heartbeatRunId: null,
|
||||
issueId: null,
|
||||
phase: overrides.phase ?? "workspace_runtime_provision",
|
||||
phase: overrides.phase ?? "workspace_seed",
|
||||
command: overrides.command ?? "bash ./scripts/provision-worktree-runtime.sh",
|
||||
cwd: null,
|
||||
status: overrides.status ?? "succeeded",
|
||||
|
|
@ -58,6 +58,15 @@ describe("resolveRuntimeProvisionStatus", () => {
|
|||
).toEqual({ kind: "provisioned", at: finishedAt });
|
||||
});
|
||||
|
||||
it("continues to recognize custom runtime dependency provisioning", () => {
|
||||
expect(
|
||||
resolveRuntimeProvisionStatus({
|
||||
runtimeProvisionCommand: "pnpm install",
|
||||
operations: [operation({ phase: "workspace_runtime_provision" })],
|
||||
}),
|
||||
).toMatchObject({ kind: "provisioned" });
|
||||
});
|
||||
|
||||
it("reports provisioning while the op is running", () => {
|
||||
const startedAt = new Date("2026-08-01T12:00:00.000Z");
|
||||
expect(
|
||||
|
|
|
|||
|
|
@ -377,6 +377,8 @@ function workspaceOperationPhaseLabel(phase: string) {
|
|||
return "Config freshness";
|
||||
case "workspace_provision":
|
||||
return "Provision";
|
||||
case "workspace_seed":
|
||||
return "Database seed";
|
||||
case "workspace_runtime_provision":
|
||||
return "Runtime provision";
|
||||
case "workspace_teardown":
|
||||
|
|
@ -399,14 +401,16 @@ export type RuntimeProvisionStatus =
|
|||
|
||||
/**
|
||||
* Derives the lazy runtime-provisioning state from the configured command and the
|
||||
* `workspace_runtime_provision` operation-log entries (most-recent first). Returns
|
||||
* database-seed or runtime-provision operation-log entries (most-recent first). Returns
|
||||
* "eager" when no runtime provision command is configured (the legacy path).
|
||||
*/
|
||||
export function resolveRuntimeProvisionStatus(input: {
|
||||
runtimeProvisionCommand: string | null | undefined;
|
||||
operations: WorkspaceOperation[] | undefined;
|
||||
}): RuntimeProvisionStatus {
|
||||
const latest = (input.operations ?? []).find((operation) => operation.phase === "workspace_runtime_provision") ?? null;
|
||||
const latest = (input.operations ?? []).find((operation) => (
|
||||
operation.phase === "workspace_seed" || operation.phase === "workspace_runtime_provision"
|
||||
)) ?? null;
|
||||
if (latest) {
|
||||
const at = latest.finishedAt ?? latest.startedAt ?? null;
|
||||
if (latest.status === "running") return { kind: "provisioning", at };
|
||||
|
|
|
|||
Loading…
Reference in New Issue