diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 500f9c89be..d4266b9395 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -311,6 +311,8 @@ The state `projectWorkspaceId` plus `executionWorkspaceId` without `projectId` i Workspace incoherence feeds into the same non-terminal liveness and stranded assigned-work model as a disappeared run. The recovery path should first fail or reject the incoherent wake, then either repair and requeue one bounded continuation for the same assignee or surface an explicit recovery action. It must not leave an agent-owned `in_progress` issue healthy solely because a wake record exists that would invoke the adapter in the wrong cwd, a non-git directory where git is required, an unrelated project workspace, or an unrecoverable missing worktree. +For runtime-created `git_worktree` execution workspaces, branch coherence is part of workspace coherence. The persisted execution workspace branch is the recorded branch for future dispatch. Reusing that workspace must verify that the worktree is still registered and that `HEAD` is on the recorded branch. Successful run finalization must perform the same check before recording `workspace_finalize=succeeded`; if the run switched to a publishing/PR branch without updating the execution workspace record, finalization records a failed workspace finalize and the run fails with the expected and actual branch. A branch change is sanctioned only when a control-plane path updates the execution workspace record before finalization, or when publishing work happens in a separate worktree and the managed issue worktree remains on its recorded branch. + ### Explicit recovery actions An explicit recovery action is a typed liveness repair path for a source issue. It is the recovery primitive; the action can be rendered directly on the source issue or backed by a separate recovery issue when the repair needs its own work item. diff --git a/server/src/__tests__/heartbeat-workspace-finalize-branch.test.ts b/server/src/__tests__/heartbeat-workspace-finalize-branch.test.ts new file mode 100644 index 0000000000..031d70f980 --- /dev/null +++ b/server/src/__tests__/heartbeat-workspace-finalize-branch.test.ts @@ -0,0 +1,427 @@ +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { and, asc, eq } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { + activityLog, + agentRuntimeState, + agentTaskSessions, + agentWakeupRequests, + agents, + companies, + companySkills, + createDb, + documentRevisions, + documents, + environmentLeases, + environments, + executionWorkspaces, + heartbeatRunEvents, + heartbeatRuns, + issueComments, + issueDocuments, + issuePlanDecompositions, + issues, + projects, + projectWorkspaces, + workspaceOperations, +} from "@paperclipai/db"; +import { + getEmbeddedPostgresTestSupport, + startEmbeddedPostgresTestDatabase, +} from "./helpers/embedded-postgres.js"; +import { heartbeatService } from "../services/heartbeat.ts"; +import { instanceSettingsService } from "../services/instance-settings.ts"; + +const execFileAsync = promisify(execFile); + +const adapterExecute = vi.hoisted(() => + vi.fn(async () => ({ + exitCode: 0, + signal: null, + timedOut: false, + summary: "Finalization branch guard test run.", + provider: "test", + model: "test-model", + })), +); + +vi.mock("../adapters/index.js", () => ({ + getServerAdapter: () => ({ + type: "codex_local", + execute: adapterExecute, + supportsLocalAgentJwt: false, + }), + findActiveServerAdapter: () => ({ + type: "codex_local", + execute: adapterExecute, + supportsLocalAgentJwt: false, + }), + listAdapterModelProfiles: async () => [], + runningProcesses: new Map(), +})); + +const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; + +if (!embeddedPostgresSupport.supported) { + console.warn( + `Skipping embedded Postgres heartbeat workspace finalize branch tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`, + ); +} + +type Db = ReturnType; +type Heartbeat = ReturnType; + +async function runGit(cwd: string, args: string[]) { + await execFileAsync("git", args, { cwd }); +} + +async function createGitRepo() { + const repoRoot = await mkdtemp(path.join(os.tmpdir(), "paperclip-finalize-branch-repo-")); + await runGit(repoRoot, ["init"]); + await runGit(repoRoot, ["config", "user.email", "paperclip-test@example.com"]); + await runGit(repoRoot, ["config", "user.name", "Paperclip Test"]); + await writeFile(path.join(repoRoot, "README.md"), "finalization branch guard\n", "utf8"); + await runGit(repoRoot, ["add", "README.md"]); + await runGit(repoRoot, ["commit", "-m", "initial"]); + return repoRoot; +} + +async function waitForRunToFinish(heartbeat: Heartbeat, runId: string, timeoutMs = 10_000) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const run = await heartbeat.getRun(runId); + if (run && run.status !== "queued" && run.status !== "running") return run; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + return heartbeat.getRun(runId); +} + +async function waitForHeartbeatIdle(db: Db, timeoutMs = 5_000) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const runs = await db.select({ status: heartbeatRuns.status }).from(heartbeatRuns); + if (!runs.some((run) => run.status === "queued" || run.status === "running")) return; + await new Promise((resolve) => setTimeout(resolve, 50)); + } +} + +async function waitForRuntimeStateLastRun(db: Db, agentId: string, runId: string, timeoutMs = 5_000) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const state = await db + .select({ lastRunId: agentRuntimeState.lastRunId }) + .from(agentRuntimeState) + .where(eq(agentRuntimeState.agentId, agentId)) + .then((rows) => rows[0] ?? null); + if (state?.lastRunId === runId) return; + await new Promise((resolve) => setTimeout(resolve, 50)); + } +} + +function readAdapterWorkspace(input: unknown) { + const context = (input as { context?: Record }).context ?? {}; + const workspace = context.paperclipWorkspace as Record | undefined; + const cwd = typeof workspace?.cwd === "string" ? workspace.cwd : null; + const branchName = typeof workspace?.branchName === "string" ? workspace.branchName : null; + const executionWorkspaceId = + typeof context.executionWorkspaceId === "string" ? context.executionWorkspaceId : null; + if (!cwd || !branchName || !executionWorkspaceId) { + throw new Error("Adapter input is missing the realized execution workspace context"); + } + return { cwd, branchName, executionWorkspaceId }; +} + +async function seedRunTarget(db: Db, repoRoot: string) { + const companyId = randomUUID(); + const projectId = randomUUID(); + const projectWorkspaceId = randomUUID(); + const issueId = randomUUID(); + const agentId = randomUUID(); + + await instanceSettingsService(db).updateExperimental({ + enableIsolatedWorkspaces: true, + }); + await db.insert(companies).values({ + id: companyId, + name: "Acme", + issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`, + status: "active", + createdAt: new Date(), + updatedAt: new Date(), + }); + await db.insert(projects).values({ + id: projectId, + companyId, + name: "Workspace Finalize Branch Guard", + status: "active", + createdAt: new Date(), + updatedAt: new Date(), + }); + await db.insert(projectWorkspaces).values({ + id: projectWorkspaceId, + companyId, + projectId, + name: "Primary", + cwd: repoRoot, + isPrimary: true, + createdAt: new Date(), + updatedAt: new Date(), + }); + await db.insert(agents).values({ + id: agentId, + companyId, + name: "CodexCoder", + role: "engineer", + status: "idle", + adapterType: "codex_local", + adapterConfig: {}, + runtimeConfig: { + heartbeat: { + wakeOnDemand: true, + maxConcurrentRuns: 1, + }, + }, + permissions: {}, + createdAt: new Date(), + updatedAt: new Date(), + }); + await db.insert(issues).values({ + id: issueId, + companyId, + projectId, + projectWorkspaceId, + title: "Publish without drifting managed workspace", + status: "in_progress", + workMode: "standard", + priority: "medium", + assigneeAgentId: agentId, + identifier: `PAP-${issueId.replace(/-/g, "").slice(0, 6).toUpperCase()}`, + executionWorkspaceSettings: { + mode: "isolated_workspace", + }, + createdAt: new Date(), + updatedAt: new Date(), + }); + + return { companyId, projectId, projectWorkspaceId, issueId, agentId }; +} + +async function wakeIssue(heartbeat: Heartbeat, agentId: string, issueId: string) { + return heartbeat.wakeup(agentId, { + source: "automation", + triggerDetail: "system", + reason: "issue_commented", + payload: { issueId }, + contextSnapshot: { + issueId, + taskId: issueId, + wakeReason: "issue_commented", + skipIssueComment: true, + }, + }); +} + +async function listFinalizeOperations(db: Db, runId: string) { + return db + .select() + .from(workspaceOperations) + .where(and( + eq(workspaceOperations.heartbeatRunId, runId), + eq(workspaceOperations.phase, "workspace_finalize"), + )) + .orderBy(asc(workspaceOperations.startedAt), asc(workspaceOperations.createdAt)); +} + +describeEmbeddedPostgres("heartbeat workspace finalization branch guard", () => { + let db!: Db; + let tempDb: Awaited> | null = null; + const tempRoots: string[] = []; + + beforeAll(async () => { + tempDb = await startEmbeddedPostgresTestDatabase("paperclip-finalize-branch-"); + db = createDb(tempDb.connectionString); + }, 20_000); + + afterEach(async () => { + await waitForHeartbeatIdle(db); + adapterExecute.mockReset(); + adapterExecute.mockImplementation(async () => ({ + exitCode: 0, + signal: null, + timedOut: false, + summary: "Finalization branch guard test run.", + provider: "test", + model: "test-model", + })); + while (tempRoots.length > 0) { + const root = tempRoots.pop(); + if (root) await rm(root, { recursive: true, force: true }).catch(() => undefined); + } + await db.delete(issuePlanDecompositions); + await db.delete(issueDocuments); + await db.delete(documentRevisions); + await db.delete(documents); + await db.delete(agentTaskSessions); + await db.delete(environmentLeases); + await db.delete(activityLog); + await db.delete(heartbeatRunEvents); + await db.delete(heartbeatRuns); + await db.delete(issueComments); + await db.delete(issues); + await db.delete(projectWorkspaces); + await db.delete(projects); + await db.delete(agentWakeupRequests); + await db.delete(agentRuntimeState); + await db.delete(agents); + await db.delete(workspaceOperations); + await db.delete(executionWorkspaces); + await db.delete(environments); + await db.delete(companySkills); + await db.delete(companies); + }); + + afterAll(async () => { + await db.$client.end(); + await tempDb?.cleanup(); + }); + + it("fails a successful adapter run when the managed worktree branch drift was not recorded", async () => { + const repoRoot = await createGitRepo(); + tempRoots.push(repoRoot); + const { agentId, issueId } = await seedRunTarget(db, repoRoot); + const publishBranch = `publish-${issueId.slice(0, 8)}`; + let recordedBranch: string | null = null; + let executionWorkspaceId: string | null = null; + + adapterExecute.mockImplementationOnce(async (input) => { + const workspace = readAdapterWorkspace(input); + recordedBranch = workspace.branchName; + executionWorkspaceId = workspace.executionWorkspaceId; + await runGit(workspace.cwd, ["checkout", "-b", publishBranch]); + await db.update(issues).set({ status: "done", updatedAt: new Date() }).where(eq(issues.id, issueId)); + return { + exitCode: 0, + signal: null, + timedOut: false, + summary: "Adapter completed after switching to a publish branch.", + provider: "test", + model: "test-model", + }; + }); + + const heartbeat = heartbeatService(db); + const run = await wakeIssue(heartbeat, agentId, issueId); + expect(run).not.toBeNull(); + + const finishedRun = await waitForRunToFinish(heartbeat, run!.id); + expect(finishedRun).toMatchObject({ + status: "failed", + errorCode: "workspace_validation_failed", + error: expect.stringContaining("Record a sanctioned execution-workspace branch transition"), + }); + const workspaceValidation = (finishedRun?.resultJson as Record | null)?.workspaceValidation; + expect(workspaceValidation).toMatchObject({ + reason: "git_worktree_branch_mismatch_after_run", + fingerprint: expect.stringMatching(/^workspace_finalize_branch_mismatch:v1:sha256:/), + issueId, + persistedExecutionWorkspaceId: executionWorkspaceId, + managedGitWorktreeBranch: expect.objectContaining({ + executionWorkspaceId, + reasonCode: "branch_mismatch", + expectedBranchName: recordedBranch, + actualBranchName: publishBranch, + }), + }); + await waitForRuntimeStateLastRun(db, agentId, run!.id); + expect(adapterExecute).toHaveBeenCalledTimes(1); + + const finalizeOps = await listFinalizeOperations(db, run!.id); + expect(finalizeOps).toHaveLength(1); + expect(finalizeOps[0]).toMatchObject({ + status: "failed", + executionWorkspaceId, + stderrExcerpt: expect.stringContaining("Managed git worktree branch check failed"), + }); + expect(finalizeOps[0]?.metadata).toMatchObject({ + managedGitWorktreeBranch: { + executionWorkspaceId, + valid: false, + reasonCode: "branch_mismatch", + expectedBranchName: recordedBranch, + actualBranchName: publishBranch, + }, + }); + }, 20_000); + + it("allows a successful adapter run when the branch transition is recorded before finalization", async () => { + const repoRoot = await createGitRepo(); + tempRoots.push(repoRoot); + const { agentId, issueId } = await seedRunTarget(db, repoRoot); + const publishBranch = `publish-${issueId.slice(0, 8)}`; + let executionWorkspaceId: string | null = null; + + adapterExecute.mockImplementationOnce(async (input) => { + const workspace = readAdapterWorkspace(input); + executionWorkspaceId = workspace.executionWorkspaceId; + await runGit(workspace.cwd, ["checkout", "-b", publishBranch]); + await db.update(issues).set({ status: "done", updatedAt: new Date() }).where(eq(issues.id, issueId)); + await db + .update(executionWorkspaces) + .set({ + branchName: publishBranch, + updatedAt: new Date(), + }) + .where(eq(executionWorkspaces.id, workspace.executionWorkspaceId)); + return { + exitCode: 0, + signal: null, + timedOut: false, + summary: "Adapter completed after recording a branch transition.", + provider: "test", + model: "test-model", + }; + }); + + const heartbeat = heartbeatService(db); + const run = await wakeIssue(heartbeat, agentId, issueId); + expect(run).not.toBeNull(); + + const finishedRun = await waitForRunToFinish(heartbeat, run!.id); + expect(finishedRun).toMatchObject({ + status: "succeeded", + errorCode: null, + error: null, + }); + await waitForRuntimeStateLastRun(db, agentId, run!.id); + expect(adapterExecute).toHaveBeenCalledTimes(1); + + const finalizedWorkspace = await db + .select({ branchName: executionWorkspaces.branchName }) + .from(executionWorkspaces) + .where(eq(executionWorkspaces.id, executionWorkspaceId!)) + .then((rows) => rows[0] ?? null); + expect(finalizedWorkspace?.branchName).toBe(publishBranch); + + const finalizeOps = await listFinalizeOperations(db, run!.id); + expect(finalizeOps).toHaveLength(1); + expect(finalizeOps[0]).toMatchObject({ + status: "succeeded", + executionWorkspaceId, + }); + expect(finalizeOps[0]?.metadata).toMatchObject({ + managedGitWorktreeBranch: { + executionWorkspaceId, + valid: true, + reasonCode: null, + expectedBranchName: publishBranch, + actualBranchName: publishBranch, + }, + }); + }, 20_000); +}); diff --git a/server/src/__tests__/heartbeat-workspace-session.test.ts b/server/src/__tests__/heartbeat-workspace-session.test.ts index cdce55dd7b..63456956b3 100644 --- a/server/src/__tests__/heartbeat-workspace-session.test.ts +++ b/server/src/__tests__/heartbeat-workspace-session.test.ts @@ -365,6 +365,51 @@ describe("assertGitSensitiveAdapterWorkspaceValid", () => { ); }); + it("rejects a git worktree persisted workspace when the checked-out branch differs from the recorded branch", async () => { + const repoRoot = await createGitCheckout({ withRemote: false }); + const worktreeParent = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-branch-worktree-")); + const worktreePath = path.join(worktreeParent, "workspace"); + const recordedBranch = "PAP-1-recorded-branch"; + const actualBranch = "PAP-1-push-pr-head"; + try { + await runGit(repoRoot, ["config", "user.email", "test@example.com"]); + await runGit(repoRoot, ["config", "user.name", "Paperclip Test"]); + await fs.writeFile(path.join(repoRoot, "README.md"), "initial\n", "utf8"); + await runGit(repoRoot, ["add", "README.md"]); + await runGit(repoRoot, ["commit", "-m", "Initial commit"]); + await runGit(repoRoot, ["worktree", "add", "-b", recordedBranch, worktreePath, "HEAD"]); + await runGit(worktreePath, ["checkout", "-b", actualBranch]); + + const input = buildWorkspaceValidationInput(); + await expectWorkspaceValidationFailure( + buildWorkspaceValidationInput({ + resolvedWorkspace: buildResolvedWorkspace({ cwd: worktreePath }), + executionWorkspace: { + ...input.executionWorkspace, + strategy: "git_worktree", + baseCwd: repoRoot, + cwd: worktreePath, + branchName: recordedBranch, + worktreePath, + }, + persistedExecutionWorkspace: { + ...input.persistedExecutionWorkspace!, + strategyType: "git_worktree", + cwd: worktreePath, + providerType: "git_worktree", + providerRef: worktreePath, + branchName: recordedBranch, + }, + }), + "git_worktree_branch_mismatch", + `expected git worktree branch "${recordedBranch}"`, + ); + } finally { + await fs.rm(repoRoot, { recursive: true, force: true }); + await fs.rm(worktreeParent, { recursive: true, force: true }); + } + }); + it("rejects a workspace-linked issue when adapter cwd has no git metadata", async () => { const input = buildWorkspaceValidationInput(); const cwd = "/tmp/paperclip-workspace-without-git-metadata"; diff --git a/server/src/__tests__/issue-recovery-actions.test.ts b/server/src/__tests__/issue-recovery-actions.test.ts index dd15e7c0cd..54a740c1cf 100644 --- a/server/src/__tests__/issue-recovery-actions.test.ts +++ b/server/src/__tests__/issue-recovery-actions.test.ts @@ -379,6 +379,101 @@ describeEmbeddedPostgres("issue recovery actions", () => { }); }); + it("deduplicates workspace-incoherence recovery actions by the typed workspace fingerprint", async () => { + const { companyId, coderId, sourceIssue } = await seedCompany(); + const enqueueWakeup = vi.fn(async () => null); + const recovery = recoveryService(db, { enqueueWakeup }); + const workspaceFingerprint = `workspace_incoherence:v1:sha256:${"a".repeat(64)}`; + const workspaceValidation = { + reason: "git_worktree_branch_incoherence", + fingerprint: workspaceFingerprint, + sourceIssueId: sourceIssue.id, + sourceIdentifier: sourceIssue.identifier, + executionWorkspaceId: "execution-workspace-1", + expectedBranch: "PAP-1-expected", + actualBranch: "PAP-1-publish", + cleanliness: "dirty", + provenance: { + expectedBranchExists: true, + actualBranchExists: true, + expectedHeadSha: "1111111111111111111111111111111111111111", + actualHeadSha: "2222222222222222222222222222222222222222", + sameHead: false, + }, + safeRepair: { + eligible: false, + attempted: false, + succeeded: false, + reason: "worktree is not clean", + }, + }; + const firstLatestRun = { + id: randomUUID(), + agentId: coderId, + status: "failed", + error: "workspace branch mismatch", + errorCode: "workspace_validation_failed", + contextSnapshot: {}, + livenessState: "failed", + resultJson: { workspaceValidation }, + } as const; + const secondLatestRun = { + ...firstLatestRun, + id: randomUUID(), + }; + + await recovery.escalateStrandedAssignedIssue({ + issue: sourceIssue, + previousStatus: "in_progress", + latestRun: firstLatestRun, + comment: "Workspace failed validation.", + recoveryCause: "workspace_validation_failed", + }); + await recovery.escalateStrandedAssignedIssue({ + issue: sourceIssue, + previousStatus: "in_progress", + latestRun: secondLatestRun, + comment: "Workspace failed validation.", + recoveryCause: "workspace_validation_failed", + }); + + const actionRows = await db + .select() + .from(issueRecoveryActions) + .where(eq(issueRecoveryActions.sourceIssueId, sourceIssue.id)); + expect(actionRows).toHaveLength(1); + expect(actionRows[0]).toMatchObject({ + companyId, + kind: "workspace_validation", + cause: "workspace_validation_failed", + status: "active", + attemptCount: 2, + fingerprint: expect.stringContaining(workspaceFingerprint), + evidence: expect.objectContaining({ + latestRunId: secondLatestRun.id, + latestRunErrorCode: "workspace_validation_failed", + workspaceValidation: expect.objectContaining({ + reason: "git_worktree_branch_incoherence", + fingerprint: workspaceFingerprint, + sourceIssueId: sourceIssue.id, + executionWorkspaceId: "execution-workspace-1", + expectedBranch: "PAP-1-expected", + actualBranch: "PAP-1-publish", + cleanliness: "dirty", + }), + }), + nextAction: expect.stringContaining("git worktree branch incoherence"), + wakePolicy: expect.objectContaining({ + type: "manual_repair_required", + reason: "workspace_validation_failed", + }), + }); + + const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, sourceIssue.id)); + expect(comments.filter((comment) => comment.body.includes(`Recovery action: \`${actionRows[0]?.id}\``))).toHaveLength(1); + expect(enqueueWakeup).not.toHaveBeenCalled(); + }); + it("keeps the source issue blocked when source-scoped wakeup is claimed synchronously", async () => { const { companyId, managerId, coderId, sourceIssue } = await seedCompany(); await db.update(agents).set({ status: "paused" }).where(eq(agents.id, managerId)); diff --git a/server/src/__tests__/workspace-runtime.test.ts b/server/src/__tests__/workspace-runtime.test.ts index 4f066822dd..4c8ac90980 100644 --- a/server/src/__tests__/workspace-runtime.test.ts +++ b/server/src/__tests__/workspace-runtime.test.ts @@ -754,8 +754,9 @@ describe("realizeExecutionWorkspace", () => { await expect(fs.realpath(realized.worktreePath ?? "")).resolves.toBe(expectedWorktreePath); }); - it("rejects reusing a linked worktree whose branch drifted from the expected issue branch", async () => { + it("repairs a clean linked worktree whose branch drifted from the expected issue branch", async () => { const repoRoot = await createTempRepo(); + const { recorder, operations } = createWorkspaceOperationRecorderDouble(); const initial = await realizeExecutionWorkspace({ base: { @@ -786,34 +787,52 @@ describe("realizeExecutionWorkspace", () => { await runGit(initial.cwd, ["checkout", "-b", "unexpected-branch"]); - await expect( - realizeExecutionWorkspace({ - base: { - baseCwd: repoRoot, - source: "project_primary", - projectId: "project-1", - workspaceId: "workspace-1", - repoUrl: null, - repoRef: "HEAD", + const repaired = await realizeExecutionWorkspace({ + base: { + baseCwd: repoRoot, + source: "project_primary", + projectId: "project-1", + workspaceId: "workspace-1", + repoUrl: null, + repoRef: "HEAD", + }, + config: { + workspaceStrategy: { + type: "git_worktree", + branchTemplate: "{{issue.identifier}}-{{slug}}", }, - config: { - workspaceStrategy: { - type: "git_worktree", - branchTemplate: "{{issue.identifier}}-{{slug}}", - }, - }, - issue: { - id: "issue-1", - identifier: "PAP-447", - title: "Add Worktree Support", - }, - agent: { - id: "agent-1", - name: "Codex Coder", - companyId: "company-1", - }, - }), - ).rejects.toThrow(/not a reusable git worktree \(worktree HEAD is on "unexpected-branch" instead of "PAP-447-add-worktree-support"\)\./); + }, + issue: { + id: "issue-1", + identifier: "PAP-447", + title: "Add Worktree Support", + }, + agent: { + id: "agent-1", + name: "Codex Coder", + companyId: "company-1", + }, + recorder, + }); + + expect(repaired.created).toBe(false); + expect(repaired.cwd).toBe(initial.cwd); + await expect(readGit(initial.cwd, ["branch", "--show-current"])).resolves.toBe("PAP-447-add-worktree-support"); + expect(operations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + phase: "worktree_prepare", + command: "git checkout PAP-447-add-worktree-support", + metadata: expect.objectContaining({ + branchIncoherenceRepair: true, + expectedBranchName: "PAP-447-add-worktree-support", + actualBranchName: "unexpected-branch", + sourceIssueId: "issue-1", + fingerprint: expect.stringMatching(/^workspace_incoherence:v1:sha256:/), + }), + }), + ]), + ); }); it("reuses an already checked out branch from git worktree metadata even when the target path differs", async () => { @@ -2057,6 +2076,290 @@ describe("realizeExecutionWorkspace", () => { expect(actualHead).toBe(expectedHead); }, 15_000); + it("repairs a clean persisted git worktree branch mismatch when both branches point at the same commit", async () => { + const repoRoot = await createTempRepo(); + const expectedBranch = "PAP-454-repair-clean-branch-mismatch"; + const actualBranch = "PAP-454-publish-head"; + const realWorktreeRoot = path.join(repoRoot, ".paperclip", "real-worktrees"); + const symlinkedWorktreeRoot = path.join(repoRoot, ".paperclip", "worktrees"); + const realWorktreePath = path.join(realWorktreeRoot, expectedBranch); + const worktreePath = path.join(symlinkedWorktreeRoot, expectedBranch); + await fs.mkdir(realWorktreeRoot, { recursive: true }); + await runGit(repoRoot, ["branch", expectedBranch]); + await runGit(repoRoot, ["worktree", "add", "-b", actualBranch, realWorktreePath, "HEAD"]); + await fs.symlink(realWorktreeRoot, symlinkedWorktreeRoot, "dir"); + const { recorder, operations } = createWorkspaceOperationRecorderDouble(); + + const restored = await ensurePersistedExecutionWorkspaceAvailable({ + base: { + baseCwd: repoRoot, + source: "project_primary", + projectId: "project-1", + workspaceId: "workspace-1", + repoUrl: null, + repoRef: "HEAD", + }, + workspace: { + id: "execution-workspace-1", + mode: "isolated_workspace", + strategyType: "git_worktree", + cwd: worktreePath, + providerRef: worktreePath, + projectId: "project-1", + projectWorkspaceId: "workspace-1", + repoUrl: null, + baseRef: "HEAD", + branchName: expectedBranch, + }, + issue: { + id: "issue-1", + identifier: "PAP-454", + title: "Repair clean branch mismatch", + }, + agent: { + id: "agent-1", + name: "Codex Coder", + companyId: "company-1", + }, + recorder, + }); + + expect(restored?.cwd).toBe(worktreePath); + await expect(readGit(worktreePath, ["branch", "--show-current"])).resolves.toBe(expectedBranch); + expect(operations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + phase: "worktree_prepare", + command: "git checkout PAP-454-repair-clean-branch-mismatch", + metadata: expect.objectContaining({ + branchIncoherenceRepair: true, + expectedBranchName: expectedBranch, + actualBranchName: actualBranch, + sourceIssueId: "issue-1", + executionWorkspaceId: "execution-workspace-1", + fingerprint: expect.stringMatching(/^workspace_incoherence:v1:sha256:/), + }), + }), + ]), + ); + }, 15_000); + + it("rejects dirty persisted git worktree branch incoherence with bounded recovery evidence", async () => { + const repoRoot = await createTempRepo(); + const expectedBranch = "PAP-455-reject-dirty-branch-mismatch"; + const actualBranch = "PAP-455-publish-head"; + const worktreePath = path.join(repoRoot, ".paperclip", "worktrees", expectedBranch); + await fs.mkdir(path.dirname(worktreePath), { recursive: true }); + await runGit(repoRoot, ["branch", expectedBranch]); + await runGit(repoRoot, ["worktree", "add", "-b", actualBranch, worktreePath, "HEAD"]); + await fs.writeFile(path.join(worktreePath, "untracked.txt"), "not safe to switch\n", "utf8"); + + await expect(ensurePersistedExecutionWorkspaceAvailable({ + base: { + baseCwd: repoRoot, + source: "project_primary", + projectId: "project-1", + workspaceId: "workspace-1", + repoUrl: null, + repoRef: "HEAD", + }, + workspace: { + id: "execution-workspace-2", + mode: "isolated_workspace", + strategyType: "git_worktree", + cwd: worktreePath, + providerRef: worktreePath, + projectId: "project-1", + projectWorkspaceId: "workspace-1", + repoUrl: null, + baseRef: "HEAD", + branchName: expectedBranch, + }, + issue: { + id: "issue-2", + identifier: "PAP-455", + title: "Reject dirty branch mismatch", + }, + agent: { + id: "agent-1", + name: "Codex Coder", + companyId: "company-1", + }, + })).rejects.toMatchObject({ + code: "workspace_validation_failed", + resultJson: { + workspaceValidation: expect.objectContaining({ + reason: "git_worktree_branch_incoherence", + fingerprint: expect.stringMatching(/^workspace_incoherence:v1:sha256:/), + sourceIssueId: "issue-2", + sourceIdentifier: "PAP-455", + executionWorkspaceId: "execution-workspace-2", + expectedBranch, + actualBranch, + cleanliness: "dirty", + provenance: expect.objectContaining({ + expectedBranchExists: true, + actualBranchExists: true, + sameHead: true, + }), + safeRepair: expect.objectContaining({ + eligible: false, + attempted: false, + succeeded: false, + reason: "worktree is not clean", + }), + }), + }, + }); + }, 15_000); + + it("routes non-reusable persisted git worktrees through workspace validation recovery", async () => { + const repoRoot = await createTempRepo(); + const expectedBranch = "PAP-455-not-registered-worktree"; + const detachedWorktreePath = path.join(repoRoot, ".paperclip", "worktrees", expectedBranch); + await fs.mkdir(path.dirname(detachedWorktreePath), { recursive: true }); + await execFileAsync("git", ["clone", repoRoot, detachedWorktreePath]); + await runGit(detachedWorktreePath, ["checkout", "-B", expectedBranch]); + + await expect(ensurePersistedExecutionWorkspaceAvailable({ + base: { + baseCwd: repoRoot, + source: "project_primary", + projectId: "project-1", + workspaceId: "workspace-1", + repoUrl: null, + repoRef: "HEAD", + }, + workspace: { + id: "execution-workspace-not-registered", + mode: "isolated_workspace", + strategyType: "git_worktree", + cwd: detachedWorktreePath, + providerRef: detachedWorktreePath, + projectId: "project-1", + projectWorkspaceId: "workspace-1", + repoUrl: null, + baseRef: "HEAD", + branchName: expectedBranch, + }, + issue: { + id: "issue-not-registered", + identifier: "PAP-455", + title: "Reject unregistered persisted worktree", + }, + agent: { + id: "agent-1", + name: "Codex Coder", + companyId: "company-1", + }, + })).rejects.toMatchObject({ + code: "workspace_validation_failed", + resultJson: { + workspaceValidation: { + reason: "git_worktree_not_reusable", + reasonCode: "not_registered", + worktreePath: detachedWorktreePath, + executionWorkspaceId: "execution-workspace-not-registered", + }, + }, + }); + }, 15_000); + + it("rejects an existing persisted git worktree when the checked-out branch changed to a different commit", async () => { + const repoRoot = await createTempRepo(); + + const initial = await realizeExecutionWorkspace({ + base: { + baseCwd: repoRoot, + source: "project_primary", + projectId: "project-1", + workspaceId: "workspace-1", + repoUrl: null, + repoRef: "HEAD", + }, + config: { + workspaceStrategy: { + type: "git_worktree", + branchTemplate: "{{issue.identifier}}-{{slug}}", + }, + }, + issue: { + id: "issue-1", + identifier: "PAP-456", + title: "Keep persisted branch coherent", + }, + agent: { + id: "agent-1", + name: "Codex Coder", + companyId: "company-1", + }, + }); + + const actualBranch = "PAP-456-push-pr-head"; + await runGit(initial.cwd, ["checkout", "-b", actualBranch]); + await fs.writeFile(path.join(initial.cwd, "publish.txt"), "publish\n", "utf8"); + await runGit(initial.cwd, ["add", "publish.txt"]); + await runGit(initial.cwd, ["commit", "-m", "Add publish branch work"]); + + await expect(ensurePersistedExecutionWorkspaceAvailable({ + base: { + baseCwd: repoRoot, + source: "project_primary", + projectId: "project-1", + workspaceId: "workspace-1", + repoUrl: null, + repoRef: "HEAD", + }, + workspace: { + id: "execution-workspace-3", + mode: "isolated_workspace", + strategyType: "git_worktree", + cwd: initial.cwd, + providerRef: initial.worktreePath, + projectId: "project-1", + projectWorkspaceId: "workspace-1", + repoUrl: null, + baseRef: "HEAD", + branchName: initial.branchName, + }, + issue: { + id: "issue-3", + identifier: "PAP-456", + title: "Keep persisted branch coherent", + }, + agent: { + id: "agent-1", + name: "Codex Coder", + companyId: "company-1", + }, + })).rejects.toMatchObject({ + code: "workspace_validation_failed", + resultJson: { + workspaceValidation: expect.objectContaining({ + reason: "git_worktree_branch_incoherence", + fingerprint: expect.stringMatching(/^workspace_incoherence:v1:sha256:/), + sourceIssueId: "issue-3", + sourceIdentifier: "PAP-456", + executionWorkspaceId: "execution-workspace-3", + expectedBranch: initial.branchName, + actualBranch, + cleanliness: "clean", + provenance: expect.objectContaining({ + expectedBranchExists: true, + actualBranchExists: true, + sameHead: false, + }), + safeRepair: expect.objectContaining({ + eligible: false, + attempted: false, + succeeded: false, + reason: "expected branch and current HEAD differ", + }), + }), + }, + }); + }, 15_000); + it("does not reuse a missing persisted local filesystem workspace", async () => { const baseCwd = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-workspace-base-")); const missingCwd = path.join(baseCwd, "missing-workspace"); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 5c6a2b17d1..63afaca38c 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -98,6 +98,8 @@ import { cleanupExecutionWorkspaceArtifacts, ensurePersistedExecutionWorkspaceAvailable, ensureRuntimeServicesForRun, + formatManagedGitWorktreeBranchInspection, + inspectManagedGitWorktreeBranch, persistAdapterManagedRuntimeServices, realizeExecutionWorkspace, releaseRuntimeServicesForRun, @@ -1153,8 +1155,21 @@ async function ensureManagedProjectWorkspace(input: { } } -function isWorkspaceValidationFailure(error: unknown): error is WorkspaceValidationFailure { - return error instanceof WorkspaceValidationFailure; +type WorkspaceValidationFailureLike = WorkspaceValidationFailure | { + code: typeof WORKSPACE_VALIDATION_FAILURE_CODE; + resultJson: Record; +}; + +function isWorkspaceValidationFailure(error: unknown): error is WorkspaceValidationFailureLike { + if (error instanceof WorkspaceValidationFailure) return true; + const maybe = error as { code?: unknown; resultJson?: unknown } | null; + return Boolean( + maybe && + maybe.code === WORKSPACE_VALIDATION_FAILURE_CODE && + maybe.resultJson && + typeof maybe.resultJson === "object" && + !Array.isArray(maybe.resultJson), + ); } function isWorkspaceValidationFailedRun( @@ -1163,6 +1178,38 @@ function isWorkspaceValidationFailedRun( return run?.errorCode === WORKSPACE_VALIDATION_FAILURE_CODE; } +function stableStringifyForFingerprint(value: unknown): string { + if (Array.isArray(value)) { + return `[${value.map((entry) => stableStringifyForFingerprint(entry)).join(",")}]`; + } + if (value && typeof value === "object") { + const rec = value as Record; + return `{${Object.keys(rec).sort().map((key) => `${JSON.stringify(key)}:${stableStringifyForFingerprint(rec[key])}`).join(",")}}`; + } + return JSON.stringify(value); +} + +function fingerprintFinalizeWorkspaceBranchValidation(input: { + issueId: string | null; + executionWorkspaceId: string; + inspection: ReturnType; +}) { + const digest = createHash("sha256") + .update(stableStringifyForFingerprint({ + version: 1, + reason: "git_worktree_branch_mismatch_after_run", + issueId: input.issueId, + executionWorkspaceId: input.executionWorkspaceId, + worktreePath: input.inspection.worktreePath ? path.resolve(input.inspection.worktreePath) : null, + repoRoot: input.inspection.repoRoot ? path.resolve(input.inspection.repoRoot) : null, + expectedBranchName: input.inspection.expectedBranchName, + actualBranchName: input.inspection.actualBranchName, + reasonCode: input.inspection.reasonCode, + })) + .digest("hex"); + return `workspace_finalize_branch_mismatch:v1:sha256:${digest}`; +} + function isConfigurationIncompleteFailure(error: unknown): error is ConfigurationIncompleteFailure { return error instanceof ConfigurationIncompleteFailure; } @@ -1386,6 +1433,27 @@ export async function assertGitSensitiveAdapterWorkspaceValid(input: { `Issue ${issue.identifier ?? issue.id} expected a git workspace for ${input.adapterType}, but "${effectiveCwd}" has no .git metadata.`, ); } + + const expectedManagedBranchName = + readNonEmptyString(input.persistedExecutionWorkspace?.branchName) ?? + readNonEmptyString(input.executionWorkspace.branchName); + if ( + input.persistedExecutionWorkspace?.strategyType === "git_worktree" && + effectiveCwd && + expectedManagedBranchName + ) { + const inspection = await inspectManagedGitWorktreeBranch({ + worktreePath: effectiveCwd, + expectedBranchName: expectedManagedBranchName, + }); + if (!inspection.valid) { + fail( + "git_worktree_branch_mismatch", + `Issue ${issue.identifier ?? issue.id} expected git worktree branch "${expectedManagedBranchName}" at "${effectiveCwd}", but ${inspection.reason ?? "the checked-out branch could not be verified"}.`, + { managedGitWorktreeBranch: formatManagedGitWorktreeBranchInspection(inspection) }, + ); + } + } } const heartbeatRunProcessGroupIdColumn = @@ -9946,6 +10014,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) ? await ensurePersistedExecutionWorkspaceAvailable({ base: executionWorkspaceBase, workspace: { + id: existingExecutionWorkspace.id, mode: existingExecutionWorkspace.mode, strategyType: existingExecutionWorkspace.strategyType, cwd: existingExecutionWorkspace.cwd, @@ -10665,11 +10734,81 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) ); } let adapterFinalizeOutcome: "succeeded" | "failed" | null = null; + const inspectFinalizeWorkspaceBranch = async () => { + const workspaceRecord = persistedExecutionWorkspace?.id + ? await executionWorkspacesSvc.getById(persistedExecutionWorkspace.id) + : persistedExecutionWorkspace; + if (workspaceRecord?.strategyType !== "git_worktree") return null; + + const worktreePath = + readNonEmptyString(workspaceRecord.providerRef) ?? + readNonEmptyString(workspaceRecord.cwd) ?? + readNonEmptyString(executionWorkspace.worktreePath) ?? + readNonEmptyString(executionWorkspace.cwd); + const expectedBranchName = + readNonEmptyString(workspaceRecord.branchName) ?? + readNonEmptyString(executionWorkspace.branchName); + if (!worktreePath || !expectedBranchName) return null; + + const inspection = await inspectManagedGitWorktreeBranch({ + worktreePath, + expectedBranchName, + }); + return { workspaceRecord, inspection }; + }; const recordWorkspaceFinalize = async ( status: "succeeded" | "failed", metadata?: Record, ) => { if (adapterFinalizeOutcome) return; + let finalizeBranchMetadata: Record | null = null; + if (status === "succeeded") { + const branchInspection = await inspectFinalizeWorkspaceBranch(); + if (branchInspection) { + const managedGitWorktreeBranch = formatManagedGitWorktreeBranchInspection(branchInspection.inspection); + finalizeBranchMetadata = { + executionWorkspaceId: branchInspection.workspaceRecord.id, + ...managedGitWorktreeBranch, + }; + if (!branchInspection.inspection.valid) { + const workspaceValidationFingerprint = fingerprintFinalizeWorkspaceBranchValidation({ + issueId: issueRef?.id ?? null, + executionWorkspaceId: branchInspection.workspaceRecord.id, + inspection: managedGitWorktreeBranch, + }); + await workspaceOperationRecorder.recordOperation({ + phase: "workspace_finalize", + cwd: executionWorkspace.cwd, + metadata: { + adapterType: agent.adapterType, + executionTargetKind: executionTarget?.kind ?? "local", + ...metadata, + managedGitWorktreeBranch: finalizeBranchMetadata, + }, + run: async () => ({ + status: "failed", + stderr: `Managed git worktree branch check failed: ${branchInspection.inspection.reason ?? "unknown branch mismatch"}\n`, + }), + }); + adapterFinalizeOutcome = "failed"; + throw new WorkspaceValidationFailure( + `Execution workspace ${branchInspection.workspaceRecord.id} expected git worktree branch "${branchInspection.inspection.expectedBranchName}" at "${branchInspection.inspection.worktreePath}", but ${branchInspection.inspection.reason ?? "the checked-out branch could not be verified"}. Record a sanctioned execution-workspace branch transition or restore the workspace branch before completing the run.`, + { + workspaceValidation: { + reason: "git_worktree_branch_mismatch_after_run", + fingerprint: workspaceValidationFingerprint, + adapterType: agent.adapterType, + issueId: issueRef?.id ?? null, + issueIdentifier: issueRef?.identifier ?? null, + persistedExecutionWorkspaceId: branchInspection.workspaceRecord.id, + executionWorkspaceCwd: executionWorkspace.cwd, + managedGitWorktreeBranch: finalizeBranchMetadata, + }, + }, + ); + } + } + } await workspaceOperationRecorder.recordOperation({ phase: "workspace_finalize", cwd: executionWorkspace.cwd, @@ -10677,6 +10816,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) adapterType: agent.adapterType, executionTargetKind: executionTarget?.kind ?? "local", ...metadata, + ...(finalizeBranchMetadata ? { managedGitWorktreeBranch: finalizeBranchMetadata } : {}), }, run: async () => ({ status }), }); @@ -11201,8 +11341,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) // A missing secret/env binding is a known pre-dispatch configuration gap, // not an opaque setup crash. Surface it with its own errorCode so the // recovery path routes it to a human owner instead of looping retries. + const workspaceValidationSetupFailure = isWorkspaceValidationFailure(outerErr) ? outerErr : null; const configurationIncompleteSetupFailure = isConfigurationIncompleteFailure(outerErr) ? outerErr : null; - const setupFailureErrorCode = configurationIncompleteSetupFailure?.code ?? "setup_failed"; + const setupFailureErrorCode = + workspaceValidationSetupFailure?.code ?? configurationIncompleteSetupFailure?.code ?? "setup_failed"; logger.error({ err: outerErr, runId }, "heartbeat execution setup failed"); const setupFailureAgent = await getAgent(run.agentId).catch(() => null); const setupFailureWrite = await setRunStatusIfRunning(runId, "failed", { @@ -11213,7 +11355,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) resultJson: mergeRunStopMetadataForAgent(setupFailureAgent, "failed", { errorCode: setupFailureErrorCode, errorMessage: message, - resultJson: configurationIncompleteSetupFailure?.resultJson ?? null, + resultJson: + workspaceValidationSetupFailure?.resultJson ?? configurationIncompleteSetupFailure?.resultJson ?? null, }), } : {}), }).catch(() => ({ run: null, updated: false as const })); diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index b98c428a2d..fa577ee791 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -113,7 +113,9 @@ type RecoveryWakeup = ( type LatestIssueRun = Pick< typeof heartbeatRuns.$inferSelect, "id" | "agentId" | "status" | "error" | "errorCode" | "contextSnapshot" | "livenessState" -> | null; +> & { + resultJson?: unknown; +} | null; type SuccessfulLatestIssueRun = NonNullable & { status: "succeeded" }; type StrandedRecoveryCause = @@ -130,6 +132,16 @@ type SuccessfulRunHandoffRecoveryEvidence = { maxHandoffAttempts: number; }; +function readWorkspaceValidationPayload(latestRun: LatestIssueRun): Record | null { + const payload = parseObject(parseObject(latestRun?.resultJson).workspaceValidation); + return Object.keys(payload).length > 0 ? payload : null; +} + +function readWorkspaceValidationFingerprint(latestRun: LatestIssueRun): string | null { + const payload = readWorkspaceValidationPayload(latestRun); + return readNonEmptyString(payload?.fingerprint); +} + type WatchdogDecisionActor = | { type: "board"; userId?: string | null; runId?: string | null } | { type: "agent"; agentId?: string | null; runId?: string | null } @@ -497,6 +509,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) errorCode: heartbeatRuns.errorCode, contextSnapshot: heartbeatRuns.contextSnapshot, livenessState: heartbeatRuns.livenessState, + resultJson: heartbeatRuns.resultJson, }) .from(heartbeatRuns) .where( @@ -2246,7 +2259,20 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) function strandedRecoveryActionFingerprint(input: { issue: typeof issues.$inferSelect; recoveryCause: StrandedRecoveryCause; + latestRun: LatestIssueRun; }) { + if (input.recoveryCause === "workspace_validation_failed") { + const workspaceFingerprint = readWorkspaceValidationFingerprint(input.latestRun); + if (workspaceFingerprint) { + return [ + "source_scoped_recovery", + input.issue.companyId, + input.issue.id, + input.recoveryCause, + workspaceFingerprint, + ].join(":"); + } + } return [ "source_scoped_recovery", input.issue.companyId, @@ -2263,6 +2289,9 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) successfulRunHandoffEvidence?: SuccessfulRunHandoffRecoveryEvidence | null; }) { const context = parseObject(input.latestRun?.contextSnapshot); + const workspaceValidation = input.recoveryCause === "workspace_validation_failed" + ? readWorkspaceValidationPayload(input.latestRun) + : null; return { sourceIssueId: input.issue.id, sourceIdentifier: input.issue.identifier, @@ -2278,6 +2307,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) missingDisposition: input.successfulRunHandoffEvidence?.missingDisposition ?? null, handoffAttempt: input.successfulRunHandoffEvidence?.handoffAttempt ?? null, maxHandoffAttempts: input.successfulRunHandoffEvidence?.maxHandoffAttempts ?? null, + ...(workspaceValidation ? { workspaceValidation } : {}), }; } @@ -2303,6 +2333,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) fingerprint: strandedRecoveryActionFingerprint({ issue: input.issue, recoveryCause, + latestRun: input.latestRun, }), evidence: buildStrandedRecoveryActionEvidence({ issue: input.issue, @@ -2314,7 +2345,9 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) nextAction: recoveryCause === SUCCESSFUL_RUN_MISSING_STATE_REASON ? "Choose and record a valid issue disposition without copying transcript content." : recoveryCause === "workspace_validation_failed" - ? "Repair the source issue workspace link, project workspace cwd, or git checkout before resuming adapter execution." + ? readWorkspaceValidationPayload(input.latestRun)?.reason === "git_worktree_branch_incoherence" + ? "Repair the source issue git worktree branch incoherence, or choose a new execution workspace, before resuming adapter execution." + : "Repair the source issue workspace link, project workspace cwd, or git checkout before resuming adapter execution." : recoveryCause === "configuration_incomplete" ? "Bind the missing secret(s) named in the run failure to the agent/project/routine env before resuming adapter execution." : "Restore a live execution path, fix the runtime/adapter failure, or record an intentional manual resolution.", diff --git a/server/src/services/workspace-runtime.ts b/server/src/services/workspace-runtime.ts index 5b29ffcfbb..d7e87effde 100644 --- a/server/src/services/workspace-runtime.ts +++ b/server/src/services/workspace-runtime.ts @@ -70,6 +70,17 @@ export interface RealizedExecutionWorkspace extends ExecutionWorkspaceInput { baseRefSha?: string | null; } +export class WorkspaceRuntimeValidationFailure extends Error { + code = "workspace_validation_failed" as const; + resultJson: Record; + + constructor(message: string, resultJson: Record) { + super(message); + this.name = "WorkspaceRuntimeValidationFailure"; + this.resultJson = resultJson; + } +} + export interface RuntimeServiceRef { id: string; companyId: string; @@ -640,6 +651,238 @@ async function remoteExists(repoRoot: string, remote: string): Promise .catch(() => false); } +const GIT_WORKTREE_BRANCH_INCOHERENCE_REASON = "git_worktree_branch_incoherence"; + +type GitWorktreeCleanliness = "clean" | "dirty" | "unknown"; + +type GitWorktreeBranchIncoherenceEvidence = { + reason: typeof GIT_WORKTREE_BRANCH_INCOHERENCE_REASON; + fingerprint: string; + sourceIssueId: string | null; + sourceIdentifier: string | null; + executionWorkspaceId: string | null; + worktreePath: string; + repoRoot: string; + expectedBranch: string; + actualBranch: string | null; + cleanliness: GitWorktreeCleanliness; + statusEntryCount: number | null; + provenance: { + expectedBranchRef: string; + actualBranchRef: string | null; + registeredBranchRef: string | null; + registeredPathFound: boolean; + registeredBranchMatchesHead: boolean; + expectedBranchExists: boolean; + actualBranchExists: boolean | null; + expectedHeadSha: string | null; + actualHeadSha: string | null; + sameHead: boolean; + }; + safeRepair: { + eligible: boolean; + attempted: boolean; + succeeded: boolean; + reason: string; + }; +}; + +function formatBranchForMessage(branch: string | null | undefined) { + return branch && branch.length > 0 ? branch : ""; +} + +function fingerprintWorkspaceBranchIncoherence(input: { + sourceIssueId: string | null; + executionWorkspaceId: string | null; + worktreePath: string; + expectedBranch: string; + actualBranch: string | null; + cleanliness: GitWorktreeCleanliness; + expectedHeadSha: string | null; + actualHeadSha: string | null; +}) { + const digest = createHash("sha256") + .update(stableStringify({ + version: 1, + reason: GIT_WORKTREE_BRANCH_INCOHERENCE_REASON, + sourceIssueId: input.sourceIssueId, + executionWorkspaceId: input.executionWorkspaceId, + worktreePath: path.resolve(input.worktreePath), + expectedBranch: input.expectedBranch, + actualBranch: input.actualBranch, + cleanliness: input.cleanliness, + expectedHeadSha: input.expectedHeadSha, + actualHeadSha: input.actualHeadSha, + })) + .digest("hex"); + return `workspace_incoherence:v1:sha256:${digest}`; +} + +async function inspectGitWorktreeBranchIncoherence(input: { + repoRoot: string; + worktreePath: string; + expectedBranchName: string; + actualBranchName: string | null; + sourceIssue: ExecutionWorkspaceIssueRef | null; + executionWorkspaceId?: string | null; +}): Promise { + const status = await runGit( + ["status", "--porcelain", "--untracked-files=all"], + input.worktreePath, + ).catch(() => null); + const statusLines = status === null + ? null + : status.split(/\r?\n/).map((line) => line.trim()).filter(Boolean); + const cleanliness: GitWorktreeCleanliness = + status === null ? "unknown" : status.trim().length > 0 ? "dirty" : "clean"; + const expectedHeadSha = await runGit( + ["rev-parse", "--verify", `refs/heads/${input.expectedBranchName}^{commit}`], + input.repoRoot, + ).catch(() => null); + const actualHeadSha = await runGit(["rev-parse", "HEAD"], input.worktreePath).catch(() => null); + const actualBranchExists = input.actualBranchName + ? await localBranchExists(input.repoRoot, input.actualBranchName) + : null; + const registered = await findRegisteredGitWorktreeByPath(input.repoRoot, input.worktreePath); + const actualBranchRef = input.actualBranchName ? `refs/heads/${input.actualBranchName}` : null; + const registeredBranchRef = registered?.branch ?? null; + const registeredBranchMatchesHead = Boolean(registered && registeredBranchRef === actualBranchRef); + const sameHead = Boolean(expectedHeadSha && actualHeadSha && expectedHeadSha === actualHeadSha); + const expectedBranchExists = Boolean(expectedHeadSha); + const eligible = cleanliness === "clean" && expectedBranchExists && sameHead && registeredBranchMatchesHead; + const safeRepairReason = eligible + ? "clean worktree and expected branch points at the current HEAD" + : cleanliness !== "clean" + ? "worktree is not clean" + : !registered + ? "worktree path is not registered" + : !registeredBranchMatchesHead + ? "registered worktree branch does not match HEAD" + : !expectedBranchExists + ? "expected branch does not exist" + : !sameHead + ? "expected branch and current HEAD differ" + : "safe repair could not be proven"; + const fingerprint = fingerprintWorkspaceBranchIncoherence({ + sourceIssueId: input.sourceIssue?.id ?? null, + executionWorkspaceId: input.executionWorkspaceId ?? null, + worktreePath: input.worktreePath, + expectedBranch: input.expectedBranchName, + actualBranch: input.actualBranchName, + cleanliness, + expectedHeadSha, + actualHeadSha, + }); + + return { + reason: GIT_WORKTREE_BRANCH_INCOHERENCE_REASON, + fingerprint, + sourceIssueId: input.sourceIssue?.id ?? null, + sourceIdentifier: input.sourceIssue?.identifier ?? null, + executionWorkspaceId: input.executionWorkspaceId ?? null, + worktreePath: path.resolve(input.worktreePath), + repoRoot: path.resolve(input.repoRoot), + expectedBranch: input.expectedBranchName, + actualBranch: input.actualBranchName, + cleanliness, + statusEntryCount: statusLines?.length ?? null, + provenance: { + expectedBranchRef: `refs/heads/${input.expectedBranchName}`, + actualBranchRef, + registeredBranchRef, + registeredPathFound: Boolean(registered), + registeredBranchMatchesHead, + expectedBranchExists, + actualBranchExists, + expectedHeadSha, + actualHeadSha, + sameHead, + }, + safeRepair: { + eligible, + attempted: false, + succeeded: false, + reason: safeRepairReason, + }, + }; +} + +function branchIncoherenceValidationFailure(evidence: GitWorktreeBranchIncoherenceEvidence) { + return new WorkspaceRuntimeValidationFailure( + `Execution workspace git worktree expected branch "${evidence.expectedBranch}" but found "${formatBranchForMessage(evidence.actualBranch)}" at "${evidence.worktreePath}". Safe repair ${evidence.safeRepair.succeeded ? "succeeded" : "was not completed"}: ${evidence.safeRepair.reason}.`, + { + workspaceValidation: evidence, + }, + ); +} + +async function ensureGitWorktreeBranchCoherent(input: { + repoRoot: string; + worktreePath: string; + expectedBranchName: string | null; + sourceIssue: ExecutionWorkspaceIssueRef | null; + executionWorkspaceId?: string | null; + actualBranchName?: string | null; + recorder?: WorkspaceOperationRecorder | null; +}) { + const expectedBranchName = input.expectedBranchName?.trim(); + if (!expectedBranchName) return; + + const currentBranch = input.actualBranchName !== undefined + ? input.actualBranchName + : await runGit(["symbolic-ref", "--quiet", "--short", "HEAD"], input.worktreePath).catch(() => null); + if (currentBranch === expectedBranchName) return; + + const evidence = await inspectGitWorktreeBranchIncoherence({ + repoRoot: input.repoRoot, + worktreePath: input.worktreePath, + expectedBranchName, + actualBranchName: currentBranch, + sourceIssue: input.sourceIssue, + executionWorkspaceId: input.executionWorkspaceId ?? null, + }); + + if (!evidence.safeRepair.eligible) { + throw branchIncoherenceValidationFailure(evidence); + } + + evidence.safeRepair.attempted = true; + try { + await recordGitOperation(input.recorder, { + phase: "worktree_prepare", + args: ["checkout", expectedBranchName], + cwd: input.worktreePath, + metadata: { + repoRoot: input.repoRoot, + worktreePath: input.worktreePath, + expectedBranchName, + actualBranchName: currentBranch, + branchIncoherenceRepair: true, + fingerprint: evidence.fingerprint, + sourceIssueId: evidence.sourceIssueId, + executionWorkspaceId: evidence.executionWorkspaceId, + }, + successMessage: `Repaired clean git worktree branch mismatch at ${input.worktreePath}: checked out ${expectedBranchName}\n`, + failureLabel: `git checkout ${expectedBranchName}`, + }); + } catch (error) { + evidence.safeRepair.succeeded = false; + evidence.safeRepair.reason = `safe checkout failed: ${error instanceof Error ? error.message : String(error)}`; + throw branchIncoherenceValidationFailure(evidence); + } + + const repairedBranch = await runGit(["symbolic-ref", "--quiet", "--short", "HEAD"], input.worktreePath) + .catch(() => null); + if (repairedBranch !== expectedBranchName) { + evidence.safeRepair.succeeded = false; + evidence.safeRepair.reason = `checkout completed but HEAD is ${formatBranchForMessage(repairedBranch)}`; + throw branchIncoherenceValidationFailure(evidence); + } + + evidence.safeRepair.succeeded = true; + evidence.safeRepair.reason = "clean worktree checked out the recorded branch"; +} + // Resolve the authoritative base ref for a fresh worktree. A configured local // branch is mapped to its `origin/` counterpart so unpushed local // divergence never leaks into the task branch; remote-tracking refs, SHAs, and @@ -751,6 +994,22 @@ type GitWorktreeListEntry = { branch: string | null; }; +export type ManagedGitWorktreeBranchInspection = { + valid: boolean; + reason: string | null; + reasonCode: + | "missing_worktree" + | "not_a_git_checkout" + | "not_registered" + | "wrong_repository_root" + | "branch_mismatch" + | null; + repoRoot: string | null; + worktreePath: string; + expectedBranchName: string | null; + actualBranchName: string | null; +}; + function parseGitWorktreeListPorcelain(raw: string): GitWorktreeListEntry[] { const entries: GitWorktreeListEntry[] = []; let current: Partial = {}; @@ -803,6 +1062,19 @@ async function findRegisteredGitWorktreeByBranch(repoRoot: string, branchName: s return null; } +async function findRegisteredGitWorktreeByPath(repoRoot: string, worktreePath: string): Promise { + const raw = await runGit(["worktree", "list", "--porcelain"], repoRoot).catch(() => null); + if (!raw) return null; + + const expectedPath = await resolvePathForWorktreeComparison(worktreePath); + for (const entry of parseGitWorktreeListPorcelain(raw)) { + if (await resolvePathForWorktreeComparison(entry.worktree) === expectedPath) { + return entry; + } + } + return null; +} + async function isGitCheckout(cwd: string): Promise { return Boolean(await runGit(["rev-parse", "--git-dir"], cwd).catch(() => null)); } @@ -846,6 +1118,11 @@ async function directoryExists(value: string) { return fs.stat(value).then((stats) => stats.isDirectory()).catch(() => false); } +async function resolvePathForWorktreeComparison(value: string): Promise { + const resolved = path.resolve(value); + return fs.realpath(resolved).then((realPath) => path.resolve(realPath)).catch(() => resolved); +} + async function listLinkedGitWorktreePaths(repoRoot: string): Promise> { const output = await runGit(["worktree", "list", "--porcelain"], repoRoot); const paths = new Set(); @@ -853,47 +1130,132 @@ async function listLinkedGitWorktreePaths(repoRoot: string): Promise if (!line.startsWith("worktree ")) continue; const worktree = line.slice("worktree ".length).trim(); if (!worktree) continue; - paths.add(path.resolve(worktree)); + paths.add(await resolvePathForWorktreeComparison(worktree)); } return paths; } +export async function inspectManagedGitWorktreeBranch(input: { + worktreePath: string; + expectedBranchName: string | null | undefined; + repoRoot?: string | null; +}): Promise { + const worktreePath = await resolvePathForWorktreeComparison(input.worktreePath); + const expectedBranchName = asString(input.expectedBranchName, "").trim() || null; + const base = { + worktreePath, + expectedBranchName, + actualBranchName: null, + }; + + if (!await directoryExists(worktreePath)) { + return { + ...base, + valid: false, + reason: `worktree path "${worktreePath}" does not exist`, + reasonCode: "missing_worktree", + repoRoot: input.repoRoot ? path.resolve(input.repoRoot) : null, + }; + } + + const repoRoot = input.repoRoot + ? path.resolve(input.repoRoot) + : await resolveGitOwnerRepoRoot(worktreePath).catch(() => null); + if (!repoRoot) { + return { + ...base, + valid: false, + reason: "path is not a git checkout", + reasonCode: "not_a_git_checkout", + repoRoot: null, + }; + } + + const listedWorktrees = await listLinkedGitWorktreePaths(repoRoot).catch(() => null); + if (!listedWorktrees?.has(worktreePath)) { + return { + ...base, + valid: false, + reason: "path is not registered in `git worktree list`", + reasonCode: "not_registered", + repoRoot, + }; + } + + const worktreeTopLevel = await runGit(["rev-parse", "--show-toplevel"], worktreePath).catch(() => null); + if (!worktreeTopLevel || path.resolve(worktreeTopLevel) !== worktreePath) { + return { + ...base, + valid: false, + reason: "git resolves this path to a different repository root", + reasonCode: "wrong_repository_root", + repoRoot, + }; + } + + const actualBranchName = await runGit( + ["symbolic-ref", "--quiet", "--short", "HEAD"], + worktreePath, + ).catch(() => null); + if (expectedBranchName && actualBranchName !== expectedBranchName) { + return { + ...base, + valid: false, + reason: `worktree HEAD is on "${actualBranchName ?? ""}" instead of "${expectedBranchName}"`, + reasonCode: "branch_mismatch", + repoRoot, + actualBranchName, + }; + } + + return { + ...base, + valid: true, + reason: null, + reasonCode: null, + repoRoot, + actualBranchName, + }; +} + async function validateLinkedGitWorktree(input: { repoRoot: string; worktreePath: string; expectedBranchName: string | null; -}): Promise<{ valid: true } | { valid: false; reason: string }> { - const resolvedWorktreePath = path.resolve(input.worktreePath); - const listedWorktrees = await listLinkedGitWorktreePaths(input.repoRoot); - if (!listedWorktrees.has(resolvedWorktreePath)) { - return { - valid: false, - reason: "path is not registered in `git worktree list`", - }; +}): Promise< + | { valid: true } + | { + valid: false; + reason: string; + reasonCode: Exclude; + actualBranchName?: string | null; } - - const worktreeTopLevel = await runGit(["rev-parse", "--show-toplevel"], resolvedWorktreePath).catch(() => null); - if (!worktreeTopLevel || path.resolve(worktreeTopLevel) !== resolvedWorktreePath) { - return { - valid: false, - reason: "git resolves this path to a different repository root", - }; - } - - if (input.expectedBranchName) { - const currentBranch = await runGit( - ["symbolic-ref", "--quiet", "--short", "HEAD"], - resolvedWorktreePath, - ).catch(() => null); - if (currentBranch !== input.expectedBranchName) { - return { +> { + const inspection = await inspectManagedGitWorktreeBranch({ + repoRoot: input.repoRoot, + worktreePath: input.worktreePath, + expectedBranchName: input.expectedBranchName, + }); + return inspection.valid + ? { valid: true } + : { valid: false, - reason: `worktree HEAD is on "${currentBranch ?? ""}" instead of "${input.expectedBranchName}"`, + reason: inspection.reason ?? "unknown git worktree mismatch", + reasonCode: inspection.reasonCode ?? "not_a_git_checkout", + actualBranchName: inspection.actualBranchName, }; - } - } +} - return { valid: true }; +export function formatManagedGitWorktreeBranchInspection(input: ManagedGitWorktreeBranchInspection) { + return { + valid: input.valid, + reason: input.reason, + reasonCode: input.reasonCode, + repoRoot: input.repoRoot, + worktreePath: input.worktreePath, + expectedBranchName: input.expectedBranchName, + actualBranchName: input.actualBranchName, + }; } function terminateChildProcess(child: ChildProcess) { @@ -1328,11 +1690,28 @@ export async function realizeExecutionWorkspace(input: { } async function validateReusableWorktree(reusablePath: string) { - return await validateLinkedGitWorktree({ + const validation = await validateLinkedGitWorktree({ repoRoot, worktreePath: reusablePath, expectedBranchName: branchName, }).catch(() => null); + if (validation && !validation.valid && validation.reasonCode === "branch_mismatch") { + await ensureGitWorktreeBranchCoherent({ + repoRoot, + worktreePath: reusablePath, + expectedBranchName: branchName, + actualBranchName: validation.actualBranchName ?? null, + sourceIssue: input.issue, + executionWorkspaceId: null, + recorder: input.recorder ?? null, + }); + return await validateLinkedGitWorktree({ + repoRoot, + worktreePath: reusablePath, + expectedBranchName: branchName, + }).catch(() => null); + } + return validation; } const existingWorktree = await directoryExists(worktreePath); @@ -1431,6 +1810,7 @@ export async function realizeExecutionWorkspace(input: { export async function ensurePersistedExecutionWorkspaceAvailable(input: { base: ExecutionWorkspaceInput; workspace: { + id?: string | null; mode: string | null | undefined; strategyType: string | null | undefined; cwd: string | null | undefined; @@ -1481,6 +1861,34 @@ export async function ensurePersistedExecutionWorkspaceAvailable(input: { if (await directoryExists(cwd)) { const reuseBaseRef = input.workspace.baseRef ?? input.base.repoRef ?? null; const reuseWorktreePath = realized.worktreePath ?? cwd; + if (await isGitCheckout(reuseWorktreePath)) { + await ensureGitWorktreeBranchCoherent({ + repoRoot, + worktreePath: reuseWorktreePath, + expectedBranchName: realized.branchName, + sourceIssue: input.issue, + executionWorkspaceId: input.workspace.id ?? null, + recorder: input.recorder ?? null, + }); + } + const validation = await validateLinkedGitWorktree({ + repoRoot, + worktreePath: reuseWorktreePath, + expectedBranchName: realized.branchName, + }); + if (!validation.valid) { + throw new WorkspaceRuntimeValidationFailure( + `Persisted git worktree "${reuseWorktreePath}" is not reusable (${validation.reason}).`, + { + workspaceValidation: { + reason: "git_worktree_not_reusable", + reasonCode: validation.reasonCode, + worktreePath: reuseWorktreePath, + executionWorkspaceId: input.workspace.id ?? null, + }, + }, + ); + } const baseRefreshWarnings = reuseBaseRef ? await refreshRemoteTrackingBaseRef(repoRoot, reuseBaseRef) : [];