diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 911c5a7980..0067de1e6f 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -819,6 +819,7 @@ export type { AgentWakeupSkipped, GitWorktreeBranchAncestryVerdict, GitWorktreeBranchIncoherenceEvidence, + GitWorktreeInProgressOperation, HeartbeatRun, HeartbeatRunEvent, HeartbeatRunStatusPhase, diff --git a/packages/shared/src/types/heartbeat.ts b/packages/shared/src/types/heartbeat.ts index 427ebab800..1c2592edd0 100644 --- a/packages/shared/src/types/heartbeat.ts +++ b/packages/shared/src/types/heartbeat.ts @@ -10,6 +10,8 @@ import type { export type GitWorktreeBranchAncestryVerdict = "ancestor" | "diverged" | "unknown"; +export type GitWorktreeInProgressOperation = "rebase" | "merge" | "cherry_pick" | "revert" | "bisect"; + export interface GitWorktreeBranchIncoherenceEvidence { reason: "git_worktree_branch_incoherence"; fingerprint: string; @@ -21,7 +23,25 @@ export interface GitWorktreeBranchIncoherenceEvidence { expectedBranch: string; actualBranch: string | null; cleanliness: "clean" | "dirty" | "unknown"; + /** + * Interrupted git operation (rebase/merge/cherry-pick/revert/bisect) whose + * state directory is still present in the worktree. Optional so previously + * persisted evidence payloads stay valid. + */ + inProgressOperation?: GitWorktreeInProgressOperation | null; statusEntryCount: number | null; + dirtyPathSample: string[]; + contention: { + claimedByWorkspaceId: string; + claimedByIssueId: string | null; + claimedByIssueIdentifier: string | null; + activeRun: { + id: string; + status: "queued" | "running"; + issueId: string | null; + issueIdentifier: string | null; + } | null; + } | null; provenance: { expectedBranchRef: string; actualBranchRef: string | null; diff --git a/packages/shared/src/types/index.ts b/packages/shared/src/types/index.ts index c7e820f8d2..a8edf41318 100644 --- a/packages/shared/src/types/index.ts +++ b/packages/shared/src/types/index.ts @@ -509,6 +509,7 @@ export type { AgentWakeupSkipped, GitWorktreeBranchAncestryVerdict, GitWorktreeBranchIncoherenceEvidence, + GitWorktreeInProgressOperation, HeartbeatRun, HeartbeatRunEvent, HeartbeatRunStatusPhase, diff --git a/packages/shared/src/types/instance.ts b/packages/shared/src/types/instance.ts index 79cbbd3d7a..10a72e0c6d 100644 --- a/packages/shared/src/types/instance.ts +++ b/packages/shared/src/types/instance.ts @@ -61,6 +61,7 @@ export interface InstanceExperimentalSettings { autoRestartDevServerWhenIdle: boolean; enableIssueGraphLivenessAutoRecovery: boolean; enableWorkspaceBranchReconcileForward: boolean; + enableWorkspaceDirtyQuarantineRepair: boolean; /** * Worktree preview instances (`PAPERCLIP_IN_WORKTREE=true`) suppress the * heartbeat run engine by default so previews never self-execute tasks. When diff --git a/packages/shared/src/validators/execution-workspace.ts b/packages/shared/src/validators/execution-workspace.ts index 234c151375..4e3e1148c5 100644 --- a/packages/shared/src/validators/execution-workspace.ts +++ b/packages/shared/src/validators/execution-workspace.ts @@ -160,6 +160,10 @@ export const reconcileExecutionWorkspaceBranchSchema = z.discriminatedUnion("mod mode: z.literal("override"), reason: branchReconcileReasonSchema, }).strict(), + z.object({ + mode: z.literal("quarantine_restore"), + reason: branchReconcileReasonSchema.optional().nullable(), + }).strict(), ]); export type UpdateExecutionWorkspace = z.infer; diff --git a/packages/shared/src/validators/instance.test.ts b/packages/shared/src/validators/instance.test.ts index eb2117a0c0..09b0c34472 100644 --- a/packages/shared/src/validators/instance.test.ts +++ b/packages/shared/src/validators/instance.test.ts @@ -11,10 +11,11 @@ describe("instance experimental settings validators", () => { expect(settings.enableServerInfoDebugView).toBe(false); }); - it("defaults workspace branch forward reconciliation off", () => { + it("defaults workspace branch repair settings on", () => { const settings = instanceExperimentalSettingsSchema.parse({}); - expect(settings.enableWorkspaceBranchReconcileForward).toBe(false); + expect(settings.enableWorkspaceBranchReconcileForward).toBe(true); + expect(settings.enableWorkspaceDirtyQuarantineRepair).toBe(true); }); it("defaults the goals sidebar link off", () => { @@ -58,10 +59,12 @@ describe("instance experimental settings validators", () => { it("accepts workspace branch forward reconciliation patches", () => { expect( patchInstanceExperimentalSettingsSchema.parse({ - enableWorkspaceBranchReconcileForward: true, + enableWorkspaceBranchReconcileForward: false, + enableWorkspaceDirtyQuarantineRepair: false, }), ).toEqual({ - enableWorkspaceBranchReconcileForward: true, + enableWorkspaceBranchReconcileForward: false, + enableWorkspaceDirtyQuarantineRepair: false, }); }); diff --git a/packages/shared/src/validators/instance.ts b/packages/shared/src/validators/instance.ts index bdd0a53a65..8798e19f69 100644 --- a/packages/shared/src/validators/instance.ts +++ b/packages/shared/src/validators/instance.ts @@ -54,7 +54,8 @@ export const instanceExperimentalSettingsSchema = z.object({ enableServerInfoDebugView: z.boolean().default(false), autoRestartDevServerWhenIdle: z.boolean().default(false), enableIssueGraphLivenessAutoRecovery: z.boolean().default(false), - enableWorkspaceBranchReconcileForward: z.boolean().default(false), + enableWorkspaceBranchReconcileForward: z.boolean().default(true), + enableWorkspaceDirtyQuarantineRepair: z.boolean().default(true), enableWorktreeRunExecution: z.boolean().default(false), issueGraphLivenessAutoRecoveryLookbackHours: z .number() diff --git a/server/src/__tests__/execution-workspaces-routes.test.ts b/server/src/__tests__/execution-workspaces-routes.test.ts index 757c4c6fe7..be7deacb2e 100644 --- a/server/src/__tests__/execution-workspaces-routes.test.ts +++ b/server/src/__tests__/execution-workspaces-routes.test.ts @@ -19,6 +19,10 @@ const mockWorkspaceOperationService = vi.hoisted(() => ({ createRecorder: vi.fn(), })); +const mockHeartbeatService = vi.hoisted(() => ({ + wakeup: vi.fn(), +})); + const mockAccessService = vi.hoisted(() => ({ decide: vi.fn(), })); @@ -27,6 +31,7 @@ const mockLogActivity = vi.hoisted(() => vi.fn(async () => undefined)); vi.mock("../services/index.js", () => ({ accessService: () => mockAccessService, executionWorkspaceService: () => mockExecutionWorkspaceService, + heartbeatService: () => mockHeartbeatService, logActivity: mockLogActivity, workspaceOperationService: () => mockWorkspaceOperationService, })); @@ -77,6 +82,7 @@ describe.sequential("execution workspace routes", () => { ]); mockExecutionWorkspaceService.getById.mockResolvedValue(null); mockExecutionWorkspaceService.reconcileExecutionWorkspaceBranch.mockResolvedValue(null); + mockHeartbeatService.wakeup.mockResolvedValue(null); }); it("uses summary mode for lightweight workspace lookups", async () => { @@ -133,6 +139,7 @@ describe.sequential("execution workspace routes", () => { it.each([ ["forward", { mode: "forward" }], ["override", { mode: "override", reason: "operator break-glass" }], + ["quarantine_restore", { mode: "quarantine_restore", reason: "rescue dirty branch" }], ])("rejects agent actors for %s branch reconciliation", async (_mode, body) => { mockExecutionWorkspaceService.getById.mockResolvedValue({ id: "workspace-1", @@ -220,4 +227,166 @@ describe.sequential("execution workspace routes", () => { }), })); }); + + it("accepts quarantine_restore, logs the rescue ref, and wakes the restored source issue", async () => { + mockExecutionWorkspaceService.getById.mockResolvedValue({ + id: "workspace-1", + companyId: "company-1", + sourceIssueId: "issue-1", + }); + mockExecutionWorkspaceService.reconcileExecutionWorkspaceBranch.mockResolvedValue({ + workspace: { + id: "workspace-1", + companyId: "company-1", + sourceIssueId: "issue-1", + branchName: "feature/recorded", + }, + inspection: { + fingerprint: "workspace_incoherence:v1:sha256:dirty", + worktreePath: "/tmp/worktree", + repoRoot: "/tmp/repo", + fromBranch: "feature/recorded", + toBranch: "feature/live", + fromSha: "1111111", + toSha: "2222222", + ancestryVerdict: "diverged", + cleanliness: "dirty", + statusEntryCount: 2, + plainLanguageReason: "dirty live branch", + }, + recoveryAction: { + id: "recovery-1", + }, + auditCommentId: "comment-1", + rescueRef: { + branchName: "paperclip/rescue/PAP-123/20260709T120000Z", + commitSha: "3333333", + fileCount: 2, + sourceAuditCommentId: "comment-0", + claimantAuditCommentId: null, + }, + restoredSourceIssue: { + id: "issue-1", + companyId: "company-1", + status: "todo", + assigneeAgentId: "agent-1", + }, + sourceIssueStatusChanged: true, + }); + + const res = await request(createApp()) + .post("/api/execution-workspaces/workspace-1/reconcile-branch") + .send({ mode: "quarantine_restore" }); + + expect(res.status).toBe(200); + expect(mockExecutionWorkspaceService.reconcileExecutionWorkspaceBranch).toHaveBeenCalledWith("workspace-1", { + mode: "quarantine_restore", + reason: null, + actor: { + actorType: "user", + actorId: "local-board", + agentId: null, + runId: null, + }, + }); + expect(mockLogActivity).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ + action: "execution_workspace.branch_reconciled", + entityType: "execution_workspace", + entityId: "workspace-1", + details: expect.objectContaining({ + mode: "quarantine_restore", + fingerprint: "workspace_incoherence:v1:sha256:dirty", + recoveryActionId: "recovery-1", + rescueRef: expect.objectContaining({ + branchName: "paperclip/rescue/PAP-123/20260709T120000Z", + commitSha: "3333333", + }), + sourceIssueStatus: "todo", + }), + })); + expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith("agent-1", expect.objectContaining({ + source: "automation", + reason: "issue_recovery_action_restored", + payload: expect.objectContaining({ + issueId: "issue-1", + recoveryActionId: "recovery-1", + executionWorkspaceId: "workspace-1", + rescueRef: "paperclip/rescue/PAP-123/20260709T120000Z", + mutation: "execution_workspace_quarantine_restore", + }), + contextSnapshot: expect.objectContaining({ + issueId: "issue-1", + taskId: "issue-1", + wakeReason: "issue_recovery_action_restored", + source: "execution_workspace.quarantine_restore", + recoveryActionId: "recovery-1", + executionWorkspaceId: "workspace-1", + rescueRef: "paperclip/rescue/PAP-123/20260709T120000Z", + }), + })); + }); + + it("wakes a restored in_review agent participant after quarantine_restore", async () => { + mockExecutionWorkspaceService.getById.mockResolvedValue({ + id: "workspace-1", + companyId: "company-1", + sourceIssueId: "issue-1", + }); + mockExecutionWorkspaceService.reconcileExecutionWorkspaceBranch.mockResolvedValue({ + workspace: { + id: "workspace-1", + companyId: "company-1", + sourceIssueId: "issue-1", + branchName: "feature/recorded", + }, + inspection: { + fingerprint: "workspace_incoherence:v1:sha256:dirty", + worktreePath: "/tmp/worktree", + repoRoot: "/tmp/repo", + fromBranch: "feature/recorded", + toBranch: "feature/live", + fromSha: "1111111", + toSha: "2222222", + ancestryVerdict: "diverged", + cleanliness: "dirty", + statusEntryCount: 2, + plainLanguageReason: "dirty live branch", + }, + recoveryAction: { + id: "recovery-1", + }, + auditCommentId: "comment-1", + rescueRef: null, + restoredSourceIssue: { + id: "issue-1", + companyId: "company-1", + status: "in_review", + assigneeAgentId: "reviewer-agent-1", + }, + sourceIssueStatusChanged: true, + }); + + const res = await request(createApp()) + .post("/api/execution-workspaces/workspace-1/reconcile-branch") + .send({ mode: "quarantine_restore" }); + + expect(res.status).toBe(200); + expect(mockLogActivity).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ + details: expect.objectContaining({ + sourceIssueStatus: "in_review", + }), + })); + expect(mockHeartbeatService.wakeup).toHaveBeenCalledWith("reviewer-agent-1", expect.objectContaining({ + reason: "issue_recovery_action_restored", + payload: expect.objectContaining({ + issueId: "issue-1", + mutation: "execution_workspace_quarantine_restore", + }), + contextSnapshot: expect.objectContaining({ + issueId: "issue-1", + wakeReason: "issue_recovery_action_restored", + source: "execution_workspace.quarantine_restore", + }), + })); + }); }); diff --git a/server/src/__tests__/execution-workspaces-service.test.ts b/server/src/__tests__/execution-workspaces-service.test.ts index 3217626851..e7fc8db03f 100644 --- a/server/src/__tests__/execution-workspaces-service.test.ts +++ b/server/src/__tests__/execution-workspaces-service.test.ts @@ -8,9 +8,12 @@ import { promisify } from "node:util"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { eq, inArray, sql } from "drizzle-orm"; import { + activityLog, + agents, companies, createDb, executionWorkspaces, + heartbeatRuns, issueComments, issueRecoveryActions, issues, @@ -226,12 +229,15 @@ describeEmbeddedPostgres("executionWorkspaceService.getCloseReadiness", () => { afterEach(async () => { await db.delete(workspaceRuntimeServices); + await db.delete(activityLog); await db.delete(issueRecoveryActions); await db.delete(issueComments); await db.delete(issues); await db.delete(executionWorkspaces); await db.delete(projectWorkspaces); await db.delete(projects); + await db.delete(heartbeatRuns); + await db.delete(agents); await db.delete(companies); for (const dir of tempDirs) { @@ -625,6 +631,640 @@ describeEmbeddedPostgres("executionWorkspaceService.getCloseReadiness", () => { }); }, 20_000); + it("quarantine_restore rescues dirty live-branch work, resolves recovery, and returns the source issue to todo", async () => { + const repoRoot = await createTempRepo(); + tempDirs.add(repoRoot); + const worktreePath = path.join(path.dirname(repoRoot), `paperclip-quarantine-restore-${randomUUID()}`); + tempDirs.add(worktreePath); + + await runGit(repoRoot, ["branch", "feature/recorded"]); + await runGit(repoRoot, ["worktree", "add", "-b", "feature/live", worktreePath, "feature/recorded"]); + await fs.appendFile(path.join(worktreePath, "README.md"), "dirty tracked work\n", "utf8"); + await fs.writeFile(path.join(worktreePath, "untracked.txt"), "dirty untracked work\n", "utf8"); + + const companyId = randomUUID(); + const agentId = randomUUID(); + const projectId = randomUUID(); + const projectWorkspaceId = randomUUID(); + const issueId = randomUUID(); + const executionWorkspaceId = randomUUID(); + const actualBranch = await readGit(worktreePath, ["symbolic-ref", "--quiet", "--short", "HEAD"]); + const fingerprint = await fingerprintWorkspaceBranchIncoherenceForTest({ + repoRoot, + worktreePath, + sourceIssueId: issueId, + executionWorkspaceId, + expectedBranch: "feature/recorded", + actualBranch, + }); + + await db.insert(companies).values({ + id: companyId, + name: "Paperclip", + issuePrefix: "PAP", + requireBoardApprovalForNewAgents: false, + }); + await db.insert(agents).values({ + id: agentId, + companyId, + name: "Codex Coder", + role: "engineer", + status: "active", + adapterType: "codex_local", + adapterConfig: {}, + runtimeConfig: {}, + permissions: {}, + }); + await db.insert(projects).values({ + id: projectId, + companyId, + name: "Branch reconcile", + status: "in_progress", + }); + await db.insert(projectWorkspaces).values({ + id: projectWorkspaceId, + companyId, + projectId, + name: "Primary", + cwd: repoRoot, + isPrimary: true, + }); + await db.insert(issues).values({ + id: issueId, + companyId, + projectId, + projectWorkspaceId, + title: "Source task", + identifier: "PAP-124", + status: "blocked", + priority: "medium", + assigneeAgentId: agentId, + }); + await db.insert(executionWorkspaces).values({ + id: executionWorkspaceId, + companyId, + projectId, + projectWorkspaceId, + sourceIssueId: issueId, + mode: "isolated_workspace", + strategyType: "git_worktree", + name: "feature/recorded", + status: "active", + providerType: "git_worktree", + cwd: worktreePath, + providerRef: worktreePath, + branchName: "feature/recorded", + baseRef: "main", + }); + await db.insert(issueRecoveryActions).values({ + companyId, + sourceIssueId: issueId, + kind: "workspace_validation", + status: "active", + ownerType: "board", + cause: "workspace_validation_failed", + fingerprint, + evidence: { + workspaceValidation: { + fingerprint, + }, + }, + nextAction: "Repair the source issue workspace link.", + }); + + const result = await svc.reconcileExecutionWorkspaceBranch(executionWorkspaceId, { + mode: "quarantine_restore", + reason: "rescue dirty work and restore recorded branch", + actor: { + actorType: "user", + actorId: "local-board", + agentId: null, + runId: null, + }, + }); + + expect(result.workspace.branchName).toBe("feature/recorded"); + expect(result.inspection).toMatchObject({ + fromBranch: "feature/recorded", + toBranch: "feature/live", + cleanliness: "dirty", + fingerprint, + }); + expect(result.rescueRef).toMatchObject({ + branchName: expect.stringMatching(/^paperclip\/rescue\/PAP-124\/\d{8}T\d{6}Z$/), + fileCount: 2, + }); + expect(result.restoredSourceIssue).toMatchObject({ + id: issueId, + status: "todo", + assigneeAgentId: agentId, + }); + expect(result.sourceIssueStatusChanged).toBe(true); + expect(result.recoveryAction).toMatchObject({ + kind: "workspace_validation", + status: "resolved", + outcome: "restored", + fingerprint, + }); + + const rescueRef = result.rescueRef!.branchName; + await expect(readGit(worktreePath, ["branch", "--show-current"])).resolves.toBe("feature/recorded"); + await expect(readGit(worktreePath, ["status", "--porcelain", "--untracked-files=all"])).resolves.toBeNull(); + await expect(readGit(repoRoot, ["show", `${rescueRef}:untracked.txt`])).resolves.toBe("dirty untracked work"); + + const [sourceIssue] = await db.select().from(issues).where(eq(issues.id, issueId)); + expect(sourceIssue).toMatchObject({ + status: "todo", + checkoutRunId: null, + executionRunId: null, + }); + + const [recoveryAction] = await db + .select() + .from(issueRecoveryActions) + .where(eq(issueRecoveryActions.sourceIssueId, issueId)); + expect(recoveryAction).toMatchObject({ + status: "resolved", + outcome: "restored", + resolutionNote: `Execution workspace dirty worktree quarantined on "${rescueRef}" and restored recorded branch "feature/recorded".`, + }); + + const comments = await db + .select() + .from(issueComments) + .where(eq(issueComments.issueId, issueId)) + .orderBy(issueComments.createdAt); + expect(comments).toHaveLength(2); + expect(comments[0]?.body).toContain("Execution workspace dirty worktree quarantined before restore."); + expect(comments[0]?.body).toContain(`Rescue branch: \`${rescueRef}\``); + expect(comments[1]?.body).toContain("Execution workspace branch reconciled."); + expect(comments[1]?.body).toContain("- Mode: `quarantine_restore`"); + expect(comments[1]?.body).toContain(`- Rescue ref: \`${rescueRef}\``); + }, 20_000); + + it("quarantine_restore rejects active runtime services before creating a rescue branch", async () => { + const repoRoot = await createTempRepo(); + tempDirs.add(repoRoot); + const worktreePath = path.join(path.dirname(repoRoot), `paperclip-quarantine-running-${randomUUID()}`); + tempDirs.add(worktreePath); + + await runGit(repoRoot, ["branch", "feature/recorded"]); + await runGit(repoRoot, ["worktree", "add", "-b", "feature/live", worktreePath, "feature/recorded"]); + await fs.appendFile(path.join(worktreePath, "README.md"), "dirty tracked work\n", "utf8"); + + const companyId = randomUUID(); + const projectId = randomUUID(); + const issueId = randomUUID(); + const executionWorkspaceId = randomUUID(); + const runtimeServiceId = randomUUID(); + + await db.insert(companies).values({ + id: companyId, + name: "Paperclip", + issuePrefix: "PAP", + requireBoardApprovalForNewAgents: false, + }); + await db.insert(projects).values({ + id: projectId, + companyId, + name: "Branch reconcile", + status: "in_progress", + }); + await db.insert(issues).values({ + id: issueId, + companyId, + projectId, + title: "Source task", + identifier: "PAP-125", + status: "blocked", + priority: "medium", + }); + await db.insert(executionWorkspaces).values({ + id: executionWorkspaceId, + companyId, + projectId, + sourceIssueId: issueId, + mode: "isolated_workspace", + strategyType: "git_worktree", + name: "feature/recorded", + status: "active", + providerType: "git_worktree", + cwd: worktreePath, + providerRef: worktreePath, + branchName: "feature/recorded", + baseRef: "main", + }); + await db.insert(workspaceRuntimeServices).values({ + id: runtimeServiceId, + companyId, + projectId, + executionWorkspaceId, + issueId, + scopeType: "execution_workspace", + serviceName: "web", + status: "running", + lifecycle: "shared", + command: "pnpm dev", + cwd: worktreePath, + provider: "local_process", + healthStatus: "healthy", + }); + + await expect(svc.reconcileExecutionWorkspaceBranch(executionWorkspaceId, { + mode: "quarantine_restore", + actor: { + actorType: "user", + actorId: "local-board", + agentId: null, + runId: null, + }, + })).rejects.toMatchObject({ + status: 422, + message: "Execution workspace branch reconciliation requires all runtime services to be stopped", + details: { + inspection: expect.objectContaining({ + cleanliness: "dirty", + fromBranch: "feature/recorded", + toBranch: "feature/live", + }), + runtimeServices: [ + { + id: runtimeServiceId, + serviceName: "web", + status: "running", + }, + ], + }, + }); + + await expect(readGit(worktreePath, ["branch", "--show-current"])).resolves.toBe("feature/live"); + await expect(readGit( + repoRoot, + ["for-each-ref", "--format=%(refname:short)", "refs/heads/paperclip/rescue"], + )).resolves.toBeNull(); + const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId)); + expect(comments).toHaveLength(0); + }, 20_000); + + it.each(["review", "approval"] as const)( + "quarantine_restore preserves pending execution-%s semantics on the source issue", + async (stageType) => { + const repoRoot = await createTempRepo(); + tempDirs.add(repoRoot); + const worktreePath = path.join(path.dirname(repoRoot), `paperclip-quarantine-${stageType}-${randomUUID()}`); + tempDirs.add(worktreePath); + + await runGit(repoRoot, ["branch", "feature/recorded"]); + await runGit(repoRoot, ["worktree", "add", "-b", "feature/live", worktreePath, "feature/recorded"]); + await fs.appendFile(path.join(worktreePath, "README.md"), "dirty tracked review work\n", "utf8"); + + const companyId = randomUUID(); + const coderAgentId = randomUUID(); + const reviewerAgentId = randomUUID(); + const projectId = randomUUID(); + const projectWorkspaceId = randomUUID(); + const issueId = randomUUID(); + const executionWorkspaceId = randomUUID(); + const reviewStageId = randomUUID(); + const actualBranch = await readGit(worktreePath, ["symbolic-ref", "--quiet", "--short", "HEAD"]); + const fingerprint = await fingerprintWorkspaceBranchIncoherenceForTest({ + repoRoot, + worktreePath, + sourceIssueId: issueId, + executionWorkspaceId, + expectedBranch: "feature/recorded", + actualBranch, + }); + + await db.insert(companies).values({ + id: companyId, + name: "Paperclip", + issuePrefix: "PAP", + requireBoardApprovalForNewAgents: false, + }); + await db.insert(agents).values([ + { + id: coderAgentId, + companyId, + name: "Codex Coder", + role: "engineer", + status: "active", + adapterType: "codex_local", + adapterConfig: {}, + runtimeConfig: {}, + permissions: {}, + }, + { + id: reviewerAgentId, + companyId, + name: "QA Reviewer", + role: "qa", + status: "active", + adapterType: "codex_local", + adapterConfig: {}, + runtimeConfig: {}, + permissions: {}, + }, + ]); + await db.insert(projects).values({ + id: projectId, + companyId, + name: "Branch reconcile", + status: "in_progress", + }); + await db.insert(projectWorkspaces).values({ + id: projectWorkspaceId, + companyId, + projectId, + name: "Primary", + cwd: repoRoot, + isPrimary: true, + }); + await db.insert(issues).values({ + id: issueId, + companyId, + projectId, + projectWorkspaceId, + title: "Source task awaiting review", + identifier: "PAP-125", + status: "blocked", + priority: "medium", + assigneeAgentId: coderAgentId, + executionPolicy: { + stages: [ + { + id: reviewStageId, + type: stageType, + participants: [{ type: "agent", agentId: reviewerAgentId }], + }, + ], + }, + executionState: { + status: "pending", + currentStageId: reviewStageId, + currentStageIndex: 0, + currentStageType: stageType, + currentParticipant: { type: "agent", agentId: reviewerAgentId }, + returnAssignee: { type: "agent", agentId: coderAgentId }, + reviewRequest: null, + completedStageIds: [], + lastDecisionId: null, + lastDecisionOutcome: null, + }, + }); + await db.insert(executionWorkspaces).values({ + id: executionWorkspaceId, + companyId, + projectId, + projectWorkspaceId, + sourceIssueId: issueId, + mode: "isolated_workspace", + strategyType: "git_worktree", + name: "feature/recorded", + status: "active", + providerType: "git_worktree", + cwd: worktreePath, + providerRef: worktreePath, + branchName: "feature/recorded", + baseRef: "main", + }); + await db.insert(issueRecoveryActions).values({ + companyId, + sourceIssueId: issueId, + kind: "workspace_validation", + status: "active", + ownerType: "board", + cause: "workspace_validation_failed", + fingerprint, + evidence: { + workspaceValidation: { + fingerprint, + }, + }, + nextAction: "Repair the source issue workspace link.", + }); + + const result = await svc.reconcileExecutionWorkspaceBranch(executionWorkspaceId, { + mode: "quarantine_restore", + reason: "rescue dirty work and restore recorded branch", + actor: { + actorType: "user", + actorId: "local-board", + agentId: null, + runId: null, + }, + }); + + expect(result.restoredSourceIssue).toMatchObject({ + id: issueId, + status: "in_review", + assigneeAgentId: reviewerAgentId, + }); + expect(result.sourceIssueStatusChanged).toBe(true); + + const [sourceIssue] = await db.select().from(issues).where(eq(issues.id, issueId)); + expect(sourceIssue).toMatchObject({ + status: "in_review", + assigneeAgentId: reviewerAgentId, + assigneeUserId: null, + checkoutRunId: null, + executionRunId: null, + }); + expect(sourceIssue?.executionState).toMatchObject({ + status: "pending", + currentStageId: reviewStageId, + currentStageType: stageType, + currentParticipant: { type: "agent", agentId: reviewerAgentId }, + returnAssignee: { type: "agent", agentId: coderAgentId }, + }); + }, 20_000); + + it.each([ + { + claimantLabel: "active", + claimantIssueIdentifier: "PAP-126", + claimantHasActiveRun: true, + expectedReason: "active run", + }, + { + claimantLabel: "idle", + claimantIssueIdentifier: "PAP-127", + claimantHasActiveRun: false, + expectedReason: "no active run", + }, + ])( + "quarantine_restore refuses dirty repair when the live branch has a $claimantLabel claimant", + async ({ claimantIssueIdentifier, claimantHasActiveRun, expectedReason }) => { + const repoRoot = await createTempRepo(); + tempDirs.add(repoRoot); + const worktreePath = path.join(path.dirname(repoRoot), `paperclip-quarantine-claimant-${randomUUID()}`); + tempDirs.add(worktreePath); + + await runGit(repoRoot, ["branch", "feature/recorded"]); + await runGit(repoRoot, ["worktree", "add", "-b", "feature/live", worktreePath, "feature/recorded"]); + await fs.appendFile(path.join(worktreePath, "README.md"), "dirty tracked work\n", "utf8"); + await fs.writeFile(path.join(worktreePath, "untracked.txt"), "dirty untracked work\n", "utf8"); + + const companyId = randomUUID(); + const agentId = randomUUID(); + const projectId = randomUUID(); + const projectWorkspaceId = randomUUID(); + const issueId = randomUUID(); + const claimantIssueId = randomUUID(); + const executionWorkspaceId = randomUUID(); + const claimantWorkspaceId = randomUUID(); + const claimantRunId = claimantHasActiveRun ? randomUUID() : null; + const claimantWorkspacePath = path.join(path.dirname(repoRoot), `paperclip-claimant-${randomUUID()}`); + const now = new Date(); + + await db.insert(companies).values({ + id: companyId, + name: "Paperclip", + issuePrefix: "PAP", + requireBoardApprovalForNewAgents: false, + }); + await db.insert(agents).values({ + id: agentId, + companyId, + name: "Codex Coder", + role: "engineer", + status: "active", + adapterType: "codex_local", + adapterConfig: {}, + runtimeConfig: {}, + permissions: {}, + }); + await db.insert(projects).values({ + id: projectId, + companyId, + name: "Branch reconcile", + status: "in_progress", + }); + await db.insert(projectWorkspaces).values({ + id: projectWorkspaceId, + companyId, + projectId, + name: "Primary", + cwd: repoRoot, + isPrimary: true, + }); + if (claimantRunId) { + await db.insert(heartbeatRuns).values({ + id: claimantRunId, + companyId, + agentId, + invocationSource: "manual", + status: "running", + startedAt: now, + updatedAt: now, + }); + } + await db.insert(issues).values([ + { + id: issueId, + companyId, + projectId, + projectWorkspaceId, + title: "Source task", + identifier: "PAP-125", + status: "blocked", + priority: "medium", + assigneeAgentId: agentId, + }, + { + id: claimantIssueId, + companyId, + projectId, + projectWorkspaceId, + title: claimantHasActiveRun ? "Active claimant" : "Idle claimant", + identifier: claimantIssueIdentifier, + status: "in_progress", + priority: "medium", + assigneeAgentId: agentId, + executionRunId: claimantRunId, + }, + ]); + await db.insert(executionWorkspaces).values([ + { + id: executionWorkspaceId, + companyId, + projectId, + projectWorkspaceId, + sourceIssueId: issueId, + mode: "isolated_workspace", + strategyType: "git_worktree", + name: "feature/recorded", + status: "active", + providerType: "git_worktree", + cwd: worktreePath, + providerRef: worktreePath, + branchName: "feature/recorded", + baseRef: "main", + }, + { + id: claimantWorkspaceId, + companyId, + projectId, + projectWorkspaceId, + sourceIssueId: claimantIssueId, + mode: "isolated_workspace", + strategyType: "git_worktree", + name: "feature/live", + status: "active", + providerType: "git_worktree", + cwd: claimantWorkspacePath, + providerRef: claimantWorkspacePath, + branchName: "feature/live", + baseRef: "main", + lastUsedAt: new Date(now.getTime() + 1_000), + updatedAt: new Date(now.getTime() + 1_000), + }, + ]); + await db + .update(issues) + .set({ executionWorkspaceId: claimantWorkspaceId }) + .where(eq(issues.id, claimantIssueId)); + + await expect(svc.reconcileExecutionWorkspaceBranch(executionWorkspaceId, { + mode: "quarantine_restore", + reason: "should refuse branch claimant", + actor: { + actorType: "user", + actorId: "local-board", + agentId: null, + runId: null, + }, + })).rejects.toMatchObject({ + status: 422, + details: { + code: "workspace_validation_failed", + workspaceValidation: expect.objectContaining({ + cleanliness: "dirty", + contention: expect.objectContaining({ + claimedByWorkspaceId: claimantWorkspaceId, + claimedByIssueIdentifier: claimantIssueIdentifier, + activeRun: claimantRunId + ? expect.objectContaining({ + id: claimantRunId, + status: "running", + }) + : null, + }), + safeRepair: expect.objectContaining({ + eligible: false, + succeeded: false, + reason: expect.stringContaining(expectedReason), + }), + }), + }, + }); + + await expect(readGit(worktreePath, ["branch", "--show-current"])).resolves.toBe("feature/live"); + await expect(readGit(worktreePath, ["status", "--porcelain", "--untracked-files=all"])).resolves.not.toBeNull(); + }, + 20_000, + ); + it("rejects branch reconciliation when the worktree is dirty", async () => { const repoRoot = await createTempRepo(); tempDirs.add(repoRoot); diff --git a/server/src/__tests__/heartbeat-auto-checkout.test.ts b/server/src/__tests__/heartbeat-auto-checkout.test.ts new file mode 100644 index 0000000000..f9b60c0e57 --- /dev/null +++ b/server/src/__tests__/heartbeat-auto-checkout.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from "vitest"; +import { shouldAutoCheckoutIssueForWake } from "../services/heartbeat.ts"; + +describe("shouldAutoCheckoutIssueForWake", () => { + it("auto-checks out an assigned todo issue for an actionable wake", () => { + expect(shouldAutoCheckoutIssueForWake({ + contextSnapshot: { wakeReason: "issue_assigned" }, + issueStatus: "todo", + issueAssigneeAgentId: "agent-1", + isDependencyReady: true, + agentId: "agent-1", + })).toBe(true); + }); + + it("does not auto-checkout pending execution-review state even if the row status is todo", () => { + const reviewerAgentId = "11111111-1111-4111-8111-111111111111"; + const coderAgentId = "22222222-2222-4222-8222-222222222222"; + expect(shouldAutoCheckoutIssueForWake({ + contextSnapshot: { wakeReason: "issue_recovery_action_restored" }, + issueStatus: "todo", + issueAssigneeAgentId: reviewerAgentId, + issueExecutionState: { + status: "pending", + currentStageId: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + currentStageIndex: 0, + currentStageType: "review", + currentParticipant: { type: "agent", agentId: reviewerAgentId }, + returnAssignee: { type: "agent", agentId: coderAgentId }, + reviewRequest: null, + completedStageIds: [], + lastDecisionId: null, + lastDecisionOutcome: null, + }, + isDependencyReady: true, + agentId: reviewerAgentId, + })).toBe(false); + }); +}); diff --git a/server/src/__tests__/instance-settings-routes.test.ts b/server/src/__tests__/instance-settings-routes.test.ts index 337551b270..5148b1abfd 100644 --- a/server/src/__tests__/instance-settings-routes.test.ts +++ b/server/src/__tests__/instance-settings-routes.test.ts @@ -87,7 +87,8 @@ describe("instance settings routes", () => { enableServerInfoDebugView: false, autoRestartDevServerWhenIdle: false, enableIssueGraphLivenessAutoRecovery: true, - enableWorkspaceBranchReconcileForward: false, + enableWorkspaceBranchReconcileForward: true, + enableWorkspaceDirtyQuarantineRepair: true, issueGraphLivenessAutoRecoveryLookbackHours: 24, }, createdAt: "2026-06-20T00:00:00.000Z", @@ -111,7 +112,8 @@ describe("instance settings routes", () => { enableServerInfoDebugView: false, autoRestartDevServerWhenIdle: false, enableIssueGraphLivenessAutoRecovery: true, - enableWorkspaceBranchReconcileForward: false, + enableWorkspaceBranchReconcileForward: true, + enableWorkspaceDirtyQuarantineRepair: true, issueGraphLivenessAutoRecoveryLookbackHours: 24, }); mockInstanceSettingsService.update.mockResolvedValue({ @@ -134,7 +136,8 @@ describe("instance settings routes", () => { enableServerInfoDebugView: false, autoRestartDevServerWhenIdle: false, enableIssueGraphLivenessAutoRecovery: true, - enableWorkspaceBranchReconcileForward: false, + enableWorkspaceBranchReconcileForward: true, + enableWorkspaceDirtyQuarantineRepair: true, issueGraphLivenessAutoRecoveryLookbackHours: 24, }, createdAt: "2026-06-20T00:00:00.000Z", @@ -163,7 +166,8 @@ describe("instance settings routes", () => { enableServerInfoDebugView: true, autoRestartDevServerWhenIdle: false, enableIssueGraphLivenessAutoRecovery: true, - enableWorkspaceBranchReconcileForward: false, + enableWorkspaceBranchReconcileForward: true, + enableWorkspaceDirtyQuarantineRepair: true, issueGraphLivenessAutoRecoveryLookbackHours: 24, }, }); @@ -220,7 +224,8 @@ describe("instance settings routes", () => { enableServerInfoDebugView: false, autoRestartDevServerWhenIdle: false, enableIssueGraphLivenessAutoRecovery: true, - enableWorkspaceBranchReconcileForward: false, + enableWorkspaceBranchReconcileForward: true, + enableWorkspaceDirtyQuarantineRepair: true, issueGraphLivenessAutoRecoveryLookbackHours: 24, }); diff --git a/server/src/__tests__/instance-settings-service.test.ts b/server/src/__tests__/instance-settings-service.test.ts index e7b45a2a44..85cb08c0e4 100644 --- a/server/src/__tests__/instance-settings-service.test.ts +++ b/server/src/__tests__/instance-settings-service.test.ts @@ -16,6 +16,7 @@ describe("instance settings service", () => { autoRestartDevServerWhenIdle: true, enableIssueGraphLivenessAutoRecovery: true, enableWorkspaceBranchReconcileForward: true, + enableWorkspaceDirtyQuarantineRepair: false, issueGraphLivenessAutoRecoveryLookbackHours: 48, enableNewestFirstIssueThread: true, })).toEqual({ @@ -35,6 +36,7 @@ describe("instance settings service", () => { autoRestartDevServerWhenIdle: true, enableIssueGraphLivenessAutoRecovery: true, enableWorkspaceBranchReconcileForward: true, + enableWorkspaceDirtyQuarantineRepair: false, enableWorktreeRunExecution: false, issueGraphLivenessAutoRecoveryLookbackHours: 48, }); @@ -73,13 +75,19 @@ describe("instance settings service", () => { ).toBe(false); }); - it("defaults enableWorkspaceBranchReconcileForward to false for empty and legacy stored settings", () => { - expect(normalizeExperimentalSettings(undefined).enableWorkspaceBranchReconcileForward).toBe(false); - expect(normalizeExperimentalSettings({}).enableWorkspaceBranchReconcileForward).toBe(false); + it("defaults workspace branch repair settings to true for empty and legacy stored settings", () => { + expect(normalizeExperimentalSettings(undefined).enableWorkspaceBranchReconcileForward).toBe(true); + expect(normalizeExperimentalSettings({}).enableWorkspaceBranchReconcileForward).toBe(true); expect( normalizeExperimentalSettings({ enableIssueGraphLivenessAutoRecovery: true }) .enableWorkspaceBranchReconcileForward, - ).toBe(false); + ).toBe(true); + expect(normalizeExperimentalSettings(undefined).enableWorkspaceDirtyQuarantineRepair).toBe(true); + expect(normalizeExperimentalSettings({}).enableWorkspaceDirtyQuarantineRepair).toBe(true); + expect( + normalizeExperimentalSettings({ enableWorkspaceBranchReconcileForward: false }) + .enableWorkspaceDirtyQuarantineRepair, + ).toBe(true); }); it("round-trips an enableConferenceRoomChat patch through the update merge", () => { diff --git a/server/src/__tests__/workspace-runtime-routes-authz.test.ts b/server/src/__tests__/workspace-runtime-routes-authz.test.ts index e78f6c181c..437821883e 100644 --- a/server/src/__tests__/workspace-runtime-routes-authz.test.ts +++ b/server/src/__tests__/workspace-runtime-routes-authz.test.ts @@ -26,6 +26,7 @@ const mockEnvironmentService = vi.hoisted(() => ({ })); const mockWorkspaceOperationService = vi.hoisted(() => ({})); +const mockHeartbeatService = vi.hoisted(() => ({})); const mockLogActivity = vi.hoisted(() => vi.fn()); const mockGetTelemetryClient = vi.hoisted(() => vi.fn()); const mockAccessService = vi.hoisted(() => ({ @@ -42,6 +43,7 @@ vi.mock("../services/index.js", () => ({ accessService: () => mockAccessService, environmentService: () => mockEnvironmentService, executionWorkspaceService: () => mockExecutionWorkspaceService, + heartbeatService: () => mockHeartbeatService, logActivity: mockLogActivity, projectService: () => mockProjectService, secretService: () => mockSecretService, @@ -69,6 +71,7 @@ function registerWorkspaceRouteMocks() { accessService: () => mockAccessService, environmentService: () => mockEnvironmentService, executionWorkspaceService: () => mockExecutionWorkspaceService, + heartbeatService: () => mockHeartbeatService, logActivity: mockLogActivity, projectService: () => mockProjectService, secretService: () => mockSecretService, diff --git a/server/src/__tests__/workspace-runtime.test.ts b/server/src/__tests__/workspace-runtime.test.ts index ff33318d5f..0bfc8dd562 100644 --- a/server/src/__tests__/workspace-runtime.test.ts +++ b/server/src/__tests__/workspace-runtime.test.ts @@ -1,5 +1,6 @@ import { execFile, spawn } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; +import { existsSync } from "node:fs"; import fs from "node:fs/promises"; import net from "node:net"; import os from "node:os"; @@ -9,11 +10,14 @@ import { promisify } from "node:util"; import { parse as parseEnvContents } from "dotenv"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; import { + activityLog, agents, companies, createDb, executionWorkspaces, heartbeatRuns, + issueComments, + issues, projectWorkspaces, projects, workspaceRuntimeServices, @@ -2432,6 +2436,7 @@ describe("realizeExecutionWorkspace", () => { name: "Codex Coder", companyId: "company-1", }, + enableWorkspaceDirtyQuarantineRepair: false, })).rejects.toMatchObject({ code: "workspace_validation_failed", resultJson: { @@ -2444,6 +2449,7 @@ describe("realizeExecutionWorkspace", () => { expectedBranch, actualBranch, cleanliness: "dirty", + dirtyPathSample: ["untracked.txt"], provenance: expect.objectContaining({ expectedBranchExists: true, actualBranchExists: true, @@ -4090,6 +4096,648 @@ describe("readLocalServicePortOwner", () => { }); }); +describeEmbeddedPostgres("workspace dirty quarantine branch repair", () => { + let db!: ReturnType; + let tempDb: Awaited> | null = null; + + beforeAll(async () => { + tempDb = await startEmbeddedPostgresTestDatabase("paperclip-workspace-dirty-quarantine-"); + db = createDb(tempDb.connectionString); + }, 20_000); + + afterAll(async () => { + await tempDb?.cleanup(); + }); + + afterEach(async () => { + await db.delete(issueComments); + await db.delete(activityLog); + await db.delete(issues); + await db.delete(workspaceRuntimeServices); + await db.delete(executionWorkspaces); + await db.delete(projectWorkspaces); + await db.delete(projects); + await db.delete(heartbeatRuns); + await db.delete(agents); + await db.delete(companies); + }); + + async function createDirtyMismatchRepo(input: { + expectedBranch: string; + actualBranch: string; + }) { + const repoRoot = await createTempRepo(); + const worktreePath = path.join(repoRoot, ".paperclip", "worktrees", input.expectedBranch); + await fs.mkdir(path.dirname(worktreePath), { recursive: true }); + await runGit(repoRoot, ["branch", input.expectedBranch]); + await runGit(repoRoot, ["worktree", "add", "-b", input.actualBranch, worktreePath, input.expectedBranch]); + const actualBranchHead = await readGit(worktreePath, ["rev-parse", input.actualBranch]); + await fs.appendFile(path.join(worktreePath, "README.md"), "dirty tracked work\n", "utf8"); + await fs.writeFile(path.join(worktreePath, "untracked.txt"), "dirty untracked work\n", "utf8"); + return { repoRoot, worktreePath, actualBranchHead }; + } + + async function seedDirtyQuarantineRecords(input: { + repoRoot: string; + worktreePath: string; + expectedBranch: string; + actualBranch: string; + sourceIdentifier?: string; + claimant?: "idle" | "active" | "none"; + }) { + const companyId = randomUUID(); + const agentId = randomUUID(); + const projectId = randomUUID(); + const projectWorkspaceId = randomUUID(); + const sourceIssueId = randomUUID(); + const sourceWorkspaceId = randomUUID(); + const runId = randomUUID(); + const now = new Date(); + + await db.insert(companies).values({ + id: companyId, + name: "Paperclip", + issuePrefix: `Q${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`, + requireBoardApprovalForNewAgents: false, + }); + await db.insert(agents).values({ + id: agentId, + companyId, + name: "Codex Coder", + role: "engineer", + status: "active", + adapterType: "codex_local", + adapterConfig: {}, + runtimeConfig: {}, + permissions: {}, + }); + await db.insert(projects).values({ + id: projectId, + companyId, + name: "Paperclip App", + status: "in_progress", + }); + await db.insert(projectWorkspaces).values({ + id: projectWorkspaceId, + companyId, + projectId, + name: "Primary", + cwd: input.repoRoot, + isPrimary: true, + }); + await db.insert(heartbeatRuns).values({ + id: runId, + companyId, + agentId, + invocationSource: "manual", + status: "running", + startedAt: now, + updatedAt: now, + }); + await db.insert(issues).values({ + id: sourceIssueId, + companyId, + projectId, + projectWorkspaceId, + title: "Repair dirty branch mismatch", + status: "in_progress", + priority: "medium", + assigneeAgentId: agentId, + identifier: input.sourceIdentifier ?? "PAP-455", + }); + await db.insert(executionWorkspaces).values({ + id: sourceWorkspaceId, + companyId, + projectId, + projectWorkspaceId, + sourceIssueId, + mode: "isolated_workspace", + strategyType: "git_worktree", + name: input.expectedBranch, + status: "active", + cwd: input.worktreePath, + providerRef: input.worktreePath, + baseRef: "HEAD", + branchName: input.expectedBranch, + providerType: "git_worktree", + lastUsedAt: now, + updatedAt: now, + }); + await db + .update(issues) + .set({ executionWorkspaceId: sourceWorkspaceId, executionRunId: runId, updatedAt: now }) + .where(eq(issues.id, sourceIssueId)); + + let claimant: + | { + issueId: string; + workspaceId: string; + runId: string | null; + identifier: string; + } + | null = null; + if (input.claimant && input.claimant !== "none") { + const claimantIssueId = randomUUID(); + const claimantWorkspaceId = randomUUID(); + const claimantRunId = input.claimant === "active" ? randomUUID() : null; + const claimantIdentifier = "PAP-999"; + await db.insert(issues).values({ + id: claimantIssueId, + companyId, + projectId, + projectWorkspaceId, + title: "Live branch claimant", + status: "in_progress", + priority: "medium", + assigneeAgentId: agentId, + identifier: claimantIdentifier, + }); + if (claimantRunId) { + await db.insert(heartbeatRuns).values({ + id: claimantRunId, + companyId, + agentId, + invocationSource: "manual", + status: "running", + startedAt: now, + updatedAt: now, + }); + } + await db.insert(executionWorkspaces).values({ + id: claimantWorkspaceId, + companyId, + projectId, + projectWorkspaceId, + sourceIssueId: claimantIssueId, + mode: "isolated_workspace", + strategyType: "git_worktree", + name: input.actualBranch, + status: "active", + cwd: path.join(input.repoRoot, ".paperclip", "claimants", claimantWorkspaceId), + providerRef: path.join(input.repoRoot, ".paperclip", "claimants", claimantWorkspaceId), + baseRef: "HEAD", + branchName: input.actualBranch, + providerType: "git_worktree", + lastUsedAt: new Date(now.getTime() + 1_000), + updatedAt: new Date(now.getTime() + 1_000), + }); + await db + .update(issues) + .set({ + executionWorkspaceId: claimantWorkspaceId, + executionRunId: claimantRunId, + updatedAt: now, + }) + .where(eq(issues.id, claimantIssueId)); + claimant = { + issueId: claimantIssueId, + workspaceId: claimantWorkspaceId, + runId: claimantRunId, + identifier: claimantIdentifier, + }; + } + + return { + companyId, + agentId, + projectId, + projectWorkspaceId, + sourceIssueId, + sourceWorkspaceId, + runId, + claimant, + sourceIdentifier: input.sourceIdentifier ?? "PAP-455", + }; + } + + async function restoreDirtyQuarantine(input: { + repoRoot: string; + worktreePath: string; + expectedBranch: string; + actualBranch: string; + ids: Awaited>; + recorder?: WorkspaceOperationRecorder | null; + }) { + return ensurePersistedExecutionWorkspaceAvailable({ + db, + base: { + baseCwd: input.repoRoot, + source: "project_primary", + projectId: input.ids.projectId, + workspaceId: input.ids.projectWorkspaceId, + repoUrl: null, + repoRef: "HEAD", + }, + workspace: { + id: input.ids.sourceWorkspaceId, + mode: "isolated_workspace", + strategyType: "git_worktree", + cwd: input.worktreePath, + providerRef: input.worktreePath, + projectId: input.ids.projectId, + projectWorkspaceId: input.ids.projectWorkspaceId, + repoUrl: null, + baseRef: "HEAD", + branchName: input.expectedBranch, + }, + issue: { + id: input.ids.sourceIssueId, + identifier: input.ids.sourceIdentifier, + title: "Repair dirty branch mismatch", + }, + agent: { + id: input.ids.agentId, + name: "Codex Coder", + companyId: input.ids.companyId, + }, + heartbeatRunId: input.ids.runId, + enableWorkspaceBranchReconcileForward: true, + enableWorkspaceDirtyQuarantineRepair: true, + recorder: input.recorder ?? null, + }); + } + + it("quarantines dirty foreign-branch work into a rescue branch before restoring the recorded branch", async () => { + const expectedBranch = "PAP-455-recorded"; + const actualBranch = "PAP-455-live"; + const { repoRoot, worktreePath, actualBranchHead } = await createDirtyMismatchRepo({ + expectedBranch, + actualBranch, + }); + const ids = await seedDirtyQuarantineRecords({ + repoRoot, + worktreePath, + expectedBranch, + actualBranch, + sourceIdentifier: "PAP-455", + claimant: "none", + }); + const { recorder, operations } = createWorkspaceOperationRecorderDouble(); + + const restored = await restoreDirtyQuarantine({ + repoRoot, + worktreePath, + expectedBranch, + actualBranch, + ids, + recorder, + }); + + expect(restored?.branchName).toBe(expectedBranch); + const warning = restored?.warnings.find((entry) => entry.includes("dirty worktree state was quarantined")); + expect(warning).toBeTruthy(); + const rescueBranch = warning?.match(/"([^"]+)"/)?.[1] ?? ""; + expect(rescueBranch).toMatch(/^paperclip\/rescue\/PAP-455\/\d{8}T\d{6}Z$/); + const rescueCommitSha = await readGit(repoRoot, ["rev-parse", rescueBranch]); + await expect(readGit(worktreePath, ["branch", "--show-current"])).resolves.toBe(expectedBranch); + await expect(readGit(worktreePath, ["status", "--porcelain", "--untracked-files=all"])).resolves.toBe(""); + await expect(readGit(repoRoot, ["rev-parse", actualBranch])).resolves.toBe(actualBranchHead); + await expect(readGit(repoRoot, ["show", `${rescueBranch}:untracked.txt`])).resolves.toBe("dirty untracked work"); + + const comments = await db + .select() + .from(issueComments) + .where(eq(issueComments.companyId, ids.companyId)); + expect(comments).toHaveLength(1); + expect(comments[0]?.issueId).toBe(ids.sourceIssueId); + expect(comments[0]?.body).toContain(`Rescue branch: \`${rescueBranch}\``); + expect(comments[0]?.body).toContain(`Rescue commit: \`${rescueCommitSha}\``); + expect(comments[0]?.body).toContain("Dirty file count: `2`"); + expect(comments[0]?.body).toContain("`untracked.txt`"); + expect(comments[0]?.body).toContain("- Claimant: none"); + + const activityRows = await db + .select() + .from(activityLog) + .where(eq(activityLog.companyId, ids.companyId)); + expect(activityRows).toEqual([ + expect.objectContaining({ + action: "execution_workspace.dirty_worktree_quarantined", + entityType: "execution_workspace", + entityId: ids.sourceWorkspaceId, + details: expect.objectContaining({ + rescueBranch, + rescueCommitSha, + fileCount: 2, + dirtyPathSample: expect.arrayContaining(["README.md", "untracked.txt"]), + }), + }), + ]); + expect(operations).toEqual(expect.arrayContaining([ + expect.objectContaining({ + command: `git checkout -b ${rescueBranch}`, + metadata: expect.objectContaining({ + branchIncoherenceDirtyQuarantineRepair: true, + rescueBranch, + fileCount: 2, + }), + }), + expect.objectContaining({ + command: null, + metadata: expect.objectContaining({ + branchIncoherenceDirtyQuarantineRepair: true, + rescueBranch, + rescueCommitSha, + }), + }), + ])); + }, 20_000); + + it("quarantines a worktree wedged mid-rebase and clears the interrupted rebase state", async () => { + const expectedBranch = "PAP-456-recorded"; + const repoRoot = await createTempRepo("master"); + const worktreePath = path.join(repoRoot, ".paperclip", "worktrees", expectedBranch); + await fs.mkdir(path.dirname(worktreePath), { recursive: true }); + await runGit(repoRoot, ["branch", expectedBranch]); + await runGit(repoRoot, ["worktree", "add", worktreePath, expectedBranch]); + await fs.writeFile(path.join(worktreePath, "README.md"), "feature change\n", "utf8"); + await runGit(worktreePath, ["commit", "-am", "Feature change"]); + const expectedBranchHead = await readGit(worktreePath, ["rev-parse", expectedBranch]); + await fs.writeFile(path.join(repoRoot, "README.md"), "master change\n", "utf8"); + await runGit(repoRoot, ["commit", "-am", "Master change"]); + await expect(runGit(worktreePath, ["rebase", "master"])).rejects.toThrow(); + const rebaseStatePath = await readGit(worktreePath, ["rev-parse", "--git-path", "rebase-merge"]); + expect(existsSync(path.resolve(worktreePath, rebaseStatePath))).toBe(true); + await expect(readGit(worktreePath, ["branch", "--show-current"])).resolves.toBe(""); + + const ids = await seedDirtyQuarantineRecords({ + repoRoot, + worktreePath, + expectedBranch, + actualBranch: "PAP-456-live", + sourceIdentifier: "PAP-456", + claimant: "none", + }); + const { recorder } = createWorkspaceOperationRecorderDouble(); + + const restored = await restoreDirtyQuarantine({ + repoRoot, + worktreePath, + expectedBranch, + actualBranch: "PAP-456-live", + ids, + recorder, + }); + + expect(restored?.branchName).toBe(expectedBranch); + const warning = restored?.warnings.find((entry) => entry.includes("dirty worktree state was quarantined")); + expect(warning).toContain("An interrupted git rebase was also cleared"); + const rescueBranch = warning?.match(/"([^"]+)"/)?.[1] ?? ""; + expect(rescueBranch).toMatch(/^paperclip\/rescue\/PAP-456\/\d{8}T\d{6}Z$/); + + await expect(readGit(worktreePath, ["branch", "--show-current"])).resolves.toBe(expectedBranch); + await expect(readGit(worktreePath, ["status", "--porcelain", "--untracked-files=all"])).resolves.toBe(""); + expect(existsSync(path.resolve(worktreePath, rebaseStatePath))).toBe(false); + await expect(readGit(repoRoot, ["rev-parse", expectedBranch])).resolves.toBe(expectedBranchHead); + await expect(readGit(repoRoot, ["show", `${rescueBranch}:README.md`])).resolves.toContain("<<<<<<<"); + + const comments = await db + .select() + .from(issueComments) + .where(eq(issueComments.companyId, ids.companyId)); + expect(comments).toHaveLength(1); + expect(comments[0]?.body).toContain("Interrupted operation: `git rebase`"); + }, 20_000); + + it("refuses dirty quarantine repair when the live branch has an active claimant", async () => { + const expectedBranch = "PAP-456-recorded"; + const actualBranch = "PAP-456-live"; + const { repoRoot, worktreePath } = await createDirtyMismatchRepo({ expectedBranch, actualBranch }); + const ids = await seedDirtyQuarantineRecords({ + repoRoot, + worktreePath, + expectedBranch, + actualBranch, + sourceIdentifier: "PAP-456", + claimant: "active", + }); + + await expect(restoreDirtyQuarantine({ + repoRoot, + worktreePath, + expectedBranch, + actualBranch, + ids, + })).rejects.toMatchObject({ + code: "workspace_validation_failed", + resultJson: { + workspaceValidation: expect.objectContaining({ + cleanliness: "dirty", + dirtyPathSample: expect.arrayContaining(["README.md", "untracked.txt"]), + contention: expect.objectContaining({ + claimedByWorkspaceId: ids.claimant!.workspaceId, + claimedByIssueIdentifier: ids.claimant!.identifier, + activeRun: expect.objectContaining({ + id: ids.claimant!.runId, + status: "running", + issueIdentifier: ids.claimant!.identifier, + }), + }), + safeRepair: expect.objectContaining({ + eligible: false, + succeeded: false, + reason: expect.stringContaining("active run"), + }), + }), + }, + }); + await expect(readGit(worktreePath, ["branch", "--show-current"])).resolves.toBe(actualBranch); + await expect(readGit(worktreePath, ["status", "--porcelain", "--untracked-files=all"])).resolves.not.toBe(""); + }, 20_000); + + it("refuses dirty quarantine repair when the live branch has an idle claimant", async () => { + const expectedBranch = "PAP-457-recorded"; + const actualBranch = "PAP-457-live"; + const { repoRoot, worktreePath } = await createDirtyMismatchRepo({ expectedBranch, actualBranch }); + const ids = await seedDirtyQuarantineRecords({ + repoRoot, + worktreePath, + expectedBranch, + actualBranch, + sourceIdentifier: "PAP-457", + claimant: "idle", + }); + + await expect(restoreDirtyQuarantine({ + repoRoot, + worktreePath, + expectedBranch, + actualBranch, + ids, + })).rejects.toMatchObject({ + code: "workspace_validation_failed", + resultJson: { + workspaceValidation: expect.objectContaining({ + cleanliness: "dirty", + dirtyPathSample: expect.arrayContaining(["README.md", "untracked.txt"]), + contention: expect.objectContaining({ + claimedByWorkspaceId: ids.claimant!.workspaceId, + claimedByIssueIdentifier: ids.claimant!.identifier, + activeRun: null, + }), + safeRepair: expect.objectContaining({ + eligible: false, + succeeded: false, + reason: expect.stringContaining("no active run"), + }), + }), + }, + }); + await expect(readGit(worktreePath, ["branch", "--show-current"])).resolves.toBe(actualBranch); + await expect(readGit(worktreePath, ["status", "--porcelain", "--untracked-files=all"])).resolves.not.toBe(""); + }, 20_000); + + it("refuses dirty quarantine repair while the execution workspace has an active runtime service", async () => { + const expectedBranch = "PAP-458-recorded"; + const actualBranch = "PAP-458-live"; + const { repoRoot, worktreePath } = await createDirtyMismatchRepo({ expectedBranch, actualBranch }); + const ids = await seedDirtyQuarantineRecords({ + repoRoot, + worktreePath, + expectedBranch, + actualBranch, + sourceIdentifier: "PAP-458", + claimant: "none", + }); + const runtimeServiceId = randomUUID(); + await db.insert(workspaceRuntimeServices).values({ + id: runtimeServiceId, + companyId: ids.companyId, + projectId: ids.projectId, + projectWorkspaceId: ids.projectWorkspaceId, + executionWorkspaceId: ids.sourceWorkspaceId, + issueId: ids.sourceIssueId, + scopeType: "execution_workspace", + scopeId: ids.sourceWorkspaceId, + serviceName: "paperclip-dev", + status: "running", + lifecycle: "shared", + reuseKey: `execution_workspace:${ids.sourceWorkspaceId}:paperclip-dev`, + command: "pnpm dev", + cwd: worktreePath, + port: 49195, + url: "http://127.0.0.1:49195", + provider: "local_process", + providerRef: "999999", + ownerAgentId: ids.agentId, + startedByRunId: ids.runId, + lastUsedAt: new Date(), + startedAt: new Date(), + stoppedAt: null, + stopPolicy: { type: "manual" }, + healthStatus: "healthy", + }); + + await expect(restoreDirtyQuarantine({ + repoRoot, + worktreePath, + expectedBranch, + actualBranch, + ids, + })).rejects.toMatchObject({ + code: "workspace_validation_failed", + resultJson: { + workspaceValidation: expect.objectContaining({ + cleanliness: "dirty", + safeRepair: expect.objectContaining({ + eligible: false, + attempted: false, + succeeded: false, + reason: expect.stringContaining("runtime service"), + }), + }), + }, + }); + await expect(readGit(worktreePath, ["branch", "--show-current"])).resolves.toBe(actualBranch); + await expect(readGit(worktreePath, ["status", "--porcelain", "--untracked-files=all"])).resolves.not.toBe(""); + await expect(readGit(repoRoot, [ + "for-each-ref", + "--format=%(refname:short)", + "refs/heads/paperclip/rescue", + ])).resolves.toBe(""); + }, 20_000); + + it("falls back to validation failure when git reports index-lock contention during quarantine", async () => { + const expectedBranch = "PAP-459-recorded"; + const actualBranch = "PAP-459-live"; + const { repoRoot, worktreePath } = await createDirtyMismatchRepo({ expectedBranch, actualBranch }); + const ids = await seedDirtyQuarantineRecords({ + repoRoot, + worktreePath, + expectedBranch, + actualBranch, + sourceIdentifier: "PAP-459", + claimant: "none", + }); + const lockPath = await readGit(worktreePath, ["rev-parse", "--git-path", "index.lock"]); + await fs.writeFile(lockPath, "locked\n", "utf8"); + try { + await expect(restoreDirtyQuarantine({ + repoRoot, + worktreePath, + expectedBranch, + actualBranch, + ids, + })).rejects.toMatchObject({ + code: "workspace_validation_failed", + resultJson: { + workspaceValidation: expect.objectContaining({ + cleanliness: "dirty", + safeRepair: expect.objectContaining({ + attempted: true, + succeeded: false, + reason: expect.stringContaining("index contention"), + }), + }), + }, + }); + } finally { + await fs.rm(lockPath, { force: true }); + } + await expect(readGit(worktreePath, ["branch", "--show-current"])).resolves.toBe(actualBranch); + }, 20_000); + + it("best-effort restores the recorded branch when the rescue commit fails", async () => { + const expectedBranch = "PAP-460-recorded"; + const actualBranch = "PAP-460-live"; + const { repoRoot, worktreePath } = await createDirtyMismatchRepo({ expectedBranch, actualBranch }); + const ids = await seedDirtyQuarantineRecords({ + repoRoot, + worktreePath, + expectedBranch, + actualBranch, + sourceIdentifier: "PAP-460", + claimant: "none", + }); + const commonDirRaw = await readGit(worktreePath, ["rev-parse", "--git-common-dir"]); + const commonDir = path.isAbsolute(commonDirRaw) ? commonDirRaw : path.resolve(worktreePath, commonDirRaw); + const hookPath = path.join(commonDir, "hooks", "commit-msg"); + await fs.mkdir(path.dirname(hookPath), { recursive: true }); + await fs.writeFile(hookPath, "#!/bin/sh\necho rescue commit blocked >&2\nexit 1\n", { mode: 0o755 }); + + await expect(restoreDirtyQuarantine({ + repoRoot, + worktreePath, + expectedBranch, + actualBranch, + ids, + })).rejects.toMatchObject({ + code: "workspace_validation_failed", + resultJson: { + workspaceValidation: expect.objectContaining({ + safeRepair: expect.objectContaining({ + attempted: true, + succeeded: false, + reason: expect.stringContaining("rescue commit blocked"), + }), + }), + }, + }); + await expect(readGit(worktreePath, ["branch", "--show-current"])).resolves.toBe(expectedBranch); + await expect(readGit(worktreePath, ["status", "--porcelain", "--untracked-files=all"])).resolves.not.toBe(""); + }, 20_000); +}); + describeEmbeddedPostgres("workspace runtime startup reconciliation", () => { let db!: ReturnType; let tempDb: Awaited> | null = null; diff --git a/server/src/routes/execution-workspaces.ts b/server/src/routes/execution-workspaces.ts index 42e16bfcc3..8c7e27d82a 100644 --- a/server/src/routes/execution-workspaces.ts +++ b/server/src/routes/execution-workspaces.ts @@ -12,7 +12,7 @@ import { } from "@paperclipai/shared"; import type { WorkspaceRuntimeDesiredState, WorkspaceRuntimeServiceStateMap } from "@paperclipai/shared"; import { validate } from "../middleware/validate.js"; -import { accessService, executionWorkspaceService, logActivity, workspaceOperationService } from "../services/index.js"; +import { accessService, executionWorkspaceService, heartbeatService, logActivity, workspaceOperationService } from "../services/index.js"; import { mergeExecutionWorkspaceConfig, readExecutionWorkspaceConfig } from "../services/execution-workspaces.js"; import { parseProjectExecutionWorkspacePolicy } from "../services/execution-workspace-policy.js"; import { readProjectWorkspaceRuntimeConfig } from "../services/project-workspace-runtime-config.js"; @@ -26,6 +26,7 @@ import { stopRuntimeServicesForExecutionWorkspace, } from "../services/workspace-runtime.js"; import { assertBoard, assertCompanyAccess, getActorInfo } from "./authz.js"; +import { logger } from "../middleware/logger.js"; import { assertNoAgentHostWorkspaceCommandMutation, collectExecutionWorkspaceCommandPaths, @@ -42,6 +43,9 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P const svc = executionWorkspaceService(db); const access = accessService(db); const workspaceOperationsSvc = workspaceOperationService(db); + const heartbeat = heartbeatService(db, { + pluginWorkerManager: opts.pluginWorkerManager, + }); const environmentRuntime = environmentRuntimeService(db, { pluginWorkerManager: opts.pluginWorkerManager, }); @@ -536,6 +540,8 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P sourceIssueId: existing.sourceIssueId, auditCommentId: result.auditCommentId, recoveryActionId: result.recoveryAction?.id ?? null, + rescueRef: result.rescueRef, + sourceIssueStatus: result.restoredSourceIssue?.status ?? null, actor: { type: actor.actorType, id: actor.actorId, @@ -544,6 +550,41 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P }, }); + if ( + result.restoredSourceIssue && + (result.restoredSourceIssue.status === "todo" || result.restoredSourceIssue.status === "in_review") && + result.sourceIssueStatusChanged && + result.restoredSourceIssue.assigneeAgentId + ) { + void heartbeat.wakeup(result.restoredSourceIssue.assigneeAgentId, { + source: "automation", + triggerDetail: "system", + reason: "issue_recovery_action_restored", + payload: { + issueId: result.restoredSourceIssue.id, + recoveryActionId: result.recoveryAction?.id ?? null, + executionWorkspaceId: existing.id, + rescueRef: result.rescueRef?.branchName ?? null, + mutation: "execution_workspace_quarantine_restore", + }, + requestedByActorType: actor.actorType, + requestedByActorId: actor.actorId, + contextSnapshot: { + issueId: result.restoredSourceIssue.id, + taskId: result.restoredSourceIssue.id, + wakeReason: "issue_recovery_action_restored", + source: "execution_workspace.quarantine_restore", + recoveryActionId: result.recoveryAction?.id ?? null, + executionWorkspaceId: existing.id, + rescueRef: result.rescueRef?.branchName ?? null, + }, + }).catch((err) => + logger.warn( + { err, issueId: result.restoredSourceIssue?.id, agentId: result.restoredSourceIssue?.assigneeAgentId }, + "failed to wake agent after execution workspace quarantine restore", + )); + } + res.json(result); }); diff --git a/server/src/services/execution-workspaces.ts b/server/src/services/execution-workspaces.ts index 13564637c4..c1dc1426b4 100644 --- a/server/src/services/execution-workspaces.ts +++ b/server/src/services/execution-workspaces.ts @@ -3,9 +3,9 @@ import { createHash } from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; import { promisify } from "node:util"; -import { and, asc, desc, eq, inArray, isNull, ne, sql } from "drizzle-orm"; +import { and, asc, desc, eq, inArray, isNull, ne, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; -import { executionWorkspaces, issueComments, issues, projects, projectWorkspaces, workspaceRuntimeServices } from "@paperclipai/db"; +import { executionWorkspaces, heartbeatRuns, issueComments, issues, projects, projectWorkspaces, workspaceRuntimeServices } from "@paperclipai/db"; import type { ExecutionWorkspace, ExecutionWorkspaceSummary, @@ -25,6 +25,11 @@ import type { } from "@paperclipai/shared"; import { deriveProjectUrlKey, WORKSPACE_OVERVIEW_LINKED_ISSUE_LIMIT } from "@paperclipai/shared"; import { conflict, notFound, unprocessable } from "../errors.js"; +import { + applyIssueExecutionPolicyTransition, + normalizeIssueExecutionPolicy, + parseIssueExecutionState, +} from "./issue-execution-policy.js"; import { parseProjectExecutionWorkspacePolicy } from "./execution-workspace-policy.js"; import { issueRecoveryActionService } from "./issue-recovery-actions.js"; import { visibleIssueCondition } from "./issue-visibility.js"; @@ -42,7 +47,7 @@ const TERMINAL_ISSUE_STATUSES = new Set(["done", "cancelled"]); const WORKSPACE_BRANCH_INCOHERENCE_REASON = "git_worktree_branch_incoherence"; const WORKSPACE_VALIDATION_RECOVERY_CAUSE = "workspace_validation_failed"; -export type ExecutionWorkspaceBranchReconcileMode = "forward" | "override"; +export type ExecutionWorkspaceBranchReconcileMode = "forward" | "override" | "quarantine_restore"; export type ExecutionWorkspaceBranchReconcileActor = { actorType: "agent" | "user" | "system"; @@ -70,8 +75,34 @@ export type ExecutionWorkspaceBranchReconcileResult = { inspection: ExecutionWorkspaceBranchReconcileInspection; recoveryAction: IssueRecoveryAction | null; auditCommentId: string | null; + rescueRef: { + branchName: string; + commitSha: string; + fileCount: number; + sourceAuditCommentId: string | null; + claimantAuditCommentId: string | null; + } | null; + restoredSourceIssue: { + id: string; + companyId: string; + status: string; + assigneeAgentId: string | null; + } | null; + sourceIssueStatusChanged: boolean; }; +export type ExecutionWorkspaceGitWorktreeContention = { + claimedByWorkspaceId: string; + claimedByIssueId: string | null; + claimedByIssueIdentifier: string | null; + activeRun: { + id: string; + status: "queued" | "running"; + issueId: string | null; + issueIdentifier: string | null; + } | null; +} | null; + function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } @@ -87,6 +118,37 @@ function cloneRecord(value: unknown): Record | null { return { ...value }; } +function assigneeMatchesExecutionPrincipal(input: { + assigneeAgentId: string | null; + assigneeUserId: string | null; +}, principal: { type: string; agentId?: string | null; userId?: string | null } | null): boolean { + if (!principal) return false; + if (principal.type === "agent") { + return input.assigneeAgentId === principal.agentId && input.assigneeUserId === null; + } + if (principal.type === "user") { + return input.assigneeAgentId === null && input.assigneeUserId === principal.userId; + } + return false; +} + +function quarantineRestoreRequestedSourceStatus(input: { + status: string; + assigneeAgentId: string | null; + assigneeUserId: string | null; + executionState: unknown; +}): "todo" | undefined { + const state = parseIssueExecutionState(input.executionState); + if ( + state?.status === "pending" && + input.status === "in_review" && + assigneeMatchesExecutionPrincipal(input, state.currentParticipant) + ) { + return undefined; + } + return "todo"; +} + function readDesiredState(value: unknown): WorkspaceRuntimeDesiredState | null { return value === "running" || value === "stopped" || value === "manual" ? value : null; } @@ -279,6 +341,7 @@ function formatBranchReconcileAuditComment(input: { workspaceId: string; inspection: ExecutionWorkspaceBranchReconcileInspection; recoveryActionId: string | null; + rescueRef: ExecutionWorkspaceBranchReconcileResult["rescueRef"]; }) { return [ "Execution workspace branch reconciled.", @@ -292,10 +355,31 @@ function formatBranchReconcileAuditComment(input: { `- Verdict: \`${input.inspection.ancestryVerdict}\``, `- Fingerprint: \`${input.inspection.fingerprint}\``, `- Recovery action: ${input.recoveryActionId ? `\`${input.recoveryActionId}\`` : "none matched"}`, + ...(input.rescueRef + ? [ + `- Rescue ref: \`${input.rescueRef.branchName}\``, + `- Rescue commit: \`${input.rescueRef.commitSha}\``, + `- Rescued file count: \`${input.rescueRef.fileCount}\``, + ] + : []), ...(input.reason ? [`- Operator reason: ${input.reason}`] : []), ].join("\n"); } +function isWorkspaceRuntimeValidationFailure(error: unknown): error is { + code: "workspace_validation_failed"; + message: string; + resultJson: Record; +} { + if (!error || typeof error !== "object") return false; + const maybe = error as { code?: unknown; resultJson?: unknown; message?: unknown }; + return maybe.code === "workspace_validation_failed" && + typeof maybe.message === "string" && + Boolean(maybe.resultJson) && + typeof maybe.resultJson === "object" && + !Array.isArray(maybe.resultJson); +} + function assertBranchReconcileWorkspaceIsSafe(input: { workspaceStatus: ExecutionWorkspace["status"]; inspection: ExecutionWorkspaceBranchReconcileInspection; @@ -316,6 +400,16 @@ function assertBranchReconcileWorkspaceIsSafe(input: { }); } + assertBranchReconcileRuntimeServicesStopped({ + inspection: input.inspection, + runtimeServices: input.runtimeServices, + }); +} + +function assertBranchReconcileRuntimeServicesStopped(input: { + inspection: ExecutionWorkspaceBranchReconcileInspection; + runtimeServices: WorkspaceRuntimeService[]; +}) { const activeRuntimeServices = input.runtimeServices.filter((service) => service.status !== "stopped"); if (activeRuntimeServices.length > 0) { throw unprocessable("Execution workspace branch reconciliation requires all runtime services to be stopped", { @@ -364,6 +458,66 @@ function assertLockedBranchReconcileWorkspaceStillMatchesInspection(input: { } } +async function quarantineRestoreDirtyWorkspaceBranch(input: { + db: Db; + workspace: Pick; + inspection: ExecutionWorkspaceBranchReconcileInspection; + actor: ExecutionWorkspaceBranchReconcileActor; +}): Promise> { + const sourceIssue = await input.db + .select({ + id: issues.id, + identifier: issues.identifier, + title: issues.title, + workMode: issues.workMode, + }) + .from(issues) + .where(eq(issues.id, input.workspace.sourceIssueId!)) + .then((rows) => rows[0] ?? null); + if (!sourceIssue) throw notFound("Source issue not found"); + + const { ensureGitWorktreeBranchCoherent } = await import("./workspace-runtime.js"); + try { + const result = await ensureGitWorktreeBranchCoherent({ + db: input.db, + repoRoot: input.inspection.repoRoot, + worktreePath: input.inspection.worktreePath, + expectedBranchName: input.inspection.fromBranch, + actualBranchName: input.inspection.toBranch, + sourceIssue, + executionWorkspaceId: input.workspace.id, + heartbeatRunId: input.actor.runId, + enableWorkspaceBranchReconcileForward: false, + enableWorkspaceDirtyQuarantineRepair: true, + persistForwardReconcile: false, + reconcileOperationPhase: "worktree_prepare", + recorder: null, + }); + + if (!result.dirtyQuarantineRepair) { + throw unprocessable("Quarantine restore requires a dirty foreign-branch worktree to repair", { + inspection: input.inspection, + }); + } + + return { + branchName: result.dirtyQuarantineRepair.rescueBranch, + commitSha: result.dirtyQuarantineRepair.rescueCommitSha, + fileCount: result.dirtyQuarantineRepair.fileCount, + sourceAuditCommentId: result.dirtyQuarantineRepair.sourceAuditCommentId, + claimantAuditCommentId: result.dirtyQuarantineRepair.claimantAuditCommentId, + }; + } catch (error) { + if (isWorkspaceRuntimeValidationFailure(error)) { + throw unprocessable(error.message, { + code: error.code, + ...error.resultJson, + }); + } + throw error; + } +} + async function inspectGitCloseReadiness(workspace: ExecutionWorkspace): Promise<{ git: ExecutionWorkspaceCloseGitReadiness | null; warnings: string[]; @@ -1049,6 +1203,124 @@ export function executionWorkspaceService(db: Db) { return rows.map((row) => toExecutionWorkspaceSummary(row)); }, + findGitWorktreeContention: async (input: { + companyId: string; + worktreePath: string; + liveBranchName: string | null; + excludingExecutionWorkspaceId?: string | null; + }): Promise => { + const resolvedWorktreePath = path.resolve(input.worktreePath); + const pathOrBranchConditions = [ + eq(executionWorkspaces.providerRef, input.worktreePath), + eq(executionWorkspaces.cwd, input.worktreePath), + ]; + if (input.liveBranchName) { + pathOrBranchConditions.push(eq(executionWorkspaces.branchName, input.liveBranchName)); + } + + const candidates = await db + .select({ + id: executionWorkspaces.id, + cwd: executionWorkspaces.cwd, + providerRef: executionWorkspaces.providerRef, + branchName: executionWorkspaces.branchName, + sourceIssueId: executionWorkspaces.sourceIssueId, + sourceIssueIdentifier: issues.identifier, + }) + .from(executionWorkspaces) + .leftJoin( + issues, + and( + eq(issues.companyId, executionWorkspaces.companyId), + eq(issues.id, executionWorkspaces.sourceIssueId), + ), + ) + .where(and( + eq(executionWorkspaces.companyId, input.companyId), + isNull(executionWorkspaces.closedAt), + ne(executionWorkspaces.status, "archived"), + input.excludingExecutionWorkspaceId + ? ne(executionWorkspaces.id, input.excludingExecutionWorkspaceId) + : sql`true`, + or(...pathOrBranchConditions), + )) + .orderBy(desc(executionWorkspaces.lastUsedAt), desc(executionWorkspaces.updatedAt)) + .limit(20); + + for (const candidate of candidates) { + const candidatePath = readNullableString(candidate.providerRef) ?? readNullableString(candidate.cwd); + const matchesPath = candidatePath ? path.resolve(candidatePath) === resolvedWorktreePath : false; + const matchesBranch = Boolean(input.liveBranchName && candidate.branchName === input.liveBranchName); + if (!matchesPath && !matchesBranch) continue; + + const linkedIssueConditions = [eq(issues.executionWorkspaceId, candidate.id)]; + if (candidate.sourceIssueId) linkedIssueConditions.push(eq(issues.id, candidate.sourceIssueId)); + const linkedIssueRows = await db + .select({ + id: issues.id, + identifier: issues.identifier, + checkoutRunId: issues.checkoutRunId, + executionRunId: issues.executionRunId, + }) + .from(issues) + .where(and( + eq(issues.companyId, input.companyId), + isNull(issues.hiddenAt), + linkedIssueConditions.length === 1 ? linkedIssueConditions[0]! : or(...linkedIssueConditions), + )) + .orderBy(desc(issues.updatedAt)) + .limit(20); + + const runToIssue = new Map(); + for (const issue of linkedIssueRows) { + if (issue.executionRunId) runToIssue.set(issue.executionRunId, { id: issue.id, identifier: issue.identifier ?? null }); + if (issue.checkoutRunId) runToIssue.set(issue.checkoutRunId, { id: issue.id, identifier: issue.identifier ?? null }); + } + + let activeRun: NonNullable["activeRun"] = null; + const runIds = [...runToIssue.keys()]; + if (runIds.length > 0) { + const [row] = await db + .select({ + id: heartbeatRuns.id, + status: heartbeatRuns.status, + }) + .from(heartbeatRuns) + .where(and( + eq(heartbeatRuns.companyId, input.companyId), + inArray(heartbeatRuns.id, runIds), + inArray(heartbeatRuns.status, ["queued", "running"]), + )) + .orderBy(desc(heartbeatRuns.startedAt), desc(heartbeatRuns.createdAt)) + .limit(1); + if (row && (row.status === "queued" || row.status === "running")) { + const issue = runToIssue.get(row.id) ?? null; + activeRun = { + id: row.id, + status: row.status, + issueId: issue?.id ?? null, + issueIdentifier: issue?.identifier ?? null, + }; + } + } + + const claimedIssue = + linkedIssueRows.find((issue) => issue.id === candidate.sourceIssueId) + ?? linkedIssueRows[0] + ?? null; + + return { + claimedByWorkspaceId: candidate.id, + claimedByIssueId: claimedIssue?.id ?? candidate.sourceIssueId ?? null, + claimedByIssueIdentifier: + claimedIssue?.identifier ?? candidate.sourceIssueIdentifier ?? null, + activeRun, + }; + } + + return null; + }, + getById: async (id: string) => { const row = await db .select() @@ -1363,6 +1635,29 @@ export function executionWorkspaceService(db: Db) { } const reason = readNullableString(input.reason); + const rescueRef = input.mode === "quarantine_restore" + ? await (async () => { + const runtimeServicesByWorkspaceId = await loadEffectiveRuntimeServicesByExecutionWorkspace( + db, + existing.companyId, + [existingRow], + ); + assertBranchReconcileRuntimeServicesStopped({ + inspection, + runtimeServices: (runtimeServicesByWorkspaceId.get(existing.id) ?? []).map(toRuntimeService), + }); + // The git rescue has to happen before the DB transaction because the + // transaction may be retried/rolled back, while git side effects cannot. + // The preflight runtime-service guard above keeps known local services + // from holding files open during the non-transactional git sequence. + return quarantineRestoreDirtyWorkspaceBranch({ + db, + workspace: existing, + inspection, + actor: input.actor, + }); + })() + : null; const now = new Date(); const allowActiveWorkspace = input.mode === "forward" && @@ -1427,57 +1722,62 @@ export function executionWorkspaceService(db: Db) { if (!lockedWorkspace.sourceIssueId) { throw unprocessable("Execution workspace needs a source issue before Paperclip can audit branch reconciliation"); } - assertBranchReconcileWorkspaceIsSafe({ - workspaceStatus: lockedWorkspace.status, - inspection, - runtimeServices: lockedRuntimeServices, - allowActiveWorkspace, - }); - if (lockedWorkspace.branchName !== inspection.fromBranch) { - throw unprocessable("Execution workspace branch changed during reconciliation; retry with a fresh inspection", { - workspaceBranch: lockedWorkspace.branchName, - inspection, - }); - } - const updatePatch: Partial = { - branchName: inspection.toBranch, - updatedAt: now, - }; - if (lockedWorkspace.name === inspection.fromBranch) { - updatePatch.name = inspection.toBranch; - } - - const [updatedRow] = await tx - .update(executionWorkspaces) - .set(updatePatch) - .where( - and( - eq(executionWorkspaces.id, lockedWorkspace.id), - allowActiveWorkspace - ? inArray(executionWorkspaces.status, ["idle", "active"]) - : eq(executionWorkspaces.status, "idle"), - eq(executionWorkspaces.branchName, inspection.fromBranch), - noActiveRuntimeServicesForWorkspaceCondition(lockedRow), - ), - ) - .returning(); - if (!updatedRow) { - const latestRuntimeServicesByWorkspaceId = await loadEffectiveRuntimeServicesByExecutionWorkspace( - txDb, - lockedRow.companyId, - [lockedRow], - ); - const latestRuntimeServices = (latestRuntimeServicesByWorkspaceId.get(lockedRow.id) ?? []).map(toRuntimeService); + let updatedRow: ExecutionWorkspaceRow = lockedRow; + if (input.mode !== "quarantine_restore") { assertBranchReconcileWorkspaceIsSafe({ workspaceStatus: lockedWorkspace.status, inspection, - runtimeServices: latestRuntimeServices, + runtimeServices: lockedRuntimeServices, allowActiveWorkspace, }); - throw unprocessable("Execution workspace branch reconciliation requires the workspace to stay idle with stopped runtime services during the update", { - inspection, - }); + if (lockedWorkspace.branchName !== inspection.fromBranch) { + throw unprocessable("Execution workspace branch changed during reconciliation; retry with a fresh inspection", { + workspaceBranch: lockedWorkspace.branchName, + inspection, + }); + } + + const updatePatch: Partial = { + branchName: inspection.toBranch, + updatedAt: now, + }; + if (lockedWorkspace.name === inspection.fromBranch) { + updatePatch.name = inspection.toBranch; + } + + const [branchUpdatedRow] = await tx + .update(executionWorkspaces) + .set(updatePatch) + .where( + and( + eq(executionWorkspaces.id, lockedWorkspace.id), + allowActiveWorkspace + ? inArray(executionWorkspaces.status, ["idle", "active"]) + : eq(executionWorkspaces.status, "idle"), + eq(executionWorkspaces.branchName, inspection.fromBranch), + noActiveRuntimeServicesForWorkspaceCondition(lockedRow), + ), + ) + .returning(); + if (!branchUpdatedRow) { + const latestRuntimeServicesByWorkspaceId = await loadEffectiveRuntimeServicesByExecutionWorkspace( + txDb, + lockedRow.companyId, + [lockedRow], + ); + const latestRuntimeServices = (latestRuntimeServicesByWorkspaceId.get(lockedRow.id) ?? []).map(toRuntimeService); + assertBranchReconcileWorkspaceIsSafe({ + workspaceStatus: lockedWorkspace.status, + inspection, + runtimeServices: latestRuntimeServices, + allowActiveWorkspace, + }); + throw unprocessable("Execution workspace branch reconciliation requires the workspace to stay idle with stopped runtime services during the update", { + inspection, + }); + } + updatedRow = branchUpdatedRow; } let recoveryAction = await recoveryActionsSvc.resolveActiveForIssue( @@ -1489,7 +1789,9 @@ export function executionWorkspaceService(db: Db) { fingerprint: inspection.fingerprint, status: "resolved", outcome: "restored", - resolutionNote: `Execution workspace branch record reconciled from "${inspection.fromBranch}" to "${inspection.toBranch}".`, + resolutionNote: input.mode === "quarantine_restore" && rescueRef + ? `Execution workspace dirty worktree quarantined on "${rescueRef.branchName}" and restored recorded branch "${inspection.fromBranch}".` + : `Execution workspace branch record reconciled from "${inspection.fromBranch}" to "${inspection.toBranch}".`, }, tx, ); @@ -1505,7 +1807,9 @@ export function executionWorkspaceService(db: Db) { fingerprint: alternateFingerprint, status: "resolved", outcome: "restored", - resolutionNote: `Execution workspace branch record reconciled from "${inspection.fromBranch}" to "${inspection.toBranch}".`, + resolutionNote: input.mode === "quarantine_restore" && rescueRef + ? `Execution workspace dirty worktree quarantined on "${rescueRef.branchName}" and restored recorded branch "${inspection.fromBranch}".` + : `Execution workspace branch record reconciled from "${inspection.fromBranch}" to "${inspection.toBranch}".`, }, tx, ); @@ -1513,6 +1817,65 @@ export function executionWorkspaceService(db: Db) { } } + let restoredSourceIssue: ExecutionWorkspaceBranchReconcileResult["restoredSourceIssue"] = null; + let sourceIssueStatusChanged = false; + if (input.mode === "quarantine_restore") { + const [sourceBefore] = await tx + .select({ + id: issues.id, + companyId: issues.companyId, + status: issues.status, + assigneeAgentId: issues.assigneeAgentId, + assigneeUserId: issues.assigneeUserId, + executionPolicy: issues.executionPolicy, + executionState: issues.executionState, + monitorNextCheckAt: issues.monitorNextCheckAt, + monitorWakeRequestedAt: issues.monitorWakeRequestedAt, + monitorLastTriggeredAt: issues.monitorLastTriggeredAt, + monitorAttemptCount: issues.monitorAttemptCount, + monitorNotes: issues.monitorNotes, + monitorScheduledBy: issues.monitorScheduledBy, + }) + .from(issues) + .where(eq(issues.id, lockedWorkspace.sourceIssueId)) + .for("update"); + if (!sourceBefore) throw notFound("Source issue not found"); + + const requestedStatus = quarantineRestoreRequestedSourceStatus(sourceBefore); + const policy = normalizeIssueExecutionPolicy(sourceBefore.executionPolicy ?? null); + const transition = applyIssueExecutionPolicyTransition({ + issue: sourceBefore, + policy, + previousPolicy: policy, + requestedStatus, + requestedAssigneePatch: {}, + actor: { + agentId: input.actor.agentId ?? null, + userId: input.actor.actorType === "user" ? input.actor.actorId : null, + }, + commentBody: null, + }); + const { issueService } = await import("./issues.js"); + const updatedIssue = await issueService(db).update( + lockedWorkspace.sourceIssueId, + { + ...(requestedStatus ? { status: requestedStatus } : {}), + ...transition.patch, + actorAgentId: input.actor.agentId ?? null, + actorUserId: input.actor.actorType === "user" ? input.actor.actorId : null, + }, + tx, + ); + if (!updatedIssue) throw notFound("Source issue not found"); + restoredSourceIssue = { + id: updatedIssue.id, + companyId: updatedIssue.companyId, + status: updatedIssue.status, + assigneeAgentId: updatedIssue.assigneeAgentId, + }; + sourceIssueStatusChanged = sourceBefore.status !== updatedIssue.status; + } + const [auditComment] = await tx .insert(issueComments) .values({ @@ -1528,6 +1891,7 @@ export function executionWorkspaceService(db: Db) { workspaceId: existing.id, inspection, recoveryActionId: recoveryAction?.id ?? null, + rescueRef, }), }) .returning({ id: issueComments.id }); @@ -1542,6 +1906,9 @@ export function executionWorkspaceService(db: Db) { inspection, recoveryAction, auditCommentId: auditComment?.id ?? null, + rescueRef, + restoredSourceIssue, + sourceIssueStatusChanged, }; }); }, diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 0b42e473ed..a97a4dadbc 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -3726,15 +3726,18 @@ export function resolveTaskSessionConfigFreshness(input: { }; } -function shouldAutoCheckoutIssueForWake(input: { +export function shouldAutoCheckoutIssueForWake(input: { contextSnapshot: Record | null | undefined; issueStatus: string | null; issueAssigneeAgentId: string | null; + issueExecutionState?: unknown; isDependencyReady: boolean; agentId: string; }) { if (input.issueAssigneeAgentId !== input.agentId) return false; if (!input.isDependencyReady) return false; + const executionState = parseIssueExecutionState(input.issueExecutionState); + if (executionState?.status === "pending") return false; const issueStatus = readNonEmptyString(input.issueStatus); if ( @@ -5231,6 +5234,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) assigneeAgentId: issues.assigneeAgentId, assigneeAdapterOverrides: issues.assigneeAdapterOverrides, executionPolicy: issues.executionPolicy, + executionState: issues.executionState, executionWorkspaceSettings: issues.executionWorkspaceSettings, parentId: issues.parentId, createdByUserId: issues.createdByUserId, @@ -10428,6 +10432,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) contextSnapshot: context, issueStatus: issueContext.status, issueAssigneeAgentId: issueContext.assigneeAgentId, + issueExecutionState: issueContext.executionState, isDependencyReady: issueDependencyReadiness?.isDependencyReady ?? true, agentId: agent.id, }) @@ -11171,6 +11176,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) heartbeatRunId: run.id, enableWorkspaceBranchReconcileForward: resolvedInstanceSettings.experimental.enableWorkspaceBranchReconcileForward, + enableWorkspaceDirtyQuarantineRepair: + resolvedInstanceSettings.experimental.enableWorkspaceDirtyQuarantineRepair, recorder: workspaceOperationRecorder, }) : null, @@ -11187,6 +11194,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) heartbeatRunId: run.id, enableWorkspaceBranchReconcileForward: resolvedInstanceSettings.experimental.enableWorkspaceBranchReconcileForward, + enableWorkspaceDirtyQuarantineRepair: + resolvedInstanceSettings.experimental.enableWorkspaceDirtyQuarantineRepair, recorder: workspaceOperationRecorder, }), }); @@ -12022,6 +12031,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) heartbeatRunId: run.id, enableWorkspaceBranchReconcileForward: resolvedInstanceSettings.experimental.enableWorkspaceBranchReconcileForward, + enableWorkspaceDirtyQuarantineRepair: + resolvedInstanceSettings.experimental.enableWorkspaceDirtyQuarantineRepair, persistForwardReconcile: false, reconcileOperationPhase: "workspace_finalize", recorder: workspaceOperationRecorder, diff --git a/server/src/services/instance-settings.ts b/server/src/services/instance-settings.ts index 9cc54d2c92..4301adf3ba 100644 --- a/server/src/services/instance-settings.ts +++ b/server/src/services/instance-settings.ts @@ -59,7 +59,8 @@ export function normalizeExperimentalSettings(raw: unknown): InstanceExperimenta enableServerInfoDebugView: parsed.data.enableServerInfoDebugView ?? false, autoRestartDevServerWhenIdle: parsed.data.autoRestartDevServerWhenIdle ?? false, enableIssueGraphLivenessAutoRecovery: parsed.data.enableIssueGraphLivenessAutoRecovery ?? false, - enableWorkspaceBranchReconcileForward: parsed.data.enableWorkspaceBranchReconcileForward ?? false, + enableWorkspaceBranchReconcileForward: parsed.data.enableWorkspaceBranchReconcileForward ?? true, + enableWorkspaceDirtyQuarantineRepair: parsed.data.enableWorkspaceDirtyQuarantineRepair ?? true, enableWorktreeRunExecution: parsed.data.enableWorktreeRunExecution ?? false, issueGraphLivenessAutoRecoveryLookbackHours: parsed.data.issueGraphLivenessAutoRecoveryLookbackHours ?? @@ -82,7 +83,8 @@ export function normalizeExperimentalSettings(raw: unknown): InstanceExperimenta enableServerInfoDebugView: false, autoRestartDevServerWhenIdle: false, enableIssueGraphLivenessAutoRecovery: false, - enableWorkspaceBranchReconcileForward: false, + enableWorkspaceBranchReconcileForward: true, + enableWorkspaceDirtyQuarantineRepair: true, enableWorktreeRunExecution: false, issueGraphLivenessAutoRecoveryLookbackHours: DEFAULT_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS, diff --git a/server/src/services/workspace-runtime.ts b/server/src/services/workspace-runtime.ts index d32ae03c6c..3960116769 100644 --- a/server/src/services/workspace-runtime.ts +++ b/server/src/services/workspace-runtime.ts @@ -7,15 +7,17 @@ import path from "node:path"; import { setTimeout as delay } from "node:timers/promises"; import type { AdapterRuntimeServiceReport } from "@paperclipai/adapter-utils"; import type { Db } from "@paperclipai/db"; -import { executionWorkspaces, projectWorkspaces, workspaceRuntimeServices } from "@paperclipai/db"; +import { executionWorkspaces, issueComments, issues, projectWorkspaces, workspaceRuntimeServices } from "@paperclipai/db"; import { listWorkspaceServiceCommandDefinitions, type GitWorktreeBranchAncestryVerdict, type GitWorktreeBranchIncoherenceEvidence as SharedGitWorktreeBranchIncoherenceEvidence, + type GitWorktreeInProgressOperation, + type WorkspaceOperationPhase, type WorkspaceRuntimeDesiredState, type WorkspaceRuntimeServiceStateMap, } from "@paperclipai/shared"; -import { and, desc, eq, inArray } from "drizzle-orm"; +import { and, desc, eq, inArray, ne } from "drizzle-orm"; import { asNumber, asString, parseObject, renderTemplate } from "../adapters/utils.js"; import { resolveHomeAwarePath } from "../home-paths.js"; import { @@ -661,13 +663,25 @@ type GitWorktreeCleanliness = SharedGitWorktreeBranchIncoherenceEvidence["cleanl type GitWorktreeBranchIncoherenceEvidence = SharedGitWorktreeBranchIncoherenceEvidence; +type GitWorktreeBranchContention = NonNullable; + type GitWorktreeBranchCoherenceResult = { branchName: string | null; reconciledForward: boolean; pendingForwardBranchReconcile?: PendingForwardBranchReconcile | null; + dirtyQuarantineRepair?: DirtyQuarantineRepairResult | null; warnings: string[]; }; +type DirtyQuarantineRepairResult = { + rescueBranch: string; + rescueCommitSha: string; + fileCount: number; + clearedInProgressOperation: GitWorktreeInProgressOperation | null; + sourceAuditCommentId: string | null; + claimantAuditCommentId: string | null; +}; + export type PendingForwardBranchReconcile = { recordedBranchName: string; adoptedBranchName: string; @@ -679,6 +693,186 @@ function formatBranchForMessage(branch: string | null | undefined) { return branch && branch.length > 0 ? branch : ""; } +const GIT_IN_PROGRESS_OPERATION_MARKERS: ReadonlyArray<{ + operation: GitWorktreeInProgressOperation; + marker: string; +}> = [ + { operation: "rebase", marker: "rebase-merge" }, + { operation: "rebase", marker: "rebase-apply" }, + { operation: "merge", marker: "MERGE_HEAD" }, + { operation: "cherry_pick", marker: "CHERRY_PICK_HEAD" }, + { operation: "revert", marker: "REVERT_HEAD" }, + { operation: "bisect", marker: "BISECT_LOG" }, +]; + +const GIT_IN_PROGRESS_OPERATION_LABELS: Record = { + rebase: "rebase", + merge: "merge", + cherry_pick: "cherry-pick", + revert: "revert", + bisect: "bisect", +}; + +// `--quit` clears the interrupted operation's state directory without touching +// the working tree or moving HEAD, unlike `--abort` which resets both. +const GIT_IN_PROGRESS_OPERATION_QUIT_ARGS: Record = { + rebase: ["rebase", "--quit"], + merge: ["merge", "--quit"], + cherry_pick: ["cherry-pick", "--quit"], + revert: ["revert", "--quit"], + bisect: ["bisect", "reset", "HEAD"], +}; + +async function detectGitWorktreeInProgressOperation( + worktreePath: string, +): Promise { + for (const { operation, marker } of GIT_IN_PROGRESS_OPERATION_MARKERS) { + const markerPath = await runGit(["rev-parse", "--git-path", marker], worktreePath).catch(() => null); + if (!markerPath) continue; + if (existsSync(path.resolve(worktreePath, markerPath))) return operation; + } + return null; +} + +const DIRTY_PATH_SAMPLE_LIMIT = 5; + +function parseGitPorcelainPath(line: string) { + const raw = line.trimEnd(); + if (raw.trim().length <= 3) return raw.trim(); + if (raw[1] === " " && raw[2] !== " ") return raw.slice(2).trim(); + return raw.slice(3).trim(); +} + +function sampleDirtyStatusPaths(statusLines: string[] | null) { + return (statusLines ?? []) + .map(parseGitPorcelainPath) + .filter((value) => value.length > 0) + .slice(0, DIRTY_PATH_SAMPLE_LIMIT); +} + +function formatUtcBranchTimestamp(date = new Date()) { + return date.toISOString().replace(/[-:]/g, "").replace(/\.\d{3}Z$/, "Z"); +} + +function buildDirtyQuarantineRescueBranch(sourceIssue: ExecutionWorkspaceIssueRef | null) { + const issueComponent = sanitizeBranchName(sourceIssue?.identifier ?? sourceIssue?.id ?? "issue"); + return sanitizeBranchName(`paperclip/rescue/${issueComponent}/${formatUtcBranchTimestamp()}`); +} + +function formatIssueReference(issueId: string | null | undefined, identifier: string | null | undefined) { + if (!identifier) return issueId ? `\`${issueId}\`` : "`unknown`"; + const match = identifier.match(/^([A-Z]+)-\d+$/); + if (!match) return `\`${identifier}\``; + return `[${identifier}](/${match[1]}/issues/${identifier})`; +} + +async function readIssueCompanyId(db: Db, issueId: string | null | undefined): Promise { + if (!issueId) return null; + return db + .select({ companyId: issues.companyId }) + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => rows[0]?.companyId ?? null); +} + +async function findGitWorktreeBranchContention(input: { + db: Db | null | undefined; + sourceIssue: ExecutionWorkspaceIssueRef | null; + executionWorkspaceId: string | null; + worktreePath: string; + actualBranchName: string | null; +}): Promise { + if (!input.db) return null; + const companyId = await readIssueCompanyId(input.db, input.sourceIssue?.id); + if (!companyId) return null; + return executionWorkspaceService(input.db).findGitWorktreeContention({ + companyId, + worktreePath: input.worktreePath, + liveBranchName: input.actualBranchName, + excludingExecutionWorkspaceId: input.executionWorkspaceId, + }); +} + +function executionWorkspaceUsesInheritedProjectRuntimeServices( + row: typeof executionWorkspaces.$inferSelect, +) { + if (row.mode !== "shared_workspace" || !row.projectWorkspaceId) return false; + return !readExecutionWorkspaceConfig((row.metadata as Record | null) ?? null)?.workspaceRuntime; +} + +async function findActiveRuntimeServiceBlockingDirtyQuarantine(input: { + db: Db; + workspace: typeof executionWorkspaces.$inferSelect; +}) { + const inheritedProjectWorkspaceId = executionWorkspaceUsesInheritedProjectRuntimeServices(input.workspace) + ? input.workspace.projectWorkspaceId + : null; + const serviceScopeCondition = inheritedProjectWorkspaceId + ? and( + eq(workspaceRuntimeServices.companyId, input.workspace.companyId), + eq(workspaceRuntimeServices.projectWorkspaceId, inheritedProjectWorkspaceId), + eq(workspaceRuntimeServices.scopeType, "project_workspace"), + ) + : and( + eq(workspaceRuntimeServices.companyId, input.workspace.companyId), + eq(workspaceRuntimeServices.executionWorkspaceId, input.workspace.id), + ); + + const [service] = await input.db + .select({ + id: workspaceRuntimeServices.id, + serviceName: workspaceRuntimeServices.serviceName, + status: workspaceRuntimeServices.status, + scopeType: workspaceRuntimeServices.scopeType, + }) + .from(workspaceRuntimeServices) + .where(and(serviceScopeCondition, ne(workspaceRuntimeServices.status, "stopped"))) + .orderBy(desc(workspaceRuntimeServices.updatedAt), desc(workspaceRuntimeServices.createdAt)) + .limit(1); + return service ?? null; +} + +async function assertDirtyQuarantineRuntimeServicesStopped(input: { + db: Db; + executionWorkspaceId: string | null; + evidence: GitWorktreeBranchIncoherenceEvidence; +}) { + if (!input.executionWorkspaceId) { + input.evidence.safeRepair.eligible = false; + input.evidence.safeRepair.reason = "dirty quarantine repair requires an execution workspace id for runtime-service checks"; + throw branchIncoherenceValidationFailure(input.evidence); + } + + const [workspace] = await input.db + .select() + .from(executionWorkspaces) + .where(eq(executionWorkspaces.id, input.executionWorkspaceId)); + if (!workspace) { + input.evidence.safeRepair.eligible = false; + input.evidence.safeRepair.reason = "dirty quarantine repair requires a persisted execution workspace for runtime-service checks"; + throw branchIncoherenceValidationFailure(input.evidence); + } + + const activeService = await findActiveRuntimeServiceBlockingDirtyQuarantine({ + db: input.db, + workspace, + }); + if (!activeService) return; + + input.evidence.safeRepair.eligible = false; + input.evidence.safeRepair.reason = + `dirty quarantine repair requires runtime service "${activeService.serviceName}" (${activeService.id}) to be stopped; current status is ${activeService.status}`; + throw branchIncoherenceValidationFailure(input.evidence); +} + +async function assertGitIndexIsUnlocked(worktreePath: string) { + const indexLockPath = await runGit(["rev-parse", "--git-path", "index.lock"], worktreePath) + .catch(() => null); + if (indexLockPath && existsSync(indexLockPath)) { + throw new Error(`git index lock exists at ${indexLockPath}`); + } +} + function fingerprintWorkspaceBranchIncoherence(input: { sourceIssueId: string | null; executionWorkspaceId: string | null; @@ -749,6 +943,7 @@ function explainGitWorktreeBranchIncoherence(input: { } async function inspectGitWorktreeBranchIncoherence(input: { + db?: Db | null; repoRoot: string; worktreePath: string; expectedBranchName: string; @@ -762,9 +957,11 @@ async function inspectGitWorktreeBranchIncoherence(input: { ).catch(() => null); const statusLines = status === null ? null - : status.split(/\r?\n/).map((line) => line.trim()).filter(Boolean); + : status.split(/\r?\n/).map((line) => line.trimEnd()).filter((line) => line.trim().length > 0); + const dirtyPathSample = sampleDirtyStatusPaths(statusLines); const cleanliness: GitWorktreeCleanliness = status === null ? "unknown" : status.trim().length > 0 ? "dirty" : "clean"; + const inProgressOperation = await detectGitWorktreeInProgressOperation(input.worktreePath); const expectedHeadSha = await runGit( ["rev-parse", "--verify", `refs/heads/${input.expectedBranchName}^{commit}`], input.repoRoot, @@ -784,7 +981,7 @@ async function inspectGitWorktreeBranchIncoherence(input: { expectedHeadSha, actualHeadSha, }); - const plainLanguageReason = explainGitWorktreeBranchIncoherence({ + const basePlainLanguageReason = explainGitWorktreeBranchIncoherence({ expectedBranchName: input.expectedBranchName, actualBranchName: input.actualBranchName, expectedHeadSha, @@ -792,6 +989,9 @@ async function inspectGitWorktreeBranchIncoherence(input: { sameHead, ancestryVerdict, }); + const plainLanguageReason = inProgressOperation + ? `${basePlainLanguageReason} An interrupted git ${GIT_IN_PROGRESS_OPERATION_LABELS[inProgressOperation]} is still in progress in this worktree.` + : basePlainLanguageReason; const canCheckoutRecordedBranch = cleanliness === "clean" && expectedBranchExists && sameHead && registeredBranchMatchesHead; const canAdoptForwardActualBranch = @@ -817,7 +1017,9 @@ async function inspectGitWorktreeBranchIncoherence(input: { ? "clean worktree and checked-out branch is forward of the recorded branch" : "clean detached worktree HEAD is forward of the recorded branch" : cleanliness !== "clean" - ? "worktree is not clean" + ? inProgressOperation + ? `worktree is not clean and a git ${GIT_IN_PROGRESS_OPERATION_LABELS[inProgressOperation]} is in progress` + : "worktree is not clean" : !registered ? "worktree path is not registered" : !registeredBranchMatchesHead @@ -837,6 +1039,13 @@ async function inspectGitWorktreeBranchIncoherence(input: { expectedHeadSha, actualHeadSha, }); + const contention = await findGitWorktreeBranchContention({ + db: input.db ?? null, + sourceIssue: input.sourceIssue, + executionWorkspaceId: input.executionWorkspaceId ?? null, + worktreePath: input.worktreePath, + actualBranchName: input.actualBranchName, + }); return { reason: GIT_WORKTREE_BRANCH_INCOHERENCE_REASON, @@ -849,7 +1058,10 @@ async function inspectGitWorktreeBranchIncoherence(input: { expectedBranch: input.expectedBranchName, actualBranch: input.actualBranchName, cleanliness, + inProgressOperation, statusEntryCount: statusLines?.length ?? null, + dirtyPathSample, + contention, provenance: { expectedBranchRef: `refs/heads/${input.expectedBranchName}`, actualBranchRef, @@ -882,6 +1094,407 @@ function branchIncoherenceValidationFailure(evidence: GitWorktreeBranchIncoheren ); } +function formatDirtyQuarantineContentionRefusal(contention: GitWorktreeBranchContention) { + const activeRunText = contention.activeRun + ? ` with active run ${contention.activeRun.id}` + : " with no active run"; + return `dirty quarantine repair refused because workspace ${contention.claimedByWorkspaceId} already claims the live branch${activeRunText}`; +} + +function formatDirtyQuarantineFailure(error: unknown) { + const message = error instanceof Error ? error.message : String(error); + if ( + gitErrorIncludes(error, "index.lock") || + gitErrorIncludes(error, "index lock") || + gitErrorIncludes(error, "another git process") || + gitErrorIncludes(error, "Unable to create") + ) { + return `dirty quarantine repair aborted because git reported index contention: ${message}`; + } + return `dirty quarantine repair failed: ${message}`; +} + +function formatDirtyQuarantineAuditComment(input: { + evidence: GitWorktreeBranchIncoherenceEvidence; + rescueBranch: string; + rescueCommitSha: string; + fileCount: number; + sourceIssue: ExecutionWorkspaceIssueRef | null; + claimant: GitWorktreeBranchContention | null; +}) { + const dirtySample = input.evidence.dirtyPathSample.length > 0 + ? input.evidence.dirtyPathSample.map((entry) => `\`${entry}\``).join(", ") + : "`none captured`"; + return [ + "Execution workspace dirty worktree quarantined before restore.", + "", + `- Source issue: ${formatIssueReference(input.evidence.sourceIssueId, input.evidence.sourceIdentifier ?? input.sourceIssue?.identifier ?? null)}`, + `- Workspace: \`${input.evidence.executionWorkspaceId ?? "unpersisted"}\``, + `- Worktree: \`${input.evidence.worktreePath}\``, + `- Recorded branch: \`${input.evidence.expectedBranch}\``, + `- Live branch: \`${formatBranchForMessage(input.evidence.actualBranch)}\``, + `- Rescue branch: \`${input.rescueBranch}\``, + `- Rescue commit: \`${input.rescueCommitSha}\``, + `- Dirty file count: \`${input.fileCount}\``, + `- Dirty path sample: ${dirtySample}`, + ...(input.evidence.inProgressOperation + ? [`- Interrupted operation: \`git ${GIT_IN_PROGRESS_OPERATION_LABELS[input.evidence.inProgressOperation]}\` (state cleared after rescue; resolution preserved on the rescue branch)`] + : []), + `- Fingerprint: \`${input.evidence.fingerprint}\``, + input.claimant + ? `- Claimant: workspace \`${input.claimant.claimedByWorkspaceId}\` on issue ${formatIssueReference(input.claimant.claimedByIssueId, input.claimant.claimedByIssueIdentifier)}${input.claimant.activeRun ? ` with active run \`${input.claimant.activeRun.id}\`` : " with no active run"}` + : "- Claimant: none", + ].join("\n"); +} + +async function writeDirtyQuarantineAuditComments(input: { + db: Db; + companyId: string; + evidence: GitWorktreeBranchIncoherenceEvidence; + sourceIssue: ExecutionWorkspaceIssueRef | null; + rescueBranch: string; + rescueCommitSha: string; + fileCount: number; + heartbeatRunId: string | null; +}): Promise<{ sourceAuditCommentId: string | null; claimantAuditCommentId: string | null }> { + const body = formatDirtyQuarantineAuditComment({ + evidence: input.evidence, + rescueBranch: input.rescueBranch, + rescueCommitSha: input.rescueCommitSha, + fileCount: input.fileCount, + sourceIssue: input.sourceIssue, + claimant: input.evidence.contention, + }); + let sourceAuditCommentId: string | null = null; + let claimantAuditCommentId: string | null = null; + const now = new Date(); + if (input.evidence.sourceIssueId) { + const [sourceComment] = await input.db + .insert(issueComments) + .values({ + companyId: input.companyId, + issueId: input.evidence.sourceIssueId, + authorAgentId: null, + authorUserId: null, + authorType: "system", + createdByRunId: input.heartbeatRunId, + body, + }) + .returning({ id: issueComments.id }); + sourceAuditCommentId = sourceComment?.id ?? null; + await input.db + .update(issues) + .set({ updatedAt: now }) + .where(eq(issues.id, input.evidence.sourceIssueId)); + } + + const claimantIssueId = input.evidence.contention?.claimedByIssueId ?? null; + if (claimantIssueId && claimantIssueId !== input.evidence.sourceIssueId) { + const [claimantComment] = await input.db + .insert(issueComments) + .values({ + companyId: input.companyId, + issueId: claimantIssueId, + authorAgentId: null, + authorUserId: null, + authorType: "system", + createdByRunId: input.heartbeatRunId, + body, + }) + .returning({ id: issueComments.id }); + claimantAuditCommentId = claimantComment?.id ?? null; + await input.db + .update(issues) + .set({ updatedAt: now }) + .where(eq(issues.id, claimantIssueId)); + } + + return { sourceAuditCommentId, claimantAuditCommentId }; +} + +async function logDirtyQuarantineActivity(input: { + db: Db; + companyId: string; + evidence: GitWorktreeBranchIncoherenceEvidence; + rescueBranch: string; + rescueCommitSha: string; + fileCount: number; + heartbeatRunId: string | null; + sourceAuditCommentId: string | null; + claimantAuditCommentId: string | null; +}) { + await logActivity(input.db, { + companyId: input.companyId, + actorType: "system", + actorId: "workspace_runtime", + runId: input.heartbeatRunId, + action: "execution_workspace.dirty_worktree_quarantined", + entityType: input.evidence.executionWorkspaceId ? "execution_workspace" : "issue", + entityId: input.evidence.executionWorkspaceId ?? input.evidence.sourceIssueId ?? input.companyId, + details: { + reason: GIT_WORKTREE_BRANCH_INCOHERENCE_REASON, + sourceIssueId: input.evidence.sourceIssueId, + executionWorkspaceId: input.evidence.executionWorkspaceId, + worktreePath: input.evidence.worktreePath, + expectedBranch: input.evidence.expectedBranch, + actualBranch: input.evidence.actualBranch, + rescueBranch: input.rescueBranch, + rescueCommitSha: input.rescueCommitSha, + fileCount: input.fileCount, + dirtyPathSample: input.evidence.dirtyPathSample, + fingerprint: input.evidence.fingerprint, + contention: input.evidence.contention, + sourceAuditCommentId: input.sourceAuditCommentId, + claimantAuditCommentId: input.claimantAuditCommentId, + actor: { + type: "system", + id: "workspace_runtime", + source: "workspace_runtime", + }, + }, + }); +} + +async function recordDirtyQuarantineOperation(input: { + recorder?: WorkspaceOperationRecorder | null; + phase?: "worktree_prepare" | "workspace_finalize"; + cwd: string; + evidence: GitWorktreeBranchIncoherenceEvidence; + rescueBranch: string; + rescueCommitSha: string; + fileCount: number; + sourceAuditCommentId: string | null; + claimantAuditCommentId: string | null; +}) { + if (!input.recorder) return; + await input.recorder.recordOperation({ + phase: input.phase ?? "worktree_prepare", + cwd: input.cwd, + metadata: { + repoRoot: input.evidence.repoRoot, + worktreePath: input.evidence.worktreePath, + expectedBranchName: input.evidence.expectedBranch, + actualBranchName: input.evidence.actualBranch, + branchIncoherenceDirtyQuarantineRepair: true, + rescueBranch: input.rescueBranch, + rescueCommitSha: input.rescueCommitSha, + fileCount: input.fileCount, + dirtyPathSample: input.evidence.dirtyPathSample, + fingerprint: input.evidence.fingerprint, + sourceIssueId: input.evidence.sourceIssueId, + executionWorkspaceId: input.evidence.executionWorkspaceId, + sourceAuditCommentId: input.sourceAuditCommentId, + claimantAuditCommentId: input.claimantAuditCommentId, + }, + run: async () => ({ + status: "succeeded", + system: + `Quarantined dirty git worktree state on ${input.rescueBranch} (${formatShortSha(input.rescueCommitSha)}) and restored recorded branch ${input.evidence.expectedBranch}.\n`, + }), + }); +} + +async function quarantineDirtyWorktreeBranchIncoherence(input: { + db: Db; + repoRoot: string; + worktreePath: string; + expectedBranchName: string; + sourceIssue: ExecutionWorkspaceIssueRef | null; + executionWorkspaceId: string | null; + heartbeatRunId: string | null; + evidence: GitWorktreeBranchIncoherenceEvidence; + phase?: "worktree_prepare" | "workspace_finalize"; + recorder?: WorkspaceOperationRecorder | null; +}): Promise { + const companyId = await readIssueCompanyId(input.db, input.evidence.sourceIssueId); + if (!companyId) { + input.evidence.safeRepair.eligible = false; + input.evidence.safeRepair.reason = "dirty quarantine repair requires a source issue company for audit"; + throw branchIncoherenceValidationFailure(input.evidence); + } + + const freshContention = await findGitWorktreeBranchContention({ + db: input.db, + sourceIssue: input.sourceIssue, + executionWorkspaceId: input.executionWorkspaceId, + worktreePath: input.worktreePath, + actualBranchName: input.evidence.actualBranch, + }); + input.evidence.contention = freshContention; + if (freshContention) { + input.evidence.safeRepair.eligible = false; + input.evidence.safeRepair.reason = formatDirtyQuarantineContentionRefusal(freshContention); + throw branchIncoherenceValidationFailure(input.evidence); + } + + const rescueBranch = buildDirtyQuarantineRescueBranch(input.sourceIssue); + const fileCount = input.evidence.statusEntryCount ?? input.evidence.dirtyPathSample.length; + const baseMetadata = { + repoRoot: input.repoRoot, + worktreePath: input.worktreePath, + expectedBranchName: input.expectedBranchName, + actualBranchName: input.evidence.actualBranch, + branchIncoherenceDirtyQuarantineRepair: true, + rescueBranch, + fingerprint: input.evidence.fingerprint, + sourceIssueId: input.evidence.sourceIssueId, + executionWorkspaceId: input.evidence.executionWorkspaceId, + fileCount, + dirtyPathSample: input.evidence.dirtyPathSample, + contention: input.evidence.contention, + }; + + let rescueBranchCreated = false; + let expectedBranchRestored = false; + try { + await assertGitIndexIsUnlocked(input.worktreePath); + await recordGitOperation(input.recorder, { + phase: input.phase ?? "worktree_prepare", + args: ["checkout", "-b", rescueBranch], + cwd: input.worktreePath, + metadata: baseMetadata, + successMessage: `Created rescue branch ${rescueBranch} for dirty git worktree state at ${input.worktreePath}\n`, + failureLabel: `git checkout -b ${rescueBranch}`, + }); + rescueBranchCreated = true; + await recordGitOperation(input.recorder, { + phase: input.phase ?? "worktree_prepare", + args: ["add", "-A"], + cwd: input.worktreePath, + metadata: baseMetadata, + successMessage: `Staged dirty git worktree state for rescue branch ${rescueBranch}\n`, + failureLabel: "git add -A", + }); + await recordGitOperation(input.recorder, { + phase: input.phase ?? "worktree_prepare", + args: [ + "commit", + "-m", + "Paperclip dirty workspace rescue", + "-m", + [ + `Source-Issue: ${input.evidence.sourceIdentifier ?? input.evidence.sourceIssueId ?? "unknown"}`, + `Run-Id: ${input.heartbeatRunId ?? "unknown"}`, + `Recorded-Branch: ${input.expectedBranchName}`, + `Live-Branch: ${formatBranchForMessage(input.evidence.actualBranch)}`, + `Fingerprint: ${input.evidence.fingerprint}`, + ].join("\n"), + ], + cwd: input.worktreePath, + metadata: baseMetadata, + successMessage: `Committed dirty git worktree state to rescue branch ${rescueBranch}\n`, + failureLabel: "git commit dirty workspace rescue", + }); + const rescueCommitSha = await runGit(["rev-parse", "HEAD"], input.worktreePath); + await recordGitOperation(input.recorder, { + phase: input.phase ?? "worktree_prepare", + args: ["checkout", input.expectedBranchName], + cwd: input.worktreePath, + metadata: { + ...baseMetadata, + rescueCommitSha, + }, + successMessage: `Restored recorded branch ${input.expectedBranchName} after dirty workspace rescue ${rescueBranch}\n`, + failureLabel: `git checkout ${input.expectedBranchName}`, + }); + expectedBranchRestored = true; + + // A run that died mid-rebase (or mid-merge/cherry-pick/revert/bisect) + // leaves the operation's state directory behind even after the recorded + // branch is checked out, which wedges the next git command in the + // worktree. The rescue commit above already preserved the in-flight + // resolution, so clearing the state metadata here loses nothing. + let clearedInProgressOperation: GitWorktreeInProgressOperation | null = null; + const lingeringOperation = await detectGitWorktreeInProgressOperation(input.worktreePath); + if (lingeringOperation) { + const operationLabel = GIT_IN_PROGRESS_OPERATION_LABELS[lingeringOperation]; + const quitArgs = GIT_IN_PROGRESS_OPERATION_QUIT_ARGS[lingeringOperation]; + await recordGitOperation(input.recorder, { + phase: input.phase ?? "worktree_prepare", + args: quitArgs, + cwd: input.worktreePath, + metadata: { + ...baseMetadata, + clearedInProgressOperation: lingeringOperation, + }, + successMessage: `Cleared interrupted git ${operationLabel} state after dirty workspace rescue ${rescueBranch}\n`, + failureLabel: `git ${quitArgs.join(" ")}`, + }); + const stillInProgress = await detectGitWorktreeInProgressOperation(input.worktreePath); + if (stillInProgress) { + input.evidence.safeRepair.succeeded = false; + input.evidence.safeRepair.reason = + `dirty quarantine repair could not clear the interrupted git ${GIT_IN_PROGRESS_OPERATION_LABELS[stillInProgress]} state`; + throw branchIncoherenceValidationFailure(input.evidence); + } + clearedInProgressOperation = lingeringOperation; + } + + const repairedBranch = await runGit(["symbolic-ref", "--quiet", "--short", "HEAD"], input.worktreePath) + .catch(() => null); + if (repairedBranch !== input.expectedBranchName) { + input.evidence.safeRepair.succeeded = false; + input.evidence.safeRepair.reason = + `dirty quarantine repair checked out ${formatBranchForMessage(repairedBranch)} instead of ${input.expectedBranchName}`; + throw branchIncoherenceValidationFailure(input.evidence); + } + const repairedStatus = await runGit(["status", "--porcelain", "--untracked-files=all"], input.worktreePath); + if (repairedStatus.trim().length > 0) { + input.evidence.safeRepair.succeeded = false; + input.evidence.safeRepair.reason = "dirty quarantine repair completed but the worktree is still dirty"; + throw branchIncoherenceValidationFailure(input.evidence); + } + + const comments = await writeDirtyQuarantineAuditComments({ + db: input.db, + companyId, + evidence: input.evidence, + sourceIssue: input.sourceIssue, + rescueBranch, + rescueCommitSha, + fileCount, + heartbeatRunId: input.heartbeatRunId, + }); + await logDirtyQuarantineActivity({ + db: input.db, + companyId, + evidence: input.evidence, + rescueBranch, + rescueCommitSha, + fileCount, + heartbeatRunId: input.heartbeatRunId, + sourceAuditCommentId: comments.sourceAuditCommentId, + claimantAuditCommentId: comments.claimantAuditCommentId, + }); + await recordDirtyQuarantineOperation({ + recorder: input.recorder, + phase: input.phase, + cwd: input.worktreePath, + evidence: input.evidence, + rescueBranch, + rescueCommitSha, + fileCount, + sourceAuditCommentId: comments.sourceAuditCommentId, + claimantAuditCommentId: comments.claimantAuditCommentId, + }); + return { + rescueBranch, + rescueCommitSha, + fileCount, + clearedInProgressOperation, + ...comments, + }; + } catch (error) { + if (rescueBranchCreated && !expectedBranchRestored) { + await runGit(["checkout", input.expectedBranchName], input.worktreePath).catch(() => null); + } + if (error instanceof WorkspaceRuntimeValidationFailure) throw error; + input.evidence.safeRepair.succeeded = false; + input.evidence.safeRepair.reason = formatDirtyQuarantineFailure(error); + throw branchIncoherenceValidationFailure(input.evidence); + } +} + async function recordForwardBranchReconcileOperation(input: { recorder?: WorkspaceOperationRecorder | null; phase?: "worktree_prepare" | "workspace_finalize"; @@ -1047,6 +1660,7 @@ export async function ensureGitWorktreeBranchCoherent(input: { actualBranchName?: string | null; heartbeatRunId?: string | null; enableWorkspaceBranchReconcileForward?: boolean; + enableWorkspaceDirtyQuarantineRepair?: boolean; persistForwardReconcile?: boolean; reconcileOperationPhase?: "worktree_prepare" | "workspace_finalize"; recorder?: WorkspaceOperationRecorder | null; @@ -1062,6 +1676,7 @@ export async function ensureGitWorktreeBranchCoherent(input: { } const evidence = await inspectGitWorktreeBranchIncoherence({ + db: input.db ?? null, repoRoot: input.repoRoot, worktreePath: input.worktreePath, expectedBranchName, @@ -1070,9 +1685,63 @@ export async function ensureGitWorktreeBranchCoherent(input: { executionWorkspaceId: input.executionWorkspaceId ?? null, }); + if (evidence.cleanliness === "dirty" && input.enableWorkspaceDirtyQuarantineRepair === true) { + if (!input.db) { + evidence.safeRepair.reason = "dirty quarantine repair requires database access for claimant checks and audit"; + throw branchIncoherenceValidationFailure(evidence); + } + if (!evidence.provenance.registeredPathFound) { + evidence.safeRepair.reason = "dirty quarantine repair requires a registered git worktree path"; + throw branchIncoherenceValidationFailure(evidence); + } + if (!evidence.provenance.expectedBranchExists) { + evidence.safeRepair.reason = "dirty quarantine repair requires the recorded branch to exist"; + throw branchIncoherenceValidationFailure(evidence); + } + if (evidence.contention) { + evidence.safeRepair.eligible = false; + evidence.safeRepair.reason = formatDirtyQuarantineContentionRefusal(evidence.contention); + throw branchIncoherenceValidationFailure(evidence); + } + await assertDirtyQuarantineRuntimeServicesStopped({ + db: input.db, + executionWorkspaceId: input.executionWorkspaceId ?? null, + evidence, + }); + evidence.safeRepair.eligible = true; + evidence.safeRepair.attempted = true; + evidence.safeRepair.reason = "dirty worktree can be quarantined on a rescue branch before restoring the recorded branch"; + const result = await quarantineDirtyWorktreeBranchIncoherence({ + db: input.db, + repoRoot: input.repoRoot, + worktreePath: input.worktreePath, + expectedBranchName, + sourceIssue: input.sourceIssue, + executionWorkspaceId: input.executionWorkspaceId ?? null, + heartbeatRunId: input.heartbeatRunId ?? null, + evidence, + phase: input.reconcileOperationPhase, + recorder: input.recorder ?? null, + }); + evidence.safeRepair.succeeded = true; + evidence.safeRepair.reason = result.clearedInProgressOperation + ? `dirty worktree quarantined on ${result.rescueBranch} at ${formatShortSha(result.rescueCommitSha)}; interrupted git ${GIT_IN_PROGRESS_OPERATION_LABELS[result.clearedInProgressOperation]} state cleared` + : `dirty worktree quarantined on ${result.rescueBranch} at ${formatShortSha(result.rescueCommitSha)}`; + return { + branchName: expectedBranchName, + reconciledForward: false, + dirtyQuarantineRepair: result, + warnings: [ + `Execution workspace dirty worktree state was quarantined on rescue branch "${result.rescueBranch}" (${formatShortSha(result.rescueCommitSha)}; ${result.fileCount} ${result.fileCount === 1 ? "file" : "files"}) before restoring recorded branch "${expectedBranchName}".${result.clearedInProgressOperation ? ` An interrupted git ${GIT_IN_PROGRESS_OPERATION_LABELS[result.clearedInProgressOperation]} was also cleared; its in-flight state is preserved on the rescue branch.` : ""}`, + ], + }; + } + if ( input.enableWorkspaceBranchReconcileForward === true && evidence.provenance.ancestryVerdict === "ancestor" && + !evidence.provenance.sameHead && + evidence.cleanliness === "clean" && currentBranch ) { const reason = "Automatic forward reconciliation: recorded branch is an ancestor of the checked-out branch."; @@ -1747,7 +2416,7 @@ async function runWorkspaceCommand(input: { async function recordGitOperation( recorder: WorkspaceOperationRecorder | null | undefined, input: { - phase: "worktree_prepare" | "worktree_cleanup"; + phase: WorkspaceOperationPhase; args: string[]; cwd: string; metadata?: Record | null; @@ -1970,6 +2639,7 @@ export async function realizeExecutionWorkspace(input: { agent: ExecutionWorkspaceAgentRef; heartbeatRunId?: string | null; enableWorkspaceBranchReconcileForward?: boolean; + enableWorkspaceDirtyQuarantineRepair?: boolean; recorder?: WorkspaceOperationRecorder | null; }): Promise { const rawStrategy = parseObject(input.config.workspaceStrategy); @@ -2100,6 +2770,7 @@ export async function realizeExecutionWorkspace(input: { executionWorkspaceId: null, heartbeatRunId: input.heartbeatRunId ?? null, enableWorkspaceBranchReconcileForward: input.enableWorkspaceBranchReconcileForward === true, + enableWorkspaceDirtyQuarantineRepair: input.enableWorkspaceDirtyQuarantineRepair === true, reconcileOperationPhase: "worktree_prepare", recorder: input.recorder ?? null, }); @@ -2240,6 +2911,7 @@ export async function ensurePersistedExecutionWorkspaceAvailable(input: { agent: ExecutionWorkspaceAgentRef; heartbeatRunId?: string | null; enableWorkspaceBranchReconcileForward?: boolean; + enableWorkspaceDirtyQuarantineRepair?: boolean; recorder?: WorkspaceOperationRecorder | null; }): Promise { const cwd = asString(input.workspace.cwd ?? input.workspace.providerRef, "").trim(); @@ -2285,6 +2957,7 @@ export async function ensurePersistedExecutionWorkspaceAvailable(input: { executionWorkspaceId: input.workspace.id ?? null, heartbeatRunId: input.heartbeatRunId ?? null, enableWorkspaceBranchReconcileForward: input.enableWorkspaceBranchReconcileForward === true, + enableWorkspaceDirtyQuarantineRepair: input.enableWorkspaceDirtyQuarantineRepair === true, persistForwardReconcile: false, reconcileOperationPhase: "worktree_prepare", recorder: input.recorder ?? null, diff --git a/ui/src/api/execution-workspaces.ts b/ui/src/api/execution-workspaces.ts index 5ca8c49f37..e6c075a86d 100644 --- a/ui/src/api/execution-workspaces.ts +++ b/ui/src/api/execution-workspaces.ts @@ -129,7 +129,11 @@ export const executionWorkspacesApi = { * never trusted); no `reason` needed. * - `mode: "override"` — audited break-glass; the server rejects agent actors, re-checks * `runtime:manage` permission, and requires a non-empty operator `reason`. + * - `mode: "quarantine_restore"` — lossless dirty-worktree repair; the server quarantines the + * dirty changes onto a rescue branch and restores the recorded branch. No `reason` needed. */ - reconcile: (id: string, body: { mode: "forward" } | { mode: "override"; reason: string }) => - api.post(`/execution-workspaces/${id}/reconcile-branch`, body), + reconcile: ( + id: string, + body: { mode: "forward" } | { mode: "override"; reason: string } | { mode: "quarantine_restore" }, + ) => api.post(`/execution-workspaces/${id}/reconcile-branch`, body), }; diff --git a/ui/src/components/IssueChatThread.tsx b/ui/src/components/IssueChatThread.tsx index 11b5f0e4f9..1b7f5ebf41 100644 --- a/ui/src/components/IssueChatThread.tsx +++ b/ui/src/components/IssueChatThread.tsx @@ -433,6 +433,8 @@ interface IssueChatThreadProps { reissueIsolatedRecoveryActionPending?: boolean; onReconcileForwardRecoveryAction?: () => void; onBreakGlassOverrideRecoveryAction?: (reason: string) => void; + onQuarantineRestoreRecoveryAction?: () => void; + quarantineRestoreRecoveryActionPending?: boolean; canBreakGlassRecoveryAction?: boolean; reconcileRecoveryActionPending?: boolean; canFalsePositiveRecoveryAction?: boolean; @@ -4176,6 +4178,8 @@ export function IssueChatThread({ reissueIsolatedRecoveryActionPending = false, onReconcileForwardRecoveryAction, onBreakGlassOverrideRecoveryAction, + onQuarantineRestoreRecoveryAction, + quarantineRestoreRecoveryActionPending = false, canBreakGlassRecoveryAction = false, reconcileRecoveryActionPending = false, canFalsePositiveRecoveryAction = false, @@ -4895,6 +4899,8 @@ export function IssueChatThread({ reissuePending={reissueIsolatedRecoveryActionPending} onReconcileForward={onReconcileForwardRecoveryAction} onBreakGlassOverride={onBreakGlassOverrideRecoveryAction} + onQuarantineRestore={onQuarantineRestoreRecoveryAction} + quarantineRestorePending={quarantineRestoreRecoveryActionPending} canBreakGlass={canBreakGlassRecoveryAction} reconcilePending={reconcileRecoveryActionPending} canFalsePositive={canFalsePositiveRecoveryAction} diff --git a/ui/src/components/IssueRecoveryActionCard.test.tsx b/ui/src/components/IssueRecoveryActionCard.test.tsx index e1c6cd665b..24efe9c8c2 100644 --- a/ui/src/components/IssueRecoveryActionCard.test.tsx +++ b/ui/src/components/IssueRecoveryActionCard.test.tsx @@ -502,3 +502,163 @@ describe("IssueRecoveryActionCard W7 reconcile actions", () => { expect(node.querySelector("[data-testid='recovery-action-breakglass-trigger']")).toBeNull(); }); }); + +function buildDirtyDivergenceAction( + overrides: { + action?: Partial; + provenance?: Record; + workspaceValidation?: Record; + } = {}, +): IssueRecoveryAction { + return buildWorkspaceValidationAction({ + ...overrides, + workspaceValidation: { + cleanliness: "dirty", + statusEntryCount: 3, + dirtyPathSample: ["src/app.ts", "README.md"], + sourceIdentifier: "PAP-1405", + ...overrides.workspaceValidation, + }, + }); +} + +describe("IssueRecoveryActionCard repair workspace (quarantine_restore)", () => { + it("offers the repair action only for a dirty divergence", () => { + const cleanNode = render( + {}} + />, + ); + expect(cleanNode.querySelector("[data-testid='recovery-action-repair-trigger']")).toBeNull(); + + const dirtyNode = render( + {}} />, + ); + expect(dirtyNode.querySelector("[data-testid='recovery-action-repair-trigger']")).not.toBeNull(); + }); + + it("does not offer the repair action without a handler or for non-workspace kinds", () => { + const noHandler = render(); + expect(noHandler.querySelector("[data-testid='recovery-action-repair-trigger']")).toBeNull(); + + const nonWorkspace = render( + {}} />, + ); + expect(nonWorkspace.querySelector("[data-testid='recovery-action-repair-trigger']")).toBeNull(); + }); + + it("confirm popover restates the dirty count, live branch, rescue branch and recorded branch, then fires the handler", () => { + const onQuarantineRestore = vi.fn(); + const node = render( + , + ); + click(node.querySelector("[data-testid='recovery-action-repair-trigger']")); + const restated = document.body.querySelector("[data-testid='recovery-repair-restated']"); + const text = restated?.textContent ?? ""; + expect(text).toContain("3 uncommitted changes"); + // live branch is explicitly left untouched + expect(text).toContain("nleach/PAP-1405-live"); + expect(text).toContain("left untouched"); + // rescue branch preview mirrors the server naming (prefix + timestamp marker) + expect( + document.body.querySelector("[data-testid='recovery-repair-rescue-branch']")?.textContent, + ).toContain("paperclip/rescue/PAP-1405/"); + // recorded branch to be restored + expect(text).toContain("PAP-522-recorded"); + + // No reason field is present — the operation is lossless. + expect(document.body.querySelector("textarea")).toBeNull(); + + click(document.body.querySelector("[data-testid='recovery-action-repair-confirm']")); + expect(onQuarantineRestore).toHaveBeenCalledTimes(1); + }); + + it("singularizes the dirty change count", () => { + const node = render( + {}} + />, + ); + click(node.querySelector("[data-testid='recovery-action-repair-trigger']")); + expect( + document.body.querySelector("[data-testid='recovery-repair-dirty-count']")?.textContent, + ).toBe("1 uncommitted change"); + }); + + it("disables the repair trigger while a quarantine-restore is pending", () => { + const node = render( + {}} + quarantineRestorePending + />, + ); + expect( + node.querySelector("[data-testid='recovery-action-repair-trigger']")?.disabled, + ).toBe(true); + }); + + it("in the contended case disables repair, explains the claimant, and recommends re-issue", () => { + const onQuarantineRestore = vi.fn(); + const node = render( + {}} + />, + ); + // Diagnosis gains a claimant line naming the issue + active run. + const notice = node.querySelector("[data-testid='recovery-contention-notice']"); + expect(notice?.textContent).toContain("PAP-9001"); + expect(notice?.textContent).toContain("active run"); + + // The repair control is present but disabled, with the claimant as the explanation. + const disabled = node.querySelector("[data-testid='recovery-action-repair-disabled']"); + expect(disabled).not.toBeNull(); + const trigger = disabled?.querySelector( + "[data-testid='recovery-action-repair-trigger']", + ); + expect(trigger?.disabled).toBe(true); + expect(disabled?.textContent).toContain("PAP-9001"); + // Clicking the disabled control never fires the repair. + click(trigger ?? null); + expect(onQuarantineRestore).not.toHaveBeenCalled(); + + // Re-issue is surfaced as the recommended action. + expect(node.querySelector("[data-testid='recovery-reissue-recommended']")).not.toBeNull(); + expect( + node + .querySelector("[data-testid='recovery-action-reissue-trigger']") + ?.getAttribute("data-recommended"), + ).toBe("true"); + }); + + it("compact variant drops the metadata table but keeps the diagnosis and repair action", () => { + const node = render( + {}} + variant="compact" + />, + ); + // Metadata rows (e.g. the Owner/Next action table) are dropped in compact mode. + expect(node.textContent).not.toContain("Choose and record a valid issue disposition."); + // The divergence diagnosis and repair action still render. + expect(node.querySelector("[data-testid='recovery-divergence-diagnosis']")).not.toBeNull(); + expect(node.querySelector("[data-testid='recovery-action-repair-trigger']")).not.toBeNull(); + }); +}); diff --git a/ui/src/components/IssueRecoveryActionCard.tsx b/ui/src/components/IssueRecoveryActionCard.tsx index adb4ef8152..45d3320087 100644 --- a/ui/src/components/IssueRecoveryActionCard.tsx +++ b/ui/src/components/IssueRecoveryActionCard.tsx @@ -12,10 +12,12 @@ import { GitBranch, GitBranchPlus, Loader2, + Lock, OctagonAlert, RefreshCw, Sparkles, TriangleAlert, + Wrench, } from "lucide-react"; import { Link } from "@/lib/router"; import { Button } from "@/components/ui/button"; @@ -90,10 +92,25 @@ export interface IssueRecoveryActionCardProps { * not rendered at all — a non-permitted user never sees the "reconcile anyway" affordance. */ canBreakGlass?: boolean; - /** Whether a reconcile (forward or override) is currently in flight (disables both actions). */ + /** + * Handler for the lossless repair — "Repair workspace — quarantine changes & restore branch" + * (workspace_validation only). Rendered only for a *dirty* divergence; the caller invokes the S4 + * reconcile op in `quarantine_restore` mode, which quarantines the dirty worktree onto a rescue + * branch and restores the recorded branch. If omitted, the repair action is not shown. + */ + onQuarantineRestore?: () => void; + /** Whether a quarantine-restore repair is currently in flight (shares the reconcile spinner). */ + quarantineRestorePending?: boolean; + /** Whether a reconcile (forward, override, or quarantine-restore) is currently in flight. */ reconcilePending?: boolean; /** Whether the viewer can run destructive board-only actions (e.g. false-positive dismissal). */ canFalsePositive?: boolean; + /** + * Rendering density. `full` (default) shows the complete metadata table; `compact` drops the + * metadata rows for embedding beside a run on the agent run page, keeping the header, divergence + * diagnosis, and action footer. + */ + variant?: "full" | "compact"; className?: string; } @@ -248,6 +265,13 @@ function formatShortSha(sha: string | null): string | null { * live ("actual"/checked-out) branch, both HEAD shas, and a server-computed ancestry verdict + * plain-language explanation of why the run was declined. */ +interface WorkspaceContention { + claimedByIssueId: string | null; + claimedByIssueIdentifier: string | null; + /** True when the claiming workspace has a queued/running run (not just a stale claim). */ + hasActiveRun: boolean; +} + interface WorkspaceDivergence { expectedBranch: string | null; liveBranch: string | null; @@ -256,10 +280,60 @@ interface WorkspaceDivergence { ancestryVerdict: GitWorktreeBranchAncestryVerdict | null; plainLanguageReason: string | null; cleanliness: "clean" | "dirty" | "unknown" | null; + /** Number of dirty (uncommitted) status entries in the live worktree, when known. */ + dirtyFileCount: number | null; + /** Sample of dirty paths (already truncated server-side) for the confirm step. */ + dirtyPathSample: string[]; + /** + * Another workspace is holding the live branch. When present, the lossless quarantine repair is + * refused server-side — re-issuing on an isolated workspace is the recommended path instead. + */ + contention: WorkspaceContention | null; + /** + * Preview of the rescue branch the quarantine repair will create. The server appends a UTC + * timestamp at repair time, so this is the stable prefix only (rendered with a trailing marker). + */ + rescueBranchPreview: string; /** Ref a re-issue should base off — the live branch when known, else the live HEAD sha. */ reissueBaseRef: string | null; } +/** Mirrors the server's `sanitizeBranchName` for a faithful rescue-branch preview. */ +function sanitizeBranchComponent(value: string): string { + return ( + value + .trim() + .replace(/[^A-Za-z0-9._/-]+/g, "-") + .replace(/-+/g, "-") + .replace(/^[-/.]+|[-/.]+$/g, "") + .slice(0, 120) || "issue" + ); +} + +function buildRescueBranchPreview(sourceIdentifier: string | null): string { + return `paperclip/rescue/${sanitizeBranchComponent(sourceIdentifier ?? "issue")}/`; +} + +function asStringArray(value: unknown): string[] { + if (!Array.isArray(value)) return []; + return value.filter((entry): entry is string => typeof entry === "string" && entry.trim().length > 0); +} + +function asNonNegativeInt(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) && value >= 0 ? Math.floor(value) : null; +} + +function readContention(value: unknown): WorkspaceContention | null { + const record = asRecord(value); + if (!record) return null; + const activeRun = asRecord(record.activeRun); + return { + claimedByIssueId: asNonEmptyString(record.claimedByIssueId), + claimedByIssueIdentifier: asNonEmptyString(record.claimedByIssueIdentifier), + hasActiveRun: activeRun !== null, + }; +} + function readWorkspaceDivergence(action: IssueRecoveryAction): WorkspaceDivergence | null { if (action.kind !== "workspace_validation") return null; const workspaceValidation = asRecord(action.evidence?.workspaceValidation); @@ -275,6 +349,7 @@ function readWorkspaceDivergence(action: IssueRecoveryAction): WorkspaceDivergen cleanlinessRaw === "clean" || cleanlinessRaw === "dirty" || cleanlinessRaw === "unknown" ? cleanlinessRaw : null; + const sourceIdentifier = asNonEmptyString(workspaceValidation.sourceIdentifier); return { expectedBranch, liveBranch, @@ -283,6 +358,10 @@ function readWorkspaceDivergence(action: IssueRecoveryAction): WorkspaceDivergen ancestryVerdict: asAncestryVerdict(provenance.ancestryVerdict), plainLanguageReason: asNonEmptyString(provenance.plainLanguageReason), cleanliness, + dirtyFileCount: asNonNegativeInt(workspaceValidation.statusEntryCount), + dirtyPathSample: asStringArray(workspaceValidation.dirtyPathSample), + contention: readContention(workspaceValidation.contention), + rescueBranchPreview: buildRescueBranchPreview(sourceIdentifier), reissueBaseRef: liveBranch ?? liveHeadSha, }; } @@ -380,10 +459,31 @@ function DivergenceDiagnosis({ {divergence.plainLanguageReason ? (

{divergence.plainLanguageReason}

) : null} + {divergence.contention ? ( +

+ + + Worktree claimed by{" "} + {contentionLabel(divergence.contention)}{" "} + {divergence.contention.hasActiveRun ? "(active run)" : "(claim held)"} — the lossless repair + can't run while another workspace holds the live branch. + +

+ ) : null} ); } +function contentionLabel(contention: WorkspaceContention): string { + return ( + contention.claimedByIssueIdentifier ?? + (contention.claimedByIssueId ? `issue ${contention.claimedByIssueId.slice(0, 8)}` : "another task") + ); +} + /** * Action 2 — the audited break-glass override. Gated by an explicit confirm step that *restates the * divergence* (both branches + short SHAs + ancestry verdict) and a required, non-empty reason: the @@ -498,6 +598,138 @@ function BreakGlassOverride({ ); } +/** + * The lossless repair — quarantine the dirty worktree onto a rescue branch, then restore the + * recorded branch. Unlike break-glass, this is *non-destructive* (no work is lost, so no reason is + * required): the confirm popover simply restates what will happen — the dirty file count, that the + * live branch is left untouched, the rescue branch that will hold the changes, and the recorded + * branch to be restored. Disabled (with an inline explanation, no popover) when the live branch is + * contended by another workspace, since the server refuses the repair in that case. + */ +function RepairWorkspace({ + divergence, + onConfirm, + pending, + disabled, + disabledReason, +}: { + divergence: WorkspaceDivergence; + onConfirm: () => void; + pending: boolean; + disabled: boolean; + disabledReason: string | null; +}) { + const dirtyCount = divergence.dirtyFileCount; + const dirtyLabel = + dirtyCount === null + ? "Uncommitted changes" + : `${dirtyCount} uncommitted ${dirtyCount === 1 ? "change" : "changes"}`; + const trigger = ( + + ); + if (disabled) { + // Contended: the server refuses the repair, so render a plainly disabled control with the reason + // inline rather than a popover the operator can't act on. + return ( +
+ {trigger} + {disabledReason ? ( + + {disabledReason} + + ) : null} +
+ ); + } + return ( + + {trigger} + +
+
+ + Repair workspace +
+

+ This is lossless — no reason required. Your uncommitted changes are committed onto a fresh + rescue branch, then the recorded branch is restored so the task can resume. The live branch + is left exactly as it is. +

+
+
+
+
Dirty changes
+
+ {dirtyLabel} +
+
+
+
Live branch
+
+ {divergence.liveBranch ?? "detached"} + (left untouched) +
+
+
+
Rescue branch
+
+ {divergence.rescueBranchPreview} + <timestamp> +
+
+
+
Restore to
+
+ {divergence.expectedBranch ?? "recorded branch"} +
+
+
+ + + + ); +} + function readWakePolicySummary(action: IssueRecoveryAction): string | null { const policy = action.wakePolicy; if (!policy) return null; @@ -671,9 +903,12 @@ export function IssueRecoveryActionCard({ reissuePending = false, onReconcileForward, onBreakGlassOverride, + onQuarantineRestore, + quarantineRestorePending = false, canBreakGlass = false, reconcilePending = false, canFalsePositive = false, + variant = "full", className, }: IssueRecoveryActionCardProps) { const cardState: RecoveryCardCardState = forcedState ?? deriveRecoveryCardState(action); @@ -741,8 +976,27 @@ export function IssueRecoveryActionCard({ cardState !== "resolved" && divergence !== null && canBreakGlass; + // The lossless repair — offered only for a *dirty* divergence (a clean one reconciles forward or + // via break-glass, with nothing to quarantine). Disabled when the live branch is contended by an + // active claimant, since the server refuses `quarantine_restore` in that case. + const repairContention = divergence?.contention ?? null; + const showRepairAction = + onQuarantineRestore !== undefined && + cardState !== "resolved" && + divergence !== null && + divergence.cleanliness === "dirty"; + const repairDisabledReason = repairContention + ? `Held by ${contentionLabel(repairContention)} — re-issue on an isolated workspace instead.` + : null; + // When contended, the re-issue is the recommended path, so it takes the primary emphasis and a + // "Recommended" hint while the repair button is disabled. + const reissueRecommended = showRepairAction && repairContention !== null; const showFooter = - showResolveActions || showReissueAction || showReconcileForward || showBreakGlass; + showResolveActions || + showReissueAction || + showReconcileForward || + showBreakGlass || + showRepairAction; return (
{headline}

+ {variant === "compact" ? null : (
@@ -852,6 +1107,7 @@ export function IssueRecoveryActionCard({ ) : null}
+ )} {divergence ? : null} {showFooter ? (
@@ -913,15 +1169,25 @@ export function IssueRecoveryActionCard({ Reconcile forward & continue ) : null} + {showRepairAction && divergence ? ( + onQuarantineRestore?.()} + /> + ) : null} {showReissueAction && divergence && reissueBaseRef ? ( diff --git a/ui/src/components/RunWorkspaceRecoverySurface.test.tsx b/ui/src/components/RunWorkspaceRecoverySurface.test.tsx new file mode 100644 index 0000000000..10aa936f0f --- /dev/null +++ b/ui/src/components/RunWorkspaceRecoverySurface.test.tsx @@ -0,0 +1,217 @@ +// @vitest-environment jsdom + +import type { ComponentProps, ReactNode } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { flushSync } from "react-dom"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import type { HeartbeatRun, Issue, IssueRecoveryAction } from "@paperclipai/shared"; +import { RunWorkspaceRecoverySurface } from "./RunWorkspaceRecoverySurface"; +import { ToastProvider } from "../context/ToastContext"; + +const navigateMock = vi.hoisted(() => vi.fn()); +const issueGetMock = vi.hoisted(() => vi.fn()); +const issueCreateMock = vi.hoisted(() => vi.fn()); +const resolveRecoveryMock = vi.hoisted(() => vi.fn()); +const reconcileMock = vi.hoisted(() => vi.fn()); +const boardAccessMock = vi.hoisted(() => vi.fn()); + +vi.mock("@/lib/router", () => ({ + useNavigate: () => navigateMock, + Link: ({ children, to, ...props }: { children: ReactNode; to: string } & ComponentProps<"a">) => ( + {children} + ), +})); + +vi.mock("../api/issues", () => ({ + issuesApi: { + get: issueGetMock, + create: issueCreateMock, + resolveRecoveryAction: resolveRecoveryMock, + }, +})); + +vi.mock("../api/execution-workspaces", () => ({ + executionWorkspacesApi: { + reconcile: reconcileMock, + }, +})); + +vi.mock("../api/access", () => ({ + accessApi: { + getCurrentBoardAccess: boardAccessMock, + }, +})); + +(globalThis as unknown as { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = true; + +// flushSync-based sync `act` shim (matches the repo pattern in IssueRecoveryActionCard.test.tsx). +function act(callback: () => void): void { + flushSync(callback); +} + +// Resolve pending mocked-promise micro/macrotasks, then flush the React state updates react-query +// queued, repeating so chained query → render → dependent-render settles fully. +async function flush() { + for (let i = 0; i < 6; i += 1) { + for (let j = 0; j < 4; j += 1) await Promise.resolve(); + await new Promise((resolve) => setTimeout(resolve, 0)); + act(() => {}); + } +} + +let container: HTMLDivElement | null = null; +let root: Root | null = null; + +afterEach(() => { + if (root) act(() => root?.unmount()); + root = null; + container?.remove(); + container = null; + vi.clearAllMocks(); +}); + +function buildRun(overrides: Partial = {}): HeartbeatRun { + return { + id: "run-aaaa1111", + companyId: "company-1", + agentId: "agent-1", + status: "failed", + errorCode: "workspace_validation_failed", + contextSnapshot: { issueId: "issue-1" }, + resultJson: null, + contextSnapshotJson: null, + ...overrides, + } as unknown as HeartbeatRun; +} + +function buildRecoveryAction(): IssueRecoveryAction { + return { + id: "action-1", + companyId: "company-1", + sourceIssueId: "issue-1", + recoveryIssueId: null, + kind: "workspace_validation", + status: "active", + ownerType: "board", + ownerAgentId: null, + ownerUserId: null, + previousOwnerAgentId: null, + returnOwnerAgentId: null, + cause: "workspace_validation_failed", + fingerprint: "fp", + evidence: { + workspaceValidation: { + reason: "git_worktree_branch_incoherence", + expectedBranch: "PAP-1405-recorded", + actualBranch: "live-branch", + cleanliness: "dirty", + statusEntryCount: 2, + dirtyPathSample: ["a.ts"], + sourceIdentifier: "PAP-1405", + persistedExecutionWorkspaceId: "ws-1", + provenance: { + expectedHeadSha: "aaaa1111bbbb", + actualHeadSha: "cccc2222dddd", + ancestryVerdict: "diverged", + plainLanguageReason: "Recorded branch is not an ancestor of the live branch.", + }, + }, + }, + nextAction: "Repair the workspace.", + wakePolicy: null, + monitorPolicy: null, + attemptCount: 1, + maxAttempts: 3, + timeoutAt: null, + lastAttemptAt: null, + outcome: null, + resolutionNote: null, + resolvedAt: null, + createdAt: "2026-07-09T00:00:00.000Z", + updatedAt: "2026-07-09T00:00:00.000Z", + }; +} + +function buildIssue(action: IssueRecoveryAction | null): Issue { + return { + id: "issue-1", + identifier: "PAP-1405", + companyId: "company-1", + title: "Do the thing", + description: "body", + priority: "medium", + projectId: null, + parentId: null, + assigneeAgentId: null, + executionWorkspaceId: "ws-1", + activeRecoveryAction: action, + } as unknown as Issue; +} + +async function renderSurface(run: HeartbeatRun) { + const client = new QueryClient({ + defaultOptions: { queries: { retry: false }, mutations: { retry: false } }, + }); + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); + act(() => { + root?.render( + + + + + , + ); + }); + // Let the issue + board-access queries settle and commit. + await flush(); + return container!; +} + +function click(element: Element | null) { + if (!element) throw new Error("Expected element to exist"); + act(() => { + element.dispatchEvent(new MouseEvent("click", { bubbles: true })); + }); +} + +describe("RunWorkspaceRecoverySurface", () => { + it("renders nothing for a run that is not a workspace-validation failure", async () => { + issueGetMock.mockResolvedValue(buildIssue(buildRecoveryAction())); + boardAccessMock.mockResolvedValue(undefined); + const node = await renderSurface(buildRun({ errorCode: "process_lost" })); + expect(node.querySelector("[data-testid='run-workspace-recovery-surface']")).toBeNull(); + expect(issueGetMock).not.toHaveBeenCalled(); + }); + + it("renders nothing when the source issue has no workspace_validation recovery action", async () => { + issueGetMock.mockResolvedValue(buildIssue(null)); + boardAccessMock.mockResolvedValue(undefined); + const node = await renderSurface(buildRun()); + expect(node.querySelector("[data-testid='run-workspace-recovery-surface']")).toBeNull(); + }); + + it("renders the compact recovery card and repair action for a dirty divergence", async () => { + issueGetMock.mockResolvedValue(buildIssue(buildRecoveryAction())); + boardAccessMock.mockResolvedValue({ source: "local_implicit", companyIds: ["company-1"] }); + const node = await renderSurface(buildRun()); + expect(node.querySelector("[data-testid='run-workspace-recovery-surface']")).not.toBeNull(); + expect(node.querySelector("[data-testid='recovery-divergence-diagnosis']")).not.toBeNull(); + expect(node.querySelector("[data-testid='recovery-action-repair-trigger']")).not.toBeNull(); + // Compact: metadata rows dropped. + expect(node.textContent).not.toContain("Repair the workspace."); + }); + + it("wires the repair confirm to reconcile in quarantine_restore mode against the pinned workspace", async () => { + issueGetMock.mockResolvedValue(buildIssue(buildRecoveryAction())); + boardAccessMock.mockResolvedValue({ source: "local_implicit", companyIds: ["company-1"] }); + reconcileMock.mockResolvedValue({ id: "ws-1" }); + const node = await renderSurface(buildRun()); + click(node.querySelector("[data-testid='recovery-action-repair-trigger']")); + click(document.body.querySelector("[data-testid='recovery-action-repair-confirm']")); + await flush(); + expect(reconcileMock).toHaveBeenCalledWith("ws-1", { mode: "quarantine_restore" }); + }); +}); diff --git a/ui/src/components/RunWorkspaceRecoverySurface.tsx b/ui/src/components/RunWorkspaceRecoverySurface.tsx new file mode 100644 index 0000000000..bfae52e276 --- /dev/null +++ b/ui/src/components/RunWorkspaceRecoverySurface.tsx @@ -0,0 +1,289 @@ +import { useCallback } from "react"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import type { HeartbeatRun } from "@paperclipai/shared"; +import { useNavigate } from "@/lib/router"; +import { issuesApi } from "../api/issues"; +import { executionWorkspacesApi } from "../api/execution-workspaces"; +import { accessApi } from "../api/access"; +import { queryKeys } from "../lib/queryKeys"; +import { useToastActions } from "../context/ToastContext"; +import { + IssueRecoveryActionCard, + type RecoveryReissueRequest, + type RecoveryResolveOutcome, +} from "./IssueRecoveryActionCard"; +import { + canBoardManageRuntime, + readRecoveryReconcileWorkspaceId, +} from "../lib/recovery-reconcile"; + +/** The run errorCode Paperclip stamps when it declines a run over a git workspace it can't validate. */ +export const WORKSPACE_VALIDATION_RUN_ERROR_CODE = "workspace_validation_failed"; + +function asRecord(value: unknown): Record | null { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : null; +} + +function asNonEmptyString(value: unknown): string | null { + if (typeof value !== "string") return null; + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : null; +} + +/** + * Reads the source issue id a failed run was working when it was declined over workspace validation. + * The run's context snapshot pins the issue the recovery action lives on. + */ +function readRunIssueId(run: HeartbeatRun): string | null { + const context = asRecord(run.contextSnapshot); + if (!context) return null; + return asNonEmptyString(context.issueId); +} + +/** + * Run-page recovery surface. When a run *failed* with workspace-validation evidence, this fetches + * the source issue's active recovery action and renders the same `IssueRecoveryActionCard` + * (compact) that `IssueDetail` shows — wired to the same reconcile-forward / repair / re-issue / + * break-glass / resolve handlers — so the divergence can be resolved from the run view directly. + * + * Renders nothing unless the run is a workspace-validation failure whose source issue still carries + * a live `workspace_validation` recovery action. + */ +export function RunWorkspaceRecoverySurface({ run }: { run: HeartbeatRun }) { + const navigate = useNavigate(); + const queryClient = useQueryClient(); + const { pushToast } = useToastActions(); + + const isWorkspaceValidationFailure = + run.status === "failed" && run.errorCode === WORKSPACE_VALIDATION_RUN_ERROR_CODE; + const issueId = readRunIssueId(run); + + const { data: issue } = useQuery({ + queryKey: queryKeys.issues.detail(issueId ?? "__none__"), + queryFn: () => issuesApi.get(issueId!), + enabled: Boolean(isWorkspaceValidationFailure && issueId), + }); + + const { data: boardAccess } = useQuery({ + queryKey: queryKeys.access.currentBoardAccess, + queryFn: () => accessApi.getCurrentBoardAccess(), + enabled: Boolean(isWorkspaceValidationFailure && issueId), + retry: false, + }); + + const recoveryAction = issue?.activeRecoveryAction ?? null; + const canManageBoardRuntime = canBoardManageRuntime(run.companyId, boardAccess); + // Prefer the workspace pinned by the recovery action's evidence (the workspace that actually + // diverged) over the page-level id, which can drift after a re-issue rebinds the issue. + const reconcileWorkspaceId = + readRecoveryReconcileWorkspaceId(recoveryAction) ?? issue?.executionWorkspaceId ?? null; + + const invalidate = useCallback(() => { + if (issueId) { + queryClient.invalidateQueries({ queryKey: queryKeys.issues.detail(issueId) }); + } + queryClient.invalidateQueries({ queryKey: queryKeys.runIssues(run.id) }); + queryClient.invalidateQueries({ queryKey: queryKeys.runDetail(run.id) }); + }, [issueId, queryClient, run.id]); + + const reconcile = useMutation({ + mutationFn: ( + input: + | { workspaceId: string; mode: "forward" } + | { workspaceId: string; mode: "override"; reason: string } + | { workspaceId: string; mode: "quarantine_restore" }, + ) => { + const { workspaceId, ...body } = input; + return executionWorkspacesApi.reconcile(workspaceId, body); + }, + onSuccess: (_result, variables) => { + invalidate(); + pushToast( + variables.mode === "quarantine_restore" + ? { + title: "Workspace repaired", + body: "Dirty changes were quarantined onto a rescue branch and the recorded branch restored; the task will resume.", + tone: "success", + } + : { + title: "Workspace branch reconciled", + body: "The recorded branch now matches the live branch; the task will resume.", + tone: "success", + }, + ); + }, + onError: (err) => { + pushToast({ + title: "Reconcile failed", + body: err instanceof Error ? err.message : "Unable to reconcile the workspace branch.", + tone: "error", + }); + }, + }); + + const reissue = useMutation({ + mutationFn: async (request: RecoveryReissueRequest) => { + if (!issue) throw new Error("Task is not loaded yet."); + const sourceLabel = issue.identifier ?? "the stalled task"; + const descriptionLines = [ + `Re-issued from ${sourceLabel} on an isolated git worktree after a workspace branch divergence.`, + "", + `- Base ref (live branch): \`${request.baseRef}\``, + ...(request.expectedBranch ? [`- Recorded branch: \`${request.expectedBranch}\``] : []), + "", + "---", + "", + issue.description ?? "", + ]; + return issuesApi.create(issue.companyId, { + title: `Re-issue (isolated): ${issue.title ?? sourceLabel}`, + description: descriptionLines.join("\n"), + priority: issue.priority, + projectId: issue.projectId ?? null, + parentId: issue.parentId ?? null, + assigneeAgentId: + recoveryAction?.returnOwnerAgentId ?? + recoveryAction?.previousOwnerAgentId ?? + issue.assigneeAgentId ?? + null, + executionWorkspacePreference: "isolated_workspace", + executionWorkspaceSettings: { + mode: "isolated_workspace", + workspaceStrategy: { type: "git_worktree", baseRef: request.baseRef }, + }, + }); + }, + onSuccess: (created) => { + invalidate(); + pushToast({ + title: "Isolated re-issue created", + body: created.identifier + ? `${created.identifier} will run on a fresh isolated workspace.` + : "A fresh isolated re-issue was created.", + tone: "success", + }); + if (created.identifier) { + navigate(`/issues/${created.identifier}`); + } + }, + onError: (err) => { + pushToast({ + title: "Re-issue failed", + body: err instanceof Error ? err.message : "Unable to create an isolated re-issue.", + tone: "error", + }); + }, + }); + + const resolve = useMutation({ + mutationFn: (data: { + outcome: "restored" | "false_positive"; + sourceIssueStatus: "todo" | "done" | "in_review"; + }) => { + if (!issueId || !recoveryAction) throw new Error("No recovery action to resolve."); + return issuesApi.resolveRecoveryAction(issueId, { + actionId: recoveryAction.id, + outcome: data.outcome, + sourceIssueStatus: data.sourceIssueStatus, + }); + }, + onSuccess: () => { + invalidate(); + }, + onError: (err) => { + pushToast({ + title: "Recovery resolution failed", + body: err instanceof Error ? err.message : "Unable to resolve recovery action", + tone: "error", + }); + }, + }); + + const handleReconcileForward = useCallback(() => { + if (!reconcileWorkspaceId) return; + void reconcile.mutateAsync({ workspaceId: reconcileWorkspaceId, mode: "forward" }); + }, [reconcile, reconcileWorkspaceId]); + + const handleBreakGlass = useCallback( + (reason: string) => { + if (!reconcileWorkspaceId) return; + void reconcile.mutateAsync({ workspaceId: reconcileWorkspaceId, mode: "override", reason }); + }, + [reconcile, reconcileWorkspaceId], + ); + + const handleQuarantineRestore = useCallback(() => { + if (!reconcileWorkspaceId) return; + void reconcile.mutateAsync({ workspaceId: reconcileWorkspaceId, mode: "quarantine_restore" }); + }, [reconcile, reconcileWorkspaceId]); + + const handleReissue = useCallback( + (request: RecoveryReissueRequest) => { + void reissue.mutateAsync(request); + }, + [reissue], + ); + + const handleResolve = useCallback( + (outcome: RecoveryResolveOutcome) => { + switch (outcome) { + case "todo": + void resolve.mutateAsync({ outcome: "restored", sourceIssueStatus: "todo" }); + return; + case "done": + void resolve.mutateAsync({ outcome: "restored", sourceIssueStatus: "done" }); + return; + case "in_review": + void resolve.mutateAsync({ outcome: "restored", sourceIssueStatus: "in_review" }); + return; + case "false_positive_done": + void resolve.mutateAsync({ outcome: "false_positive", sourceIssueStatus: "done" }); + return; + case "false_positive_in_review": + void resolve.mutateAsync({ outcome: "false_positive", sourceIssueStatus: "in_review" }); + return; + } + }, + [resolve], + ); + + if (!isWorkspaceValidationFailure || !issueId) return null; + if (!recoveryAction || recoveryAction.kind !== "workspace_validation") return null; + + return ( + + ); +} + +export default RunWorkspaceRecoverySurface; diff --git a/ui/src/lib/recovery-reconcile.ts b/ui/src/lib/recovery-reconcile.ts new file mode 100644 index 0000000000..db57b8416e --- /dev/null +++ b/ui/src/lib/recovery-reconcile.ts @@ -0,0 +1,62 @@ +import type { IssueRecoveryAction } from "@paperclipai/shared"; +import type { CurrentBoardAccess } from "../api/access"; + +function asRecord(value: unknown): Record | null { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : null; +} + +/** + * Best-effort client mirror of the backend `runtime:manage` gate that the break-glass override + * reconcile (`POST /execution-workspaces/:id/reconcile-branch` in `override` mode) actually + * enforces. The server re-checks `runtime:manage` for every reconcile and is authoritative, so + * this is defense-in-depth: it hides the "reconcile anyway" affordance from viewers rather than + * showing a button that always 403s. For human board members `runtime:manage` grants on the + * same non-viewer, active-membership condition as recovery resolution (see + * `server/src/services/authorization.ts`), so the shape matches; per-permission-key overrides + * are not surfaced to the client and remain the server's call. + */ +export function canBoardManageRuntime( + companyId: string | null | undefined, + boardAccess: CurrentBoardAccess | undefined, +) { + if (!companyId || !boardAccess) return false; + if (boardAccess.source === "local_implicit" || boardAccess.isInstanceAdmin) return true; + if (!boardAccess.memberships || boardAccess.memberships.length === 0) { + return boardAccess.companyIds.includes(companyId); + } + + const membership = boardAccess.memberships.find( + (item) => item.companyId === companyId && item.status === "active", + ); + if (!membership) return false; + return membership.membershipRole !== "viewer" && membership.membershipRole !== null; +} + +/** + * The execution workspace a reconcile action should target. The recovery card is rendered from a + * specific `workspace_validation` recovery action whose evidence pins the workspace that diverged; + * that workspace — not the page-level `issue.executionWorkspaceId` — is the authoritative target. + * The page-level id can drift (e.g. a re-issue rebinds the issue to a new workspace) while the card + * still shows the older action, so we prefer the action's evidence and only fall back to the + * page-level id when the evidence carries no workspace reference. + * + * The branch-incoherence failure (the one that renders the reconcile-forward / break-glass actions) + * records the workspace under `persistedExecutionWorkspaceId`; the not-reusable failure records it + * under `executionWorkspaceId`. We accept either key so both divergence shapes pin correctly. + */ +export function readRecoveryReconcileWorkspaceId( + action: IssueRecoveryAction | null | undefined, +): string | null { + if (!action || action.kind !== "workspace_validation") return null; + const workspaceValidation = asRecord(action.evidence?.workspaceValidation); + if (!workspaceValidation) return null; + const persisted = workspaceValidation.persistedExecutionWorkspaceId; + if (typeof persisted === "string" && persisted.length > 0) return persisted; + const executionWorkspaceId = workspaceValidation.executionWorkspaceId; + if (typeof executionWorkspaceId === "string" && executionWorkspaceId.length > 0) { + return executionWorkspaceId; + } + return null; +} diff --git a/ui/src/pages/AgentDetail.tsx b/ui/src/pages/AgentDetail.tsx index 981f3ffe0b..6006c3152c 100644 --- a/ui/src/pages/AgentDetail.tsx +++ b/ui/src/pages/AgentDetail.tsx @@ -101,6 +101,7 @@ import { responsibleUserLabel, } from "@paperclipai/shared"; import { ResponsibleUserDenialNotice } from "../components/ResponsibleUserDenialNotice"; +import { RunWorkspaceRecoverySurface } from "../components/RunWorkspaceRecoverySurface"; import { buildPermissionsForTrustPreset, getTrustPreset } from "../lib/trust-policy-ui"; import { redactHomePathUserSegments, redactHomePathUserSegmentsInValue } from "@paperclipai/adapter-utils"; import { agentRouteRef } from "../lib/utils"; @@ -3049,6 +3050,10 @@ function RunDetail({ run: initialRun, agentRouteId, adapterType, adapterConfig } return (
+ {/* Workspace-validation recovery: surfaces the recovery card when this run was declined over a + git workspace it could not validate, wired to the same reconcile / repair / re-issue / + break-glass handlers as the task detail page. */} + {/* Run summary card */}
diff --git a/ui/src/pages/InstanceExperimentalSettings.test.tsx b/ui/src/pages/InstanceExperimentalSettings.test.tsx index ce9227ea56..d9bd975f1b 100644 --- a/ui/src/pages/InstanceExperimentalSettings.test.tsx +++ b/ui/src/pages/InstanceExperimentalSettings.test.tsx @@ -69,7 +69,8 @@ function defaultExperimentalSettings(): InstanceExperimentalSettingsPayload { autoRestartDevServerWhenIdle: false, enableIssueGraphLivenessAutoRecovery: false, issueGraphLivenessAutoRecoveryLookbackHours: 24, - enableWorkspaceBranchReconcileForward: false, + enableWorkspaceBranchReconcileForward: true, + enableWorkspaceDirtyQuarantineRepair: true, enableWorktreeRunExecution: false, }; } diff --git a/ui/src/pages/IssueDetail.tsx b/ui/src/pages/IssueDetail.tsx index 4c6be5c085..d95ecff4f5 100644 --- a/ui/src/pages/IssueDetail.tsx +++ b/ui/src/pages/IssueDetail.tsx @@ -11,6 +11,10 @@ import { activityApi, type RunForIssue } from "../api/activity"; import { heartbeatsApi, type ActiveRunForIssue, type LiveRunForIssue } from "../api/heartbeats"; import { instanceSettingsApi } from "../api/instanceSettings"; import { accessApi, type CurrentBoardAccess } from "../api/access"; +import { + canBoardManageRuntime, + readRecoveryReconcileWorkspaceId, +} from "../lib/recovery-reconcile"; import { agentsApi } from "../api/agents"; import { authApi } from "../api/auth"; import { projectsApi } from "../api/projects"; @@ -295,59 +299,11 @@ export function canBoardResolveRecoveryAction( return membership.membershipRole !== "viewer" && membership.membershipRole !== null; } -/** - * Best-effort client mirror of the backend `runtime:manage` gate that the break-glass override - * reconcile (`POST /execution-workspaces/:id/reconcile-branch` in `override` mode) actually - * enforces. The server re-checks `runtime:manage` for every reconcile and is authoritative, so - * this is defense-in-depth: it hides the "reconcile anyway" affordance from viewers rather than - * showing a button that always 403s. For human board members `runtime:manage` grants on the - * same non-viewer, active-membership condition as recovery resolution (see - * `server/src/services/authorization.ts`), so the shape matches; per-permission-key overrides - * are not surfaced to the client and remain the server's call. - */ -export function canBoardManageRuntime( - companyId: string | null | undefined, - boardAccess: CurrentBoardAccess | undefined, -) { - if (!companyId || !boardAccess) return false; - if (boardAccess.source === "local_implicit" || boardAccess.isInstanceAdmin) return true; - if (!boardAccess.memberships || boardAccess.memberships.length === 0) { - return boardAccess.companyIds.includes(companyId); - } - - const membership = boardAccess.memberships.find( - (item) => item.companyId === companyId && item.status === "active", - ); - if (!membership) return false; - return membership.membershipRole !== "viewer" && membership.membershipRole !== null; -} - -/** - * The execution workspace a reconcile action should target. The recovery card is rendered from a - * specific `workspace_validation` recovery action whose evidence pins the workspace that diverged; - * that workspace — not the page-level `issue.executionWorkspaceId` — is the authoritative target. - * The page-level id can drift (e.g. a re-issue rebinds the issue to a new workspace) while the card - * still shows the older action, so we prefer the action's evidence and only fall back to the - * page-level id when the evidence carries no workspace reference. - * - * The branch-incoherence failure (the one that renders the reconcile-forward / break-glass actions) - * records the workspace under `persistedExecutionWorkspaceId`; the not-reusable failure records it - * under `executionWorkspaceId`. We accept either key so both divergence shapes pin correctly. - */ -export function readRecoveryReconcileWorkspaceId( - action: IssueRecoveryAction | null | undefined, -): string | null { - if (!action || action.kind !== "workspace_validation") return null; - const workspaceValidation = asRecord(action.evidence?.workspaceValidation); - if (!workspaceValidation) return null; - const persisted = workspaceValidation.persistedExecutionWorkspaceId; - if (typeof persisted === "string" && persisted.length > 0) return persisted; - const executionWorkspaceId = workspaceValidation.executionWorkspaceId; - if (typeof executionWorkspaceId === "string" && executionWorkspaceId.length > 0) { - return executionWorkspaceId; - } - return null; -} +// `canBoardManageRuntime` and `readRecoveryReconcileWorkspaceId` moved to `@/lib/recovery-reconcile` +// so the run-page recovery surface can reuse them without importing this page module. Re-exported +// here (from the top-of-file import) to keep existing import sites — and their tests — stable, while +// the imported bindings stay usable within this module. +export { canBoardManageRuntime, readRecoveryReconcileWorkspaceId }; export function shouldScrollIssueDetailToTopOnNavigation(input: { previousIssueId: string | undefined; @@ -899,6 +855,8 @@ type IssueDetailChatTabProps = { reissueIsolatedRecoveryActionPending?: boolean; onReconcileForwardRecoveryAction?: () => void; onBreakGlassOverrideRecoveryAction?: (reason: string) => void; + onQuarantineRestoreRecoveryAction?: () => void; + quarantineRestoreRecoveryActionPending?: boolean; canBreakGlassRecoveryAction?: boolean; reconcileRecoveryActionPending?: boolean; canFalsePositiveRecoveryAction?: boolean; @@ -983,6 +941,8 @@ const IssueDetailChatTab = memo(function IssueDetailChatTab({ reissueIsolatedRecoveryActionPending, onReconcileForwardRecoveryAction, onBreakGlassOverrideRecoveryAction, + onQuarantineRestoreRecoveryAction, + quarantineRestoreRecoveryActionPending, canBreakGlassRecoveryAction, reconcileRecoveryActionPending, canFalsePositiveRecoveryAction, @@ -1207,6 +1167,8 @@ const IssueDetailChatTab = memo(function IssueDetailChatTab({ reissueIsolatedRecoveryActionPending={reissueIsolatedRecoveryActionPending} onReconcileForwardRecoveryAction={onReconcileForwardRecoveryAction} onBreakGlassOverrideRecoveryAction={onBreakGlassOverrideRecoveryAction} + onQuarantineRestoreRecoveryAction={onQuarantineRestoreRecoveryAction} + quarantineRestoreRecoveryActionPending={quarantineRestoreRecoveryActionPending} canBreakGlassRecoveryAction={canBreakGlassRecoveryAction} reconcileRecoveryActionPending={reconcileRecoveryActionPending} canFalsePositiveRecoveryAction={canFalsePositiveRecoveryAction} @@ -3729,21 +3691,30 @@ export function IssueDetail() { mutationFn: async ( input: | { workspaceId: string; mode: "forward" } - | { workspaceId: string; mode: "override"; reason: string }, + | { workspaceId: string; mode: "override"; reason: string } + | { workspaceId: string; mode: "quarantine_restore" }, ) => { const { workspaceId, ...body } = input; return executionWorkspacesApi.reconcile(workspaceId, body); }, - onSuccess: () => { + onSuccess: (_result, variables) => { // Refresh the detail card itself (not just the list collections): a successful reconcile // clears the active recovery action, so the card must re-fetch to stop showing stale actions. invalidateIssueDetail(); invalidateIssueCollections(); - pushToast({ - title: "Workspace branch reconciled", - body: "The recorded branch now matches the live branch; the task will resume.", - tone: "success", - }); + pushToast( + variables.mode === "quarantine_restore" + ? { + title: "Workspace repaired", + body: "Dirty changes were quarantined onto a rescue branch and the recorded branch restored; the task will resume.", + tone: "success", + } + : { + title: "Workspace branch reconciled", + body: "The recorded branch now matches the live branch; the task will resume.", + tone: "success", + }, + ); }, onError: (err) => { pushToast({ @@ -3793,6 +3764,22 @@ export function IssueDetail() { }, [reconcileExecutionWorkspaceId, reconcileRecoveryAction.mutateAsync, pushToast], ); + // Repair action (workspace_validation, dirty divergence): quarantine the dirty worktree onto a + // rescue branch and restore the recorded branch. Lossless — no reason required. + const handleQuarantineRestoreRecoveryAction = useCallback(() => { + if (!reconcileExecutionWorkspaceId) { + pushToast({ + title: "Repair failed", + body: "This task has no execution workspace to repair.", + tone: "error", + }); + return; + } + void reconcileRecoveryAction.mutateAsync({ + workspaceId: reconcileExecutionWorkspaceId, + mode: "quarantine_restore", + }); + }, [reconcileExecutionWorkspaceId, reconcileRecoveryAction.mutateAsync, pushToast]); const treePreviewAffectedIssues = useMemo( () => (treeControlPreview?.issues ?? []).filter((candidate) => !candidate.skipped), @@ -4690,6 +4677,8 @@ export function IssueDetail() { reissueIsolatedRecoveryActionPending={reissueIsolatedRecoveryAction.isPending} onReconcileForwardRecoveryAction={handleReconcileForwardRecoveryAction} onBreakGlassOverrideRecoveryAction={handleBreakGlassOverrideRecoveryAction} + onQuarantineRestoreRecoveryAction={handleQuarantineRestoreRecoveryAction} + quarantineRestoreRecoveryActionPending={reconcileRecoveryAction.isPending} canBreakGlassRecoveryAction={canManageBoardRuntime} reconcileRecoveryActionPending={reconcileRecoveryAction.isPending} canFalsePositiveRecoveryAction={canResolveBoardRecoveryAction}