diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index c75625d096..7ef51ddca6 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -126,6 +126,7 @@ import { writeHotRestartIntent, } from "../services/hot-restart.ts"; import { secretService } from "../services/secrets.ts"; +import { WorkspaceRuntimeValidationFailure } from "../services/workspace-runtime.js"; import { SUCCESSFUL_RUN_HANDOFF_EXHAUSTED_NOTICE_BODY, SUCCESSFUL_RUN_HANDOFF_REQUIRED_NOTICE_BODY, @@ -4197,6 +4198,41 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(validationComment).toBeTruthy(); }); + it.each([false, true])("blocks repository preparation without an automatic continuation (resolved interaction: %s)", async (resolvedInteraction) => { + const { companyId, agentId, runId, wakeupRequestId, issueId } = await seedQueuedIssueRunFixture(); + if (resolvedInteraction) { + const interactionId = randomUUID(); + await db.insert(issueThreadInteractions).values({ id: interactionId, companyId, issueId, + kind: "request_confirmation", status: "accepted", continuationPolicy: "wake_assignee_on_accept", + createdByAgentId: agentId, resolvedByUserId: "responsible-user", resolvedAt: new Date(), + payload: { version: 1, prompt: "Approve the plan?", target: { type: "issue_document", issueId, key: "plan", revisionId: randomUUID() } }, + result: { version: 1, outcome: "accepted" } }); + const context = { issueId, taskId: issueId, wakeReason: "issue_commented", mutation: "interaction", + interactionId, interactionKind: "request_confirmation", interactionStatus: "accepted" }; + await db.update(agentWakeupRequests).set({ source: "automation", reason: "issue_commented", payload: context }).where(eq(agentWakeupRequests.id, wakeupRequestId)); + await db.update(heartbeatRuns).set({ invocationSource: "automation", contextSnapshot: context }).where(eq(heartbeatRuns.id, runId)); + } + const payload = { workspaceValidation: { reason: "sandbox_repository_preparation_failed", operation: "clone", + issueId, repositoryName: "missing-required-repository", fingerprint: `sandbox_repository:${randomUUID()}:clone` } }; + // Exercise terminal recovery with the structured failure emitted by the + // shared sandbox coordinator; the lifecycle test exercises the real clone. + mockAdapterExecute.mockRejectedValueOnce(new WorkspaceRuntimeValidationFailure("Required repository missing-required-repository could not be cloned", payload)); + const heartbeat = heartbeatService(db); + await heartbeat.resumeQueuedRuns(); + await waitForRunToSettle(heartbeat, runId, 5_000); + await heartbeat.waitForRunExecutionDrain(runId); + await heartbeat.reconcileStrandedAssignedIssues(); + expect(await heartbeat.getRun(runId)).toMatchObject({ status: "failed", errorCode: "workspace_validation_failed", resultJson: payload }); + const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.agentId, agentId)); + expect(runs.map((run) => run.id)).toEqual([runId]); + expect(mockAdapterExecute).toHaveBeenCalledTimes(1); + const [issue] = await db.select().from(issues).where(eq(issues.id, issueId)); + expect(issue).toMatchObject({ status: "blocked", executionRunId: null, assigneeAgentId: agentId }); + const [action] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId)); + expect(action).toMatchObject({ status: "active", ownerType: "board", recoveryIssueId: null }); + if (!resolvedInteraction) expect(action?.nextAction).toContain("Completed checkouts and unsaved files remain"); + }); + it("blocks before dispatch when a declared secret ref has no binding instead of emitting an opaque setup failure", async () => { const { companyId, agentId, runId, issueId } = await seedQueuedIssueRunFixture(); diff --git a/server/src/__tests__/sandbox-work-folders.test.ts b/server/src/__tests__/sandbox-work-folders.test.ts index 7641e7c0d0..36304bbe2c 100644 --- a/server/src/__tests__/sandbox-work-folders.test.ts +++ b/server/src/__tests__/sandbox-work-folders.test.ts @@ -162,6 +162,59 @@ describe("shared sandbox work-folder lifecycle", () => { runner: { supportsSingleStreamStdinProgress: options.bulkStdin, execute: (input) => localTestWorkFolderRunner.execute({ ...input, env: { ...input.env, HOME: home } }) } } }); active.push(run); return run; } + it("blocks a required clone failure and reuses completed checkouts on explicit retry", async () => { + const task = randomUUID(), leaseId = randomUUID(), invalidWorkspaceId = randomUUID(); + const home = path.join(root, `failed-clone-${task}`); + await db.insert(issues).values({ id: task, companyId, projectId, title: "Required clone failure", assigneeAgentId: agentId }); + await db.insert(projectWorkspaces).values({ id: invalidWorkspaceId, companyId, projectId, + name: "Missing required repository", repoUrl: path.join(root, "missing-required-repository"), + sourceType: "git_repo", isPrimary: false }); + try { + const error = await prepare(home, leaseId, leaseId, null, { taskId: task }).catch((failure: unknown) => failure); + expect(error).toMatchObject({ + message: expect.stringContaining("could not be cloned"), + code: "workspace_validation_failed", + resultJson: { workspaceValidation: { + reason: "sandbox_repository_preparation_failed", operation: "clone", + issueId: task, projectId, projectWorkspaceId: invalidWorkspaceId, + fingerprint: expect.stringMatching(/^sandbox_repository:/), + } }, + }); + const primary = path.join(home, "repos", "repo-one"); + expect(await fs.readFile(path.join(primary, ".setup-count"), "utf8")).toBe("initialized\n"); + await fs.writeFile(path.join(primary, "retained-staged"), "staged"); + await exec("git", ["-C", primary, "add", "retained-staged"]); + await fs.writeFile(path.join(primary, "retained-staged"), "dirty"); + await fs.writeFile(path.join(primary, "retained-untracked"), "untracked"); + await db.delete(projectWorkspaces).where(eq(projectWorkspaces.id, invalidWorkspaceId)); + const retry = await prepare(home, leaseId, leaseId, null, { taskId: task }); + expect(await fs.readFile(path.join(primary, ".setup-count"), "utf8")).toBe("initialized\n"); + expect((await exec("git", ["-C", primary, "show", ":retained-staged"])).stdout).toBe("staged"); + expect(await fs.readFile(path.join(primary, "retained-staged"), "utf8")).toBe("dirty"); + expect(await fs.readFile(path.join(primary, "retained-untracked"), "utf8")).toBe("untracked"); + await retry.stop(); active.splice(active.indexOf(retry), 1); + } finally { + await db.delete(projectWorkspaces).where(eq(projectWorkspaces.id, invalidWorkspaceId)); + } + }, 120_000); + it.each([ + { operation: "checkout", repoRef: "missing-required-ref", setupCommand: null, branchName: undefined }, + { operation: "setup", repoRef: null, setupCommand: "exit 7", branchName: undefined }, + { operation: "branch", repoRef: null, setupCommand: null, branchName: "invalid branch name" }, + ])("classifies required repository $operation failures as workspace blockers", async ({ operation, repoRef, setupCommand, branchName }) => { + const task = randomUUID(), workspaceId = randomUUID(); + await db.insert(issues).values({ id: task, companyId, projectId, title: "Required repository preparation", assigneeAgentId: agentId }); + await db.insert(projectWorkspaces).values({ id: workspaceId, companyId, projectId, + name: "Additional required repository", repoUrl: path.join(root, "repo-one"), repoRef, setupCommand, + sourceType: "git_repo", isPrimary: false }); + try { + await expect(prepare(path.join(root, `repository-${task}`), randomUUID(), undefined, null, { taskId: task, branchName })) + .rejects.toMatchObject({ code: "workspace_validation_failed", + resultJson: { workspaceValidation: { reason: "sandbox_repository_preparation_failed", operation, issueId: task } } }); + } finally { + await db.delete(projectWorkspaces).where(eq(projectWorkspaces.id, workspaceId)); + } + }); it("records scoped startup and final checkpoint stages without private identities", async () => { const task = randomUUID(); await db.insert(issues).values({ id: task, companyId, projectId, title: "Instrumented task", assigneeAgentId: agentId }); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 5609104621..d6593a0c0e 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -964,6 +964,13 @@ function isRetryableInteractionContinuationInfrastructureFailure( "error" | "errorCode" | "resultJson" >, ) { + if ( + run.errorCode === WORKSPACE_VALIDATION_FAILURE_CODE && + parseObject(parseObject(run.resultJson).workspaceValidation).reason === + "sandbox_repository_preparation_failed" + ) { + return false; + } if ( run.errorCode === WORKSPACE_VALIDATION_FAILURE_CODE || run.errorCode === "process_lost" diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 12357e55a4..26f415c0a5 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -1473,7 +1473,9 @@ export function recoveryService( : recoveryCause === "codex_output_inactivity_monitor" ? "Board operator: inspect the inactivity evidence, then explicitly retry the original owner, reassign, or intentionally resolve the task." : recoveryCause === "workspace_validation_failed" - ? readWorkspaceValidationPayload(input.latestRun)?.reason === "git_worktree_branch_incoherence" + ? readWorkspaceValidationPayload(input.latestRun)?.reason === "sandbox_repository_preparation_failed" + ? "Board operator: repair the project repository URL, clone access, ref, or setup command, then explicitly retry the original owner. Completed checkouts and unsaved files remain in the retained sandbox." + : readWorkspaceValidationPayload(input.latestRun)?.reason === "git_worktree_branch_incoherence" ? "Board operator: repair the source task git worktree branch incoherence or choose a new execution workspace, then explicitly retry or reassign." : readWorkspaceValidationPayload(input.latestRun)?.reason === "git_worktree_base_materialization_failed" ? "Board operator: repair the project workspace repository URL or clone access, or configure a local checkout cwd, then explicitly retry or reassign." diff --git a/server/src/services/recovery/stranded-notice.ts b/server/src/services/recovery/stranded-notice.ts index 75cbaace6f..d37c531927 100644 --- a/server/src/services/recovery/stranded-notice.ts +++ b/server/src/services/recovery/stranded-notice.ts @@ -64,7 +64,7 @@ export function buildImmediateExecutionPathRecoveryNoticeSeed(input: { export function buildWorkspaceValidationRecoveryNoticeSeed(): StrandedRecoveryNoticeSeed { return { body: - "Paperclip stopped before launching the local adapter because the issue workspace failed validation. " + + "Paperclip stopped before launching the adapter because the issue workspace failed validation. " + "Moving it to `blocked` so the workspace link, cwd, or git checkout can be repaired before resuming.", title: "Workspace validation failed", tone: "danger", diff --git a/server/src/services/sandbox-work-folders.ts b/server/src/services/sandbox-work-folders.ts index 8c312b3e0d..9529e851e8 100644 --- a/server/src/services/sandbox-work-folders.ts +++ b/server/src/services/sandbox-work-folders.ts @@ -19,6 +19,7 @@ import { workFolderRepositoryService } from "./work-folder-repositories.js"; import { startWorkFolderCheckpointer } from "./work-folder-checkpointer.js"; import { logActivity } from "./activity-log.js"; import { assertWorkFolderAccess } from "./work-folder-access.js"; +import { WorkspaceRuntimeValidationFailure } from "./workspace-runtime.js"; function signature(entry: WorkTreeEntry | undefined) { return entry ? JSON.stringify([entry.kind, entry.sha256, entry.executable]) : "missing"; @@ -293,8 +294,21 @@ export async function prepareSandboxWorkFolders(input: { workspaceId: workspace.id, name, repoUrl: workspace.repoUrl, repoRef: workspace.repoRef ?? workspace.defaultRef }).returning())); } if (!binding) throw new Error("Repository binding could not be created"); - if (binding.repoUrl !== workspace.repoUrl) throw new Error(`Repository ${binding.name} configuration changed; saved work was retained`); - if (binding.repoRef !== (workspace.repoRef ?? workspace.defaultRef)) throw new Error(`Repository ${binding.name} starting ref changed; saved work was retained`); + const repositoryBinding = binding; + const failPreparation = (operation: string, message: string): never => { + // No adapter has started and a continuation cannot repair these + // inputs. Preserve completed checkouts for an explicit retry. + throw new WorkspaceRuntimeValidationFailure(message, { + workspaceValidation: { + reason: "sandbox_repository_preparation_failed", operation, + issueId: taskId, projectId, projectWorkspaceId: workspace.id, + repositoryBindingId: repositoryBinding.id, repositoryName: repositoryBinding.name, + fingerprint: `sandbox_repository:${repositoryBinding.id}:${operation}`, + }, + }); + }; + if (binding.repoUrl !== workspace.repoUrl) failPreparation("configuration", `Repository ${binding.name} configuration changed; saved work was retained`); + if (binding.repoRef !== (workspace.repoRef ?? workspace.defaultRef)) failPreparation("configuration", `Repository ${binding.name} starting ref changed; saved work was retained`); const root = path.posix.join(paths.repos!, binding.name); const probe = await measureSandboxOperation("work_folder.repository.probe", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: ["-C", root, "rev-parse", "--git-dir"], bypassSession: true, timeoutMs: 10_000 }))); const freshCheckout = probe.exitCode !== 0; @@ -312,22 +326,22 @@ export async function prepareSandboxWorkFolders(input: { const result = await measureSandboxOperation("work_folder.repository.clone", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: [...(auth?.configArgs ?? []), "clone", "--no-hardlinks", "--", workspace.repoUrl!, temporary], env: { GIT_TERMINAL_PROMPT: "0", ...(auth?.env ?? {}) }, bypassSession: true, timeoutMs: 300_000 }))); - if (result.exitCode !== 0 || result.timedOut) throw new Error(`Required repository ${binding.name} could not be cloned`); + if (result.exitCode !== 0 || result.timedOut) failPreparation("clone", `Required repository ${binding.name} could not be cloned`); const repoRef = binding.repoRef; if (repoRef) { const checkout = await measureSandboxOperation("work_folder.repository.checkout", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: ["-C", temporary, "checkout", repoRef, "--"], bypassSession: true, timeoutMs: 60_000 }))); - if (checkout.exitCode !== 0 || checkout.timedOut) throw new Error(`Required repository ${binding.name} ref could not be checked out`); + if (checkout.exitCode !== 0 || checkout.timedOut) failPreparation("checkout", `Required repository ${binding.name} ref could not be checked out`); } if (primary && input.primaryBranchName) { const branch = input.primaryBranchName; const valid = await measureSandboxOperation("work_folder.repository.validate_branch", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: ["check-ref-format", "--branch", branch], bypassSession: true, timeoutMs: 10_000 }))); - if (valid.exitCode !== 0 || valid.stdout.trim() !== branch) throw new Error(`Required repository ${binding.name} branch is invalid`); + if (valid.exitCode !== 0 || valid.stdout.trim() !== branch) failPreparation("branch", `Required repository ${binding.name} branch is invalid`); // Honor the task's existing branch policy on the initial clone. // Restores and warm starts keep the saved HEAD and index untouched. const checkout = await measureSandboxOperation("work_folder.repository.checkout", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: ["-C", temporary, "checkout", branch, "--"], bypassSession: true, timeoutMs: 60_000 }))); if (checkout.exitCode !== 0) { const create = await measureSandboxOperation("work_folder.repository.create_branch", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: ["-C", temporary, "checkout", "-b", branch], bypassSession: true, timeoutMs: 60_000 }))); - if (create.exitCode !== 0 || create.timedOut) throw new Error(`Required repository ${binding.name} task branch could not be created`); + if (create.exitCode !== 0 || create.timedOut) failPreparation("branch", `Required repository ${binding.name} task branch could not be created`); } } } else { @@ -344,7 +358,7 @@ export async function prepareSandboxWorkFolders(input: { const setupCommand = workspace.setupCommand; if ((!binding.setupComplete || freshCheckout) && setupCommand) { const setup = await measureSandboxOperation("work_folder.repository.setup", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "sh", args: ["-c", setupCommand], cwd: root, bypassSession: true, timeoutMs: 300_000 }))); - if (setup.exitCode !== 0 || setup.timedOut) throw new Error(`Repository ${binding.name} setup failed`); + if (setup.exitCode !== 0 || setup.timedOut) failPreparation("setup", `Repository ${binding.name} setup failed`); } await measureSandboxOperation("work_folder.db.query", { operation: "update_task_repository_bindings", requestCount: 1 }, async () => (db.update(taskRepositoryBindings).set({ setupComplete: true, retiredAt: null }).where(eq(taskRepositoryBindings.id, binding.id)))); bindings.push({ binding, root });