From 27e60991274516e5a79d6d7a956fe714b7e5ea60 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 10 Sep 2026 07:32:58 -0500 Subject: [PATCH] fix(sandbox): block invalid repository setup before continuation Classify required repository preparation failures as workspace validation blockers and retain completed checkouts for explicit retry. Do not schedule interaction infrastructure retries for these failures. Co-Authored-By: Paperclip --- .../heartbeat-process-recovery.test.ts | 36 +++++++++++++ .../__tests__/sandbox-work-folders.test.ts | 53 +++++++++++++++++++ server/src/services/heartbeat.ts | 7 +++ server/src/services/recovery/service.ts | 4 +- .../src/services/recovery/stranded-notice.ts | 2 +- server/src/services/sandbox-work-folders.ts | 28 +++++++--- 6 files changed, 121 insertions(+), 9 deletions(-) diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index c75625d096..7ef51ddca6 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -126,6 +126,7 @@ import { writeHotRestartIntent, } from "../services/hot-restart.ts"; import { secretService } from "../services/secrets.ts"; +import { WorkspaceRuntimeValidationFailure } from "../services/workspace-runtime.js"; import { SUCCESSFUL_RUN_HANDOFF_EXHAUSTED_NOTICE_BODY, SUCCESSFUL_RUN_HANDOFF_REQUIRED_NOTICE_BODY, @@ -4197,6 +4198,41 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(validationComment).toBeTruthy(); }); + it.each([false, true])("blocks repository preparation without an automatic continuation (resolved interaction: %s)", async (resolvedInteraction) => { + const { companyId, agentId, runId, wakeupRequestId, issueId } = await seedQueuedIssueRunFixture(); + if (resolvedInteraction) { + const interactionId = randomUUID(); + await db.insert(issueThreadInteractions).values({ id: interactionId, companyId, issueId, + kind: "request_confirmation", status: "accepted", continuationPolicy: "wake_assignee_on_accept", + createdByAgentId: agentId, resolvedByUserId: "responsible-user", resolvedAt: new Date(), + payload: { version: 1, prompt: "Approve the plan?", target: { type: "issue_document", issueId, key: "plan", revisionId: randomUUID() } }, + result: { version: 1, outcome: "accepted" } }); + const context = { issueId, taskId: issueId, wakeReason: "issue_commented", mutation: "interaction", + interactionId, interactionKind: "request_confirmation", interactionStatus: "accepted" }; + await db.update(agentWakeupRequests).set({ source: "automation", reason: "issue_commented", payload: context }).where(eq(agentWakeupRequests.id, wakeupRequestId)); + await db.update(heartbeatRuns).set({ invocationSource: "automation", contextSnapshot: context }).where(eq(heartbeatRuns.id, runId)); + } + const payload = { workspaceValidation: { reason: "sandbox_repository_preparation_failed", operation: "clone", + issueId, repositoryName: "missing-required-repository", fingerprint: `sandbox_repository:${randomUUID()}:clone` } }; + // Exercise terminal recovery with the structured failure emitted by the + // shared sandbox coordinator; the lifecycle test exercises the real clone. + mockAdapterExecute.mockRejectedValueOnce(new WorkspaceRuntimeValidationFailure("Required repository missing-required-repository could not be cloned", payload)); + const heartbeat = heartbeatService(db); + await heartbeat.resumeQueuedRuns(); + await waitForRunToSettle(heartbeat, runId, 5_000); + await heartbeat.waitForRunExecutionDrain(runId); + await heartbeat.reconcileStrandedAssignedIssues(); + expect(await heartbeat.getRun(runId)).toMatchObject({ status: "failed", errorCode: "workspace_validation_failed", resultJson: payload }); + const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.agentId, agentId)); + expect(runs.map((run) => run.id)).toEqual([runId]); + expect(mockAdapterExecute).toHaveBeenCalledTimes(1); + const [issue] = await db.select().from(issues).where(eq(issues.id, issueId)); + expect(issue).toMatchObject({ status: "blocked", executionRunId: null, assigneeAgentId: agentId }); + const [action] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId)); + expect(action).toMatchObject({ status: "active", ownerType: "board", recoveryIssueId: null }); + if (!resolvedInteraction) expect(action?.nextAction).toContain("Completed checkouts and unsaved files remain"); + }); + it("blocks before dispatch when a declared secret ref has no binding instead of emitting an opaque setup failure", async () => { const { companyId, agentId, runId, issueId } = await seedQueuedIssueRunFixture(); diff --git a/server/src/__tests__/sandbox-work-folders.test.ts b/server/src/__tests__/sandbox-work-folders.test.ts index 7641e7c0d0..36304bbe2c 100644 --- a/server/src/__tests__/sandbox-work-folders.test.ts +++ b/server/src/__tests__/sandbox-work-folders.test.ts @@ -162,6 +162,59 @@ describe("shared sandbox work-folder lifecycle", () => { runner: { supportsSingleStreamStdinProgress: options.bulkStdin, execute: (input) => localTestWorkFolderRunner.execute({ ...input, env: { ...input.env, HOME: home } }) } } }); active.push(run); return run; } + it("blocks a required clone failure and reuses completed checkouts on explicit retry", async () => { + const task = randomUUID(), leaseId = randomUUID(), invalidWorkspaceId = randomUUID(); + const home = path.join(root, `failed-clone-${task}`); + await db.insert(issues).values({ id: task, companyId, projectId, title: "Required clone failure", assigneeAgentId: agentId }); + await db.insert(projectWorkspaces).values({ id: invalidWorkspaceId, companyId, projectId, + name: "Missing required repository", repoUrl: path.join(root, "missing-required-repository"), + sourceType: "git_repo", isPrimary: false }); + try { + const error = await prepare(home, leaseId, leaseId, null, { taskId: task }).catch((failure: unknown) => failure); + expect(error).toMatchObject({ + message: expect.stringContaining("could not be cloned"), + code: "workspace_validation_failed", + resultJson: { workspaceValidation: { + reason: "sandbox_repository_preparation_failed", operation: "clone", + issueId: task, projectId, projectWorkspaceId: invalidWorkspaceId, + fingerprint: expect.stringMatching(/^sandbox_repository:/), + } }, + }); + const primary = path.join(home, "repos", "repo-one"); + expect(await fs.readFile(path.join(primary, ".setup-count"), "utf8")).toBe("initialized\n"); + await fs.writeFile(path.join(primary, "retained-staged"), "staged"); + await exec("git", ["-C", primary, "add", "retained-staged"]); + await fs.writeFile(path.join(primary, "retained-staged"), "dirty"); + await fs.writeFile(path.join(primary, "retained-untracked"), "untracked"); + await db.delete(projectWorkspaces).where(eq(projectWorkspaces.id, invalidWorkspaceId)); + const retry = await prepare(home, leaseId, leaseId, null, { taskId: task }); + expect(await fs.readFile(path.join(primary, ".setup-count"), "utf8")).toBe("initialized\n"); + expect((await exec("git", ["-C", primary, "show", ":retained-staged"])).stdout).toBe("staged"); + expect(await fs.readFile(path.join(primary, "retained-staged"), "utf8")).toBe("dirty"); + expect(await fs.readFile(path.join(primary, "retained-untracked"), "utf8")).toBe("untracked"); + await retry.stop(); active.splice(active.indexOf(retry), 1); + } finally { + await db.delete(projectWorkspaces).where(eq(projectWorkspaces.id, invalidWorkspaceId)); + } + }, 120_000); + it.each([ + { operation: "checkout", repoRef: "missing-required-ref", setupCommand: null, branchName: undefined }, + { operation: "setup", repoRef: null, setupCommand: "exit 7", branchName: undefined }, + { operation: "branch", repoRef: null, setupCommand: null, branchName: "invalid branch name" }, + ])("classifies required repository $operation failures as workspace blockers", async ({ operation, repoRef, setupCommand, branchName }) => { + const task = randomUUID(), workspaceId = randomUUID(); + await db.insert(issues).values({ id: task, companyId, projectId, title: "Required repository preparation", assigneeAgentId: agentId }); + await db.insert(projectWorkspaces).values({ id: workspaceId, companyId, projectId, + name: "Additional required repository", repoUrl: path.join(root, "repo-one"), repoRef, setupCommand, + sourceType: "git_repo", isPrimary: false }); + try { + await expect(prepare(path.join(root, `repository-${task}`), randomUUID(), undefined, null, { taskId: task, branchName })) + .rejects.toMatchObject({ code: "workspace_validation_failed", + resultJson: { workspaceValidation: { reason: "sandbox_repository_preparation_failed", operation, issueId: task } } }); + } finally { + await db.delete(projectWorkspaces).where(eq(projectWorkspaces.id, workspaceId)); + } + }); it("records scoped startup and final checkpoint stages without private identities", async () => { const task = randomUUID(); await db.insert(issues).values({ id: task, companyId, projectId, title: "Instrumented task", assigneeAgentId: agentId }); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 5609104621..d6593a0c0e 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -964,6 +964,13 @@ function isRetryableInteractionContinuationInfrastructureFailure( "error" | "errorCode" | "resultJson" >, ) { + if ( + run.errorCode === WORKSPACE_VALIDATION_FAILURE_CODE && + parseObject(parseObject(run.resultJson).workspaceValidation).reason === + "sandbox_repository_preparation_failed" + ) { + return false; + } if ( run.errorCode === WORKSPACE_VALIDATION_FAILURE_CODE || run.errorCode === "process_lost" diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 12357e55a4..26f415c0a5 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -1473,7 +1473,9 @@ export function recoveryService( : recoveryCause === "codex_output_inactivity_monitor" ? "Board operator: inspect the inactivity evidence, then explicitly retry the original owner, reassign, or intentionally resolve the task." : recoveryCause === "workspace_validation_failed" - ? readWorkspaceValidationPayload(input.latestRun)?.reason === "git_worktree_branch_incoherence" + ? readWorkspaceValidationPayload(input.latestRun)?.reason === "sandbox_repository_preparation_failed" + ? "Board operator: repair the project repository URL, clone access, ref, or setup command, then explicitly retry the original owner. Completed checkouts and unsaved files remain in the retained sandbox." + : readWorkspaceValidationPayload(input.latestRun)?.reason === "git_worktree_branch_incoherence" ? "Board operator: repair the source task git worktree branch incoherence or choose a new execution workspace, then explicitly retry or reassign." : readWorkspaceValidationPayload(input.latestRun)?.reason === "git_worktree_base_materialization_failed" ? "Board operator: repair the project workspace repository URL or clone access, or configure a local checkout cwd, then explicitly retry or reassign." diff --git a/server/src/services/recovery/stranded-notice.ts b/server/src/services/recovery/stranded-notice.ts index 75cbaace6f..d37c531927 100644 --- a/server/src/services/recovery/stranded-notice.ts +++ b/server/src/services/recovery/stranded-notice.ts @@ -64,7 +64,7 @@ export function buildImmediateExecutionPathRecoveryNoticeSeed(input: { export function buildWorkspaceValidationRecoveryNoticeSeed(): StrandedRecoveryNoticeSeed { return { body: - "Paperclip stopped before launching the local adapter because the issue workspace failed validation. " + + "Paperclip stopped before launching the adapter because the issue workspace failed validation. " + "Moving it to `blocked` so the workspace link, cwd, or git checkout can be repaired before resuming.", title: "Workspace validation failed", tone: "danger", diff --git a/server/src/services/sandbox-work-folders.ts b/server/src/services/sandbox-work-folders.ts index 8c312b3e0d..9529e851e8 100644 --- a/server/src/services/sandbox-work-folders.ts +++ b/server/src/services/sandbox-work-folders.ts @@ -19,6 +19,7 @@ import { workFolderRepositoryService } from "./work-folder-repositories.js"; import { startWorkFolderCheckpointer } from "./work-folder-checkpointer.js"; import { logActivity } from "./activity-log.js"; import { assertWorkFolderAccess } from "./work-folder-access.js"; +import { WorkspaceRuntimeValidationFailure } from "./workspace-runtime.js"; function signature(entry: WorkTreeEntry | undefined) { return entry ? JSON.stringify([entry.kind, entry.sha256, entry.executable]) : "missing"; @@ -293,8 +294,21 @@ export async function prepareSandboxWorkFolders(input: { workspaceId: workspace.id, name, repoUrl: workspace.repoUrl, repoRef: workspace.repoRef ?? workspace.defaultRef }).returning())); } if (!binding) throw new Error("Repository binding could not be created"); - if (binding.repoUrl !== workspace.repoUrl) throw new Error(`Repository ${binding.name} configuration changed; saved work was retained`); - if (binding.repoRef !== (workspace.repoRef ?? workspace.defaultRef)) throw new Error(`Repository ${binding.name} starting ref changed; saved work was retained`); + const repositoryBinding = binding; + const failPreparation = (operation: string, message: string): never => { + // No adapter has started and a continuation cannot repair these + // inputs. Preserve completed checkouts for an explicit retry. + throw new WorkspaceRuntimeValidationFailure(message, { + workspaceValidation: { + reason: "sandbox_repository_preparation_failed", operation, + issueId: taskId, projectId, projectWorkspaceId: workspace.id, + repositoryBindingId: repositoryBinding.id, repositoryName: repositoryBinding.name, + fingerprint: `sandbox_repository:${repositoryBinding.id}:${operation}`, + }, + }); + }; + if (binding.repoUrl !== workspace.repoUrl) failPreparation("configuration", `Repository ${binding.name} configuration changed; saved work was retained`); + if (binding.repoRef !== (workspace.repoRef ?? workspace.defaultRef)) failPreparation("configuration", `Repository ${binding.name} starting ref changed; saved work was retained`); const root = path.posix.join(paths.repos!, binding.name); const probe = await measureSandboxOperation("work_folder.repository.probe", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: ["-C", root, "rev-parse", "--git-dir"], bypassSession: true, timeoutMs: 10_000 }))); const freshCheckout = probe.exitCode !== 0; @@ -312,22 +326,22 @@ export async function prepareSandboxWorkFolders(input: { const result = await measureSandboxOperation("work_folder.repository.clone", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: [...(auth?.configArgs ?? []), "clone", "--no-hardlinks", "--", workspace.repoUrl!, temporary], env: { GIT_TERMINAL_PROMPT: "0", ...(auth?.env ?? {}) }, bypassSession: true, timeoutMs: 300_000 }))); - if (result.exitCode !== 0 || result.timedOut) throw new Error(`Required repository ${binding.name} could not be cloned`); + if (result.exitCode !== 0 || result.timedOut) failPreparation("clone", `Required repository ${binding.name} could not be cloned`); const repoRef = binding.repoRef; if (repoRef) { const checkout = await measureSandboxOperation("work_folder.repository.checkout", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: ["-C", temporary, "checkout", repoRef, "--"], bypassSession: true, timeoutMs: 60_000 }))); - if (checkout.exitCode !== 0 || checkout.timedOut) throw new Error(`Required repository ${binding.name} ref could not be checked out`); + if (checkout.exitCode !== 0 || checkout.timedOut) failPreparation("checkout", `Required repository ${binding.name} ref could not be checked out`); } if (primary && input.primaryBranchName) { const branch = input.primaryBranchName; const valid = await measureSandboxOperation("work_folder.repository.validate_branch", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: ["check-ref-format", "--branch", branch], bypassSession: true, timeoutMs: 10_000 }))); - if (valid.exitCode !== 0 || valid.stdout.trim() !== branch) throw new Error(`Required repository ${binding.name} branch is invalid`); + if (valid.exitCode !== 0 || valid.stdout.trim() !== branch) failPreparation("branch", `Required repository ${binding.name} branch is invalid`); // Honor the task's existing branch policy on the initial clone. // Restores and warm starts keep the saved HEAD and index untouched. const checkout = await measureSandboxOperation("work_folder.repository.checkout", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: ["-C", temporary, "checkout", branch, "--"], bypassSession: true, timeoutMs: 60_000 }))); if (checkout.exitCode !== 0) { const create = await measureSandboxOperation("work_folder.repository.create_branch", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: ["-C", temporary, "checkout", "-b", branch], bypassSession: true, timeoutMs: 60_000 }))); - if (create.exitCode !== 0 || create.timedOut) throw new Error(`Required repository ${binding.name} task branch could not be created`); + if (create.exitCode !== 0 || create.timedOut) failPreparation("branch", `Required repository ${binding.name} task branch could not be created`); } } } else { @@ -344,7 +358,7 @@ export async function prepareSandboxWorkFolders(input: { const setupCommand = workspace.setupCommand; if ((!binding.setupComplete || freshCheckout) && setupCommand) { const setup = await measureSandboxOperation("work_folder.repository.setup", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "sh", args: ["-c", setupCommand], cwd: root, bypassSession: true, timeoutMs: 300_000 }))); - if (setup.exitCode !== 0 || setup.timedOut) throw new Error(`Repository ${binding.name} setup failed`); + if (setup.exitCode !== 0 || setup.timedOut) failPreparation("setup", `Repository ${binding.name} setup failed`); } await measureSandboxOperation("work_folder.db.query", { operation: "update_task_repository_bindings", requestCount: 1 }, async () => (db.update(taskRepositoryBindings).set({ setupComplete: true, retiredAt: null }).where(eq(taskRepositoryBindings.id, binding.id)))); bindings.push({ binding, root });