RomanNum3ral
/
paperclip
mirror of https://github.com/paperclipai/paperclip.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix(sandbox): block invalid repository setup before continuation 

Classify required repository preparation failures as workspace validation blockers and retain completed checkouts for explicit retry. Do not schedule interaction infrastructure retries for these failures.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta 2026-09-10 07:32:58 -05:00
parent 65127dff98
commit 27e6099127
6 changed files with 121 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
server/src/__tests__/heartbeat-process-recovery.test.ts
View File

@ -126,6 +126,7 @@ import {
writeHotRestartIntent,
} from "../services/hot-restart.ts";
import { secretService } from "../services/secrets.ts";
import { WorkspaceRuntimeValidationFailure } from "../services/workspace-runtime.js";
import {
SUCCESSFUL_RUN_HANDOFF_EXHAUSTED_NOTICE_BODY,
SUCCESSFUL_RUN_HANDOFF_REQUIRED_NOTICE_BODY,
@ -4197,6 +4198,41 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
expect(validationComment).toBeTruthy();
});
it.each([false, true])("blocks repository preparation without an automatic continuation (resolved interaction: %s)", async (resolvedInteraction) => {
const { companyId, agentId, runId, wakeupRequestId, issueId } = await seedQueuedIssueRunFixture();
if (resolvedInteraction) {
const interactionId = randomUUID();
await db.insert(issueThreadInteractions).values({ id: interactionId, companyId, issueId,
kind: "request_confirmation", status: "accepted", continuationPolicy: "wake_assignee_on_accept",
createdByAgentId: agentId, resolvedByUserId: "responsible-user", resolvedAt: new Date(),
payload: { version: 1, prompt: "Approve the plan?", target: { type: "issue_document", issueId, key: "plan", revisionId: randomUUID() } },
result: { version: 1, outcome: "accepted" } });
const context = { issueId, taskId: issueId, wakeReason: "issue_commented", mutation: "interaction",
interactionId, interactionKind: "request_confirmation", interactionStatus: "accepted" };
await db.update(agentWakeupRequests).set({ source: "automation", reason: "issue_commented", payload: context }).where(eq(agentWakeupRequests.id, wakeupRequestId));
await db.update(heartbeatRuns).set({ invocationSource: "automation", contextSnapshot: context }).where(eq(heartbeatRuns.id, runId));
}
const payload = { workspaceValidation: { reason: "sandbox_repository_preparation_failed", operation: "clone",
issueId, repositoryName: "missing-required-repository", fingerprint: `sandbox_repository:${randomUUID()}:clone` } };
// Exercise terminal recovery with the structured failure emitted by the
// shared sandbox coordinator; the lifecycle test exercises the real clone.
mockAdapterExecute.mockRejectedValueOnce(new WorkspaceRuntimeValidationFailure("Required repository missing-required-repository could not be cloned", payload));
const heartbeat = heartbeatService(db);
await heartbeat.resumeQueuedRuns();
await waitForRunToSettle(heartbeat, runId, 5_000);
await heartbeat.waitForRunExecutionDrain(runId);
await heartbeat.reconcileStrandedAssignedIssues();
expect(await heartbeat.getRun(runId)).toMatchObject({ status: "failed", errorCode: "workspace_validation_failed", resultJson: payload });
const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.agentId, agentId));
expect(runs.map((run) => run.id)).toEqual([runId]);
expect(mockAdapterExecute).toHaveBeenCalledTimes(1);
const [issue] = await db.select().from(issues).where(eq(issues.id, issueId));
expect(issue).toMatchObject({ status: "blocked", executionRunId: null, assigneeAgentId: agentId });
const [action] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId));
expect(action).toMatchObject({ status: "active", ownerType: "board", recoveryIssueId: null });
if (!resolvedInteraction) expect(action?.nextAction).toContain("Completed checkouts and unsaved files remain");
});
it("blocks before dispatch when a declared secret ref has no binding instead of emitting an opaque setup failure", async () => {
const { companyId, agentId, runId, issueId } =
await seedQueuedIssueRunFixture();

53
server/src/__tests__/sandbox-work-folders.test.ts
View File

@ -162,6 +162,59 @@ describe("shared sandbox work-folder lifecycle", () => {
runner: { supportsSingleStreamStdinProgress: options.bulkStdin, execute: (input) => localTestWorkFolderRunner.execute({ ...input, env: { ...input.env, HOME: home } }) } } });
active.push(run); return run;
}
it("blocks a required clone failure and reuses completed checkouts on explicit retry", async () => {
const task = randomUUID(), leaseId = randomUUID(), invalidWorkspaceId = randomUUID();
const home = path.join(root, `failed-clone-${task}`);
await db.insert(issues).values({ id: task, companyId, projectId, title: "Required clone failure", assigneeAgentId: agentId });
await db.insert(projectWorkspaces).values({ id: invalidWorkspaceId, companyId, projectId,
name: "Missing required repository", repoUrl: path.join(root, "missing-required-repository"),
sourceType: "git_repo", isPrimary: false });
try {
const error = await prepare(home, leaseId, leaseId, null, { taskId: task }).catch((failure: unknown) => failure);
expect(error).toMatchObject({
message: expect.stringContaining("could not be cloned"),
code: "workspace_validation_failed",
resultJson: { workspaceValidation: {
reason: "sandbox_repository_preparation_failed", operation: "clone",
issueId: task, projectId, projectWorkspaceId: invalidWorkspaceId,
fingerprint: expect.stringMatching(/^sandbox_repository:/),
} },
});
const primary = path.join(home, "repos", "repo-one");
expect(await fs.readFile(path.join(primary, ".setup-count"), "utf8")).toBe("initialized\n");
await fs.writeFile(path.join(primary, "retained-staged"), "staged");
await exec("git", ["-C", primary, "add", "retained-staged"]);
await fs.writeFile(path.join(primary, "retained-staged"), "dirty");
await fs.writeFile(path.join(primary, "retained-untracked"), "untracked");
await db.delete(projectWorkspaces).where(eq(projectWorkspaces.id, invalidWorkspaceId));
const retry = await prepare(home, leaseId, leaseId, null, { taskId: task });
expect(await fs.readFile(path.join(primary, ".setup-count"), "utf8")).toBe("initialized\n");
expect((await exec("git", ["-C", primary, "show", ":retained-staged"])).stdout).toBe("staged");
expect(await fs.readFile(path.join(primary, "retained-staged"), "utf8")).toBe("dirty");
expect(await fs.readFile(path.join(primary, "retained-untracked"), "utf8")).toBe("untracked");
await retry.stop(); active.splice(active.indexOf(retry), 1);
} finally {
await db.delete(projectWorkspaces).where(eq(projectWorkspaces.id, invalidWorkspaceId));
}
}, 120_000);
it.each([
{ operation: "checkout", repoRef: "missing-required-ref", setupCommand: null, branchName: undefined },
{ operation: "setup", repoRef: null, setupCommand: "exit 7", branchName: undefined },
{ operation: "branch", repoRef: null, setupCommand: null, branchName: "invalid branch name" },
])("classifies required repository $operation failures as workspace blockers", async ({ operation, repoRef, setupCommand, branchName }) => {
const task = randomUUID(), workspaceId = randomUUID();
await db.insert(issues).values({ id: task, companyId, projectId, title: "Required repository preparation", assigneeAgentId: agentId });
await db.insert(projectWorkspaces).values({ id: workspaceId, companyId, projectId,
name: "Additional required repository", repoUrl: path.join(root, "repo-one"), repoRef, setupCommand,
sourceType: "git_repo", isPrimary: false });
try {
await expect(prepare(path.join(root, `repository-${task}`), randomUUID(), undefined, null, { taskId: task, branchName }))
.rejects.toMatchObject({ code: "workspace_validation_failed",
resultJson: { workspaceValidation: { reason: "sandbox_repository_preparation_failed", operation, issueId: task } } });
} finally {
await db.delete(projectWorkspaces).where(eq(projectWorkspaces.id, workspaceId));
}
});
it("records scoped startup and final checkpoint stages without private identities", async () => {
const task = randomUUID();
await db.insert(issues).values({ id: task, companyId, projectId, title: "Instrumented task", assigneeAgentId: agentId });

7
server/src/services/heartbeat.ts
View File

@ -964,6 +964,13 @@ function isRetryableInteractionContinuationInfrastructureFailure(
"error" | "errorCode" | "resultJson"
>,
) {
if (
run.errorCode === WORKSPACE_VALIDATION_FAILURE_CODE &&
parseObject(parseObject(run.resultJson).workspaceValidation).reason ===
"sandbox_repository_preparation_failed"
) {
return false;
}
if (
run.errorCode === WORKSPACE_VALIDATION_FAILURE_CODE ||
run.errorCode === "process_lost"

4
server/src/services/recovery/service.ts
View File

@ -1473,7 +1473,9 @@ export function recoveryService(
: recoveryCause === "codex_output_inactivity_monitor"
? "Board operator: inspect the inactivity evidence, then explicitly retry the original owner, reassign, or intentionally resolve the task."
: recoveryCause === "workspace_validation_failed"
? readWorkspaceValidationPayload(input.latestRun)?.reason === "git_worktree_branch_incoherence"
? readWorkspaceValidationPayload(input.latestRun)?.reason === "sandbox_repository_preparation_failed"
? "Board operator: repair the project repository URL, clone access, ref, or setup command, then explicitly retry the original owner. Completed checkouts and unsaved files remain in the retained sandbox."
: readWorkspaceValidationPayload(input.latestRun)?.reason === "git_worktree_branch_incoherence"
? "Board operator: repair the source task git worktree branch incoherence or choose a new execution workspace, then explicitly retry or reassign."
: readWorkspaceValidationPayload(input.latestRun)?.reason === "git_worktree_base_materialization_failed"
? "Board operator: repair the project workspace repository URL or clone access, or configure a local checkout cwd, then explicitly retry or reassign."

2
server/src/services/recovery/stranded-notice.ts
View File

@ -64,7 +64,7 @@ export function buildImmediateExecutionPathRecoveryNoticeSeed(input: {
export function buildWorkspaceValidationRecoveryNoticeSeed(): StrandedRecoveryNoticeSeed {
return {
body:
"Paperclip stopped before launching the local adapter because the issue workspace failed validation. " +
"Paperclip stopped before launching the adapter because the issue workspace failed validation. " +
"Moving it to `blocked` so the workspace link, cwd, or git checkout can be repaired before resuming.",
title: "Workspace validation failed",
tone: "danger",

28
server/src/services/sandbox-work-folders.ts
View File

@ -19,6 +19,7 @@ import { workFolderRepositoryService } from "./work-folder-repositories.js";
import { startWorkFolderCheckpointer } from "./work-folder-checkpointer.js";
import { logActivity } from "./activity-log.js";
import { assertWorkFolderAccess } from "./work-folder-access.js";
import { WorkspaceRuntimeValidationFailure } from "./workspace-runtime.js";
function signature(entry: WorkTreeEntry | undefined) {
return entry ? JSON.stringify([entry.kind, entry.sha256, entry.executable]) : "missing";
@ -293,8 +294,21 @@ export async function prepareSandboxWorkFolders(input: {
workspaceId: workspace.id, name, repoUrl: workspace.repoUrl, repoRef: workspace.repoRef ?? workspace.defaultRef }).returning()));
}
if (!binding) throw new Error("Repository binding could not be created");
if (binding.repoUrl !== workspace.repoUrl) throw new Error(`Repository ${binding.name} configuration changed; saved work was retained`);
if (binding.repoRef !== (workspace.repoRef ?? workspace.defaultRef)) throw new Error(`Repository ${binding.name} starting ref changed; saved work was retained`);
const repositoryBinding = binding;
const failPreparation = (operation: string, message: string): never => {
// No adapter has started and a continuation cannot repair these
// inputs. Preserve completed checkouts for an explicit retry.
throw new WorkspaceRuntimeValidationFailure(message, {
workspaceValidation: {
reason: "sandbox_repository_preparation_failed", operation,
issueId: taskId, projectId, projectWorkspaceId: workspace.id,
repositoryBindingId: repositoryBinding.id, repositoryName: repositoryBinding.name,
fingerprint: `sandbox_repository:${repositoryBinding.id}:${operation}`,
},
});
};
if (binding.repoUrl !== workspace.repoUrl) failPreparation("configuration", `Repository ${binding.name} configuration changed; saved work was retained`);
if (binding.repoRef !== (workspace.repoRef ?? workspace.defaultRef)) failPreparation("configuration", `Repository ${binding.name} starting ref changed; saved work was retained`);
const root = path.posix.join(paths.repos!, binding.name);
const probe = await measureSandboxOperation("work_folder.repository.probe", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: ["-C", root, "rev-parse", "--git-dir"], bypassSession: true, timeoutMs: 10_000 })));
const freshCheckout = probe.exitCode !== 0;
@ -312,22 +326,22 @@ export async function prepareSandboxWorkFolders(input: {
const result = await measureSandboxOperation("work_folder.repository.clone", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: [...(auth?.configArgs ?? []), "clone", "--no-hardlinks",
"--", workspace.repoUrl!, temporary],
env: { GIT_TERMINAL_PROMPT: "0", ...(auth?.env ?? {}) }, bypassSession: true, timeoutMs: 300_000 })));
if (result.exitCode !== 0 || result.timedOut) throw new Error(`Required repository ${binding.name} could not be cloned`);
if (result.exitCode !== 0 || result.timedOut) failPreparation("clone", `Required repository ${binding.name} could not be cloned`);
const repoRef = binding.repoRef;
if (repoRef) {
const checkout = await measureSandboxOperation("work_folder.repository.checkout", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: ["-C", temporary, "checkout", repoRef, "--"], bypassSession: true, timeoutMs: 60_000 })));
if (checkout.exitCode !== 0 || checkout.timedOut) throw new Error(`Required repository ${binding.name} ref could not be checked out`);
if (checkout.exitCode !== 0 || checkout.timedOut) failPreparation("checkout", `Required repository ${binding.name} ref could not be checked out`);
}
if (primary && input.primaryBranchName) {
const branch = input.primaryBranchName;
const valid = await measureSandboxOperation("work_folder.repository.validate_branch", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: ["check-ref-format", "--branch", branch], bypassSession: true, timeoutMs: 10_000 })));
if (valid.exitCode !== 0 || valid.stdout.trim() !== branch) throw new Error(`Required repository ${binding.name} branch is invalid`);
if (valid.exitCode !== 0 || valid.stdout.trim() !== branch) failPreparation("branch", `Required repository ${binding.name} branch is invalid`);
// Honor the task's existing branch policy on the initial clone.
// Restores and warm starts keep the saved HEAD and index untouched.
const checkout = await measureSandboxOperation("work_folder.repository.checkout", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: ["-C", temporary, "checkout", branch, "--"], bypassSession: true, timeoutMs: 60_000 })));
if (checkout.exitCode !== 0) {
const create = await measureSandboxOperation("work_folder.repository.create_branch", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: ["-C", temporary, "checkout", "-b", branch], bypassSession: true, timeoutMs: 60_000 })));
if (create.exitCode !== 0 || create.timedOut) throw new Error(`Required repository ${binding.name} task branch could not be created`);
if (create.exitCode !== 0 || create.timedOut) failPreparation("branch", `Required repository ${binding.name} task branch could not be created`);
}
}
} else {
@ -344,7 +358,7 @@ export async function prepareSandboxWorkFolders(input: {
const setupCommand = workspace.setupCommand;
if ((!binding.setupComplete || freshCheckout) && setupCommand) {
const setup = await measureSandboxOperation("work_folder.repository.setup", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "sh", args: ["-c", setupCommand], cwd: root, bypassSession: true, timeoutMs: 300_000 })));
if (setup.exitCode !== 0 || setup.timedOut) throw new Error(`Repository ${binding.name} setup failed`);
if (setup.exitCode !== 0 || setup.timedOut) failPreparation("setup", `Repository ${binding.name} setup failed`);
}
await measureSandboxOperation("work_folder.db.query", { operation: "update_task_repository_bindings", requestCount: 1 }, async () => (db.update(taskRepositoryBindings).set({ setupComplete: true, retiredAt: null }).where(eq(taskRepositoryBindings.id, binding.id))));
bindings.push({ binding, root });