diff --git a/packages/adapter-utils/src/index.ts b/packages/adapter-utils/src/index.ts index 4a34aaa24d..5d1979c826 100644 --- a/packages/adapter-utils/src/index.ts +++ b/packages/adapter-utils/src/index.ts @@ -114,6 +114,18 @@ export type { LoginRunnerDisposable, LoginRunnerRaceResult, } from "./login-runner-lifecycle.js"; +export { + PAPERCLIP_RUNNER_PERMISSION_CAPABILITIES, + isPaperclipRunnerProvider, + resolvePaperclipRunnerPermissionMode, +} from "./paperclip-runner-permissions.js"; +export type { + CodexPermissionMode, + PaperclipRunnerPermissionCapability, + PaperclipRunnerPermissionMode, + PaperclipRunnerPermissionOption, + PaperclipRunnerProvider, +} from "./paperclip-runner-permissions.js"; // Keep the root adapter-utils entry browser-safe because the UI imports it. // The sandbox callback bridge stays available via its dedicated subpath export. export type { diff --git a/packages/adapter-utils/src/paperclip-runner-permissions.ts b/packages/adapter-utils/src/paperclip-runner-permissions.ts new file mode 100644 index 0000000000..40f2bc3638 --- /dev/null +++ b/packages/adapter-utils/src/paperclip-runner-permissions.ts @@ -0,0 +1,51 @@ +export type PaperclipRunnerProvider = "codex"; + +export type CodexPermissionMode = "never" | "on-request" | "untrusted"; +export type PaperclipRunnerPermissionMode = CodexPermissionMode; + +export interface PaperclipRunnerPermissionOption { + value: TMode; + label: string; + description: string; +} + +export interface PaperclipRunnerPermissionCapability { + configurable: true; + configKey: "codexPermissionMode"; + defaultMode: PaperclipRunnerPermissionMode; + options: readonly PaperclipRunnerPermissionOption[]; + description: string; +} + +/** + * Control-plane catalog for Paperclip Runner permission UX and validation. + * Runtime contracts validate the same native values again at the process + * boundary; this catalog must remain browser-safe. + */ +export const PAPERCLIP_RUNNER_PERMISSION_CAPABILITIES = { + codex: { + configurable: true, + configKey: "codexPermissionMode", + defaultMode: "never", + description: "Controls when Codex asks before an operation inside the assigned Paperclip environment.", + options: [ + { value: "never", label: "Full auto (never ask)", description: "Run without Codex approval pauses." }, + { value: "on-request", label: "Ask when requested", description: "Prompt when Codex requests approval." }, + { value: "untrusted", label: "Ask for untrusted operations", description: "Prompt for operations Codex does not classify as trusted." }, + ], + }, +} as const satisfies Record; + +export function isPaperclipRunnerProvider(value: unknown): value is PaperclipRunnerProvider { + return typeof value === "string" && value in PAPERCLIP_RUNNER_PERMISSION_CAPABILITIES; +} + +export function resolvePaperclipRunnerPermissionMode( + provider: PaperclipRunnerProvider, + value: unknown, +): PaperclipRunnerPermissionMode { + const capability = PAPERCLIP_RUNNER_PERMISSION_CAPABILITIES[provider]; + return capability.options.some((option) => option.value === value) + ? value as PaperclipRunnerPermissionMode + : capability.defaultMode; +} diff --git a/packages/paperclip-runner/src/testing.ts b/packages/paperclip-runner/src/testing.ts index d0ba4ba8e8..b220b2ea73 100644 --- a/packages/paperclip-runner/src/testing.ts +++ b/packages/paperclip-runner/src/testing.ts @@ -7,6 +7,7 @@ */ export * from "./index.js"; export * from "./conformance/control-plane-port.js"; +export * from "./conformance/capability-semantic-conformance.js"; export * from "./conformance/harness-driver.js"; export * from "./conformance/semantic-conformance.js"; export * from "./mock-core/deterministic-harness-driver.js"; diff --git a/scripts/dev-runner-native-binary.mjs b/scripts/dev-runner-native-binary.mjs new file mode 100644 index 0000000000..ecf9cabede --- /dev/null +++ b/scripts/dev-runner-native-binary.mjs @@ -0,0 +1,67 @@ +import { readdirSync, statSync } from "node:fs"; +import path from "node:path"; + +export function resolveNativeRunnerRequirement({ exitCode, stdout }) { + if (exitCode !== 0) { + return { nativeRunnerRequired: true, valid: false }; + } + + const jsonLines = stdout + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line.startsWith("{")); + for (let index = jsonLines.length - 1; index >= 0; index -= 1) { + try { + const payload = JSON.parse(jsonLines[index]); + if (typeof payload?.nativeRunnerRequired === "boolean") { + return { nativeRunnerRequired: payload.nativeRunnerRequired, valid: true }; + } + } catch { + // Keep scanning earlier JSON-looking output from pnpm and runtime logging. + } + } + + // An unknown state must not strand a persisted native run without runnerd. + return { nativeRunnerRequired: true, valid: false }; +} + +function newestMtimeMs(target) { + const stat = statSync(target, { throwIfNoEntry: false }); + if (!stat) return 0; + if (!stat.isDirectory()) return stat.mtimeMs; + + let newest = stat.mtimeMs; + for (const entry of readdirSync(target)) { + const childNewest = newestMtimeMs(path.join(target, entry)); + if (childNewest > newest) newest = childNewest; + } + return newest; +} + +export function paperclipRunnerBinaryNeedsBuild({ + repoRoot, + nativeRunnerRequired, + configuredBinary, + platform = process.platform, +}) { + if (!nativeRunnerRequired) return false; + if (configuredBinary?.trim()) return false; + + const executable = platform === "win32" ? "paperclip-runnerd.exe" : "paperclip-runnerd"; + const packageRoot = path.join(repoRoot, "packages", "paperclip-runner"); + const stagedBinary = path.join(packageRoot, "dist", "bin", executable); + const binaryStat = statSync(stagedBinary, { throwIfNoEntry: false }); + if (!binaryStat?.isFile()) return true; + + const runnerRoot = path.join(packageRoot, "runner"); + const buildInputs = [ + path.join(runnerRoot, "Cargo.toml"), + path.join(runnerRoot, "Cargo.lock"), + path.join(runnerRoot, ".cargo"), + path.join(runnerRoot, "rust-toolchain"), + path.join(runnerRoot, "rust-toolchain.toml"), + path.join(runnerRoot, "crates"), + ]; + + return buildInputs.some((input) => newestMtimeMs(input) > binaryStat.mtimeMs); +} diff --git a/scripts/dev-runner.ts b/scripts/dev-runner.ts index 61ac55c38d..f68e4e4ecd 100644 --- a/scripts/dev-runner.ts +++ b/scripts/dev-runner.ts @@ -6,6 +6,10 @@ import path from "node:path"; import { createInterface } from "node:readline/promises"; import { stdin, stdout } from "node:process"; import { createCapturedOutputBuffer, parseJsonResponseWithLimit } from "./dev-runner-output.ts"; +import { + paperclipRunnerBinaryNeedsBuild, + resolveNativeRunnerRequirement, +} from "./dev-runner-native-binary.mjs"; import { applyDevRunnerOptions } from "./dev-runner-options.ts"; import { collectWatchedSnapshot as collectDevServerWatchedSnapshot, diffSnapshots } from "./dev-runner-snapshot.mjs"; import { createDevServiceIdentity, repoRoot } from "./dev-service-profile.ts"; @@ -522,6 +526,75 @@ async function buildPluginSdk() { } } +async function getNativeRunnerRequired(): Promise { + const status = await runPnpm( + [ + "--silent", + "--filter", + "@paperclipai/server", + "exec", + "tsx", + "src/dev-native-runner-status.ts", + ], + { env }, + ); + if (status.signal) { + exitForSignal(status.signal); + return true; + } + const requirement = resolveNativeRunnerRequirement({ + exitCode: status.code, + stdout: status.stdout, + }); + if (!requirement.valid) { + const detail = status.stderr || status.stdout; + process.stderr.write( + `[paperclip] unable to determine the native runner requirement; conservatively preparing the native runner${detail ? `\n${detail}` : "\n"}`, + ); + } + return requirement.nativeRunnerRequired; +} + +async function buildPaperclipRunner() { + console.log("[paperclip] building paperclip runner..."); + const typescriptResult = await runPnpm( + ["--filter", "@paperclipai/paperclip-runner", "build:typescript"], + { stdio: "inherit" }, + ); + if (typescriptResult.signal) { + exitForSignal(typescriptResult.signal); + return; + } + if (typescriptResult.code !== 0) { + console.error("[paperclip] paperclip runner build failed"); + process.exit(typescriptResult.code); + } + + if ( + !paperclipRunnerBinaryNeedsBuild({ + repoRoot, + nativeRunnerRequired: await getNativeRunnerRequired(), + configuredBinary: env.PAPERCLIP_RUNNER_BINARY, + }) + ) { + return; + } + + console.log("[paperclip] building paperclip runner native binary..."); + const binaryResult = await runPnpm( + ["--filter", "@paperclipai/paperclip-runner", "build:binary"], + { stdio: "inherit" }, + ); + if (binaryResult.signal) { + exitForSignal(binaryResult.signal); + return; + } + if (binaryResult.code !== 0) { + console.error("[paperclip] paperclip runner native binary build failed"); + process.exit(binaryResult.code); + } +} + function newestMtimeMs(target: string): number { const stat = statSync(target, { throwIfNoEntry: false }); if (!stat) return 0; @@ -631,6 +704,7 @@ async function stopChildForRestart() { } async function startServerChild() { + await buildPaperclipRunner(); await buildPluginSdk(); const serverScript = mode === "watch" ? "dev:watch" : "dev"; diff --git a/server/src/__tests__/adapter-registry.test.ts b/server/src/__tests__/adapter-registry.test.ts index eb95aa673e..383c76adc8 100644 --- a/server/src/__tests__/adapter-registry.test.ts +++ b/server/src/__tests__/adapter-registry.test.ts @@ -257,6 +257,24 @@ describe("server adapter registry", () => { expect(adapter!.supportsLocalAgentJwt).toBe(true); }); + it("rejects an unsupported persisted runner provider before probing Codex", async () => { + const adapter = requireServerAdapter("paperclip_runner"); + const result = await adapter.testEnvironment({ + companyId: "company-1", + adapterType: "paperclip_runner", + config: { provider: "opencode" }, + }); + + expect(result).toMatchObject({ + adapterType: "paperclip_runner", + status: "fail", + checks: [{ + code: "paperclip_runner_provider_unsupported", + level: "error", + }], + }); + }); + it("built-in local adapters declare cheap model profile defaults where supported", async () => { await expect(listAdapterModelProfiles("claude_local")).resolves.toEqual([ expect.objectContaining({ diff --git a/server/src/__tests__/adapter-routes-authz.test.ts b/server/src/__tests__/adapter-routes-authz.test.ts index 3596d8d38e..c817a011b3 100644 --- a/server/src/__tests__/adapter-routes-authz.test.ts +++ b/server/src/__tests__/adapter-routes-authz.test.ts @@ -249,8 +249,8 @@ describe.sequential("adapter management route authorization", () => { vi.doMock("../routes/authz.js", async () => vi.importActual("../routes/authz.js")); const [routes, middleware, registry] = await Promise.all([ - vi.importActual("../routes/adapters.js"), - vi.importActual("../middleware/index.js"), + import("../routes/adapters.js"), + import("../middleware/index.js"), vi.importActual("../adapters/registry.js"), ]); adapterRoutes = routes.adapterRoutes; diff --git a/server/src/__tests__/adapter-routes.test.ts b/server/src/__tests__/adapter-routes.test.ts index db93a0fb36..a6ac7455c9 100644 --- a/server/src/__tests__/adapter-routes.test.ts +++ b/server/src/__tests__/adapter-routes.test.ts @@ -164,7 +164,7 @@ describe("adapter routes", () => { disabled: false, capabilities: { supportsInstructionsBundle: false, - supportsModelProfiles: false, + supportsModelProfiles: true, }, }); }); diff --git a/server/src/__tests__/company-portability.test.ts b/server/src/__tests__/company-portability.test.ts index 0d04930ddc..a3ab5d0a52 100644 --- a/server/src/__tests__/company-portability.test.ts +++ b/server/src/__tests__/company-portability.test.ts @@ -5906,6 +5906,20 @@ describe("company portability", () => { expect(agentSvc.create).not.toHaveBeenCalled(); instanceSettingsSvc.getExperimental.mockResolvedValue({ enableNativeRunner: true }); + await expect(portability.importBundle({ + ...request, + adapterOverrides: { + claudecoder: { + adapterType: "paperclip_runner", + adapterConfig: { provider: "opencode" }, + }, + }, + }, "user-1")).rejects.toMatchObject({ + status: 422, + details: { code: "paperclip_runner_provider_unavailable" }, + }); + expect(agentSvc.create).not.toHaveBeenCalled(); + await portability.importBundle(request, "user-1"); expect(agentSvc.create).toHaveBeenCalledWith("company-1", expect.objectContaining({ adapterType: "paperclip_runner", diff --git a/server/src/__tests__/dev-runner-native-binary.test.ts b/server/src/__tests__/dev-runner-native-binary.test.ts new file mode 100644 index 0000000000..705ea8cf76 --- /dev/null +++ b/server/src/__tests__/dev-runner-native-binary.test.ts @@ -0,0 +1,122 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +import { + paperclipRunnerBinaryNeedsBuild, + resolveNativeRunnerRequirement, +} from "../../../scripts/dev-runner-native-binary.mjs"; + +const tempRoots = new Set(); + +afterEach(() => { + for (const root of tempRoots) { + fs.rmSync(root, { recursive: true, force: true }); + } + tempRoots.clear(); +}); + +function createRunnerCheckout(): { root: string; source: string; binary: string } { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-dev-runner-binary-")); + tempRoots.add(root); + + const runnerRoot = path.join(root, "packages", "paperclip-runner", "runner"); + const source = path.join(runnerRoot, "crates", "runner-core", "src", "main.rs"); + const binary = path.join( + root, + "packages", + "paperclip-runner", + "dist", + "bin", + process.platform === "win32" ? "paperclip-runnerd.exe" : "paperclip-runnerd", + ); + fs.mkdirSync(path.dirname(source), { recursive: true }); + fs.mkdirSync(path.dirname(binary), { recursive: true }); + fs.writeFileSync(path.join(runnerRoot, "Cargo.toml"), "[workspace]\n", "utf8"); + fs.writeFileSync(path.join(runnerRoot, "Cargo.lock"), "", "utf8"); + fs.writeFileSync(source, "fn main() {}\n", "utf8"); + fs.writeFileSync(binary, "runnerd", "utf8"); + return { root, source, binary }; +} + +describe("paperclip runner native dev prerequisite", () => { + it("uses an explicit status response and fails safe when status is unknown", () => { + expect( + resolveNativeRunnerRequirement({ + exitCode: 0, + stdout: "pnpm warning\n{\"nativeRunnerRequired\":false}\n", + }), + ).toEqual({ nativeRunnerRequired: false, valid: true }); + expect( + resolveNativeRunnerRequirement({ + exitCode: 0, + stdout: "{\"nativeRunnerRequired\":true}\n", + }), + ).toEqual({ nativeRunnerRequired: true, valid: true }); + expect( + resolveNativeRunnerRequirement({ + exitCode: 1, + stdout: "", + }), + ).toEqual({ nativeRunnerRequired: true, valid: false }); + expect( + resolveNativeRunnerRequirement({ + exitCode: 0, + stdout: "{\"unexpected\":true}\n", + }), + ).toEqual({ nativeRunnerRequired: true, valid: false }); + }); + + it("builds only when the staged binary is missing or older than Rust inputs", () => { + const checkout = createRunnerCheckout(); + const now = Date.now(); + const old = new Date(now - 2_000); + const current = new Date(now + 2_000); + const next = new Date(now + 4_000); + + fs.utimesSync(checkout.source, old, old); + fs.utimesSync(checkout.binary, current, current); + expect( + paperclipRunnerBinaryNeedsBuild({ + repoRoot: checkout.root, + nativeRunnerRequired: true, + }), + ).toBe(false); + + fs.utimesSync(checkout.source, next, next); + expect( + paperclipRunnerBinaryNeedsBuild({ + repoRoot: checkout.root, + nativeRunnerRequired: true, + }), + ).toBe(true); + + fs.rmSync(checkout.binary); + expect( + paperclipRunnerBinaryNeedsBuild({ + repoRoot: checkout.root, + nativeRunnerRequired: true, + }), + ).toBe(true); + }); + + it("keeps default-off legacy development Node-only", () => { + expect( + paperclipRunnerBinaryNeedsBuild({ + repoRoot: "/checkout/without/a/staged/binary", + nativeRunnerRequired: false, + }), + ).toBe(false); + }); + + it("does not build a workspace binary when an explicit binary is configured", () => { + expect( + paperclipRunnerBinaryNeedsBuild({ + repoRoot: "/checkout/without/a/staged/binary", + nativeRunnerRequired: true, + configuredBinary: "/opt/paperclip/paperclip-runnerd", + }), + ).toBe(false); + }); +}); diff --git a/server/src/__tests__/fixtures/plugin-worker-duplex-channel.cjs b/server/src/__tests__/fixtures/plugin-worker-duplex-channel.cjs index aebae1cb32..27d8da6752 100644 --- a/server/src/__tests__/fixtures/plugin-worker-duplex-channel.cjs +++ b/server/src/__tests__/fixtures/plugin-worker-duplex-channel.cjs @@ -50,6 +50,9 @@ // scripted data and exit in one stdout write. The host then reads the open // reply and the notifications in one batch, so a test proves the host holds // and replays a frame that arrives before the route binds. +// - `emitScriptedFramesAfterFirstWrite`: when true, the fixture holds the +// scripted data and exit until it has acknowledged the first channel write. +// This gives tests a deterministic post-bind trigger without timing delays. const readline = require("node:readline"); function send(message) { @@ -169,6 +172,10 @@ rl.on("line", (line) => { noWriteReply: mode === "no-write-reply", writeReplyDelayMs: typeof directive.writeReplyDelayMs === "number" ? directive.writeReplyDelayMs : 0, + scriptedFramesAfterFirstWrite: + directive.emitScriptedFramesAfterFirstWrite === true + ? scriptedFrameLines(directive, hostRouteId, workerSessionId) + : null, emitAfterCloseChunk: typeof directive.emitAfterCloseChunk === "string" ? directive.emitAfterCloseChunk : null, }); @@ -222,9 +229,11 @@ rl.on("line", (line) => { // Emit the scripted data and the exit after the open reply, so the host // binds the route first. Each frame echoes the exact pair; a test overrides // `sid` or `rid` to force a mismatch. - setImmediate(() => { - process.stdout.write(scriptedFrameLines(directive, hostRouteId, workerSessionId)); - }); + if (directive.emitScriptedFramesAfterFirstWrite !== true) { + setImmediate(() => { + process.stdout.write(scriptedFrameLines(directive, hostRouteId, workerSessionId)); + }); + } return; } @@ -257,7 +266,16 @@ rl.on("line", (line) => { }, }); } - const replyWrite = () => send({ jsonrpc: "2.0", id: message.id, result: null }); + const replyWrite = () => { + send({ jsonrpc: "2.0", id: message.id, result: null }); + const deferredFrames = entry.scriptedFramesAfterFirstWrite; + entry.scriptedFramesAfterFirstWrite = null; + if (deferredFrames) { + // Emit only after acknowledging the host trigger. The trigger can only be + // sent through a returned session, so the route is definitively bound. + setImmediate(() => process.stdout.write(deferredFrames)); + } + }; if (entry.writeReplyDelayMs > 0) { // Delay the write reply, so the host holds the pending-write reservation for // a measurable time before the RPC settles. diff --git a/server/src/__tests__/heartbeat-accepted-plan-workspace-refresh.test.ts b/server/src/__tests__/heartbeat-accepted-plan-workspace-refresh.test.ts index 416577e4db..e7d6c933ba 100644 --- a/server/src/__tests__/heartbeat-accepted-plan-workspace-refresh.test.ts +++ b/server/src/__tests__/heartbeat-accepted-plan-workspace-refresh.test.ts @@ -1132,6 +1132,8 @@ describeEmbeddedPostgres("accepted plan workspace refresh", () => { }; expect(adapterInput.runtime.sessionId).toBe("accepted-plan-retry-session"); expect(adapterInput.context.acceptedPlanWakeRouting).toBeUndefined(); - expect(adapterInput.context.paperclipTaskMarkdown).toContain("Create child issues from the approved plan only"); + expect(adapterInput.context.paperclipTaskMarkdown).toContain( + "Implement the accepted plan on this issue when the work is small and cohesive.", + ); }, 20_000); }); diff --git a/server/src/__tests__/heartbeat-context-summary.test.ts b/server/src/__tests__/heartbeat-context-summary.test.ts index 543cca4d47..8033ac71ec 100644 --- a/server/src/__tests__/heartbeat-context-summary.test.ts +++ b/server/src/__tests__/heartbeat-context-summary.test.ts @@ -51,7 +51,9 @@ describe("buildPaperclipTaskMarkdown", () => { }, }); - expect(acceptedConfirmation).toContain("Create child issues from the approved plan only"); + expect(acceptedConfirmation).toContain( + "Implement the accepted plan on this issue when the work is small and cohesive.", + ); expect(acceptedConfirmation).not.toContain("Make the plan only."); }); @@ -68,7 +70,9 @@ describe("buildPaperclipTaskMarkdown", () => { }); expect(acceptedConfirmation).toContain("Accepted plan directive:"); - expect(acceptedConfirmation).toContain("Create child issues from the approved plan only"); + expect(acceptedConfirmation).toContain( + "Implement the accepted plan on this issue when the work is small and cohesive.", + ); expect(acceptedConfirmation).not.toContain("- Work mode: \"planning\""); }); diff --git a/server/src/__tests__/heartbeat-native-runner-cancellation.test.ts b/server/src/__tests__/heartbeat-native-runner-cancellation.test.ts new file mode 100644 index 0000000000..3b01bdf9a9 --- /dev/null +++ b/server/src/__tests__/heartbeat-native-runner-cancellation.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it, vi } from "vitest"; +import type { Db } from "@paperclipai/db"; + +import { cancelHeartbeatNativeRun } from "../services/heartbeat.js"; + +describe("native heartbeat cancellation authority", () => { + it("does not enter native cancellation for a direct-adapter run", async () => { + const cancel = vi.fn(); + + await expect(cancelHeartbeatNativeRun({ + db: {} as Db, + runId: "legacy-run", + reason: "Cancelled by control plane", + runtimeMode: "legacy", + cancel, + })).resolves.toEqual({ decision: null, auditId: null }); + + expect(cancel).not.toHaveBeenCalled(); + }); + + it("dispatches pause and bulk cancellation through the audited run scope", async () => { + const db = {} as Db; + const cancel = vi.fn(async () => ({ + decision: { reasonCode: "cancellation_run_only" }, + auditId: "audit-1", + })); + + await expect(cancelHeartbeatNativeRun({ + db, + runId: "run-1", + reason: "Cancelled due to agent pause", + runtimeMode: "native", + cancel, + })).resolves.toMatchObject({ auditId: "audit-1" }); + + expect(cancel).toHaveBeenCalledWith( + "run-1", + "Cancelled due to agent pause", + { db, scope: "run" }, + ); + }); + + it("fails closed when a native cancellation lacks its decision audit", async () => { + const cancel = vi.fn(async () => ({ + decision: null, + auditId: null, + })); + + await expect(cancelHeartbeatNativeRun({ + db: {} as Db, + runId: "run-2", + reason: "Cancelled because the agent was terminated", + runtimeMode: "native", + cancel, + })).rejects.toThrow("native_cancellation_outcome_not_audited"); + }); +}); diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index c3aa6dc5a2..687e90923b 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -1274,7 +1274,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { const run = await heartbeat.getRun(runId); expect(run).toMatchObject({ status: "running", - errorCode: "process_detached", + errorCode: "native_execution_ownership_unverified", processPid: child.pid, }); const retries = await db diff --git a/server/src/__tests__/heartbeat-runtime-mcp-servers.test.ts b/server/src/__tests__/heartbeat-runtime-mcp-servers.test.ts index c1e8baacce..d0dc7082dc 100644 --- a/server/src/__tests__/heartbeat-runtime-mcp-servers.test.ts +++ b/server/src/__tests__/heartbeat-runtime-mcp-servers.test.ts @@ -164,6 +164,26 @@ describeEmbeddedPostgres("heartbeat runtime MCP servers", () => { await expect( buildPaperclipRuntimeMcpServers({ db, agent: agent!, runId: randomUUID() }), ).resolves.toEqual([]); + await expect( + buildPaperclipRuntimeMcpServers({ + db, + agent: agent!, + runId: randomUUID(), + failOnUnavailableAssignedConnection: true, + }), + ).rejects.toThrow( + `assigned native MCP connection is unavailable: ${installedConnection!.id}`, + ); + await expect( + createManagedMcpRunConfig({ + db, + agent: agent!, + runId: randomUUID(), + config: {}, + projectId: null, + issueId: null, + }), + ).resolves.toBeNull(); }); it("audits permitted remote MCP connections that were not installed when delivery is empty", async () => { diff --git a/server/src/__tests__/helpers/paperclip-semantic-conformance.ts b/server/src/__tests__/helpers/paperclip-semantic-conformance.ts new file mode 100644 index 0000000000..4912afbcc2 --- /dev/null +++ b/server/src/__tests__/helpers/paperclip-semantic-conformance.ts @@ -0,0 +1,589 @@ +import express, { type Application } from "express"; +import request from "supertest"; +import { and, asc, eq } from "drizzle-orm"; +import { + activityLog, + agents, + companies, + createDb, + documentRevisions, + documents, + heartbeatRuns, + issueComments, + issueDocuments, + issueRelations, + issues, + issueThreadInteractions, +} from "@paperclipai/db"; +import { + CapabilitySemanticDispatcher, + createCapabilityFixtureState, + normalizeCapabilitySemanticObservation, + type CapabilityCommandEnvelope, + type CapabilityCommandOutcome, + type CapabilityCommandResult, + type CapabilityFixtureState, + type CapabilityRunContext, + type CapabilitySemanticCommand, + type SemanticConformanceAdapter, + type SemanticConformanceObservation, + type SemanticConformanceVector, +} from "../../vendor/paperclip-runner/testing.js"; + +import { errorHandler } from "../../middleware/index.js"; +import { issueRoutes } from "../../routes/issues.js"; + +type Db = ReturnType; + +export interface PaperclipSemanticConformanceIds { + readonly companyId: string; + readonly actorId: string; + readonly foreignCompanyId: string; + readonly foreignTaskId: string; + readonly blockerTaskId: string; + readonly worlds: Readonly>; +} + +interface IdempotencyRecord { + readonly canonicalCommand: string; + readonly result: CapabilityCommandResult; +} + +interface ProductionWorld { + readonly port: PaperclipRouteSemanticPort; + readonly dispatcher: CapabilitySemanticDispatcher; + readonly taskId: string; + readonly runId: string; +} + +/** + * Controlled production binding for the shared kit. It delegates writes to + * issueRoutes, so authorization, service transactions, audit logging, document + * revisions, interactions, and terminal arbitration remain production-owned. + */ +export class PaperclipProductionSemanticConformanceAdapter implements SemanticConformanceAdapter { + readonly id = "paperclip-production-services"; + readonly kind = "production_binding" as const; + + private constructor(readonly worlds: ReadonlyMap) {} + + static async create(db: Db, ids: PaperclipSemanticConformanceIds): Promise { + const app = createProductionAuthorityApp(db, ids); + const worlds = new Map(); + for (const [id, binding] of Object.entries(ids.worlds)) { + const port = new PaperclipRouteSemanticPort(db, app, ids, binding); + await port.refresh(); + worlds.set(id, { + port, + dispatcher: new CapabilitySemanticDispatcher(port), + taskId: binding.taskId, + runId: binding.runId, + }); + } + return new PaperclipProductionSemanticConformanceAdapter(worlds); + } + + async execute(vector: SemanticConformanceVector): Promise { + const worldId = vector.worldId ?? "default"; + const world = this.worlds.get(worldId); + if (world === undefined) throw new Error(`semantic_conformance_world_missing:${worldId}`); + const before = world.port.snapshot(); + const result = await world.dispatcher.dispatch({ + runId: world.runId, + callId: vector.id, + operationId: vector.operationId, + input: vector.input, + }); + const after = world.port.snapshot(); + return normalizeCapabilitySemanticObservation({ + result, + before, + after, + taskId: world.taskId, + semanticInput: vector.input, + }); + } +} + +export async function seedPaperclipSemanticConformance( + db: Db, + ids: PaperclipSemanticConformanceIds, +): Promise { + await db.insert(companies).values([ + { id: ids.companyId, name: "Semantic Conformance", issuePrefix: "SCF" }, + { id: ids.foreignCompanyId, name: "Foreign Conformance", issuePrefix: "FRN" }, + ]); + await db.insert(agents).values({ + id: ids.actorId, + companyId: ids.companyId, + name: "Conformance Approver", + role: "approver", + status: "active", + adapterType: "codex_local", + adapterConfig: {}, + runtimeConfig: {}, + permissions: {}, + }); + let sequence = 0; + for (const [worldId, world] of Object.entries(ids.worlds)) { + sequence += 1; + await db.insert(issues).values({ + id: world.taskId, + companyId: ids.companyId, + identifier: `SCF-${sequence}`, + title: `${worldId} semantic conformance`, + status: "in_progress", + workMode: "standard", + priority: "medium", + assigneeAgentId: ids.actorId, + }); + await db.insert(heartbeatRuns).values({ + id: world.runId, + companyId: ids.companyId, + agentId: ids.actorId, + status: "running", + invocationSource: "assignment", + triggerDetail: "system", + contextSnapshot: { issueId: world.taskId, capabilities: world.capabilities }, + }); + await db.update(issues).set({ + checkoutRunId: world.runId, + executionRunId: world.runId, + }).where(eq(issues.id, world.taskId)); + } + await db.insert(issues).values({ + id: ids.blockerTaskId, + companyId: ids.companyId, + identifier: "SCF-90", + title: "Unresolved semantic blocker", + status: "todo", + workMode: "standard", + priority: "medium", + }); + const blockedTerminal = ids.worlds["terminal-blocked"]; + if (blockedTerminal === undefined) throw new Error("semantic_conformance_blocked_terminal_world_missing"); + await db.insert(issueRelations).values({ + companyId: ids.companyId, + issueId: ids.blockerTaskId, + relatedIssueId: blockedTerminal.taskId, + type: "blocks", + }); + await db.insert(issues).values({ + id: ids.foreignTaskId, + companyId: ids.foreignCompanyId, + identifier: "FRN-1", + title: "Foreign semantic task", + status: "todo", + workMode: "standard", + priority: "medium", + }); +} + +class PaperclipRouteSemanticPort { + readonly #idempotency = new Map(); + #state: CapabilityFixtureState; + + constructor( + readonly db: Db, + readonly app: Application, + readonly ids: PaperclipSemanticConformanceIds, + readonly binding: { taskId: string; runId: string; capabilities: readonly string[] }, + ) { + this.#state = createCapabilityFixtureState(); + } + + context(runId: string): CapabilityRunContext { + if (runId !== this.binding.runId) throw new Error("semantic_conformance_run_binding_mismatch"); + const task = this.#state.tasks.find((candidate) => candidate.id === this.binding.taskId); + if (task === undefined) throw new Error("semantic_conformance_task_missing"); + return { + schema: "paperclip.capability.run-context.v1", + company: { + id: this.ids.companyId, + name: "Semantic Conformance", + issuePrefix: "SCF", + status: "active", + }, + actor: { + id: this.ids.actorId, + name: "Conformance Approver", + role: "approver", + status: "active", + capabilityGrants: [...this.binding.capabilities], + }, + activeTask: structuredClone(task), + ancestors: [], + wake: { reason: "manual", payload: {} }, + capabilities: [...this.binding.capabilities], + budget: { limitCents: 10_000, spentCents: 0, remainingCents: 10_000 }, + interactionResults: [], + }; + } + + snapshot(): Readonly { + return structuredClone(this.#state); + } + + async tryApplyCommand(envelope: CapabilityCommandEnvelope): Promise { + const canonicalCommand = canonicalJson(envelope.command); + const prior = this.#idempotency.get(envelope.idempotencyKey); + if (prior !== undefined) { + if (prior.canonicalCommand !== canonicalCommand) { + return this.denial(envelope.command, "idempotency_conflict", "Idempotency key was reused with different input"); + } + return { + ok: true, + result: { ...structuredClone(prior.result), disposition: "duplicate" }, + }; + } + + const outcome = await this.applyThroughProductionRoutes(envelope.command); + await this.refresh(); + if (outcome.ok) { + this.#idempotency.set(envelope.idempotencyKey, { + canonicalCommand, + result: structuredClone(outcome.result), + }); + } + return outcome; + } + + async refresh(): Promise { + const [issue] = await this.db.select().from(issues).where(eq(issues.id, this.binding.taskId)); + if (issue === undefined) throw new Error("semantic_conformance_task_missing"); + const [run] = await this.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, this.binding.runId)); + if (run === undefined) throw new Error("semantic_conformance_run_missing"); + const comments = await this.db.select().from(issueComments) + .where(eq(issueComments.issueId, issue.id)) + .orderBy(asc(issueComments.createdAt), asc(issueComments.id)); + const linkedDocuments = await this.db.select({ link: issueDocuments, document: documents }) + .from(issueDocuments) + .innerJoin(documents, eq(issueDocuments.documentId, documents.id)) + .where(eq(issueDocuments.issueId, issue.id)); + const revisions = linkedDocuments.length === 0 + ? [] + : await this.db.select().from(documentRevisions) + .where(eq(documentRevisions.companyId, this.ids.companyId)) + .orderBy(asc(documentRevisions.revisionNumber)); + const interactions = await this.db.select().from(issueThreadInteractions) + .where(eq(issueThreadInteractions.issueId, issue.id)) + .orderBy(asc(issueThreadInteractions.createdAt), asc(issueThreadInteractions.id)); + const relations = await this.db.select().from(issueRelations) + .where(and(eq(issueRelations.relatedIssueId, issue.id), eq(issueRelations.type, "blocks"))); + const audit = await this.db.select().from(activityLog) + .where(and( + eq(activityLog.companyId, this.ids.companyId), + eq(activityLog.runId, this.binding.runId), + )) + .orderBy(asc(activityLog.createdAt), asc(activityLog.id)); + + const state = createCapabilityFixtureState({ + company: { id: this.ids.companyId, name: "Semantic Conformance", issuePrefix: "SCF" }, + actors: [{ + id: this.ids.actorId, + companyId: this.ids.companyId, + name: "Conformance Approver", + role: "approver", + status: "active", + budgetId: "production-binding-budget", + capabilityGrants: [...this.binding.capabilities], + }], + tasks: [{ + id: issue.id, + companyId: issue.companyId, + identifier: issue.identifier, + title: issue.title, + description: issue.description, + status: issue.status as CapabilityFixtureState["tasks"][number]["status"], + priority: issue.priority as CapabilityFixtureState["tasks"][number]["priority"], + workMode: issue.workMode as CapabilityFixtureState["tasks"][number]["workMode"], + parentId: issue.parentId, + assigneeActorId: issue.assigneeAgentId, + checkoutRunId: issue.checkoutRunId, + executionRunId: issue.executionRunId, + startedAt: issue.startedAt?.toISOString() ?? null, + completedAt: issue.completedAt?.toISOString() ?? null, + }], + comments: comments.map((comment) => ({ + id: comment.id, + taskId: comment.issueId, + authorActorId: comment.authorAgentId, + body: comment.body, + createdAt: comment.createdAt.toISOString(), + })), + documents: linkedDocuments.map(({ link, document }) => ({ + id: document.id, + taskId: link.issueId, + key: link.key, + title: document.title ?? "", + format: "markdown" as const, + latestRevisionId: document.latestRevisionId ?? "", + revisions: revisions + .filter((revision) => revision.documentId === document.id) + .map((revision) => ({ + id: revision.id, + documentId: revision.documentId, + revision: revision.revisionNumber, + body: revision.body, + changeSummary: revision.changeSummary, + createdAt: revision.createdAt.toISOString(), + })), + })), + interactions: interactions.map((interaction) => ({ + id: interaction.id, + taskId: interaction.issueId, + kind: toFixtureInteractionKind(interaction.kind), + status: interaction.status as CapabilityFixtureState["interactions"][number]["status"], + title: interaction.title ?? "", + prompt: readPrompt(interaction.payload), + payload: interaction.payload, + targetRevisionId: readTargetRevisionId(interaction.payload), + continuationPolicy: interaction.continuationPolicy as CapabilityFixtureState["interactions"][number]["continuationPolicy"], + result: interaction.result ?? null, + createdAt: interaction.createdAt.toISOString(), + resolvedAt: interaction.resolvedAt?.toISOString() ?? null, + })), + blockers: relations.map((relation) => ({ + id: relation.id, + taskId: relation.relatedIssueId, + blockedByTaskId: relation.issueId, + createdAt: relation.createdAt.toISOString(), + })), + }); + state.lifecycle = "running"; + state.activeRunId = run.id; + state.runs = [{ + id: run.id, + companyId: run.companyId, + actorId: run.agentId, + taskId: issue.id, + sessionId: run.sessionIdAfter ?? run.sessionIdBefore ?? run.externalRunId ?? run.id, + backendKind: "runner", + sourceInstanceId: "paperclip-production-services", + status: toFixtureRunStatus(run.status), + attempt: run.scheduledRetryAttempt + 1, + openedAt: (run.startedAt ?? run.createdAt).toISOString(), + finishedAt: run.finishedAt?.toISOString() ?? null, + wake: { reason: "manual", payload: {} }, + capabilities: [...this.binding.capabilities], + events: [], + result: null, + sessionCheckpoint: null, + }]; + state.revision = this.#state.revision + 1; + state.audit = audit.map((entry) => ({ + id: entry.id, + at: entry.createdAt.toISOString(), + runId: entry.runId, + actorId: entry.agentId, + action: entry.action, + entityType: entry.entityType, + entityId: entry.entityId, + details: (entry.details ?? {}) as CapabilityFixtureState["audit"][number]["details"], + })); + this.#state = state; + } + + private async applyThroughProductionRoutes(command: CapabilitySemanticCommand): Promise { + const taskId = this.binding.taskId; + switch (command.kind) { + case "report_progress": { + const response = await this.post(`/api/issues/${taskId}/comments`, { body: command.body }); + if (!isSuccess(response.status)) return this.routeDenial(command, response.status, response.body); + return this.success(command, [`task:${taskId}`, `comment:${String(response.body.id)}`]); + } + case "write_document": { + const response = await this.put(`/api/issues/${taskId}/documents/${encodeURIComponent(command.key)}`, { + title: command.title, + format: "markdown", + body: command.body, + changeSummary: command.changeSummary ?? null, + baseRevisionId: command.baseRevisionId, + }); + if (!isSuccess(response.status)) return this.routeDenial(command, response.status, response.body); + return this.success(command, [ + `task:${taskId}`, + `document:${String(response.body.id)}`, + `revision:${String(response.body.latestRevisionId)}`, + ]); + } + case "request_human_input": { + if (command.interactionKind !== "confirmation") { + return this.denial(command, "operation_unavailable", "Only confirmation is bound in this controlled adapter"); + } + const response = await this.post(`/api/issues/${taskId}/interactions`, { + kind: "request_confirmation", + idempotencyKey: `semantic:${command.title}`, + title: command.title, + summary: command.prompt, + continuationPolicy: command.continuationPolicy, + payload: { + version: 1, + prompt: command.prompt, + detailsMarkdown: "", + acceptLabel: "Confirm", + rejectLabel: "Request changes", + rejectRequiresReason: false, + supersedeOnUserComment: true, + }, + }); + if (!isSuccess(response.status)) return this.routeDenial(command, response.status, response.body); + const transition = await this.patch(`/api/issues/${taskId}`, { status: "in_review" }); + if (!isSuccess(transition.status)) return this.routeDenial(command, transition.status, transition.body); + return this.success(command, [`task:${taskId}`, `interaction:${String(response.body.id)}`]); + } + case "set_dependencies": { + const response = await this.patch(`/api/issues/${taskId}`, { + blockedByIssueIds: command.blockedByTaskIds, + }); + if (!isSuccess(response.status)) return this.routeDenial(command, response.status, response.body); + return this.success(command, [ + `task:${taskId}`, + ...command.blockedByTaskIds.map((id) => `blocker:${id}`), + ]); + } + case "finish_task": { + const response = await this.patch(`/api/issues/${taskId}`, { + status: "done", + comment: command.summary, + }); + if (!isSuccess(response.status)) return this.routeDenial(command, response.status, response.body); + return this.success(command, [`task:${taskId}`, `comment:${taskId}`]); + } + default: + return this.denial(command, "operation_unavailable", "Operation is not bound in this controlled adapter"); + } + } + + private success(command: CapabilitySemanticCommand, entityRefs: string[]): CapabilityCommandOutcome { + return { + ok: true, + result: { + commandId: `production:${this.binding.runId}:${this.#idempotency.size + 1}`, + commandKind: command.kind, + disposition: "applied", + stateRevision: this.#state.revision + 1, + entityRefs, + scheduledWakeIds: [], + }, + }; + } + + private routeDenial(command: CapabilitySemanticCommand, status: number, body: unknown): CapabilityCommandOutcome { + const message = errorMessage(body); + if (command.kind === "write_document" && status === 409) { + return this.denial(command, "document_revision_conflict", message); + } + if (command.kind === "set_dependencies" && status === 422) { + return this.denial(command, "company_scope_violation", message); + } + const code = status === 403 + ? "operation_not_authorized" + : status === 409 + ? "state_conflict" + : status === 422 + ? "semantic_rule_violation" + : "production_service_error"; + return this.denial(command, code, message, status >= 500); + } + + private denial( + command: CapabilitySemanticCommand, + code: string, + message: string, + retryable = false, + ): CapabilityCommandOutcome { + return { + ok: false, + commandKind: command.kind, + stateRevision: this.#state.revision, + error: { code, message, retryable }, + }; + } + + private post(path: string, body: unknown) { + return request(this.app).post(path).set("X-Paperclip-Run-Id", this.binding.runId).send(body); + } + + private put(path: string, body: unknown) { + return request(this.app).put(path).set("X-Paperclip-Run-Id", this.binding.runId).send(body); + } + + private patch(path: string, body: unknown) { + return request(this.app).patch(path).set("X-Paperclip-Run-Id", this.binding.runId).send(body); + } +} + +function toFixtureRunStatus(status: string): CapabilityFixtureState["runs"][number]["status"] { + switch (status) { + case "running": + case "succeeded": + case "failed": + case "cancelled": + return status; + default: + throw new Error(`semantic_conformance_run_status_unmapped:${status}`); + } +} + +function createProductionAuthorityApp(db: Db, ids: PaperclipSemanticConformanceIds): Application { + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + req.actor = { + type: "agent", + agentId: ids.actorId, + companyId: ids.companyId, + runId: req.header("X-Paperclip-Run-Id") ?? undefined, + source: "agent_jwt", + }; + next(); + }); + app.use("/api", issueRoutes(db, {} as never)); + app.use(errorHandler); + return app; +} + +function toFixtureInteractionKind(kind: string): CapabilityFixtureState["interactions"][number]["kind"] { + switch (kind) { + case "request_confirmation": return "confirmation"; + case "request_checkbox_confirmation": return "checkbox"; + case "ask_user_questions": return "questions"; + case "suggest_tasks": return "suggest_tasks"; + case "request_item_verdicts": return "item_verdicts"; + default: throw new Error(`semantic_conformance_interaction_kind_unmapped:${kind}`); + } +} + +function readPrompt(payload: unknown): string { + return typeof payload === "object" && payload !== null && "prompt" in payload + ? String(payload.prompt) + : ""; +} + +function readTargetRevisionId(payload: unknown): string | null { + if (typeof payload !== "object" || payload === null || !("target" in payload)) return null; + const target = payload.target; + return typeof target === "object" && target !== null && "revisionId" in target + ? String(target.revisionId) + : null; +} + +function isSuccess(status: number): boolean { + return status >= 200 && status < 300; +} + +function errorMessage(body: unknown): string { + return typeof body === "object" && body !== null && "error" in body + ? String(body.error) + : "Production service denied the semantic command"; +} + +function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (typeof value === "object" && value !== null) { + const record = value as Record; + return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`).join(",")}}`; + } + return JSON.stringify(value) ?? "undefined"; +} diff --git a/server/src/__tests__/legacy-finalization-regression.test.ts b/server/src/__tests__/legacy-finalization-regression.test.ts index fd440c26a5..fc15ec9b30 100644 --- a/server/src/__tests__/legacy-finalization-regression.test.ts +++ b/server/src/__tests__/legacy-finalization-regression.test.ts @@ -123,7 +123,7 @@ describe("P6-32 legacy finalization regression", () => { id: runId, status: "succeeded", runtimeMode: "legacy", - runtimeModeReason: "instance_flag_disabled", + runtimeModeReason: "direct_adapter", resultJson: { summary: "Legacy bytes", nested: { count: 1, ok: true } }, }); await expect(reconcileNativeFinalizations(db, [runId])).resolves.toEqual([]); diff --git a/server/src/__tests__/native-session-resumption.test.ts b/server/src/__tests__/native-session-resumption.test.ts new file mode 100644 index 0000000000..42d5491ef3 --- /dev/null +++ b/server/src/__tests__/native-session-resumption.test.ts @@ -0,0 +1,894 @@ +import { spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { once } from "node:events"; +import { fileURLToPath } from "node:url"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { eq } from "drizzle-orm"; +import { + activityLog, + agents, + companies, + completionContracts, + createDb, + executionWorkspaces, + heartbeatRunEvents, + heartbeatRuns, + issueRecoveryActions, + issueWorkProducts, + issues, + nativeRunFinalizations, + nativeRunResults, + projectWorkspaces, + projects, + statusDecisionEffects, + statusDecisions, + workAssessments, +} from "@paperclipai/db"; +import { + type NativeExecutionInputV1, + type NativeSession, + type NativeSessionBackend, + type PersistedNativeSession, + type PrpEvent, +} from "@paperclipai/paperclip-runner"; +import { + CONTROL_PLANE_CONFORMANCE_RESULT, + CONTROL_PLANE_CONFORMANCE_TERMINAL, +} from "../vendor/paperclip-runner/testing.js"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { drainHeartbeatRunsToQuiescence } from "./helpers/drain-heartbeat-runs.js"; +import { + claimNativeSessionResumptions, + dispatchNativeSessionResumptions, +} from "../services/native-runtime/native-finalization-reconciler.js"; + +const legacyAdapterExecute = vi.hoisted(() => vi.fn(async () => ({ + exitCode: 0, + signal: null, + timedOut: false, + summary: "Fresh flag-off run completed through legacy.", + resultJson: { summary: "fresh legacy after persisted native recovery" }, + provider: "test", + model: "legacy-test", +}))); + +vi.mock("../adapters/index.js", async () => { + const actual = await vi.importActual("../adapters/index.js"); + return { + ...actual, + getServerAdapter: vi.fn(() => ({ + type: "codex_local", + execute: legacyAdapterExecute, + supportsLocalAgentJwt: false, + })), + }; +}); + +import { heartbeatService } from "../services/heartbeat.js"; +import { instanceSettingsService } from "../services/instance-settings.js"; + +describe("P6-25 pre-result native session recovery", () => { + let temporary: Awaited> | null = null; + let db: ReturnType; + const companyId = "79000000-0000-4000-8000-000000000001"; + const agentId = "79000000-0000-4000-8000-000000000002"; + const issueId = "79000000-0000-4000-8000-000000000003"; + const runId = "79000000-0000-4000-8000-000000000004"; + const cancelledRunId = "79000000-0000-4000-8000-000000000005"; + const exhaustedRunId = "79000000-0000-4000-8000-000000000006"; + const missingCheckpointRunId = "79000000-0000-4000-8000-000000000007"; + const initialRunId = "79000000-0000-4000-8000-000000000008"; + const bootstrapRetryRunId = "79000000-0000-4000-8000-000000000009"; + const observedExpiredRunId = "79000000-0000-4000-8000-000000000010"; + const observedLivePidRunId = "79000000-0000-4000-8000-000000000011"; + const persistedProfile = { + mode: "native", + nativeExecutionInput: { schema: "paperclip.native-execution-input.v1", binding: { runId } }, + sessionCheckpoint: { + backendKind: "codex_app_server", + sessionId: "persisted-session", + identity: { runId }, + providerSessionId: "provider-session", + activeTurnId: "active-turn", + }, + }; + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-resume-"); + db = createDb(temporary.connectionString); + await db.insert(companies).values({ id: companyId, name: "Native resume", issuePrefix: "NRR" }); + await db.insert(agents).values({ + id: agentId, + companyId, + name: "Native resume agent", + adapterType: "codex_local", + status: "running", + }); + await db.insert(issues).values({ + id: issueId, + companyId, + title: "Resume the same native run", + status: "in_progress", + assigneeAgentId: agentId, + workMode: "standard", + }); + await db.insert(heartbeatRuns).values([ + { + id: runId, + companyId, + agentId, + nativeIssueId: issueId, + status: "running", + runtimeMode: "native", + runtimeModeResolvedAt: new Date(), + runnerProfileJson: persistedProfile, + contextSnapshot: { issueId }, + }, + { + id: cancelledRunId, + companyId, + agentId, + nativeIssueId: issueId, + status: "cancelled", + runtimeMode: "native", + runtimeModeResolvedAt: new Date(), + runnerProfileJson: persistedProfile, + contextSnapshot: { issueId }, + }, + { + id: exhaustedRunId, + companyId, + agentId, + nativeIssueId: issueId, + status: "failed", + runtimeMode: "native", + runtimeModeResolvedAt: new Date(), + runnerProfileJson: persistedProfile, + contextSnapshot: { issueId }, + }, + { + id: missingCheckpointRunId, + companyId, + agentId, + nativeIssueId: issueId, + status: "running", + runtimeMode: "native", + runtimeModeResolvedAt: new Date(), + runnerProfileJson: { + nativeExecutionInput: { + ...persistedProfile.nativeExecutionInput, + binding: { runId: missingCheckpointRunId }, + }, + }, + contextSnapshot: { issueId }, + }, + { + id: initialRunId, + companyId, + agentId, + nativeIssueId: issueId, + status: "running", + runtimeMode: "native", + runtimeModeResolvedAt: new Date(), + runnerProfileJson: { + nativeExecutionInput: { + ...persistedProfile.nativeExecutionInput, + binding: { runId: initialRunId }, + }, + }, + contextSnapshot: { issueId }, + }, + { + id: bootstrapRetryRunId, + companyId, + agentId, + nativeIssueId: issueId, + status: "failed", + runtimeMode: "native", + runtimeModeResolvedAt: new Date(), + runnerProfileJson: { + nativeExecutionInput: { + ...persistedProfile.nativeExecutionInput, + binding: { runId: bootstrapRetryRunId }, + }, + }, + errorCode: "provider_initialize_timeout", + contextSnapshot: { issueId }, + }, + ...[observedExpiredRunId, observedLivePidRunId].map((observedRunId) => ({ + id: observedRunId, + companyId, + agentId, + nativeIssueId: issueId, + status: "running" as const, + runtimeMode: "native" as const, + runtimeModeResolvedAt: new Date(), + runnerProfileJson: { + ...persistedProfile, + nativeExecutionInput: { + ...persistedProfile.nativeExecutionInput, + binding: { runId: observedRunId }, + }, + sessionCheckpoint: { + ...persistedProfile.sessionCheckpoint, + identity: { runId: observedRunId }, + }, + }, + contextSnapshot: { issueId }, + })), + ]); + await db.insert(nativeRunFinalizations).values([ + { runId, companyId, issueId, phase: "retryable_failure", attempt: 1, nextAttemptAt: new Date(0) }, + { runId: cancelledRunId, companyId, issueId, phase: "retryable_failure", attempt: 1, nextAttemptAt: new Date(0) }, + { runId: exhaustedRunId, companyId, issueId, phase: "terminal_failure", attempt: 3 }, + { runId: missingCheckpointRunId, companyId, issueId, phase: "retryable_failure", attempt: 1 }, + { runId: initialRunId, companyId, issueId, phase: "observed", attempt: 0 }, + { + runId: bootstrapRetryRunId, + companyId, + issueId, + phase: "retryable_failure", + attempt: 1, + nextAttemptAt: new Date(0), + failureCode: "native_session_interrupted", + failureDetail: { + message: "provider_initialize_timeout: provider=codex stage=health", + originalFailureCode: "provider_initialize_timeout", + recoveryMode: "bootstrap_retry", + providerSessionEstablished: false, + providerEventsExist: false, + checkpointExists: false, + }, + }, + ...[observedExpiredRunId, observedLivePidRunId].map((observedRunId) => ({ + runId: observedRunId, + companyId, + issueId, + phase: "observed" as const, + attempt: 2, + leaseOwner: "prior-native-owner", + leaseExpiresAt: new Date(0), + })), + ]); + }, 30_000); + + afterAll(async () => temporary?.cleanup()); + + it("wins one database lease for the original result-less run without consulting the flag", async () => { + const results = await Promise.all([ + claimNativeSessionResumptions({ db, runnerInstanceId: "reaper-a", runIds: [runId] }), + claimNativeSessionResumptions({ db, runnerInstanceId: "reaper-b", runIds: [runId] }), + ]); + expect(results.flat()).toHaveLength(1); + expect(results.flat()[0]).toMatchObject({ runId }); + await expect(db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId))).resolves.toEqual([ + expect.objectContaining({ id: runId, status: "running", runtimeMode: "native" }), + ]); + await expect(db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId))).resolves.toEqual([ + expect.objectContaining({ runId, phase: "observed", resultId: null, attempt: 1 }), + ]); + }); + + it("dispatches the persisted run id and lease to the live same-run resume consumer", async () => { + await db.update(nativeRunFinalizations).set({ + phase: "retryable_failure", + leaseOwner: null, + leaseExpiresAt: null, + nextAttemptAt: new Date(0), + }).where(eq(nativeRunFinalizations.runId, runId)); + const dispatched: Array<{ runId: string; leaseOwner: string }> = []; + await expect(dispatchNativeSessionResumptions({ + db, + runnerInstanceId: "heartbeat-reaper", + runIds: [runId], + dispatch: (claim) => dispatched.push(claim), + })).resolves.toHaveLength(1); + expect(dispatched).toEqual([{ runId, leaseOwner: expect.stringContaining("heartbeat-reaper:resume:") }]); + await expect(db.select().from(heartbeatRuns)).resolves.toHaveLength(8); + await expect(db.select().from(nativeRunFinalizations)).resolves.toHaveLength(8); + }); + + it("uses checkpoint-free bootstrap retry only when durable evidence proves no provider session existed", async () => { + await expect(claimNativeSessionResumptions({ + db, + runnerInstanceId: "reaper", + runIds: [bootstrapRetryRunId], + })).resolves.toEqual([ + { runId: bootstrapRetryRunId, leaseOwner: expect.stringContaining("reaper:resume:") }, + ]); + }); + + it("never claims an expired observed coordinator without explicit retryable failure", async () => { + const dispatched: Array<{ runId: string; leaseOwner: string }> = []; + await expect(dispatchNativeSessionResumptions({ + db, + runnerInstanceId: "replacement-reaper", + runIds: [observedExpiredRunId], + dispatch: (claim) => dispatched.push(claim), + })).resolves.toEqual([]); + expect(dispatched).toEqual([]); + await expect(db.select().from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, observedExpiredRunId))).resolves.toEqual([ + expect.objectContaining({ + phase: "observed", + attempt: 2, + leaseOwner: "prior-native-owner", + leaseExpiresAt: new Date(0), + }), + ]); + }); + + it("blocks ambiguous observed ownership and a live unrelated persisted PID without replacement effects", async () => { + const unrelatedProcess = spawn( + process.execPath, + ["-e", "setInterval(() => {}, 1_000)"], + { stdio: "ignore" }, + ); + await once(unrelatedProcess, "spawn"); + try { + await db.update(heartbeatRuns).set({ + processPid: unrelatedProcess.pid!, + processStartedAt: new Date("2026-08-09T04:00:00.000Z"), + }).where(eq(heartbeatRuns.id, observedLivePidRunId)); + const backendFactory = vi.fn((): NativeSessionBackend => ({ + async descriptor() { + return { + kind: "mock", + name: "unexpected-observed-recovery", + version: "1", + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, + }; + }, + async openSession() { + throw new Error("observed ownership must not open a provider session"); + }, + async recoverSession() { + throw new Error("observed ownership must not recover a provider session"); + }, + })); + const heartbeat = heartbeatService(db, { + runtimeEnv: { PAPERCLIP_INSTANCE_ID: "observed-owner-test" }, + nativeSessionBackendFactory: backendFactory, + }); + + const reaped = await heartbeat.reapOrphanedRuns({ staleThresholdMs: 0 }); + expect(reaped.runIds).not.toContain(observedExpiredRunId); + expect(reaped.runIds).not.toContain(observedLivePidRunId); + await heartbeat.drainActiveRunExecutions(); + + expect(backendFactory).not.toHaveBeenCalled(); + expect(() => process.kill(unrelatedProcess.pid!, 0)).not.toThrow(); + await expect(db.select().from(heartbeatRuns).where(eq( + heartbeatRuns.id, + observedExpiredRunId, + ))).resolves.toEqual([ + expect.objectContaining({ + status: "running", + errorCode: "native_execution_ownership_unverified", + }), + ]); + await expect(db.select().from(heartbeatRuns).where(eq( + heartbeatRuns.id, + observedLivePidRunId, + ))).resolves.toEqual([ + expect.objectContaining({ + status: "running", + processPid: unrelatedProcess.pid, + errorCode: "native_execution_ownership_unverified", + }), + ]); + for (const observedRunId of [observedExpiredRunId, observedLivePidRunId]) { + await expect(db.select().from(nativeRunFinalizations).where(eq( + nativeRunFinalizations.runId, + observedRunId, + ))).resolves.toEqual([ + expect.objectContaining({ + phase: "observed", + attempt: 2, + leaseOwner: "prior-native-owner", + leaseExpiresAt: new Date(0), + }), + ]); + await expect(db.select().from(nativeRunResults).where(eq( + nativeRunResults.runId, + observedRunId, + ))).resolves.toHaveLength(0); + await expect(db.select().from(workAssessments).where(eq( + workAssessments.runId, + observedRunId, + ))).resolves.toHaveLength(0); + } + } finally { + if ( + unrelatedProcess.exitCode === null && + unrelatedProcess.signalCode === null + ) { + const exited = once(unrelatedProcess, "exit"); + unrelatedProcess.kill("SIGKILL"); + await exited; + } + } + }); + + it("does not resume cancelled or exhausted runs and fails closed without a checkpoint", async () => { + await expect(claimNativeSessionResumptions({ + db, + runnerInstanceId: "reaper", + runIds: [cancelledRunId, exhaustedRunId, missingCheckpointRunId], + })).resolves.toEqual([]); + await expect(db.select().from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, missingCheckpointRunId))).resolves.toEqual([ + expect.objectContaining({ phase: "terminal_failure", failureCode: "native_session_interrupted" }), + ]); + await expect(db.select().from(issueRecoveryActions) + .where(eq(issueRecoveryActions.sourceIssueId, issueId))).resolves.toEqual([ + expect.objectContaining({ cause: "native_session_interrupted", wakePolicy: null }), + ]); + }); + + it("does not mistake the pre-first-attempt observed coordinator for an orphan", async () => { + await expect(claimNativeSessionResumptions({ + db, + runnerInstanceId: "reaper", + runIds: [initialRunId], + })).resolves.toEqual([]); + await expect(db.select().from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, initialRunId))).resolves.toEqual([ + expect.objectContaining({ phase: "observed", attempt: 0, failureCode: null }), + ]); + }); +}); + +describe("P6-25 persisted reaper-to-finalization recovery", () => { + let temporary: Awaited> | null = null; + let db: ReturnType; + let staleProviderProcess: ReturnType | null = null; + const companyId = randomUUID(); + const agentId = randomUUID(); + const projectId = randomUUID(); + const projectWorkspaceId = randomUUID(); + const executionWorkspaceId = randomUUID(); + const newerExecutionWorkspaceId = randomUUID(); + const issueId = randomUUID(); + const freshIssueId = randomUUID(); + const runId = randomUUID(); + const contractId = randomUUID(); + const workProductId = randomUUID(); + const sessionId = randomUUID(); + const runnerInstanceId = randomUUID(); + const turnId = "provider-active-turn"; + const providerSessionId = "provider-existing-session"; + const repoRoot = fileURLToPath(new URL("../../../", import.meta.url)); + const contract = { + revision: "phase6-recovery-v1", + objective: "Recover the persisted provider turn", + criteria: [{ id: "objective", requirement: "Complete through same-run recovery" }], + }; + const contractSha = "phase6-recovery-contract"; + const evidenceRef = `work_product:${workProductId}`; + const result = structuredClone(CONTROL_PLANE_CONFORMANCE_RESULT); + result.completionClaim.contractRevision = contract.revision; + result.completionClaim.criteria[0]!.evidenceRefs = [evidenceRef]; + result.evidence = [{ kind: "work_product", ref: evidenceRef }]; + result.verification[0]!.artifactRef = evidenceRef; + result.summary = "Recovered the already-active provider turn."; + const terminal = { + ...CONTROL_PLANE_CONFORMANCE_TERMINAL, + reportedWorkDisposition: result.reportedWorkDisposition, + }; + const execution: NativeExecutionInputV1 = { + schema: "paperclip.native-execution-input.v1", + binding: { companyId, runId, issueId, agentId, executionWorkspaceId }, + task: { + identifier: "NRR-1", + title: "Recover one native heartbeat", + description: null, + workMode: "standard", + }, + workspace: { cwd: repoRoot, repoUrl: null, repoRef: null, branchName: null }, + session: { normalizedSessionId: sessionId, driverKind: "codex_app_server", protocolVersion: 1 }, + provider: { kind: "codex", model: null }, + completionContract: { + id: contractId, + sha256: contractSha, + schemaVersion: "paperclip.completion-contract.v1", + contract, + }, + interactionResponses: [], + credentialBindings: [], + }; + const checkpoint: PersistedNativeSession = { + backendKind: "mock", + sessionId: "driver-existing-session", + identity: { companyId, runId, issueId, agentId, sessionId }, + providerSessionId, + cursor: "1", + activeTurnId: turnId, + pendingRuntimeRequests: [], + lineage: [], + }; + const providerTerminalEvent: PrpEvent = { + schema: "paperclip.prp.event.v1", + sourceEventId: `${runnerInstanceId}:provider-terminal`, + sourceSeq: 1, + sourceInstanceId: runnerInstanceId, + sourceKind: "runner", + runId, + normalizedSessionId: sessionId, + turnId, + eventType: "turn.completed", + schemaVersion: 1, + priority: 0, + emittedAt: "2026-08-09T04:30:00.000Z", + payload: {}, + }; + const openSession = vi.fn(async () => { + throw new Error("same-run recovery must not open a second provider session"); + }); + const startTurn = vi.fn(async () => ({ turnId: "duplicate-turn" })); + const close = vi.fn(async () => undefined); + const recoverSession = vi.fn(async (persisted: PersistedNativeSession) => { + expect(persisted).toMatchObject({ providerSessionId, activeTurnId: turnId }); + expect(staleProviderProcess).not.toBeNull(); + expect( + staleProviderProcess!.exitCode !== null || + staleProviderProcess!.signalCode !== null, + ).toBe(true); + const recoveredSnapshot: PersistedNativeSession = { ...structuredClone(checkpoint), cursor: "2" }; + const session: NativeSession = { + identity: () => structuredClone(checkpoint.identity), + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + async *events() { yield providerTerminalEvent; }, + startTurn, + async result() { return { result, terminal, turnId }; }, + async snapshot() { return structuredClone(recoveredSnapshot); }, + close, + }; + return { recovered: true, session }; + }); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "persisted-recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + openSession, + recoverSession, + }; + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-reaper-e2e-"); + db = createDb(temporary.connectionString); + await instanceSettingsService(db).updateExperimental({ enableNativeRunner: false }); + await db.insert(companies).values({ + id: companyId, + name: "Native same-run recovery", + issuePrefix: "NRR", + status: "active", + defaultResponsibleUserId: "responsible-user", + }); + await db.insert(projects).values({ id: projectId, companyId, name: "Recovery project", status: "active" }); + await db.insert(projectWorkspaces).values({ + id: projectWorkspaceId, + companyId, + projectId, + name: "Recovery workspace", + cwd: repoRoot, + isPrimary: true, + }); + await db.insert(agents).values({ + id: agentId, + companyId, + name: "Native recovery agent", + adapterType: "paperclip_runner", + status: "active", + runtimeConfig: { + heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 }, + nativeRunner: { mode: "native", backend: "codex_app_server", protocolVersion: 1 }, + }, + }); + await db.insert(issues).values({ + id: issueId, + companyId, + projectId, + projectWorkspaceId, + issueNumber: 1, + identifier: "NRR-1", + title: "Recover one native heartbeat", + status: "in_progress", + assigneeAgentId: agentId, + workMode: "standard", + }); + await db.insert(executionWorkspaces).values({ + id: executionWorkspaceId, + companyId, + projectId, + projectWorkspaceId, + sourceIssueId: issueId, + mode: "shared_workspace", + strategyType: "project_primary", + name: "Persisted recovery workspace", + status: "active", + cwd: repoRoot, + providerType: "local_fs", + }); + await db.insert(executionWorkspaces).values({ + id: newerExecutionWorkspaceId, + companyId, + projectId, + projectWorkspaceId, + sourceIssueId: issueId, + mode: "shared_workspace", + strategyType: "project_primary", + name: "Newer issue workspace", + status: "active", + cwd: repoRoot, + providerType: "local_fs", + }); + await db.update(issues).set({ + // Simulate a newer run moving the issue-level pointer before the older native run is + // recovered. The older run must still restore its own immutable workspace binding. + executionWorkspaceId: newerExecutionWorkspaceId, + executionWorkspacePreference: "reuse_existing", + executionWorkspaceSettings: { mode: "shared_workspace" }, + }).where(eq(issues.id, issueId)); + await db.insert(completionContracts).values({ + id: contractId, + companyId, + issueId, + revision: 1, + schemaVersion: "paperclip.completion-contract.v1", + policyVersion: "phase6-v1", + risk: "standard", + completionAuthority: "server_arbiter", + incompleteCriteriaPolicy: "preserve_non_terminal", + contractJson: contract, + canonicalSha256: contractSha, + createdByActorType: "system", + createdByActorId: "test", + }); + await db.insert(issueWorkProducts).values({ + id: workProductId, + companyId, + issueId, + type: "artifact", + provider: "paperclip", + title: "Recovered result evidence", + status: "ready_for_review", + reviewState: "approved", + }); + await db.insert(heartbeatRuns).values({ + id: runId, + companyId, + agentId, + nativeIssueId: issueId, + status: "running", + runtimeMode: "native", + runtimeModeResolverVersion: "phase6-v1", + runtimeModeReason: "eligible_opt_in", + runtimeModeResolvedAt: new Date("2026-08-09T04:00:00.000Z"), + runnerProfileJson: { + mode: "native", + backend: "codex_app_server", + protocolVersion: 1, + nativeExecutionInput: execution, + sessionCheckpoint: checkpoint, + }, + runnerInstanceId, + nativeSessionId: sessionId, + driverKind: "codex_app_server", + driverVersion: "phase6-v1", + completionContractId: contractId, + completionContractSha256: contractSha, + nativePhase: "retryable_failure", + nativePhaseUpdatedAt: new Date("2026-08-09T04:00:00.000Z"), + contextSnapshot: { issueId, taskId: issueId, skipIssueComment: true }, + }); + await db.insert(nativeRunFinalizations).values({ + runId, + companyId, + issueId, + phase: "retryable_failure", + attempt: 1, + failureCode: "native_session_interrupted", + nextAttemptAt: new Date(0), + }); + }, 30_000); + + afterAll(async () => { + if ( + staleProviderProcess && + staleProviderProcess.exitCode === null && + staleProviderProcess.signalCode === null + ) { + staleProviderProcess.kill("SIGKILL"); + } + if (temporary) { + await drainHeartbeatRunsToQuiescence(db, heartbeatService(db, { + runtimeEnv: { PAPERCLIP_INSTANCE_ID: "phase6-recovery-test" }, + nativeSessionBackendFactory: () => backend, + })); + await temporary.cleanup(); + } + }); + + it("does not kill an unowned persisted PID, then recovers after it exits while flag-off", async () => { + legacyAdapterExecute.mockClear(); + staleProviderProcess = spawn( + process.execPath, + ["-e", "setInterval(() => {}, 1_000)"], + { stdio: "ignore" }, + ); + await once(staleProviderProcess, "spawn"); + expect(staleProviderProcess.pid).toEqual(expect.any(Number)); + await db.update(heartbeatRuns).set({ + processPid: staleProviderProcess.pid!, + processStartedAt: new Date("2026-08-09T04:00:00.000Z"), + }).where(eq(heartbeatRuns.id, runId)); + const backendFactory = vi.fn(() => backend); + const heartbeat = heartbeatService(db, { + runtimeEnv: { PAPERCLIP_INSTANCE_ID: "phase6-recovery-test" }, + nativeSessionBackendFactory: backendFactory, + }); + + await expect(heartbeat.reapOrphanedRuns({ staleThresholdMs: 0 })).resolves.not.toContain(runId); + await heartbeat.drainActiveRunExecutions(); + + expect(backendFactory).not.toHaveBeenCalled(); + expect(recoverSession).not.toHaveBeenCalled(); + expect(() => process.kill(staleProviderProcess!.pid!, 0)).not.toThrow(); + await expect(db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId))).resolves.toEqual([ + expect.objectContaining({ + id: runId, + status: "running", + processPid: staleProviderProcess.pid, + errorCode: "native_execution_ownership_unverified", + }), + ]); + await expect(db.select().from(nativeRunFinalizations).where(eq( + nativeRunFinalizations.runId, + runId, + ))).resolves.toEqual([ + expect.objectContaining({ + phase: "retryable_failure", + attempt: 1, + leaseOwner: null, + }), + ]); + + const unrelatedProcessExit = once(staleProviderProcess, "exit"); + staleProviderProcess.kill("SIGKILL"); + await unrelatedProcessExit; + await db.update(nativeRunFinalizations).set({ + leaseOwner: null, + leaseExpiresAt: null, + }).where(eq(nativeRunFinalizations.runId, runId)); + + await expect(heartbeat.reapOrphanedRuns({ staleThresholdMs: 0 })).resolves.not.toContain(runId); + await heartbeat.drainActiveRunExecutions(); + + const recoveryState = { + run: await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)), + coordinator: await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)), + }; + expect( + backendFactory.mock.calls.length, + JSON.stringify(recoveryState), + ).toBe(1); + expect(recoverSession).toHaveBeenCalledOnce(); + expect(openSession).not.toHaveBeenCalled(); + expect(startTurn).not.toHaveBeenCalled(); + expect(close).toHaveBeenCalledOnce(); + expect(legacyAdapterExecute).not.toHaveBeenCalled(); + + await expect(db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId))).resolves.toEqual([ + expect.objectContaining({ + id: runId, + runtimeMode: "native", + status: "succeeded", + nativePhase: "committed", + processPid: null, + processGroupId: null, + processStartedAt: null, + }), + ]); + await expect(db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, runId))).resolves.toHaveLength(1); + await expect(db.select().from(workAssessments).where(eq(workAssessments.runId, runId))).resolves.toHaveLength(1); + const decisions = await db.select().from(statusDecisions).where(eq(statusDecisions.issueId, issueId)); + expect(decisions).toEqual([ + expect.objectContaining({ reasonCode: "completion_contract_satisfied", toStatus: "done", applicationState: "applied" }), + ]); + const effects = await db.select().from(statusDecisionEffects).where(eq(statusDecisionEffects.issueId, issueId)); + expect(new Set(effects.map((effect) => effect.decisionId))).toEqual(new Set([decisions[0]!.id])); + expect(effects.map((effect) => effect.effectKind).sort()).toEqual(["issue_status_projection", "release_checkout"]); + await expect(db.select().from(issues).where(eq(issues.id, issueId))).resolves.toEqual([ + expect.objectContaining({ + status: "done", + statusVersion: 1, + lastStatusDecisionId: decisions[0]!.id, + executionWorkspaceId: newerExecutionWorkspaceId, + }), + ]); + await expect(db.select().from(executionWorkspaces).where(eq(executionWorkspaces.companyId, companyId))) + .resolves.toHaveLength(2); + await expect(db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId))).resolves.toEqual([ + expect.objectContaining({ phase: "committed", resultId: expect.any(String), assessmentId: expect.any(String), decisionId: decisions[0]!.id }), + ]); + await expect(db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, runId))).resolves.toEqual( + expect.arrayContaining([ + expect.objectContaining({ eventType: "turn.completed" }), + expect.objectContaining({ eventType: "run.result.accepted" }), + expect.objectContaining({ eventType: "run.terminal" }), + ]), + ); + await expect(db.select().from(activityLog).where(eq(activityLog.entityId, issueId))).resolves.toEqual( + expect.arrayContaining([expect.objectContaining({ action: "issue.updated" })]), + ); + + await heartbeat.reapOrphanedRuns({ staleThresholdMs: 0 }); + await heartbeat.drainActiveRunExecutions(); + expect(backendFactory).toHaveBeenCalledOnce(); + await expect(db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId))).resolves.toHaveLength(1); + await expect(db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, runId))).resolves.toHaveLength(1); + await expect(db.select().from(workAssessments).where(eq(workAssessments.runId, runId))).resolves.toHaveLength(1); + await expect(db.select().from(statusDecisions).where(eq(statusDecisions.issueId, issueId))).resolves.toHaveLength(1); + + // The persisted Paperclip Runner run above remains recoverable while the + // flag is off. Switching the agent back to a direct adapter now proves a + // fresh run ignores the stale native profile and stays on the legacy path. + await db + .update(agents) + .set({ adapterType: "codex_local" }) + .where(eq(agents.id, agentId)); + await db.insert(issues).values({ + id: freshIssueId, + companyId, + projectId, + projectWorkspaceId, + issueNumber: 2, + identifier: "NRR-2", + title: "Start only after the native kill switch is off", + status: "in_progress", + assigneeAgentId: agentId, + workMode: "standard", + }); + const fresh = await heartbeat.wakeup(agentId, { + source: "automation", + triggerDetail: "system", + reason: "issue_commented", + payload: { issueId: freshIssueId }, + contextSnapshot: { issueId: freshIssueId, taskId: freshIssueId, skipIssueComment: true }, + }); + expect(fresh).not.toBeNull(); + await drainHeartbeatRunsToQuiescence(db, heartbeat); + expect(legacyAdapterExecute).toHaveBeenCalledOnce(); + await expect(db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fresh!.id))).resolves.toEqual([ + expect.objectContaining({ + agentId, + runtimeMode: "legacy", + runtimeModeReason: "direct_adapter", + status: "succeeded", + }), + ]); + await expect(db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, fresh!.id))).resolves.toHaveLength(0); + await expect(db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, fresh!.id))).resolves.toHaveLength(0); + await expect(db.select({ runtimeConfig: agents.runtimeConfig }).from(agents).where(eq(agents.id, agentId))).resolves.toEqual([ + expect.objectContaining({ + runtimeConfig: expect.objectContaining({ + nativeRunner: { mode: "native", backend: "codex_app_server", protocolVersion: 1 }, + }), + }), + ]); + }, 30_000); +}); diff --git a/server/src/__tests__/native-status-arbiter-corpus.test.ts b/server/src/__tests__/native-status-arbiter-corpus.test.ts new file mode 100644 index 0000000000..f67c5e17d0 --- /dev/null +++ b/server/src/__tests__/native-status-arbiter-corpus.test.ts @@ -0,0 +1,2231 @@ +import { randomUUID } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { and, eq } from "drizzle-orm"; +import { + agents, + agentWakeupRequests, + companies, + completionContracts, + createDb, + heartbeatRuns, + issueThreadInteractions, + issueRecoveryActions, + issueWorkProducts, + issues, + nativeRunFinalizations, + nativeRunResults, + statusDecisionEffects, + statusDecisions, + workAssessments, + workspaceOperations, +} from "@paperclipai/db"; +import type { NativeEvidenceAssessment } from "../services/native-runtime/evidence-classifier.js"; +import { classifyNativeEvidence } from "../services/native-runtime/evidence-classifier.js"; +import { + applyNativeAttentionStatusDecision, + materializeNativeInteractionResponses, + rejectUnsupportedNativeRuntimeRequest, + resolveNativeAttentionStatus, + routePersistedNativeResultAttention, +} from "../services/native-runtime/native-interaction-bridge.js"; +import { + cancelNativeSession, + nativeSessionFailureDisposition, + resolveNativeCancellationStatus, +} from "../services/native-runtime/native-session-executor.js"; +import { + inspectNativeCompatibilityState, + inspectNativeMigrationState, + resolveHeartbeatNativeRuntimeMode, + resolveNativeCompatibilityStatus, + resolveNativeMigrationStatus, + resolveNativeRuntimeMode, +} from "../services/native-runtime/runtime-mode.js"; +import { + arbitrateNativeStatus, + NATIVE_STATUS_ARBITER_POLICY_VERSION, + type NativeStatusDecision, + type NativeStatusEffect, +} from "../services/native-runtime/status-arbiter.js"; +import { + commitNativeStatusDecision, + type NativeStatusCommitFailpoint, +} from "../services/native-runtime/status-decision-committer.js"; +import { + projectNativeTerminalRunStatus, + recordNativeFinalizationFailure, + finalizeNativeRun, + resolveNativeFinalizerStatus, +} from "../services/native-runtime/native-run-finalizer.js"; +import { + reconcileNativeFinalizations, + resolveNativeReconciliationStatus, +} from "../services/native-runtime/native-finalization-reconciler.js"; +import { issueService } from "../services/issues.js"; +import { issueThreadInteractionService } from "../services/issue-thread-interactions.js"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +type Fixture = { + id: string; + mode: "native" | "legacy"; + covers: Record; + given: Record; + expected: { + statusAction: string; + runStatus: string; + reasonCode: string | null; + requiredEffects: string[]; + forbiddenEffects: string[]; + livePathKind: string | null; + preserveClaim: boolean; + nativeRecords: boolean; + decisionCount: number; + maxWakeCount: number; + maxNotificationCount: number; + }; +}; + +type Corpus = { schema: string; corpusRevision: number; fixtures: Fixture[] }; + +type PolicyObservation = { + runStatus: string; + statusAction: string; + reasonCode: string | null; + effects: string[]; + livePathKind: string | null; + preserveClaim: boolean; + nativeRecords: boolean; + decisionCount: number; + wakeCount: number; + notificationCount: number; +}; + +type ConsumerExecution = { + consumer: string; + observed: Record; +}; + +type FixtureObservation = PolicyObservation & { + fixtureId: string; + consumerExecutions: ConsumerExecution[]; + consumerEvidenceByRow: Map; +}; + +type DisabledLiveEntrypoint = "attention" | "cancellation" | "reconciliation" | "rollout"; + +const corpusPath = fileURLToPath(new URL( + "../../../packages/paperclip-runner/spec/fixtures/status-authority-sdk.json", + import.meta.url, +)); + +const corpus = JSON.parse(readFileSync(corpusPath, "utf8")) as Corpus; + +const matrixRowConsumers: Record = { + "SD-01": "native-finalizer-status", + "SD-02": "native-finalizer-status", + "SD-03": "native-finalizer-status", + "SD-04": "native-finalizer-status", + "SD-05": "native-finalizer-status", + "SD-06": "native-finalizer-status", + "SD-07": "native-finalizer-status", + "SD-08": "native-finalizer-status", + "SD-09": "native-attention-resolver", + "SD-10": "native-attention-resolver", + "SD-11": "native-attention-resolver", + "SD-12": "native-attention-resolver", + "SD-13": "native-attention-resolver", + "SD-14": "native-finalizer-status", + "SD-15": "native-finalizer-status", + "SD-16": "native-cancellation-authority", + "SD-17": "native-cancellation-authority", + "SD-18": "native-cancellation-authority", + "SD-19": "native-reconciliation-consumer", + "TC-01": "native-run-terminal-projection", + "TC-02": "native-run-terminal-projection", + "TC-03": "native-run-terminal-projection", + "TC-04": "native-run-terminal-projection", + "TC-05": "native-run-terminal-projection", + "TC-06": "native-run-terminal-projection", + "TC-07": "native-run-terminal-projection", + "TC-08": "native-run-terminal-projection", + "ATT-01": "native-attention-resolver", + "ATT-02": "native-attention-resolver", + "ATT-03": "native-attention-resolver", + "ATT-04": "native-attention-resolver", + "ATT-05": "native-attention-resolver", + "ATT-06": "native-attention-resolver", + "ATT-07": "native-attention-resolver", + "ATT-08": "native-attention-resolver", + "ATT-09": "native-attention-resolver", + "ATT-10": "native-attention-resolver", + "ATT-11": "native-attention-resolver", + "ATT-12": "native-attention-resolver", + "LIVE-01": "status-decision-committer", + "LIVE-02": "status-decision-committer", + "LIVE-03": "status-decision-committer", + "LIVE-04": "status-decision-committer", + "LIVE-05": "status-decision-committer", + "LIVE-06": "status-decision-committer", + "REC-01": "native-reconciliation-consumer", + "REC-02": "native-reconciliation-consumer", + "REC-03": "native-reconciliation-consumer", + "REC-04": "native-reconciliation-consumer", + "REC-05": "native-reconciliation-consumer", + "REC-06": "native-reconciliation-consumer", + "REC-07": "native-reconciliation-consumer", + "REC-08": "native-reconciliation-consumer", + "COMP-01": "native-compatibility-read-model", + "COMP-02": "native-compatibility-status", + "COMP-03": "native-compatibility-read-model", + "COMP-04": "native-compatibility-read-model", + "COMP-05": "native-compatibility-status", + "COMP-06": "native-compatibility-status", + "COMP-07": "native-compatibility-read-model", + "COMP-08": "native-compatibility-status", + "MIG-01": "native-migration-read-model", + "MIG-02": "native-migration-read-model", + "MIG-03": "native-migration-read-model", + "MIG-04": "native-migration-status", + "MIG-05": "native-migration-status", + "MIG-06": "native-migration-status", + "MIG-07": "native-migration-status", + "MIG-08": "heartbeat-runtime-selection", + "MIG-09": "native-migration-read-model", +}; + +function requiredConsumerForMatrixRow(matrixRow: string) { + const consumer = matrixRowConsumers[matrixRow]; + if (!consumer) throw new Error(`No production consumer assertion for ${matrixRow}`); + return consumer; +} + +const noNativeRecordStates = new Set([ + "legacy_exit_zero", + "native_field_only_in_result_json", + "preexisting_open_issue", + "production_shaped_upgrade", + "authorized_status_write", + "no_native_rows", + "no_reviewed_adapter_contract_migration", +]); + +const completeEvidenceStates = new Set([ + "mechanically_satisfied", + "low_risk_policy_claim", + "new_evidence_satisfies_contract", + "identical_result_before_ack", + "result_preserved", + "shadow_application_disabled", + "mixed_ledger", + "shadow_compute", + "cohort_policy_pinned", +]); + +const zeroDecisionStates = new Set([ + "safe_partial_parse", "equivalent_attention_family", + "response_after_supersession", "reused_id_changed_material", "decision_committed_delivery_pending", +]); + +const nativeStatusEffectKinds = new Set([ + "create_interaction", "bind_reviewer", "notify_owner", "enqueue_continuation", + "bind_blocker", "schedule_retry", "record_finalization_error", "release_run_resources", + "create_delegated_issue", "accept_replacement_turn", "cancel_continuations", + "append_superseding_assessment", "dispatch_pending_effect", "increment_status_version", + "schedule_reconciliation", "record_shadow_decision", "render_four_layers", + "materialize_contract", "record_mode_labeled_divergence", "record_mode_native", + "record_policy_version", "finish_as_native", + "resume_workspace_operation", "record_expiry", "record_stale_response", + "link_canonical_request", "record_recovery", "release_checkout", +]); + +const supersedingDecisionStates = new Set([ + "new_evidence_satisfies_contract", "dependency_now_done", "explicit_resume_capability", + "board_cancelled_before_cas", "new_policy_requires_review", "authorized_writer_incremented_version", +]); + +const liveReconciliationStates = new Set([ + "board_cancelled_before_cas", "new_evidence_satisfies_contract", "new_policy_requires_review", +]); + +function initialRunStatus(fixture: Fixture) { + const terminalState = fixture.given.runTerminalState; + if (["succeeded", "failed", "cancelled", "active"].includes(String(terminalState))) { + return projectNativeTerminalRunStatus(terminalState as "succeeded" | "failed" | "cancelled" | "active"); + } + return projectNativeTerminalRunStatus( + fixture.given.nativeFinalization === "present" ? "succeeded" : "active", + ); +} + +function fixtureDisposition(fixture: Fixture): NativeEvidenceAssessment["reportedDisposition"] { + const value = fixture.given.reportedWorkDisposition; + return ["done", "blocked", "needs_review", "yielded"].includes(String(value)) + ? value as NativeEvidenceAssessment["reportedDisposition"] + : "yielded"; +} + +function failpointFor(fixture: Fixture): NativeStatusCommitFailpoint | undefined { + if (fixture.given.fault === "continuation_insert_failure") return "continuation_materialization"; + if (fixture.given.fault === "reviewer_insert_failure") return "interaction_materialization"; + if (fixture.given.fault === "blocker_insert_failure") return "blocker_materialization"; + return undefined; +} + +function attentionFactsFor(completionState: string, summary: string, governanceGate: { kind: "interaction"; id: string } | null) { + switch (completionState) { + case "alternate_track_runnable": + return { companyScopeValid: true, responseState: "none" as const, route: "alternate_track" as const, summary }; + case "context_answer_current": + return { companyScopeValid: true, responseState: "resolved" as const, route: "context" as const, summary }; + case "ordinary_domain_expertise": + return { companyScopeValid: true, responseState: "none" as const, route: "agent" as const, summary }; + case "intentional_human_judgment": + return { companyScopeValid: true, responseState: "none" as const, route: "human" as const, summary }; + case "equivalent_attention_family": + return { companyScopeValid: true, responseState: "none" as const, route: "duplicate" as const, summary }; + case "resolver_budget_exhausted": + return { companyScopeValid: true, responseState: "none" as const, route: "recovery" as const, summary, budgetExhausted: true }; + case "transient_retry_then_success": + return { companyScopeValid: true, responseState: "resolved" as const, route: "retry" as const, summary }; + case "cross_company_target": + return { companyScopeValid: false, responseState: "none" as const, route: "agent" as const, summary }; + case "response_after_supersession": + return { companyScopeValid: true, responseState: "stale" as const, route: "context" as const, summary }; + case "interaction_expired": + return { companyScopeValid: true, responseState: "expired" as const, route: "human" as const, summary }; + case "governed_gate_pending": + return { + companyScopeValid: true, + responseState: "none" as const, + route: "human" as const, + summary, + governanceGate, + }; + default: + return null; + } +} + +function reconciliationFactsFor(completionState: string) { + switch (completionState) { + case "equivalent_attention_family": return { equivalentAttentionFamily: true }; + case "identical_result_before_ack": return { canonicalReplay: true }; + case "reused_id_changed_material": return { callerMaterialConflict: true }; + case "result_preserved": return { workspaceOperationPending: true }; + case "decision_committed_delivery_pending": return { undeliveredEffectCount: 1 }; + case "board_cancelled_before_cas": return { authoritativeStatusChanged: true }; + case "new_evidence_satisfies_contract": return { newEvidenceSatisfiesContract: true }; + case "dependency_now_done": return { dependencyResolved: true }; + case "explicit_resume_capability": return { authorizedResume: true }; + case "new_policy_requires_review": return { policyVersionChanged: true }; + case "authorized_writer_incremented_version": return { statusVersionAdvanced: true }; + default: return null; + } +} + +function compatibilityFactsFor(completionState: string) { + switch (completionState) { + case "safe_partial_parse": return { invalidNativeFinalization: true }; + case "explicit_resume_capability": return { terminalResumeAuthorized: true }; + case "shadow_application_disabled": return { shadowApplicationDisabled: true }; + case "mixed_ledger": return { mixedLedger: true }; + case "authorized_writer_incremented_version": return { statusWriterAdvancedVersion: true }; + default: return null; + } +} + +function migrationFactsFor(completionState: string) { + switch (completionState) { + case "shadow_compute": return { shadowMaterialization: true }; + case "classified_native_legacy_divergence": return { classifiedDivergence: true }; + case "allowlisted_company_adapter_policy": return { applicationEnabled: true }; + case "cohort_policy_pinned": return { policyPinned: true }; + case "kill_switch_during_active_native_run": return { killSwitchActiveForNewRuns: true }; + default: return null; + } +} + +function comparisonFailures(fixture: Fixture, observed: FixtureObservation): string[] { + const failures: string[] = []; + if (observed.runStatus !== fixture.expected.runStatus) failures.push("runStatus"); + if (observed.statusAction !== fixture.expected.statusAction) failures.push("statusAction"); + if (observed.reasonCode !== fixture.expected.reasonCode) failures.push("reasonCode"); + for (const effect of fixture.expected.requiredEffects) { + if (!observed.effects.includes(effect)) failures.push(`requiredEffects:${effect}`); + } + for (const effect of fixture.expected.forbiddenEffects) { + if (observed.effects.includes(effect)) failures.push(`forbiddenEffects:${effect}`); + } + if (observed.livePathKind !== fixture.expected.livePathKind) failures.push("livePathKind"); + if (observed.preserveClaim !== fixture.expected.preserveClaim) failures.push("preserveClaim"); + if (observed.nativeRecords !== fixture.expected.nativeRecords) failures.push("nativeRecords"); + if (observed.decisionCount !== fixture.expected.decisionCount) failures.push("decisionCount"); + if (observed.wakeCount > fixture.expected.maxWakeCount) failures.push("maxWakeCount"); + if (observed.notificationCount > fixture.expected.maxNotificationCount) failures.push("maxNotificationCount"); + return failures; +} + +describe("P6-31 Section 18.13 executable status-authority corpus", () => { + let temporary: Awaited> | null = null; + let db: ReturnType; + const companyId = randomUUID(); + const agentId = randomUUID(); + const delegateAgentId = randomUUID(); + const outsideCompanyId = randomUUID(); + const outsideAgentId = randomUUID(); + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("paperclip-status-corpus-"); + db = createDb(temporary.connectionString); + await db.insert(companies).values({ id: companyId, name: "Status corpus", issuePrefix: "PSC" }); + await db.insert(companies).values({ id: outsideCompanyId, name: "Outside corpus", issuePrefix: "OUT" }); + await db.insert(agents).values([ + { + id: agentId, + companyId, + name: "Status corpus agent", + adapterType: "codex_local", + status: "running", + }, + { + id: delegateAgentId, + companyId, + name: "Eligible status corpus delegate", + adapterType: "codex_local", + status: "idle", + capabilities: "Provide production status-authority expertise", + }, + { + id: outsideAgentId, + companyId: outsideCompanyId, + name: "Outside-company agent", + adapterType: "codex_local", + status: "idle", + }, + ]); + }, 30_000); + + afterAll(async () => temporary?.cleanup()); + + async function seedFixture(fixture: Fixture) { + const issueId = randomUUID(); + const runId = randomUUID(); + const contractId = randomUUID(); + const resultId = randomUUID(); + const assessmentId = randomUUID(); + const workProductId = randomUUID(); + const assessmentCreatedAt = new Date(); + const priorStatus = String(fixture.given.priorIssueStatus ?? "in_progress"); + const completionState = String(fixture.given.completionState ?? ""); + const nativeRecords = fixture.mode === "native" && !noNativeRecordStates.has(completionState); + + await db.insert(issues).values({ + id: issueId, + companyId, + title: fixture.id, + status: priorStatus, + assigneeAgentId: agentId, + workMode: "standard", + }); + if (!nativeRecords) return { + issueId, + runId, + workProductId, + nativeRecords, + assessmentId, + contractId: null, + resultId: null, + }; + + await db.insert(heartbeatRuns).values({ + id: runId, + companyId, + agentId, + status: initialRunStatus(fixture), + runtimeMode: "native", + runtimeModeResolvedAt: new Date(), + nativeIssueId: issueId, + contextSnapshot: { issueId, fixtureId: fixture.id }, + completionContractId: contractId, + completionContractSha256: `contract:${fixture.id}`, + }); + await db.insert(completionContracts).values({ + id: contractId, + companyId, + issueId, + revision: 1, + schemaVersion: "paperclip.completion-contract.v1", + policyVersion: "phase6-v1", + risk: "standard", + completionAuthority: "server_arbiter", + incompleteCriteriaPolicy: "preserve_non_terminal", + contractJson: { revision: "corpus-v1", criteria: [{ id: "objective", requirement: fixture.id }] }, + canonicalSha256: `contract:${fixture.id}`, + createdByActorType: "system", + createdByActorId: "status-corpus", + }); + await db.insert(nativeRunResults).values({ + id: resultId, + companyId, + issueId, + runId, + completionContractId: contractId, + serverFingerprint: `fingerprint:${fixture.id}`, + schemaStatus: "accepted", + resultJson: { + fixtureId: fixture.id, + result: { + reportedWorkDisposition: fixtureDisposition(fixture), + summary: fixture.id, + completionClaim: { + contractRevision: "corpus-v1", + objectiveSatisfied: true, + criteria: [{ + criterionId: "objective", + status: "satisfied", + evidenceRefs: [`work_product:${workProductId}`], + }], + remainingWork: [], + }, + verification: [{ + commandOrCheck: "fixture", + status: "passed", + artifactRef: `work_product:${workProductId}`, + }], + }, + terminal: { + runTerminalState: fixture.given.runTerminalState === "failed" + ? "failed" + : fixture.given.runTerminalState === "cancelled" ? "cancelled" : "succeeded", + }, + ...(fixture.given.reportedWorkDisposition === null && fixture.given.nativeFinalization !== "invalid" + ? {} + : { completionClaim: { fixtureId: fixture.id, preserved: true } }), + }, + canonicalSha256: `result:${fixture.id}`, + }); + await db.insert(issueWorkProducts).values({ + id: workProductId, + companyId, + issueId, + type: "artifact", + provider: "paperclip", + title: `${fixture.id} evidence`, + status: "ready_for_review", + reviewState: completionState === "new_evidence_satisfies_contract" ? "none" : "approved", + createdAt: new Date(assessmentCreatedAt.getTime() - 1_000), + updatedAt: new Date(assessmentCreatedAt.getTime() - 1_000), + }); + await db.insert(workAssessments).values({ + id: assessmentId, + companyId, + issueId, + runId, + contractId, + resultId, + triggerKind: "native_result", + triggerActorCompanyId: companyId, + priorIssueStatus: completionState === "board_cancelled_before_cas" ? "in_progress" : priorStatus, + priorStatusVersion: 0, + policyVersion: completionState === "new_policy_requires_review" + ? "phase6-v1" + : NATIVE_STATUS_ARBITER_POLICY_VERSION, + assessmentJson: { + fixtureId: fixture.id, + ...(completionState === "new_evidence_satisfies_contract" ? { allCriteriaSatisfied: false } : {}), + }, + inputDigest: `assessment:${fixture.id}`, + createdAt: assessmentCreatedAt, + }); + await db.insert(nativeRunFinalizations).values({ + runId, + companyId, + issueId, + phase: "assessing", + resultId, + assessmentId, + }); + return { issueId, runId, workProductId, nativeRecords, assessmentId, contractId, resultId }; + } + + async function executeFixture( + fixture: Fixture, + options: { disableLiveEntrypoint?: DisabledLiveEntrypoint } = {}, + ): Promise { + const completionState = String(fixture.given.completionState ?? ""); + const seeded = await seedFixture(fixture); + const consumerExecutions: ConsumerExecution[] = []; + const materializedEffects = new Set(); + const operationalEffects = new Set(); + const priorIssueStatus = String(fixture.given.priorIssueStatus ?? "in_progress") as Parameters[0]["priorIssueStatus"]; + const mode = resolveNativeRuntimeMode({ + enabled: fixture.mode === "native", + runtimeConfig: { nativeRunner: { mode: "native", backend: "codex_app_server", protocolVersion: 1 } }, + adapterConfig: { provider: "codex" }, + agent: { + id: agentId, + status: "running", + adapterType: fixture.mode === "native" ? "paperclip_runner" : "codex_local", + }, + issue: { id: seeded.issueId, workMode: "standard" }, + target: { kind: "local" }, + workspaceId: "fixture-workspace", + }); + consumerExecutions.push({ consumer: "runtime-mode", observed: { kind: mode.kind, reason: mode.reason } }); + if (mode.kind === "native") { + expect(mode.authorityDecision.effects.some((effect) => effect.kind === "record_mode_native"), `${fixture.id}:native runtime authority`).toBe(true); + } + + let governanceGate: { kind: "interaction"; id: string } | null = null; + if (seeded.nativeRecords && completionState === "governed_gate_pending") { + const interactionId = randomUUID(); + await db.insert(issueThreadInteractions).values({ + id: interactionId, + companyId, + issueId: seeded.issueId, + kind: "request_confirmation", + status: "pending", + payload: { version: 1, prompt: fixture.id }, + }); + governanceGate = { kind: "interaction", id: interactionId }; + } + + const pushDecisionConsumer = ( + consumer: string, + decision: NativeStatusDecision, + ) => { + consumerExecutions.push({ + consumer, + observed: { + statusAction: decision.statusAction, + toStatus: decision.toStatus, + reasonCode: decision.reasonCode, + effects: decision.effects.map((effect) => effect.kind), + }, + }); + return decision; + }; + + let semanticConsumer: string | null = null; + let consumerDecision: NativeStatusDecision | null = null; + let liveEntrypointCommitted = false; + let liveAttentionInteractionId: string | null = null; + const attentionFacts = attentionFactsFor(completionState, fixture.id, governanceGate); + if ( + seeded.nativeRecords + && options.disableLiveEntrypoint !== "attention" + && attentionFacts + && ["attention_response", "attention_candidate", "interaction", "monitor"].includes(String(fixture.given.trigger)) + ) { + let canonicalRequestId: string | undefined; + if (attentionFacts.route === "duplicate") { + canonicalRequestId = randomUUID(); + liveAttentionInteractionId = randomUUID(); + await db.insert(issueThreadInteractions).values([ + { + id: canonicalRequestId, + companyId, + issueId: seeded.issueId, + kind: "request_confirmation", + status: "pending", + idempotencyKey: `canonical:${fixture.id}`, + payload: { version: 1, prompt: `Canonical ${fixture.id}` }, + }, + { + id: liveAttentionInteractionId, + companyId, + issueId: seeded.issueId, + kind: "request_confirmation", + status: "expired", + resolvedAt: new Date(), + idempotencyKey: `duplicate:${fixture.id}`, + payload: { version: 1, prompt: `Duplicate ${fixture.id}` }, + result: { version: 1, outcome: "superseded_by_newer_request", supersededByInteractionId: canonicalRequestId }, + }, + ]); + } else if (attentionFacts.responseState === "stale") { + liveAttentionInteractionId = randomUUID(); + await db.insert(issueThreadInteractions).values({ + id: liveAttentionInteractionId, + companyId, + issueId: seeded.issueId, + kind: "request_confirmation", + status: "expired", + resolvedAt: new Date(), + payload: { version: 1, prompt: `Stale ${fixture.id}` }, + result: { version: 1, outcome: "superseded_by_comment", commentId: randomUUID() }, + }); + } else if (attentionFacts.responseState === "expired") { + liveAttentionInteractionId = randomUUID(); + await db.insert(issueThreadInteractions).values({ + id: liveAttentionInteractionId, + companyId, + issueId: seeded.issueId, + kind: "ask_user_questions", + status: "pending", + payload: { version: 1, questions: [] }, + }); + } + const target = attentionFacts.route === "agent" + ? { + ownerClass: "agent", + agentId: completionState === "cross_company_target" ? outsideAgentId : delegateAgentId, + companyId: completionState === "cross_company_target" ? outsideCompanyId : companyId, + } + : attentionFacts.route === "human" + ? { ownerClass: "board_user", companyId } + : { ownerClass: "current_agent", companyId }; + const requestedCapability = attentionFacts.route === "context" + ? "context_lookup" + : attentionFacts.route === "retry" + ? "retry" + : attentionFacts.route === "duplicate" + ? "duplicate" + : attentionFacts.route === "alternate_track" + ? "alternate_track" + : attentionFacts.route === "human" + ? "subjective_decision" + : "domain_expertise"; + const resultRow = await db.select({ resultJson: nativeRunResults.resultJson }) + .from(nativeRunResults).where(eq(nativeRunResults.id, seeded.resultId!)) + .then((rows) => rows[0] ?? null); + if (!resultRow) throw new Error(`${fixture.id}: persisted result missing`); + const resultEnvelope = resultRow.resultJson as Record; + const persistedResult = resultEnvelope.result as Record; + await db.update(nativeRunResults).set({ + resultJson: { + ...resultEnvelope, + result: { + ...persistedResult, + attentionRequests: [{ + id: `attention:${fixture.id}`, + requestedCapability, + requiredAuthority: attentionFacts.route === "human" ? "board" : "agent", + target, + summary: attentionFacts.summary, + responseState: attentionFacts.responseState, + budgetExhausted: attentionFacts.budgetExhausted === true, + governanceGate: attentionFacts.governanceGate, + targetInteractionId: liveAttentionInteractionId, + canonicalRequestId, + }], + }, + }, + }).where(eq(nativeRunResults.id, seeded.resultId!)); + const attentionReceipts = await routePersistedNativeResultAttention({ db, runId: seeded.runId }); + const finalized = await db.select().from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, seeded.runId)).then((rows) => rows[0] ?? null); + const [receipt] = attentionReceipts; + if (!receipt) throw new Error(`${fixture.id}: persisted attention route missing`); + const persistedDecision = receipt.decisionId + ? await db.select().from(statusDecisions).where(eq(statusDecisions.id, receipt.decisionId)) + .then((rows) => rows[0] ?? null) + : null; + const persistedDecisionJson = persistedDecision?.decisionJson as Record | undefined; + const routedDecision = persistedDecision + ? { + policyVersion: persistedDecision.policyVersion, + statusAction: persistedDecisionJson?.statusAction, + toStatus: persistedDecision.toStatus, + reasonCode: persistedDecision.reasonCode, + unblockDescriptor: persistedDecisionJson?.unblockDescriptor ?? null, + effects: persistedDecisionJson?.effects, + } as NativeStatusDecision + : resolveNativeAttentionStatus({ facts: attentionFacts, priorIssueStatus, agentId }); + semanticConsumer = "native-attention-resolver"; + consumerDecision = pushDecisionConsumer(semanticConsumer, routedDecision); + liveEntrypointCommitted = true; + for (const effect of routedDecision.effects) materializedEffects.add(effect.kind); + consumerExecutions.push({ + consumer: "native-attention-finalizer", + observed: { + phase: finalized?.phase, + decisionId: receipt.decisionId, + resolvedTargetAgentId: receipt.resolvedTargetAgentId, + reasonCode: receipt.reasonCode, + materializedTargets: receipt.materializedTargets, + persistedResultId: seeded.resultId, + }, + }); + if (receipt.decisionId === null) { + const beforeReplay = liveAttentionInteractionId + ? await db.select({ + summary: issueThreadInteractions.summary, + updatedAt: issueThreadInteractions.updatedAt, + }).from(issueThreadInteractions).where(eq(issueThreadInteractions.id, liveAttentionInteractionId)) + .then((rows) => rows[0] ?? null) + : null; + const assessmentCount = await db.select().from(workAssessments) + .where(eq(workAssessments.runId, seeded.runId)).then((rows) => rows.length); + await routePersistedNativeResultAttention({ db, runId: seeded.runId }); + const replayed = await db.select().from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, seeded.runId)).then((rows) => rows[0] ?? null); + expect(replayed, `${fixture.id}:audit replay coordinator`).toMatchObject({ phase: "assessing", decisionId: null }); + if (liveAttentionInteractionId) { + await expect(db.select({ + summary: issueThreadInteractions.summary, + updatedAt: issueThreadInteractions.updatedAt, + }).from(issueThreadInteractions).where(eq(issueThreadInteractions.id, liveAttentionInteractionId)) + .then((rows) => rows[0] ?? null)).resolves.toEqual(beforeReplay); + } + await expect(db.select().from(workAssessments) + .where(eq(workAssessments.runId, seeded.runId)).then((rows) => rows.length)).resolves.toBe(assessmentCount); + } + if (attentionFacts.route === "agent" && completionState !== "cross_company_target") { + expect(receipt.resolvedTargetAgentId, `${fixture.id}:eligible delegate`).toBe(delegateAgentId); + } + } + if (["turn_scope", "run_scope", "issue_scope_authorized", "replacement_turn_accepted"].includes(completionState)) { + semanticConsumer = "native-cancellation-authority"; + const scope = completionState === "run_scope" ? "run" : completionState === "issue_scope_authorized" ? "issue" : "turn"; + if (options.disableLiveEntrypoint === "cancellation") { + consumerDecision = pushDecisionConsumer(semanticConsumer, resolveNativeCancellationStatus({ + scope, + priorIssueStatus, + agentId, + replacementAccepted: completionState === "replacement_turn_accepted", + })); + } else { + const cancellation = await cancelNativeSession(seeded.runId, `fixture:${fixture.id}`, { + db, + scope, + replacementAccepted: completionState === "replacement_turn_accepted", + }); + if (typeof cancellation === "boolean" || !cancellation.decision || !cancellation.auditId) { + throw new Error(`${fixture.id}: live cancellation did not persist an authoritative outcome`); + } + consumerDecision = pushDecisionConsumer(semanticConsumer, cancellation.decision); + liveEntrypointCommitted = true; + for (const effect of cancellation.decision.effects) materializedEffects.add(effect.kind); + consumerExecutions.push({ + consumer: "native-session-cancellation", + observed: { + dispatched: cancellation.dispatched, + scope, + reasonCode: cancellation.decision.reasonCode, + decisionId: cancellation.decisionId, + auditId: cancellation.auditId, + }, + }); + } + } else { + const reconciliationFacts = reconciliationFactsFor(completionState); + const compatibilityFacts = compatibilityFactsFor(completionState); + const migrationFacts = migrationFactsFor(completionState); + if ( + reconciliationFacts + && !liveReconciliationStates.has(completionState) + && ( + supersedingDecisionStates.has(completionState) + || (fixture.covers.reconciliationRows ?? []).length > 0 + || String(fixture.given.trigger) === "dependency" + ) + ) { + semanticConsumer = "native-reconciliation-consumer"; + consumerDecision = pushDecisionConsumer(semanticConsumer, resolveNativeReconciliationStatus({ + facts: reconciliationFacts, + priorIssueStatus, + agentId, + })); + } else if (!consumerDecision && attentionFacts && ["attention_response", "attention_candidate", "interaction", "monitor"].includes(String(fixture.given.trigger))) { + semanticConsumer = "native-attention-resolver"; + consumerDecision = pushDecisionConsumer(semanticConsumer, resolveNativeAttentionStatus({ + facts: options.disableLiveEntrypoint === "attention" && attentionFacts.route === "agent" + ? { ...attentionFacts, resolvedTargetAgentId: delegateAgentId } + : attentionFacts, + priorIssueStatus, + agentId, + })); + } else if ( + completionState === "kill_switch_during_active_native_run" + && (fixture.covers.migrationRows ?? []).includes("MIG-08") + ) { + const runtimeConfig = { + nativeRunner: { mode: "native", backend: "codex_app_server", protocolVersion: 1 }, + }; + const enabled = options.disableLiveEntrypoint === "rollout"; + const activeResolution = resolveHeartbeatNativeRuntimeMode({ + persisted: { + runtimeMode: "native", + runtimeModeReason: "eligible_opt_in", + runtimeModeResolvedAt: new Date(), + }, + enabled, + runtimeConfig, + adapterConfig: { provider: "codex" }, + agent: { id: agentId, status: "running", adapterType: "paperclip_runner" }, + issue: { id: seeded.issueId, workMode: "standard" }, + target: { kind: "local" }, + workspaceId: "fixture-workspace", + }); + let freshReason: string | null = null; + let freshMode: string | null = null; + try { + const freshResolution = resolveHeartbeatNativeRuntimeMode({ + persisted: { runtimeMode: null, runtimeModeReason: null, runtimeModeResolvedAt: null }, + enabled, + runtimeConfig, + adapterConfig: { provider: "codex" }, + agent: { id: agentId, status: "running", adapterType: "paperclip_runner" }, + issue: { id: seeded.issueId, workMode: "standard" }, + target: { kind: "local" }, + workspaceId: "fixture-workspace", + }); + freshMode = freshResolution.kind; + freshReason = freshResolution.reason; + } catch (error) { + freshMode = "rejected"; + freshReason = error instanceof Error && "code" in error + ? String(error.code) + : null; + } + if ( + activeResolution.kind !== "native" + || freshMode !== "rejected" + || freshReason !== "paperclip_runner_rollout_disabled" + || runtimeConfig.nativeRunner.mode !== "native" + ) { + throw new Error(`${fixture.id}: global kill-switch transition missing`); + } + semanticConsumer = "native-migration-status"; + consumerDecision = pushDecisionConsumer(semanticConsumer, activeResolution.authorityDecision); + operationalEffects.add("fresh_flag_off_run_rejected"); + consumerExecutions.push({ + consumer: "heartbeat-runtime-selection", + observed: { + activeMode: activeResolution.kind, + freshMode, + freshReason, + profileMode: runtimeConfig.nativeRunner.mode, + }, + }); + } else if (migrationFacts && (fixture.covers.migrationRows ?? []).length > 0 && (fixture.covers.decisionRows ?? []).length === 0) { + semanticConsumer = "native-migration-status"; + consumerDecision = pushDecisionConsumer( + semanticConsumer, + completionState === "allowlisted_company_adapter_policy" && mode.kind === "native" + ? mode.authorityDecision + : resolveNativeMigrationStatus({ facts: migrationFacts, priorIssueStatus, agentId }), + ); + } else if (compatibilityFacts && (fixture.covers.compatibilityRows ?? []).length > 0 && (fixture.covers.decisionRows ?? []).length === 0) { + semanticConsumer = "native-compatibility-status"; + consumerDecision = pushDecisionConsumer(semanticConsumer, resolveNativeCompatibilityStatus({ + facts: compatibilityFacts, + priorIssueStatus, + agentId, + })); + } else if (completionState === "safe_partial_parse" && compatibilityFacts) { + semanticConsumer = "native-finalizer-status"; + consumerDecision = pushDecisionConsumer(semanticConsumer, resolveNativeCompatibilityStatus({ + facts: compatibilityFacts, + priorIssueStatus, + agentId, + })); + } + + if (attentionFacts && (fixture.covers.attentionRows ?? []).length > 0 && semanticConsumer !== "native-attention-resolver") { + pushDecisionConsumer("native-attention-resolver", resolveNativeAttentionStatus({ facts: attentionFacts, priorIssueStatus, agentId })); + } + if ( + reconciliationFacts + && !liveReconciliationStates.has(completionState) + && (fixture.covers.reconciliationRows ?? []).length > 0 + && semanticConsumer !== "native-reconciliation-consumer" + ) { + pushDecisionConsumer("native-reconciliation-consumer", resolveNativeReconciliationStatus({ facts: reconciliationFacts, priorIssueStatus, agentId })); + } + if (compatibilityFacts && (fixture.covers.compatibilityRows ?? []).some((row) => ["COMP-02", "COMP-05", "COMP-06", "COMP-07", "COMP-08"].includes(row)) && semanticConsumer !== "native-compatibility-status") { + pushDecisionConsumer("native-compatibility-status", resolveNativeCompatibilityStatus({ facts: compatibilityFacts, priorIssueStatus, agentId })); + } + if (migrationFacts && (fixture.covers.migrationRows ?? []).some((row) => ["MIG-04", "MIG-05", "MIG-06", "MIG-07", "MIG-08"].includes(row)) && semanticConsumer !== "native-migration-status") { + pushDecisionConsumer("native-migration-status", resolveNativeMigrationStatus({ facts: migrationFacts, priorIssueStatus, agentId })); + } + } + + if ( + seeded.nativeRecords + && options.disableLiveEntrypoint !== "reconciliation" + && liveReconciliationStates.has(completionState) + ) { + const [priorDecision] = await db.insert(statusDecisions).values({ + companyId, + runId: seeded.runId, + issueId: seeded.issueId, + assessmentId: seeded.assessmentId, + decisionVersion: 1, + policyVersion: completionState === "new_policy_requires_review" + ? "phase6-v1" + : NATIVE_STATUS_ARBITER_POLICY_VERSION, + fromStatus: completionState === "board_cancelled_before_cas" ? "in_progress" : priorIssueStatus, + toStatus: completionState === "board_cancelled_before_cas" ? "in_progress" : priorIssueStatus, + reasonCode: "prior_fixture_decision", + decisionJson: { superseded: true }, + decisionDigest: `prior-decision:${fixture.id}`, + applicationState: "applied", + appliedAt: new Date(), + }).returning({ id: statusDecisions.id }); + await db.update(issues).set({ + statusVersion: 1, + lastStatusDecisionId: priorDecision!.id, + }).where(eq(issues.id, seeded.issueId)); + await db.update(nativeRunFinalizations).set({ + phase: "committed", + decisionId: priorDecision!.id, + }).where(eq(nativeRunFinalizations.runId, seeded.runId)); + if (completionState === "new_evidence_satisfies_contract") { + await db.update(issueWorkProducts).set({ + reviewState: "approved", + updatedAt: new Date(Date.now() + 1_000), + }).where(eq(issueWorkProducts.id, seeded.workProductId)); + } + const [reconciled] = await reconcileNativeFinalizations(db, [seeded.runId]); + if (!reconciled?.reconciliationDecision || !reconciled.decisionId) { + throw new Error(`${fixture.id}: live reconciliation did not commit an authoritative decision`); + } + semanticConsumer = "native-reconciliation-consumer"; + consumerDecision = pushDecisionConsumer(semanticConsumer, reconciled.reconciliationDecision); + liveEntrypointCommitted = true; + consumerExecutions.push({ + consumer: "native-reconciliation-entrypoint", + observed: { + action: reconciled.reconciliationAction, + decisionId: reconciled.decisionId, + }, + }); + } + if ( + seeded.nativeRecords + && options.disableLiveEntrypoint === "reconciliation" + && liveReconciliationStates.has(completionState) + ) { + const facts = reconciliationFactsFor(completionState); + if (!facts) throw new Error(`${fixture.id}: missing reconciliation facts`); + semanticConsumer = "native-reconciliation-consumer"; + consumerDecision = pushDecisionConsumer(semanticConsumer, resolveNativeReconciliationStatus({ + facts, + priorIssueStatus, + agentId, + })); + } + + let assessment: NativeEvidenceAssessment | null = null; + if (seeded.nativeRecords && ["runner_finalizer", "dependency", "shadow_comparator", "read_model", "authorized_agent"].includes(String(fixture.given.trigger))) { + const accepted = completeEvidenceStates.has(completionState); + const evidenceRef = accepted + ? `work_product:${seeded.workProductId}` + : `work_product:${randomUUID()}`; + assessment = await classifyNativeEvidence({ + db, + companyId, + issueId: seeded.issueId, + runId: seeded.runId, + contract: { revision: "corpus-v1", criteria: [{ id: "objective" }] }, + result: { + reportedWorkDisposition: fixtureDisposition(fixture), + summary: fixture.id, + completionClaim: { + contractRevision: "corpus-v1", + objectiveSatisfied: true, + criteria: [{ criterionId: "objective", status: "satisfied", evidenceRefs: [evidenceRef] }], + remainingWork: accepted ? [] : [{ blocksCompletion: true }], + }, + verification: [{ commandOrCheck: "fixture", status: "passed", artifactRef: evidenceRef }], + blocker: fixture.given.reportedWorkDisposition === "blocked" + ? { + scope: completionState === "task_wide_owner_action_bound" ? "task_wide" : "current_track", + owner: { kind: "board" }, + unblockAction: `Resolve ${fixture.id}`, + } + : null, + continuation: fixture.given.reportedWorkDisposition === "yielded" && completionState !== "no_durable_continuation" + ? { kind: "same_agent", summary: fixture.id, idempotencyKey: `fixture:${fixture.id}` } + : null, + }, + }); + consumerExecutions.push({ + consumer: "evidence-classifier", + observed: { + allCriteriaSatisfied: assessment.allCriteriaSatisfied, + verificationPassed: assessment.verificationPassed, + acceptedEvidenceCount: assessment.acceptedEvidenceRefs.length, + missingRequirementCount: assessment.missingRequirements.length, + }, + }); + + const liveDecision = resolveNativeFinalizerStatus({ + assessment, + terminalState: fixture.given.runTerminalState === "failed" + ? "failed" + : fixture.given.runTerminalState === "cancelled" ? "cancelled" : "succeeded", + workspaceFinalizeStatus: fixture.given.fault === "workspace_finalize_failure" ? "failed" : "succeeded", + governanceGate, + completionClaimPolicyAccepted: completionState === "low_risk_policy_claim", + allowIncompleteContinuation: completionState !== "no_durable_continuation", + agentId, + priorIssueStatus, + }); + consumerExecutions.push({ + consumer: "status-arbiter", + observed: { + toStatus: liveDecision.toStatus, + reasonCode: liveDecision.reasonCode, + effects: liveDecision.effects.map((effect) => effect.kind), + }, + }); + if (!consumerDecision) { + semanticConsumer = "native-finalizer-status"; + consumerDecision = pushDecisionConsumer(semanticConsumer, liveDecision); + } + } + + if ( + seeded.nativeRecords + && options.disableLiveEntrypoint !== "reconciliation" + && consumerDecision?.effects.some((effect) => effect.kind === "resume_workspace_operation") + ) { + await db.insert(workspaceOperations).values({ + companyId, + heartbeatRunId: seeded.runId, + issueId: seeded.issueId, + phase: "workspace_finalize", + status: "failed", + exitCode: 1, + cwd: process.cwd(), + finishedAt: new Date(), + }); + const [reconciled] = await reconcileNativeFinalizations(db, [seeded.runId]); + if (!reconciled || reconciled.reconciliationAction !== "resume_workspace_operation") { + throw new Error(`${fixture.id}: live workspace reconciliation did not execute (${JSON.stringify(reconciled)})`); + } + liveEntrypointCommitted = true; + materializedEffects.add("resume_workspace_operation"); + consumerExecutions.push({ + consumer: "native-reconciliation-entrypoint", + observed: { + action: reconciled.reconciliationAction, + workspaceOperationId: reconciled.workspaceOperationId, + workspaceFinalizeStatus: reconciled.workspaceFinalizeStatus, + }, + }); + } + if (!liveEntrypointCommitted && seeded.nativeRecords && consumerDecision?.effects.some((effect) => effect.kind === "link_canonical_request")) { + const canonicalInteractionId = randomUUID(); + liveAttentionInteractionId = randomUUID(); + await db.insert(issueThreadInteractions).values([ + { + id: canonicalInteractionId, + companyId, + issueId: seeded.issueId, + kind: "request_confirmation", + status: "pending", + idempotencyKey: `canonical:${fixture.id}`, + payload: { version: 1, prompt: `Canonical ${fixture.id}` }, + }, + { + id: liveAttentionInteractionId, + companyId, + issueId: seeded.issueId, + kind: "request_confirmation", + status: "expired", + resolvedAt: new Date(), + idempotencyKey: `duplicate:${fixture.id}`, + payload: { version: 1, prompt: `Duplicate ${fixture.id}` }, + result: { + version: 1, + outcome: "superseded_by_newer_request", + supersededByInteractionId: canonicalInteractionId, + }, + }, + ]); + } + if (!liveEntrypointCommitted && seeded.nativeRecords && consumerDecision?.effects.some((effect) => effect.kind === "record_stale_response")) { + liveAttentionInteractionId = randomUUID(); + await db.insert(issueThreadInteractions).values({ + id: liveAttentionInteractionId, + companyId, + issueId: seeded.issueId, + kind: "request_confirmation", + status: "expired", + resolvedAt: new Date(), + payload: { version: 1, prompt: `Stale ${fixture.id}` }, + result: { version: 1, outcome: "superseded_by_comment", commentId: randomUUID() }, + }); + } + if (!liveEntrypointCommitted && seeded.nativeRecords && consumerDecision?.effects.some((effect) => effect.kind === "record_expiry")) { + await db.insert(issueThreadInteractions).values({ + companyId, + issueId: seeded.issueId, + kind: "ask_user_questions", + status: "pending", + payload: { version: 1, questions: [] }, + }); + } + if (seeded.nativeRecords && completionState === "decision_committed_delivery_pending") { + const [priorDecision] = await db.insert(statusDecisions).values({ + companyId, + runId: seeded.runId, + issueId: seeded.issueId, + assessmentId: seeded.assessmentId, + decisionVersion: 1, + policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, + fromStatus: priorIssueStatus, + toStatus: priorIssueStatus, + reasonCode: "live_continuation_registered", + decisionJson: { persistedBeforeDelivery: true }, + decisionDigest: `delivery-pending:${fixture.id}`, + applicationState: "applied", + appliedAt: new Date(), + }).returning({ id: statusDecisions.id }); + const [pendingWake] = await db.insert(agentWakeupRequests).values({ + companyId, + agentId, + source: "automation", + triggerDetail: "system", + reason: "issue_status_changed", + payload: { + issueId: seeded.issueId, + taskId: seeded.issueId, + nativeDecisionId: priorDecision!.id, + continuationKind: "same_agent", + continuationSummary: fixture.id, + }, + requestedByActorType: "system", + requestedByActorId: "native-status-committer", + idempotencyKey: `delivery-pending:${seeded.issueId}`, + }).returning({ id: agentWakeupRequests.id }); + await db.insert(statusDecisionEffects).values({ + companyId, + issueId: seeded.issueId, + decisionId: priorDecision!.id, + ordinal: 1, + effectKind: "enqueue_continuation", + targetType: "agent_wakeup_request", + targetId: pendingWake!.id, + idempotencyKey: `delivery-pending:${seeded.issueId}`, + payload: { fixtureId: fixture.id }, + deliveryState: "pending", + attemptCount: 0, + }); + await db.update(nativeRunFinalizations).set({ + phase: "committed", + decisionId: priorDecision!.id, + }).where(eq(nativeRunFinalizations.runId, seeded.runId)); + await reconcileNativeFinalizations(db, [seeded.runId]); + } + if ( + seeded.nativeRecords + && zeroDecisionStates.has(completionState) + && !liveEntrypointCommitted + && consumerDecision + && consumerDecision.effects.some((effect) => [ + "link_canonical_request", + "record_stale_response", + "record_finalization_error", + ].includes(effect.kind)) + ) { + const interactionEffect = consumerDecision.effects.find((effect) => ["link_canonical_request", "record_stale_response"].includes(effect.kind)); + if (interactionEffect && liveAttentionInteractionId) { + const projection = await materializeNativeInteractionResponses({ + db, + companyId, + issueId: seeded.issueId, + runId: seeded.runId, + agentId, + interactionIds: [liveAttentionInteractionId], + }).then(() => ({ code: null })).catch((error) => ({ + code: error instanceof Error && "code" in error ? String(error.code) : String(error), + })); + expect(projection.code, `${fixture.id}:live attention terminal`).toBe("native_interaction_missing"); + materializedEffects.add(interactionEffect.kind); + consumerExecutions.push({ consumer: "native-attention-effect-materializer", observed: projection }); + } else { + const targets = await applyNativeAttentionStatusDecision({ + db, + companyId, + issueId: seeded.issueId, + runId: seeded.runId, + decision: consumerDecision, + }); + for (const target of targets) materializedEffects.add(target.effectKind); + consumerExecutions.push({ + consumer: "native-attention-effect-materializer", + observed: { targets }, + }); + } + } + + let rolledBack = false; + if ( + seeded.nativeRecords + && consumerDecision + && consumerDecision.reasonCode !== null + && completionState !== "replacement_turn_accepted" + && !zeroDecisionStates.has(completionState) + && !liveEntrypointCommitted + ) { + let priorStatusVersion = 0; + let priorDecisionId: string | null = null; + if (supersedingDecisionStates.has(completionState)) { + const priorAssessmentId = randomUUID(); + await db.insert(workAssessments).values({ + id: priorAssessmentId, + companyId, + issueId: seeded.issueId, + runId: seeded.runId, + contractId: seeded.contractId!, + resultId: seeded.resultId!, + triggerKind: "prior_fixture_fact", + triggerActorCompanyId: companyId, + priorIssueStatus, + priorStatusVersion: 0, + policyVersion: "phase6-v1", + assessmentJson: { fixtureId: fixture.id, superseded: true }, + inputDigest: `prior-assessment:${fixture.id}`, + }); + const [priorDecision] = await db.insert(statusDecisions).values({ + companyId, + runId: seeded.runId, + issueId: seeded.issueId, + assessmentId: priorAssessmentId, + decisionVersion: 1, + policyVersion: "phase6-v1", + fromStatus: priorIssueStatus, + toStatus: priorIssueStatus, + reasonCode: "prior_fixture_decision", + decisionJson: { superseded: true }, + decisionDigest: `prior-decision:${fixture.id}`, + applicationState: "applied", + appliedAt: new Date(), + }).returning({ id: statusDecisions.id }); + priorDecisionId = priorDecision!.id; + priorStatusVersion = 1; + await db.update(issues).set({ + statusVersion: priorStatusVersion, + lastStatusDecisionId: priorDecisionId, + }).where(eq(issues.id, seeded.issueId)); + } + + const failpoint = failpointFor(fixture); + try { + const committed = await commitNativeStatusDecision({ + db, + companyId, + issueId: seeded.issueId, + runId: seeded.runId, + assessmentId: seeded.assessmentId, + priorStatus: priorIssueStatus, + priorStatusVersion, + priorDecisionId, + decision: consumerDecision, + failpoint, + }); + const replayed = await commitNativeStatusDecision({ + db, + companyId, + issueId: seeded.issueId, + runId: seeded.runId, + assessmentId: seeded.assessmentId, + priorStatus: priorIssueStatus, + priorStatusVersion, + priorDecisionId, + decision: consumerDecision, + }); + expect(replayed.replayed, `${fixture.id} decision replay`).toBe(true); + expect(replayed.decision.id, `${fixture.id} replay decision identity`).toBe(committed.decision.id); + consumerExecutions.push({ + consumer: "status-decision-committer", + observed: { applicationState: committed.decision.applicationState, failpoint: null, replayed: replayed.replayed }, + }); + } catch (error) { + if (!failpoint) throw error; + rolledBack = true; + consumerExecutions.push({ + consumer: "status-decision-committer", + observed: { applicationState: "rolled_back", failpoint, error: String(error) }, + }); + } + } + + if (options.disableLiveEntrypoint === "rollout" && completionState === "kill_switch_during_active_native_run") { + await db.update(agents).set({ + runtimeConfig: { nativeRunner: { mode: "native", backend: "codex_app_server", protocolVersion: 1 } }, + }).where(eq(agents.id, agentId)); + } + + if (seeded.nativeRecords && (rolledBack || completionState === "safe_partial_parse")) { + const failure = await recordNativeFinalizationFailure({ + db, + runId: seeded.runId, + error: new Error(rolledBack ? "side_effect_planning_failed" : "native_finalization_invalid"), + projectRunStatus: true, + }); + materializedEffects.add("record_finalization_error"); + consumerExecutions.push({ + consumer: "native-finalization-failure", + observed: { phase: failure.phase, failureCode: failure.failureCode }, + }); + } + + if (["attention_response", "attention_candidate", "interaction", "monitor"].includes(String(fixture.given.trigger))) { + const interactionId = randomUUID(); + const resolved = ["attention_response", "interaction"].includes(String(fixture.given.trigger)); + await db.insert(issueThreadInteractions).values({ + id: interactionId, + companyId, + issueId: seeded.issueId, + kind: "ask_user_questions", + status: resolved ? "answered" : completionState === "interaction_expired" ? "expired" : "pending", + resolvedByUserId: resolved ? "board-user" : null, + resolvedAt: resolved ? new Date() : null, + payload: { + version: 1, + questions: [{ id: "answer", prompt: fixture.id, selectionMode: "single", options: [{ id: "continue", label: "Continue" }] }], + }, + result: resolved ? { version: 1, answers: [{ questionId: "answer", optionIds: ["continue"] }] } : null, + }); + const projected = await materializeNativeInteractionResponses({ + db, + companyId, + issueId: seeded.issueId, + runId: seeded.runId, + agentId, + interactionIds: [interactionId], + }).then((responses) => ({ responseCount: responses.length, code: null })) + .catch((error) => ({ responseCount: 0, code: error instanceof Error && "code" in error ? String(error.code) : String(error) })); + consumerExecutions.push({ consumer: "interaction-lifecycle", observed: projected }); + const denied = rejectUnsupportedNativeRuntimeRequest(`fixture:${fixture.id}`); + consumerExecutions.push({ + consumer: "native-runtime-request-boundary", + observed: { accepted: denied.accepted, credentialsInjected: denied.credentialsInjected, selfApproval: denied.selfApproval }, + }); + } + + const boardTarget = completionState === "explicit_resume_capability" + ? "in_progress" + : completionState === "authorized_status_write" + ? "in_review" + : completionState === "authorized_writer_incremented_version" + ? String(fixture.given.priorIssueStatus ?? "in_progress") + : null; + if (boardTarget) { + const currentVersion = await db.select({ statusVersion: issues.statusVersion }) + .from(issues).where(eq(issues.id, seeded.issueId)) + .then((rows) => Number(rows[0]?.statusVersion ?? 0)); + const updated = await issueService(db).update(seeded.issueId, { + status: boardTarget, + statusVersion: currentVersion + 1, + actorUserId: "status-corpus-board", + actorAgentId: null, + }); + consumerExecutions.push({ + consumer: "authorized-issue-writer", + observed: { status: updated.status, statusVersion: updated.statusVersion }, + }); + if (completionState === "authorized_status_write") { + const reviewer = await issueThreadInteractionService(db).create( + updated, + { + kind: "request_confirmation", + idempotencyKey: `migration-review:${fixture.id}`, + sourceRunId: null, + title: "Migration writer review", + summary: "Preserve the existing review liveness path.", + continuationPolicy: "wake_assignee", + payload: { + version: 1, + prompt: "Review the migrated issue status.", + acceptLabel: "Approve", + rejectLabel: "Continue", + supersedeOnUserComment: false, + }, + }, + { systemId: "status-corpus" }, + ); + materializedEffects.add("bind_reviewer"); + consumerExecutions.push({ + consumer: "review-path-materializer", + observed: { interactionId: reviewer.id, status: reviewer.status }, + }); + } + } + + const reconciliationRows = fixture.covers.reconciliationRows ?? []; + if (reconciliationRows.length > 0 || ["authorized_agent", "board_user"].includes(String(fixture.given.trigger))) { + const disposition = nativeSessionFailureDisposition( + completionState === "resolver_budget_exhausted" ? 3 : 1, + new Date("2026-08-09T00:00:00.000Z"), + ); + consumerExecutions.push({ + consumer: "native-recovery-policy", + observed: { phase: disposition.phase, failureCode: disposition.failureCode, retryScheduled: disposition.nextAttemptAt !== null }, + }); + } + + if (["migration", "read_model", "shadow_comparator"].includes(String(fixture.given.trigger))) { + const persistedIssue = await db.select({ status: issues.status, statusVersion: issues.statusVersion }) + .from(issues).where(and(eq(issues.id, seeded.issueId), eq(issues.companyId, companyId))) + .then((rows) => rows[0] ?? null); + consumerExecutions.push({ + consumer: "migration-compatibility-read", + observed: { found: persistedIssue !== null, status: persistedIssue?.status, statusVersion: persistedIssue?.statusVersion }, + }); + } + + const [decisionRows, effectRows, nativeRows, wakeRows, recoveryRows, interactionRows, persistedIssue, persistedRun] = await Promise.all([ + db.select({ id: statusDecisions.id }).from(statusDecisions).where(eq(statusDecisions.issueId, seeded.issueId)), + db.select({ + id: statusDecisionEffects.id, + effectKind: statusDecisionEffects.effectKind, + targetType: statusDecisionEffects.targetType, + targetId: statusDecisionEffects.targetId, + deliveryState: statusDecisionEffects.deliveryState, + attemptCount: statusDecisionEffects.attemptCount, + }).from(statusDecisionEffects).where(eq(statusDecisionEffects.issueId, seeded.issueId)), + db.select({ id: nativeRunResults.id, resultJson: nativeRunResults.resultJson }) + .from(nativeRunResults).where(eq(nativeRunResults.issueId, seeded.issueId)), + db.select({ id: agentWakeupRequests.id, payload: agentWakeupRequests.payload }) + .from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId)), + db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions) + .where(eq(issueRecoveryActions.sourceIssueId, seeded.issueId)), + db.select({ + id: issueThreadInteractions.id, + status: issueThreadInteractions.status, + summary: issueThreadInteractions.summary, + }) + .from(issueThreadInteractions).where(eq(issueThreadInteractions.issueId, seeded.issueId)), + db.select({ + status: issues.status, + statusVersion: issues.statusVersion, + checkoutRunId: issues.checkoutRunId, + executionRunId: issues.executionRunId, + unblockDescriptor: issues.unblockDescriptor, + }) + .from(issues).where(eq(issues.id, seeded.issueId)).then((rows) => rows[0] ?? null), + db.select({ + status: heartbeatRuns.status, + runtimeMode: heartbeatRuns.runtimeMode, + completionContractId: heartbeatRuns.completionContractId, + continuationAttempt: heartbeatRuns.continuationAttempt, + processPid: heartbeatRuns.processPid, + processGroupId: heartbeatRuns.processGroupId, + resultJson: heartbeatRuns.resultJson, + runnerProfileJson: heartbeatRuns.runnerProfileJson, + }).from(heartbeatRuns) + .where(eq(heartbeatRuns.id, seeded.runId)).then((rows) => rows[0] ?? null), + ]); + const persistedEffects = effectRows + .map((row) => row.effectKind) + .filter((effect) => effect !== "issue_status_projection"); + for (const effect of persistedEffects) materializedEffects.add(effect); + for (const effectRow of effectRows.filter((row) => row.effectKind !== "issue_status_projection")) { + expect(effectRow.deliveryState, `${fixture.id}:${effectRow.effectKind}:delivery`).toBe("delivered"); + expect(effectRow.attemptCount, `${fixture.id}:${effectRow.effectKind}:at-most-once`).toBe(1); + expect(effectRow.targetId, `${fixture.id}:${effectRow.effectKind}:target`).not.toBeNull(); + if (effectRow.effectKind !== "release_checkout") { + expect(effectRow.targetType, `${fixture.id}:${effectRow.effectKind}:no-synthetic-checkout`).not.toBe("issue_checkout"); + } + if (effectRow.targetType === "agent_wakeup_request") { + const target = await db.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests) + .where(eq(agentWakeupRequests.id, effectRow.targetId!)).then((rows) => rows[0] ?? null); + expect(target, `${fixture.id}:${effectRow.effectKind}:wake-target`).not.toBeNull(); + } else if (effectRow.targetType === "agent") { + const target = await db.select({ id: agents.id }).from(agents) + .where(eq(agents.id, effectRow.targetId!)).then((rows) => rows[0] ?? null); + expect(target, `${fixture.id}:${effectRow.effectKind}:agent-target`).not.toBeNull(); + } else if (effectRow.targetType === "delegated_issue" || effectRow.targetType === "issue" || effectRow.targetType === "issue_checkout" || effectRow.targetType === "issue_unblock_descriptor") { + const target = await db.select({ id: issues.id }).from(issues) + .where(eq(issues.id, effectRow.targetId!)).then((rows) => rows[0] ?? null); + expect(target, `${fixture.id}:${effectRow.effectKind}:issue-target`).not.toBeNull(); + } else if (effectRow.targetType === "issue_recovery_action") { + const target = await db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions) + .where(eq(issueRecoveryActions.id, effectRow.targetId!)).then((rows) => rows[0] ?? null); + expect(target, `${fixture.id}:${effectRow.effectKind}:recovery-target`).not.toBeNull(); + } else if (effectRow.targetType === "issue_thread_interaction") { + const target = await db.select({ id: issueThreadInteractions.id }).from(issueThreadInteractions) + .where(eq(issueThreadInteractions.id, effectRow.targetId!)).then((rows) => rows[0] ?? null); + expect(target, `${fixture.id}:${effectRow.effectKind}:interaction-target`).not.toBeNull(); + } else if (effectRow.targetType === "heartbeat_run") { + expect(effectRow.targetId, `${fixture.id}:${effectRow.effectKind}:run-target`).toBe(seeded.runId); + } else if (effectRow.targetType === "workspace_operation") { + const target = await db.select({ status: workspaceOperations.status }).from(workspaceOperations) + .where(eq(workspaceOperations.id, effectRow.targetId!)).then((rows) => rows[0] ?? null); + expect(target?.status, `${fixture.id}:${effectRow.effectKind}:workspace-target`).toBe("succeeded"); + } else if (effectRow.targetType === "completion_contract") { + expect(persistedRun?.completionContractId, `${fixture.id}:${effectRow.effectKind}:contract-link`).toBe(effectRow.targetId); + } else if (effectRow.targetType === "status_decision") { + const target = await db.select({ id: statusDecisions.id }).from(statusDecisions) + .where(eq(statusDecisions.id, effectRow.targetId!)).then((rows) => rows[0] ?? null); + expect(target, `${fixture.id}:${effectRow.effectKind}:decision-target`).not.toBeNull(); + } else if (effectRow.targetType === "status_decision_effect") { + const target = await db.select({ deliveryState: statusDecisionEffects.deliveryState }).from(statusDecisionEffects) + .where(eq(statusDecisionEffects.id, effectRow.targetId!)).then((rows) => rows[0] ?? null); + expect(target?.deliveryState, `${fixture.id}:${effectRow.effectKind}:effect-target`).toBe("delivered"); + } else if (!["approval", "interaction", "execution_stage"].includes(effectRow.targetType)) { + throw new Error(`${fixture.id}:${effectRow.effectKind}:unknown target type ${effectRow.targetType}`); + } + } + const persistedRunResult = persistedRun?.resultJson && typeof persistedRun.resultJson === "object" ? persistedRun.resultJson : {}; + const persistedRunnerProfile = persistedRun?.runnerProfileJson && typeof persistedRun.runnerProfileJson === "object" ? persistedRun.runnerProfileJson : {}; + if (Array.isArray(persistedRunResult.nativeDispatchedEffectIds) && persistedRunResult.nativeDispatchedEffectIds.length > 0) { + materializedEffects.add("dispatch_pending_effect"); + expect(persistedRunResult.nativeDispatchedEffectIds, `${fixture.id}:dispatched effect identity`) + .toEqual(expect.arrayContaining(effectRows.map((row) => row.id))); + } + if (persistedEffects.includes("release_checkout")) { + expect(persistedIssue?.checkoutRunId, `${fixture.id}:checkout released`).toBeNull(); + expect(persistedIssue?.executionRunId, `${fixture.id}:execution released`).toBeNull(); + } + if (persistedEffects.includes("bind_blocker")) expect(persistedIssue?.unblockDescriptor, `${fixture.id}:blocker target`).not.toBeNull(); + if (materializedEffects.has("accept_replacement_turn")) expect(persistedRun?.continuationAttempt, `${fixture.id}:replacement target`).toBeGreaterThan(0); + if (persistedEffects.includes("release_run_resources")) { + expect(persistedRun?.processPid, `${fixture.id}:pid released`).toBeNull(); + expect(persistedRun?.processGroupId, `${fixture.id}:process group released`).toBeNull(); + } + if (persistedEffects.includes("record_shadow_decision")) expect(persistedRunResult.nativeShadowDecision, `${fixture.id}:shadow target`).toBeDefined(); + if (persistedEffects.includes("render_four_layers")) expect(persistedRunResult.nativeOutcomeLayers, `${fixture.id}:four-layer target`).toBeDefined(); + if (persistedEffects.includes("record_mode_labeled_divergence")) expect(persistedRunResult.nativeLegacyDivergence, `${fixture.id}:divergence target`).toBeDefined(); + if (persistedEffects.includes("record_mode_native")) expect(persistedRun?.runtimeMode, `${fixture.id}:mode target`).toBe("native"); + if (persistedEffects.includes("record_policy_version")) { + expect(persistedRunnerProfile.nativeStatusPolicyVersion, `${fixture.id}:policy target`) + .toBe(NATIVE_STATUS_ARBITER_POLICY_VERSION); + } + if (persistedEffects.includes("finish_as_native")) expect(persistedRunResult.nativeKillSwitchDisposition, `${fixture.id}:finish-native target`).toBe("finish_as_native"); + if (materializedEffects.has("link_canonical_request")) { + expect(interactionRows.some((row) => row.summary?.startsWith("Canonical native attention request:")), `${fixture.id}:canonical link target`).toBe(true); + } + if (materializedEffects.has("record_stale_response")) { + expect(interactionRows.some((row) => row.summary === "Response retained for audit after native supersession."), `${fixture.id}:stale audit target`).toBe(true); + } + if ( + materializedEffects.has("record_finalization_error") + && fixture.given.nativeFinalization !== "invalid" + && fixture.expected.runStatus === initialRunStatus(fixture) + ) { + expect(persistedRun?.status, `${fixture.id}:issue finalization must not rewrite provider run status`) + .toBe(initialRunStatus(fixture)); + } + const observedNativeRecords = nativeRows.length > 0; + const compatibilityState = (fixture.covers.compatibilityRows ?? []).length > 0 + ? inspectNativeCompatibilityState({ + resolution: mode, + nativeRecordCount: nativeRows.length, + decisionCount: decisionRows.length, + issueStatus: persistedIssue?.status ?? priorIssueStatus, + statusVersion: Number(persistedIssue?.statusVersion ?? 0), + persistedEffectKinds: persistedEffects, + }) + : null; + const migrationState = (fixture.covers.migrationRows ?? []).length > 0 + ? inspectNativeMigrationState({ + resolution: mode, + nativeRecordCount: nativeRows.length, + decisionCount: decisionRows.length, + issueStatusBefore: priorIssueStatus, + issueStatusAfter: persistedIssue?.status ?? priorIssueStatus, + statusVersion: Number(persistedIssue?.statusVersion ?? 0), + hasPendingReview: interactionRows.some((row) => row.status === "pending"), + }) + : null; + let effects = [...new Set([ + ...persistedEffects, + ...(consumerDecision?.effects.map((effect) => effect.kind) ?? []), + ...operationalEffects, + ])]; + if (rolledBack) effects = ["record_finalization_error"]; + if (!consumerDecision) { + effects = migrationState?.effects.length + ? [...migrationState.effects] + : compatibilityState?.effects.length ? [...compatibilityState.effects] : []; + } + for (const effect of effects) { + if (nativeStatusEffectKinds.has(effect as NativeStatusEffect["kind"])) { + expect(materializedEffects.has(effect), `${fixture.id}:${effect}:materialized-target`).toBe(true); + } + } + + const issueWakeRows = wakeRows.filter((row) => { + const payload = row.payload && typeof row.payload === "object" ? row.payload : {}; + return payload.issueId === seeded.issueId || payload.taskId === seeded.issueId; + }); + const statusAction = rolledBack + ? "preserve" + : consumerDecision + ? consumerDecision.statusAction + : migrationState?.statusAction ?? compatibilityState?.statusAction ?? "preserve"; + const livePathKind = rolledBack + ? null + : effects.includes("create_delegated_issue") ? "delegated_issue" + : effects.includes("bind_reviewer") ? "review" + : effects.includes("create_interaction") ? "interaction" + : effects.includes("bind_blocker") ? "blocker" + : effects.includes("schedule_retry") ? "retry" + : effects.includes("enqueue_continuation") || effects.includes("accept_replacement_turn") ? "continuation" + : recoveryRows.length > 0 ? "recovery" + : completionState === "preexisting_open_issue" && persistedIssue?.status === "in_review" ? "review" + : null; + if ((fixture.covers.terminalRows ?? []).length > 0) { + const terminalInput = persistedRun?.status === "running" + ? "active" + : persistedRun?.status === "cancelled" + ? "cancelled" + : persistedRun?.status === "failed" ? "failed" : "succeeded"; + consumerExecutions.push({ + consumer: "native-run-terminal-projection", + observed: { status: projectNativeTerminalRunStatus(terminalInput) }, + }); + } + if (compatibilityState) { + consumerExecutions.push({ + consumer: "native-compatibility-read-model", + observed: compatibilityState, + }); + } + if (migrationState) { + consumerExecutions.push({ + consumer: "native-migration-read-model", + observed: migrationState, + }); + } + const consumerEvidenceByRow = new Map(); + for (const matrixRow of Object.values(fixture.covers).flat()) { + const consumer = requiredConsumerForMatrixRow(matrixRow); + const execution = consumerExecutions.find((candidate) => candidate.consumer === consumer); + if (!execution) continue; + const semanticRow = matrixRow.startsWith("SD-") + || matrixRow.startsWith("ATT-") + || matrixRow.startsWith("REC-") + || consumer.endsWith("-status"); + if (semanticRow) { + const returnedEffects = Array.isArray(execution.observed.effects) + ? execution.observed.effects.map(String) + : []; + const returnedStatusAction = execution.observed.statusAction + ?? (execution.observed.toStatus === priorIssueStatus ? "preserve" : execution.observed.toStatus); + if ( + returnedStatusAction !== statusAction + || execution.observed.reasonCode !== (rolledBack ? "side_effect_planning_failed" : consumerDecision?.reasonCode ?? null) + || !effects.every((effect) => returnedEffects.includes(effect)) + ) continue; + } else if (matrixRow.startsWith("TC-")) { + if (execution.observed.status !== (persistedRun?.status ?? initialRunStatus(fixture))) continue; + } else if (consumer.endsWith("-read-model")) { + const returnedEffects = Array.isArray(execution.observed.effects) + ? execution.observed.effects.map(String) + : []; + if ( + execution.observed.statusAction !== statusAction + || execution.observed.native !== observedNativeRecords + || !effects.every((effect) => returnedEffects.includes(effect)) + ) continue; + } else if (consumer === "status-decision-committer") { + const expectedApplicationState = rolledBack ? "rolled_back" : "applied"; + if (execution.observed.applicationState !== expectedApplicationState) continue; + } else if (consumer === "heartbeat-runtime-selection") { + if ( + execution.observed.activeMode !== "native" + || execution.observed.freshMode !== "rejected" + || execution.observed.freshReason !== "paperclip_runner_rollout_disabled" + || execution.observed.profileMode !== "native" + ) continue; + } + consumerEvidenceByRow.set(matrixRow, consumer); + } + consumerExecutions.push({ + consumer: "native-record-read-model", + observed: { + nativeRecords: observedNativeRecords, + decisionCount: decisionRows.length, + effectCount: effectRows.length, + wakeCount: issueWakeRows.length, + recoveryCount: recoveryRows.length, + }, + }); + if (consumerExecutions.length < 2) throw new Error(`${fixture.id} did not execute a concern consumer`); + if ( + ["turn_scope", "run_scope", "issue_scope_authorized", "replacement_turn_accepted"].includes(completionState) + && !consumerExecutions.some((execution) => + execution.consumer === "native-session-cancellation" && typeof execution.observed.auditId === "string" + ) + ) { + throw new Error(`${fixture.id}: live cancellation proof missing`); + } + if ( + attentionFacts + && ["attention_response", "attention_candidate", "interaction", "monitor"].includes(String(fixture.given.trigger)) + && !consumerExecutions.some((execution) => execution.consumer === "native-attention-finalizer") + ) { + throw new Error(`${fixture.id}: live attention finalizer proof missing`); + } + if ( + ["REC-04", "REC-06", "REC-07", "REC-08"].some((row) => (fixture.covers.reconciliationRows ?? []).includes(row)) + && !consumerExecutions.some((execution) => execution.consumer === "native-reconciliation-entrypoint") + ) { + throw new Error(`${fixture.id}: live reconciliation proof missing`); + } + + return { + fixtureId: fixture.id, + runStatus: mode.kind === "legacy" + ? "legacy_derived" + : persistedRun?.status ?? initialRunStatus(fixture), + statusAction, + reasonCode: rolledBack ? "side_effect_planning_failed" : consumerDecision?.reasonCode ?? null, + effects, + livePathKind, + preserveClaim: nativeRows.some((row) => { + const result = row.resultJson && typeof row.resultJson === "object" ? row.resultJson : {}; + return result.completionClaim !== undefined; + }), + nativeRecords: observedNativeRecords, + decisionCount: decisionRows.length, + wakeCount: issueWakeRows.length, + notificationCount: effectRows.filter((row) => ["notify_owner", "create_delegated_issue", "cancel_continuations"].includes(row.effectKind)).length, + consumerExecutions, + consumerEvidenceByRow, + }; + } + + it("executes all 52 fixtures in their production consumers and joins all 70 matrix rows", async () => { + expect(corpus.schema).toBe("paperclip.status-authority-conformance.v1"); + expect(corpus.fixtures).toHaveLength(52); + + const observations = new Map(); + for (const fixture of corpus.fixtures) observations.set(fixture.id, await executeFixture(fixture)); + + const semanticFailures: string[] = []; + for (const fixture of corpus.fixtures) { + const observed = observations.get(fixture.id)!; + semanticFailures.push(...comparisonFailures(fixture, observed).map((failure) => `${fixture.id}:${failure}`)); + expect(observed.consumerExecutions.length, `${fixture.id} consumer execution`).toBeGreaterThan(1); + for (const matrixRow of Object.values(fixture.covers).flat()) { + expect(observed.consumerEvidenceByRow.get(matrixRow), `${fixture.id}:${matrixRow}`) + .toBe(requiredConsumerForMatrixRow(matrixRow)); + } + } + expect(semanticFailures).toEqual([]); + + const matrixByRow = new Map>(); + for (const fixture of corpus.fixtures) { + for (const matrixRow of Object.values(fixture.covers).flat()) { + const joined = { fixtureId: fixture.id, observation: observations.get(fixture.id)! }; + const existing = matrixByRow.get(matrixRow); + if (existing) existing.push(joined); + else matrixByRow.set(matrixRow, [joined]); + } + } + const matrixResults = [...matrixByRow].map(([matrixRow, joinedFixtures]) => ({ matrixRow, joinedFixtures })); + expect(matrixResults).toHaveLength(70); + expect(new Set(matrixResults.map((result) => result.matrixRow))).toEqual(new Set([ + ...Array.from({ length: 19 }, (_, index) => `SD-${String(index + 1).padStart(2, "0")}`), + ...Array.from({ length: 8 }, (_, index) => `TC-${String(index + 1).padStart(2, "0")}`), + ...Array.from({ length: 12 }, (_, index) => `ATT-${String(index + 1).padStart(2, "0")}`), + ...Array.from({ length: 6 }, (_, index) => `LIVE-${String(index + 1).padStart(2, "0")}`), + ...Array.from({ length: 8 }, (_, index) => `REC-${String(index + 1).padStart(2, "0")}`), + ...Array.from({ length: 8 }, (_, index) => `COMP-${String(index + 1).padStart(2, "0")}`), + ...Array.from({ length: 9 }, (_, index) => `MIG-${String(index + 1).padStart(2, "0")}`), + ])); + for (const result of matrixResults) { + expect(result.joinedFixtures.length, result.matrixRow).toBeGreaterThan(0); + for (const joined of result.joinedFixtures) { + expect(joined.observation.fixtureId).toBe(joined.fixtureId); + expect(joined.observation.consumerEvidenceByRow.get(result.matrixRow), result.matrixRow) + .toBe(requiredConsumerForMatrixRow(result.matrixRow)); + } + } + }, 60_000); + + it("fails independently when any asserted field category is mutated", async () => { + const observations = new Map(); + for (const fixture of corpus.fixtures) observations.set(fixture.id, await executeFixture(fixture)); + + for (const fixture of corpus.fixtures) { + const observed = observations.get(fixture.id)!; + const observedEffect = observed.effects[0] ?? "__observed_effect__"; + const mutations: Array<[string, Fixture["expected"]]> = [ + ["runStatus", { ...fixture.expected, runStatus: `mutated:${fixture.expected.runStatus}` }], + ["statusAction", { ...fixture.expected, statusAction: `mutated:${fixture.expected.statusAction}` }], + ["reasonCode", { ...fixture.expected, reasonCode: fixture.expected.reasonCode === null ? "mutated" : null }], + ["requiredEffects", { ...fixture.expected, requiredEffects: [...fixture.expected.requiredEffects, "__missing_effect__"] }], + ["forbiddenEffects", { ...fixture.expected, forbiddenEffects: [...fixture.expected.forbiddenEffects, observedEffect] }], + ["livePathKind", { ...fixture.expected, livePathKind: fixture.expected.livePathKind === null ? "continuation" : null }], + ["preserveClaim", { ...fixture.expected, preserveClaim: !fixture.expected.preserveClaim }], + ["nativeRecords", { ...fixture.expected, nativeRecords: !fixture.expected.nativeRecords }], + ["decisionCount", { ...fixture.expected, decisionCount: fixture.expected.decisionCount + 1 }], + ["maxWakeCount", { ...fixture.expected, maxWakeCount: observed.wakeCount - 1 }], + ["maxNotificationCount", { ...fixture.expected, maxNotificationCount: observed.notificationCount - 1 }], + ]; + for (const [field, expected] of mutations) { + const mutated = { ...fixture, expected }; + expect(comparisonFailures(mutated, observed), `${fixture.id}:${field}`).not.toEqual([]); + } + } + }, 60_000); + + it("fails mapped fixtures when live entrypoints or owning actions are removed despite matching policy labels", async () => { + const byId = (id: string) => { + const fixture = corpus.fixtures.find((candidate) => candidate.id === id); + if (!fixture) throw new Error(`fixture missing: ${id}`); + return fixture; + }; + + expect(resolveNativeCancellationStatus({ + scope: "turn", + priorIssueStatus: "in_progress", + agentId, + }).reasonCode).toBe("cancellation_turn_only"); + await expect(executeFixture(byId("turn-only-cancellation"), { + disableLiveEntrypoint: "cancellation", + })).rejects.toThrow("live cancellation proof missing"); + + expect(resolveNativeAttentionStatus({ + facts: { + companyScopeValid: true, + responseState: "none", + route: "agent", + summary: "delegate", + resolvedTargetAgentId: delegateAgentId, + }, + priorIssueStatus: "in_progress", + agentId, + }).reasonCode).toBe("attention_routed_to_agent"); + await expect(executeFixture(byId("human-request-routed-to-agent"), { + disableLiveEntrypoint: "attention", + })).rejects.toThrow("live attention finalizer proof missing"); + await expect(executeFixture(byId("duplicate-and-fresh-key-question"), { + disableLiveEntrypoint: "attention", + })).rejects.toThrow("live attention finalizer proof missing"); + await expect(executeFixture(byId("stale-attention-response"), { + disableLiveEntrypoint: "attention", + })).rejects.toThrow("live attention finalizer proof missing"); + + expect(resolveNativeReconciliationStatus({ + facts: { workspaceOperationPending: true }, + priorIssueStatus: "in_progress", + agentId, + }).effects.map((effect) => effect.kind)).toContain("resume_workspace_operation"); + await expect(executeFixture(byId("crash-before-workspace-finalization"), { + disableLiveEntrypoint: "reconciliation", + })).rejects.toThrow("native_workspace_operation_not_executed"); + + expect(resolveNativeMigrationStatus({ + facts: { killSwitchActiveForNewRuns: true }, + priorIssueStatus: "in_progress", + agentId, + }).effects.map((effect) => effect.kind)).toContain("finish_as_native"); + await expect(executeFixture(byId("migration-kill-switch-rollback"), { + disableLiveEntrypoint: "rollout", + })).rejects.toThrow("global kill-switch transition missing"); + + const pendingFixture = byId("crash-after-decision-commit"); + const seeded = await seedFixture(pendingFixture); + const [decision] = await db.insert(statusDecisions).values({ + companyId, + runId: seeded.runId, + issueId: seeded.issueId, + assessmentId: seeded.assessmentId, + decisionVersion: 1, + policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, + fromStatus: "in_progress", + toStatus: "in_progress", + reasonCode: "live_continuation_registered", + decisionJson: { fixtureId: pendingFixture.id }, + decisionDigest: `pending-negative:${seeded.issueId}`, + applicationState: "applied", + appliedAt: new Date(), + }).returning({ id: statusDecisions.id }); + const [wake] = await db.insert(agentWakeupRequests).values({ + companyId, + agentId, + source: "automation", + triggerDetail: "system", + reason: "issue_status_changed", + payload: { issueId: seeded.issueId, nativeDecisionId: decision!.id }, + requestedByActorType: "system", + requestedByActorId: "native-status-committer", + idempotencyKey: `pending-negative:${seeded.issueId}`, + }).returning({ id: agentWakeupRequests.id }); + await db.insert(statusDecisionEffects).values({ + companyId, + issueId: seeded.issueId, + decisionId: decision!.id, + ordinal: 1, + effectKind: "enqueue_continuation", + targetType: "agent_wakeup_request", + targetId: wake!.id, + idempotencyKey: `pending-negative:${seeded.issueId}`, + payload: { fixtureId: pendingFixture.id }, + deliveryState: "pending", + attemptCount: 0, + }); + await db.update(nativeRunFinalizations).set({ + phase: "committed", + decisionId: decision!.id, + }).where(eq(nativeRunFinalizations.runId, seeded.runId)); + await db.delete(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake!.id)); + expect(resolveNativeReconciliationStatus({ + facts: { undeliveredEffectCount: 1 }, + priorIssueStatus: "in_progress", + agentId, + }).effects.map((effect) => effect.kind)).toContain("dispatch_pending_effect"); + await expect(reconcileNativeFinalizations(db, [seeded.runId])) + .rejects.toThrow("native_pending_effect_target_missing:enqueue_continuation"); + }, 30_000); + + it("preserves terminal issues when a newer reconciliation policy is available", () => { + expect(resolveNativeReconciliationStatus({ + facts: { policyVersionChanged: true }, + priorIssueStatus: "done", + agentId, + })).toMatchObject({ + statusAction: "preserve", + toStatus: "done", + reasonCode: "prior_status_terminal_preserved", + effects: [{ kind: "append_superseding_assessment" }], + }); + expect(resolveNativeReconciliationStatus({ + facts: { authoritativeStatusChanged: true, policyVersionChanged: true }, + priorIssueStatus: "blocked", + agentId, + })).toMatchObject({ + statusAction: "preserve", + toStatus: "blocked", + reasonCode: "prior_status_terminal_preserved", + }); + }); + + it("supersedes the exact committed coordinator decision and sequences preserve decisions independently of issue status versions", async () => { + const fixture = corpus.fixtures.find((candidate) => candidate.mode === "native"); + if (!fixture) throw new Error("native corpus fixture missing"); + const seeded = await seedFixture(fixture); + const priorStatus = String(fixture.given.priorIssueStatus ?? "in_progress") as NativeStatusDecision["toStatus"]; + const preserveDecision = (reasonCode: string): NativeStatusDecision => ({ + policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, + statusAction: "preserve", + toStatus: priorStatus, + reasonCode, + unblockDescriptor: null, + effects: [], + }); + + const first = await commitNativeStatusDecision({ + db, + companyId, + issueId: seeded.issueId, + runId: seeded.runId, + assessmentId: seeded.assessmentId, + priorStatus, + priorStatusVersion: 0, + priorDecisionId: null, + decision: preserveDecision("preserve_sequence_one"), + }); + const supersedingAssessmentId = randomUUID(); + await db.insert(workAssessments).values({ + id: supersedingAssessmentId, + companyId, + issueId: seeded.issueId, + runId: seeded.runId, + contractId: seeded.contractId!, + resultId: seeded.resultId!, + triggerKind: "reconciliation", + triggerActorCompanyId: companyId, + priorIssueStatus: priorStatus, + priorStatusVersion: 0, + priorDecisionId: null, + policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, + assessmentJson: { reason: "preserve_sequence_two" }, + inputDigest: `preserve-sequence:${supersedingAssessmentId}`, + supersedesAssessmentId: seeded.assessmentId, + }); + const second = await commitNativeStatusDecision({ + db, + companyId, + issueId: seeded.issueId, + runId: seeded.runId, + assessmentId: supersedingAssessmentId, + priorStatus, + priorStatusVersion: 0, + priorDecisionId: null, + decision: preserveDecision("preserve_sequence_two"), + supersedesCommittedDecisionId: first.decision.id, + }); + + expect(second.decision.decisionVersion).toBe(2); + expect(second.decision.decisionJson).toMatchObject({ + priorStatusVersion: 0, + projectedStatusVersion: 0, + }); + await expect(db.select({ + status: issues.status, + statusVersion: issues.statusVersion, + lastStatusDecisionId: issues.lastStatusDecisionId, + }).from(issues).where(eq(issues.id, seeded.issueId))).resolves.toEqual([ + { status: priorStatus, statusVersion: 0, lastStatusDecisionId: null }, + ]); + }); + + it("ignores historical committed finalizations superseded by a newer authoritative decision", async () => { + const fixture = corpus.fixtures.find((candidate) => candidate.mode === "native"); + if (!fixture) throw new Error("native corpus fixture missing"); + const seeded = await seedFixture(fixture); + const priorStatus = String(fixture.given.priorIssueStatus ?? "in_progress"); + const [historicalDecision] = await db.insert(statusDecisions).values({ + companyId, + runId: seeded.runId, + issueId: seeded.issueId, + assessmentId: seeded.assessmentId, + decisionVersion: 1, + policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, + fromStatus: priorStatus, + toStatus: priorStatus, + reasonCode: "historical_decision", + decisionJson: { statusAction: "in_progress" }, + decisionDigest: `historical-decision:${seeded.issueId}`, + applicationState: "applied", + appliedAt: new Date(), + }).returning({ id: statusDecisions.id }); + const authoritativeAssessmentId = randomUUID(); + await db.insert(workAssessments).values({ + id: authoritativeAssessmentId, + companyId, + issueId: seeded.issueId, + runId: seeded.runId, + contractId: seeded.contractId!, + resultId: seeded.resultId!, + triggerKind: "reconciliation", + triggerActorCompanyId: companyId, + priorIssueStatus: priorStatus, + priorStatusVersion: 1, + priorDecisionId: historicalDecision!.id, + policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, + assessmentJson: { reason: "authoritative_decision" }, + inputDigest: `authoritative-assessment:${seeded.issueId}`, + supersedesAssessmentId: seeded.assessmentId, + }); + const [authoritativeDecision] = await db.insert(statusDecisions).values({ + companyId, + runId: seeded.runId, + issueId: seeded.issueId, + assessmentId: authoritativeAssessmentId, + decisionVersion: 2, + policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, + fromStatus: priorStatus, + toStatus: priorStatus, + reasonCode: "authoritative_decision", + decisionJson: { statusAction: "in_progress" }, + decisionDigest: `authoritative-decision:${seeded.issueId}`, + applicationState: "applied", + appliedAt: new Date(), + }).returning({ id: statusDecisions.id }); + await db.update(issues).set({ + statusVersion: 2, + lastStatusDecisionId: authoritativeDecision!.id, + }).where(eq(issues.id, seeded.issueId)); + await db.update(nativeRunFinalizations).set({ + phase: "committed", + decisionId: historicalDecision!.id, + }).where(eq(nativeRunFinalizations.runId, seeded.runId)); + + await expect(reconcileNativeFinalizations(db, [seeded.runId])).resolves.toEqual([]); + await expect(db.select({ + decisionId: nativeRunFinalizations.decisionId, + }).from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, seeded.runId))).resolves.toEqual([ + { decisionId: historicalDecision!.id }, + ]); + }); + + it("records superseding assessment lineage when a board transition has no native decision predecessor", async () => { + const fixture = corpus.fixtures.find((candidate) => candidate.mode === "native"); + if (!fixture) throw new Error("native corpus fixture missing"); + const seeded = await seedFixture(fixture); + await db.update(issues).set({ + status: "blocked", + statusVersion: 1, + lastStatusDecisionId: null, + }).where(eq(issues.id, seeded.issueId)); + const supersedingAssessmentId = randomUUID(); + await db.insert(workAssessments).values({ + id: supersedingAssessmentId, + companyId, + issueId: seeded.issueId, + runId: seeded.runId, + contractId: seeded.contractId!, + resultId: seeded.resultId!, + triggerKind: "reconciliation", + triggerActorCompanyId: companyId, + priorIssueStatus: "blocked", + priorStatusVersion: 1, + priorDecisionId: null, + policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, + assessmentJson: { reason: "board_transition_without_native_decision" }, + inputDigest: `board-transition-assessment:${seeded.issueId}`, + supersedesAssessmentId: seeded.assessmentId, + }); + + const committed = await commitNativeStatusDecision({ + db, + companyId, + issueId: seeded.issueId, + runId: seeded.runId, + assessmentId: supersedingAssessmentId, + priorStatus: "blocked", + priorStatusVersion: 1, + priorDecisionId: null, + decision: { + policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, + statusAction: "preserve", + toStatus: "blocked", + reasonCode: "prior_status_terminal_preserved", + unblockDescriptor: null, + effects: [{ kind: "append_superseding_assessment" }], + }, + }); + + expect(committed.decision.supersedesDecisionId).toBeNull(); + await expect(db.select({ + supersedesAssessmentId: workAssessments.supersedesAssessmentId, + }).from(workAssessments).where(eq(workAssessments.id, supersedingAssessmentId))).resolves.toEqual([ + { supersedesAssessmentId: seeded.assessmentId }, + ]); + }); + + it("fails the transaction closed for an unknown status effect", async () => { + const fixture = corpus.fixtures.find((candidate) => candidate.mode === "native"); + if (!fixture) throw new Error("native corpus fixture missing"); + const seeded = await seedFixture(fixture); + const decision: NativeStatusDecision = { + policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, + statusAction: "preserve", + toStatus: String(fixture.given.priorIssueStatus ?? "in_progress") as NativeStatusDecision["toStatus"], + reasonCode: "unknown_effect_test", + unblockDescriptor: null, + effects: [{ kind: "unknown_effect" } as never], + }; + + await expect(commitNativeStatusDecision({ + db, + companyId, + issueId: seeded.issueId, + runId: seeded.runId, + assessmentId: seeded.assessmentId, + priorStatus: decision.toStatus, + priorStatusVersion: 0, + priorDecisionId: null, + decision, + })).rejects.toThrow("native_status_effect_unimplemented:unknown_effect"); + + const [decisionRows, effectRows, persistedIssue, coordinator] = await Promise.all([ + db.select({ id: statusDecisions.id }).from(statusDecisions).where(eq(statusDecisions.issueId, seeded.issueId)), + db.select({ id: statusDecisionEffects.id }).from(statusDecisionEffects).where(eq(statusDecisionEffects.issueId, seeded.issueId)), + db.select({ status: issues.status, statusVersion: issues.statusVersion, lastStatusDecisionId: issues.lastStatusDecisionId }) + .from(issues).where(eq(issues.id, seeded.issueId)).then((rows) => rows[0]!), + db.select({ phase: nativeRunFinalizations.phase, decisionId: nativeRunFinalizations.decisionId }) + .from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, seeded.runId)).then((rows) => rows[0]!), + ]); + expect(decisionRows).toHaveLength(0); + expect(effectRows).toHaveLength(0); + expect(persistedIssue).toMatchObject({ status: decision.toStatus, statusVersion: 0, lastStatusDecisionId: null }); + expect(coordinator).toMatchObject({ phase: "assessing", decisionId: null }); + }); +}); diff --git a/server/src/__tests__/paperclip-semantic-conformance.test.ts b/server/src/__tests__/paperclip-semantic-conformance.test.ts new file mode 100644 index 0000000000..8a54c76aad --- /dev/null +++ b/server/src/__tests__/paperclip-semantic-conformance.test.ts @@ -0,0 +1,100 @@ +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { createDb } from "@paperclipai/db"; +import { + CAPABILITY_HIGH_RISK_SEMANTIC_VECTORS, + CAPABILITY_SEMANTIC_CONFORMANCE_IDS, + CapabilityMockSemanticConformanceAdapter, + runSemanticConformanceKit, +} from "../vendor/paperclip-runner/testing.js"; + +import { + getEmbeddedPostgresTestSupport, + startEmbeddedPostgresTestDatabase, +} from "./helpers/embedded-postgres.js"; +import { + PaperclipProductionSemanticConformanceAdapter, + seedPaperclipSemanticConformance, + type PaperclipSemanticConformanceIds, +} from "./helpers/paperclip-semantic-conformance.js"; + +vi.hoisted(() => { + process.env.PAPERCLIP_HOME = "/tmp/paperclip-semantic-conformance-home"; + process.env.PAPERCLIP_INSTANCE_ID = "semantic-conformance"; + process.env.PAPERCLIP_LOG_DIR = "/tmp/paperclip-semantic-conformance-home/logs"; + process.env.PAPERCLIP_IN_WORKTREE = "false"; +}); + +const embeddedSupport = await getEmbeddedPostgresTestSupport(); +const describeEmbedded = embeddedSupport.supported ? describe : describe.skip; + +if (!embeddedSupport.supported) { + console.warn(`Skipping semantic production conformance: ${embeddedSupport.reason ?? "unsupported host"}`); +} + +describeEmbedded("Paperclip semantic mock/production conformance", () => { + let temporary: Awaited> | null = null; + let mock: CapabilityMockSemanticConformanceAdapter | null = null; + + const base = CAPABILITY_SEMANTIC_CONFORMANCE_IDS; + const ids: PaperclipSemanticConformanceIds = { + companyId: base.companyId, + actorId: base.actorId, + foreignCompanyId: "20000000-0000-4000-8000-000000000002", + foreignTaskId: base.foreignTaskId, + blockerTaskId: base.blockerTaskId, + worlds: { + default: { taskId: base.defaultTaskId, runId: base.defaultRunId, capabilities: [] }, + "cross-company": { + taskId: base.crossCompanyTaskId, + runId: base.crossCompanyRunId, + capabilities: ["dependencies:write"], + }, + interaction: { taskId: base.interactionTaskId, runId: base.interactionRunId, capabilities: [] }, + "terminal-blocked": { + taskId: base.blockedTerminalTaskId, + runId: base.blockedTerminalRunId, + capabilities: [], + }, + terminal: { taskId: base.terminalTaskId, runId: base.terminalRunId, capabilities: [] }, + }, + }; + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("paperclip-semantic-conformance-"); + const db = createDb(temporary.connectionString); + await seedPaperclipSemanticConformance(db, ids); + mock = await CapabilityMockSemanticConformanceAdapter.create(); + }, 30_000); + + afterAll(async () => { + await mock?.stop(); + await temporary?.cleanup(); + }); + + it("matches authorization, state, audit, retry, document, continuation, and terminal semantics", async () => { + if (!temporary || !mock) throw new Error("semantic_conformance_fixture_not_started"); + const production = await PaperclipProductionSemanticConformanceAdapter.create( + createDb(temporary.connectionString), + ids, + ); + + const report = await runSemanticConformanceKit({ + vectors: CAPABILITY_HIGH_RISK_SEMANTIC_VECTORS, + adapters: [mock, production], + }); + + expect(report.rows).toHaveLength(CAPABILITY_HIGH_RISK_SEMANTIC_VECTORS.length); + expect(report.rows.every((row) => row.adapterIds.join(",") === "capability-mock,paperclip-production-services")) + .toBe(true); + expect(report.rows.find((row) => row.vectorId === "progress-duplicate-retry")?.observation.audit) + .toEqual([]); + expect(report.rows.find((row) => row.vectorId === "document-stale-revision")?.observation.authorization) + .toEqual({ outcome: "denied", code: "document_revision_conflict" }); + expect(report.rows.find((row) => row.vectorId === "continuation-request")?.observation.state) + .toMatchObject({ interactions: [{ continuationPolicy: "wake_assignee" }] }); + expect(report.rows.find((row) => row.vectorId === "terminal-finish")?.observation.state) + .toMatchObject({ task: { status: "done" } }); + expect(report.rows.every((row) => row.observation.receipt?.operationReceiptPresent === true)) + .toBe(true); + }, 30_000); +}); diff --git a/server/src/__tests__/plugin-worker-manager-duplex.test.ts b/server/src/__tests__/plugin-worker-manager-duplex.test.ts index bf7d01ab57..d0c28781d3 100644 --- a/server/src/__tests__/plugin-worker-manager-duplex.test.ts +++ b/server/src/__tests__/plugin-worker-manager-duplex.test.ts @@ -182,6 +182,9 @@ describe("plugin worker manager duplex channel route", () => { await handle.start(); const session = await handle.openDuplexChannel( duplexOpenInput({ + // Batch the exit with the open response to exercise the pre-bind hold + // and prove its normalized representation retains the discriminator. + batchWithOpenReply: true, workerSessionId: "ws-A", data: [{ chunk: "one" }], // The worker reports a reason-less transport close with no exit code. @@ -360,7 +363,7 @@ describe("plugin worker manager duplex channel route", () => { // The five explicit bounds. Each bound ends the route when it is exceeded. // ------------------------------------------------------------------------- - it("ends the route when the pre-bind buffered bytes pass the bound", async () => { + it("ends the route when the post-bind buffered bytes pass the bound", async () => { const handle = makeDuplexHandle({ duplexChannelLimits: { maxPreBindBufferedChars: 10 }, }); @@ -368,6 +371,10 @@ describe("plugin worker manager duplex channel route", () => { await handle.start(); const session = await handle.openDuplexChannel( duplexOpenInput({ + // Hold these frames until the fixture acknowledges a host write. A + // write can only come from the returned session, so this makes the + // post-bind path deterministic instead of depending on pipe batching. + emitScriptedFramesAfterFirstWrite: true, data: [ { chunk: "aaaaa" }, // total 5 → buffered { chunk: "bbbbb" }, // total 10 → buffered @@ -375,8 +382,10 @@ describe("plugin worker manager duplex channel route", () => { ], }), ); - // No listener attaches, so the data buffers. The cumulative bytes pass the - // bound and the route ends. The login wait resolves with a null exit code. + session.write(new TextEncoder().encode("emit")); + // No listener attaches, so the post-bind data buffers. The cumulative bytes + // pass the bound and the route ends. The channel wait resolves with a null + // exit code. await expect(session.wait()).resolves.toEqual({ exitCode: null }); } finally { await handle.stop().catch(() => undefined); diff --git a/server/src/adapters/registry.ts b/server/src/adapters/registry.ts index 603665bb95..5bdba04386 100644 --- a/server/src/adapters/registry.ts +++ b/server/src/adapters/registry.ts @@ -8,6 +8,8 @@ import { stampClaudeAgentIdHeader } from "./claude-agent-id-header.js"; import { buildSandboxNpmInstallCommand, getAdapterSessionManagement, + PAPERCLIP_RUNNER_PERMISSION_CAPABILITIES, + resolvePaperclipRunnerPermissionMode, } from "@paperclipai/adapter-utils"; import type { AdapterLoginCapability } from "@paperclipai/adapter-utils"; import { @@ -379,6 +381,48 @@ const paperclipRunnerAdapter: ServerAdapterModule = { }; }, async testEnvironment(context) { + const configuredProvider = context.config.provider ?? "codex"; + if (configuredProvider !== "codex") { + return { + adapterType: "paperclip_runner", + status: "fail" as const, + testedAt: new Date().toISOString(), + checks: [{ + code: "paperclip_runner_provider_unsupported", + level: "error" as const, + message: "Paperclip Runner currently supports only the Codex provider.", + }], + }; + } + if (context.executionTarget?.kind === "remote") { + return { + adapterType: "paperclip_runner", + status: "fail" as const, + testedAt: new Date().toISOString(), + checks: [{ + code: "paperclip_runner_environment_unsupported", + level: "error" as const, + message: "Paperclip Runner currently requires a local execution environment.", + }], + }; + } + const configuredPermission = context.config.codexPermissionMode; + if ( + configuredPermission !== undefined + && resolvePaperclipRunnerPermissionMode("codex", configuredPermission) + !== configuredPermission + ) { + return { + adapterType: "paperclip_runner", + status: "fail" as const, + testedAt: new Date().toISOString(), + checks: [{ + code: "runner_permission_mode_invalid", + level: "error" as const, + message: "codexPermissionMode is not supported by Codex.", + }], + }; + } const result = await codexTestEnvironment(context); return { ...result, adapterType: "paperclip_runner" }; }, @@ -386,6 +430,7 @@ const paperclipRunnerAdapter: ServerAdapterModule = { syncSkills: syncCodexSkills, sessionCodec: codexSessionCodec, models: codexModels, + modelProfiles: codexModelProfiles, listModels: listCodexModels, refreshModels: refreshCodexModels, supportsLocalAgentJwt: false, @@ -394,7 +439,38 @@ const paperclipRunnerAdapter: ServerAdapterModule = { getRuntimeCommandSpec: (config) => buildNpmRuntimeCommandSpec(config, "codex", "@openai/codex"), agentConfigurationDoc: "# Paperclip Runner\n\nAdapter: paperclip_runner\n\nRuns Codex through the Rust Paperclip runner and authenticated PRP transport.\n", - getConfigSchema: getCodexConfigSchema, + getConfigSchema: () => ({ + fields: [ + { + key: "codexPermissionMode", + label: "Codex permission mode", + type: "select" as const, + default: PAPERCLIP_RUNNER_PERMISSION_CAPABILITIES.codex.defaultMode, + options: PAPERCLIP_RUNNER_PERMISSION_CAPABILITIES.codex.options.map( + ({ value, label }) => ({ value, label }), + ), + hint: PAPERCLIP_RUNNER_PERMISSION_CAPABILITIES.codex.description, + }, + { + key: "lifecycleMode", + label: "Runner lifecycle", + type: "select" as const, + default: "per_turn", + options: [ + { value: "per_turn", label: "Turn by turn" }, + { value: "warm", label: "Warm session" }, + ], + hint: "Warm sessions retain runnerd and Codex between governed runs.", + }, + { + key: "idleTimeoutMs", + label: "Warm idle timeout (ms)", + type: "number" as const, + default: 300_000, + hint: "Warm sessions suspend after this much inactivity.", + }, + ], + }), loginCapability: codexLoginCapability, }; diff --git a/server/src/dev-native-runner-status.ts b/server/src/dev-native-runner-status.ts new file mode 100644 index 0000000000..6054c0dc48 --- /dev/null +++ b/server/src/dev-native-runner-status.ts @@ -0,0 +1,70 @@ +import { and, eq, inArray, isNull } from "drizzle-orm"; +import { + closeRegisteredClients, + createDb, + heartbeatRuns, + nativeRunFinalizations, +} from "@paperclipai/db"; +import { resolveMigrationConnection } from "@paperclipai/db/migration-runtime"; + +import { instanceSettingsService } from "./services/instance-settings.js"; + +async function main(): Promise { + const connection = await resolveMigrationConnection(); + const db = createDb(connection.connectionString, { maxConnections: 1 }); + + try { + const experimental = await instanceSettingsService(db).getExperimental(); + const persistedActiveNativeRun = await db + .select({ id: heartbeatRuns.id }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.runtimeMode, "native"), + inArray(heartbeatRuns.status, ["queued", "running", "scheduled_retry"]), + ), + ) + .limit(1) + .then((rows) => rows.length > 0); + const persistedRetryableFailedNativeRun = await db + .select({ id: heartbeatRuns.id }) + .from(heartbeatRuns) + .innerJoin( + nativeRunFinalizations, + eq(nativeRunFinalizations.runId, heartbeatRuns.id), + ) + .where( + and( + eq(heartbeatRuns.runtimeMode, "native"), + eq(heartbeatRuns.status, "failed"), + eq(nativeRunFinalizations.phase, "retryable_failure"), + isNull(nativeRunFinalizations.resultId), + ), + ) + .limit(1) + .then((rows) => rows.length > 0); + const persistedNativeRun = + persistedActiveNativeRun || persistedRetryableFailedNativeRun; + + console.log( + JSON.stringify({ + nativeRunnerRequired: + experimental.enableNativeRunner === true || persistedNativeRun, + rolloutEnabled: experimental.enableNativeRunner === true, + persistedNativeRun, + persistedActiveNativeRun, + persistedRetryableFailedNativeRun, + }), + ); + } finally { + await closeRegisteredClients(connection.connectionString); + await connection.stop(); + } +} + +main().catch((error) => { + const message = + error instanceof Error ? (error.stack ?? error.message) : String(error); + process.stderr.write(`${message}\n`); + process.exit(1); +}); diff --git a/server/src/routes/agents.ts b/server/src/routes/agents.ts index 70907ac5e8..37988c52cd 100644 --- a/server/src/routes/agents.ts +++ b/server/src/routes/agents.ts @@ -121,6 +121,10 @@ import { readPendingNativeRuntimeRequest, type NativeRuntimeRequestResolver, } from "../services/native-runtime/runtime-request-resolution-authority.js"; +import { + NativeRuntimeRequestResolutionError, + resolveNativeRuntimeRequest, +} from "../services/native-runtime/native-session-executor.js"; import { renderOrgChartSvg, renderOrgChartPng, type OrgNode, type OrgChartStyle, ORG_CHART_STYLES } from "./org-chart-svg.js"; import { instanceSettingsService, @@ -1673,6 +1677,19 @@ export function agentRoutes( ); } + function assertFreshPaperclipRunnerProvider( + adapterType: string, + adapterConfig: Record, + ): void { + if (adapterType !== "paperclip_runner") return; + const provider = adapterConfig.provider; + if (provider === undefined || provider === "codex") return; + throw unprocessable( + "Paperclip Runner currently supports Codex for new or changed agent configurations.", + { code: "paperclip_runner_provider_unavailable" }, + ); + } + async function assertAgentDefaultEnvironmentSelection( companyId: string, environmentId: string | null | undefined, @@ -3477,6 +3494,36 @@ export function agentRoutes( if (!existing) return; await assertCanUpdateAgent(req, existing); + const revision = await svc.getConfigRevision(id, revisionId); + if (!revision) { + res.status(404).json({ error: "Revision not found" }); + return; + } + const rollbackConfig = asRecord(revision.afterConfig); + if (!rollbackConfig) { + throw unprocessable("Invalid revision snapshot"); + } + const rollbackAdapterType = assertKnownAdapterType( + typeof rollbackConfig.adapterType === "string" + ? rollbackConfig.adapterType + : null, + ); + if (rollbackAdapterType !== existing.adapterType) { + await assertSelectableAdapterType(rollbackAdapterType); + } + const rollbackAdapterConfig = asRecord(rollbackConfig.adapterConfig) ?? {}; + const existingAdapterConfig = asRecord(existing.adapterConfig) ?? {}; + if ( + rollbackAdapterType !== existing.adapterType || + (rollbackAdapterType === "paperclip_runner" && + rollbackAdapterConfig.provider !== existingAdapterConfig.provider) + ) { + assertFreshPaperclipRunnerProvider( + rollbackAdapterType, + rollbackAdapterConfig, + ); + } + const actor = getActorInfo(req); const updated = await svc.rollbackConfigRevision(id, revisionId, { agentId: actor.agentId, @@ -3576,6 +3623,10 @@ export function agentRoutes( } = req.body; hireInput.adapterType = await assertSelectableAdapterType(hireInput.adapterType); const rawHireAdapterConfig = (hireInput.adapterConfig ?? {}) as Record; + assertFreshPaperclipRunnerProvider( + hireInput.adapterType, + rawHireAdapterConfig, + ); assertNoNewAgentLegacyPromptTemplate( hireInput.adapterType, rawHireAdapterConfig, @@ -3796,6 +3847,10 @@ export function agentRoutes( } = req.body; createInput.adapterType = await assertSelectableAdapterType(createInput.adapterType); const rawCreateAdapterConfig = (createInput.adapterConfig ?? {}) as Record; + assertFreshPaperclipRunnerProvider( + createInput.adapterType, + rawCreateAdapterConfig, + ); assertNoNewAgentLegacyPromptTemplate( createInput.adapterType, rawCreateAdapterConfig, @@ -4287,6 +4342,20 @@ export function agentRoutes( rawEffectiveAdapterConfig, ); } + const existingRunnerProvider = + existing.adapterType === "paperclip_runner" + ? existingAdapterConfig.provider + : undefined; + if ( + changingAdapterType || + (requestedAdapterType === "paperclip_runner" && + rawEffectiveAdapterConfig.provider !== existingRunnerProvider) + ) { + assertFreshPaperclipRunnerProvider( + requestedAdapterType, + rawEffectiveAdapterConfig, + ); + } const effectiveAdapterConfig = applyCodexLocalKeyIsolation( existing.companyId, existing.id, @@ -5665,13 +5734,71 @@ export function agentRoutes( ) { throw conflict("This runtime request is stale or is no longer pending."); } - const queued = queueRunnerPrpRuntimeRequestResolution({ - companyId: existing.companyId, - runId, - pendingRequest: currentPendingRequest, - actor: resolutionActor, - resolution, - }); + let queued: { commandId: string }; + try { + queued = await resolveNativeRuntimeRequest({ + runId, + requestId, + turnId: currentPendingRequest.turnId, + resolution, + authorizeBeforeDispatch: async () => { + const dispatchPendingRequest = + await readPendingNativeRuntimeRequest(db, { + companyId: existing.companyId, + runId, + requestId, + }); + if ( + !dispatchPendingRequest + || dispatchPendingRequest.requestKind !== + currentPendingRequest.requestKind + || dispatchPendingRequest.turnId !== + currentPendingRequest.turnId + ) { + throw conflict( + "This runtime request is stale or is no longer pending.", + ); + } + assertNativeRuntimeRequestResolverAuthorized( + dispatchPendingRequest, + resolutionActor, + ); + }, + }); + } catch (error) { + if ( + error instanceof NativeRuntimeRequestResolutionError && + error.code === "runtime_request_resolution_conflict" + ) { + throw conflict( + "A different response was already submitted for this runtime request.", + ); + } + if ( + !(error instanceof NativeRuntimeRequestResolutionError) || + ![ + "native_session_not_active", + "runtime_request_resolution_unsupported", + ].includes(error.code) + ) { + if ( + error instanceof NativeRuntimeRequestResolutionError && + error.code === "runtime_request_stale_turn" + ) { + throw conflict( + "The runner session is no longer accepting runtime responses.", + ); + } + throw error; + } + queued = queueRunnerPrpRuntimeRequestResolution({ + companyId: existing.companyId, + runId, + pendingRequest: currentPendingRequest, + actor: resolutionActor, + resolution, + }); + } await logActivity(db, { companyId: existing.companyId, actorType: "user", diff --git a/server/src/services/company-portability.ts b/server/src/services/company-portability.ts index 562b4edffa..0a2e1ac8b9 100644 --- a/server/src/services/company-portability.ts +++ b/server/src/services/company-portability.ts @@ -3566,6 +3566,16 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { adapterType: string, adapterConfig: Record, ) { + if (adapterType === "paperclip_runner") { + const provider = adapterConfig.provider ?? "codex"; + if (provider !== "codex") { + throw unprocessable( + "Imported Paperclip Runner agents currently support only the Codex provider.", + { code: "paperclip_runner_provider_unavailable" }, + ); + } + return; + } if (adapterType !== "opencode_local") return; try { requireOpenCodeModelId(adapterConfig.model); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index a0112943e5..ee51cb9372 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -3,7 +3,23 @@ import path from "node:path"; import { execFile as execFileCallback } from "node:child_process"; import { promisify } from "node:util"; import { createHash, randomUUID } from "node:crypto"; -import { and, asc, desc, eq, getTableColumns, gt, gte, inArray, isNull, lt, lte, ne, notInArray, or, sql } from "drizzle-orm"; +import { + and, + asc, + desc, + eq, + getTableColumns, + gt, + gte, + inArray, + isNull, + lt, + lte, + ne, + notInArray, + or, + sql, +} from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { AGENT_DEFAULT_MAX_CONCURRENT_RUNS, @@ -43,6 +59,7 @@ import { companySkillVersions, companySkills as companySkillsTable, companies, + completionContracts, costEvents, documentAnnotationComments, documentAnnotationThreads, @@ -90,17 +107,51 @@ import { // git-credentials module became its canonical home; existing importers keep working. export { scrubGitCredentialText }; import { publishLiveEvent } from "./live-events.js"; -import { allocateHeartbeatRunEventSeq } from "./heartbeat-run-events.js"; +import { + allocateHeartbeatRunEventSeq, + appendHeartbeatRunEvent, +} from "./heartbeat-run-events.js"; import { queuedCommentIdsFromWakePayload, queuedCommentIdsFromRunContext, withQueuedCommentIdsInWakePayload, withQueuedCommentIdsInRunContext, } from "./issue-queued-comment-queue.js"; -import { materializeLegacyQuestionResponseWakeProjection } from "./native-runtime/native-interaction-bridge.js"; +import { documentService } from "./documents.js"; +import { + buildNativeProviderEnvironment, + buildNativeExecutionInput, + buildNativeRuntimeContext, + cancelNativeSession, + dispatchNativeSessionResumptions, + ensureNativeCompletionContract, + executePaperclipNativeSession, + finalizeNativeRun, + isNativeSessionId, + materializeLegacyQuestionResponseWakeProjection, + materializeNativeInteractionResponses, + NativeCancellationPendingRecoveryError, + rebindNativeSessionCheckpoint, + reconcileNativeFinalizations, + resolveHeartbeatNativeRuntimeMode, +} from "./native-runtime/index.js"; +import type { NativeRunHistoricalSpan } from "./native-runtime/native-run-trace.js"; +import { + parseNativeExecutionInput, + type NativeExecutionInput, + type NativeSessionBackend, +} from "../vendor/paperclip-runner/index.js"; import { normalizeResponsibleUserDenialCode } from "./responsible-user-denial-run-outcomes.js"; import { getRunLogStore, type RunLogHandle } from "./run-log-store.js"; -import { getServerAdapter, listAdapterModelProfiles, runningProcesses } from "../adapters/index.js"; +import { + providerTraceStore, + PROVIDER_TRACE_MAX_BYTES, +} from "./provider-trace-store.js"; +import { + getServerAdapter, + listAdapterModelProfiles, + runningProcesses, +} from "../adapters/index.js"; import type { AdapterExecutionResult, AdapterInvocationMeta, @@ -121,13 +172,21 @@ import { getTelemetryClient } from "../telemetry.js"; import { companySkillService } from "./company-skills.js"; import { budgetService, type BudgetEnforcementScope } from "./budgets.js"; import { secretService, type MissingRuntimeBinding } from "./secrets.js"; -import { resolveDefaultAgentWorkspaceDir, resolveManagedProjectWorkspaceDir } from "../home-paths.js"; +import { + resolveDefaultAgentWorkspaceDir, + resolveManagedProjectWorkspaceDir, +} from "../home-paths.js"; import { buildHeartbeatRunIssueComment, + findHeartbeatRunCompletionComment, HEARTBEAT_RUN_RESULT_OUTPUT_MAX_CHARS, HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS, HEARTBEAT_RUN_SAFE_RESULT_JSON_MAX_BYTES, + hasAcceptedSemanticResult, mergeHeartbeatRunResultJson, + resolveHeartbeatRunResponse, + selectHeartbeatRunFinalAgentMessage, + type RunPresentationDecision, } from "./heartbeat-run-summary.js"; import { buildHeartbeatRunStopMetadata, @@ -146,7 +205,11 @@ import { evaluateIssueRewakeThrottle, isThrottleCandidateIssueRewake, } from "./issue-rewake-throttle.js"; -import { logActivity, publishPluginDomainEvent, type LogActivityInput } from "./activity-log.js"; +import { + logActivity, + publishPluginDomainEvent, + type LogActivityInput, +} from "./activity-log.js"; import { buildWorkspaceReadyComment, buildWorkspaceReadyMetadata, @@ -173,8 +236,10 @@ import { } from "./workspace-instance-cleanup.js"; import { issueService } from "./issues.js"; import { projectService } from "./projects.js"; -import { getEnvironmentDriverTraits } from "./environment-driver-traits.js"; -import { authorizationService, type AuthorizationActor } from "./authorization.js"; +import { + authorizationService, + type AuthorizationActor, +} from "./authorization.js"; import { createToolGatewayService } from "./tool-gateway.js"; import { toolAccessService } from "./tool-access.js"; import { visibleIssueCondition } from "./issue-visibility.js"; @@ -195,15 +260,27 @@ import { getIssueContinuationSummaryDocument, refreshIssueContinuationSummary, } from "./issue-continuation-summary.js"; -import { buildDocumentReviewContext, buildPlanReviewContext } from "./plan-review-context.js"; -import { executionWorkspaceService, mergeExecutionWorkspaceConfig } from "./execution-workspaces.js"; +import { + buildDocumentReviewContext, + buildPlanReviewContext, +} from "./plan-review-context.js"; +import { + executionWorkspaceService, + mergeExecutionWorkspaceConfig, +} from "./execution-workspaces.js"; +import { + workspaceOperationService, + type WorkspaceOperationRecorder, +} from "./workspace-operations.js"; +import { + isProcessGroupAlive, + terminateLocalService, +} from "./local-service-supervisor.js"; import { GIT_BRANCH_OWNERSHIP_METADATA_KEY, GIT_BRANCH_OWNERSHIP_METADATA_VERSION, isRuntimeOwnedGitBranch, } from "./execution-workspace-branch-ownership.js"; -import { workspaceOperationService, type WorkspaceOperationRecorder } from "./workspace-operations.js"; -import { isProcessGroupAlive, terminateLocalService } from "./local-service-supervisor.js"; import { HEARTBEAT_RUN_SCRATCH_MARKER, buildHeartbeatRunScratchEnv, @@ -262,7 +339,10 @@ import { recoveryAssigneeAdapterOverrides, withRecoveryModelProfileHint, } from "./recovery/model-profile-hint.js"; -import { ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS as RECOVERY_ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS, recoveryService } from "./recovery/service.js"; +import { + ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS as RECOVERY_ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS, + recoveryService, +} from "./recovery/service.js"; import { collectDispositionRepairSourceState } from "./recovery/disposition-repair.js"; import { buildIssueReviewPathLostIdempotencyKey, @@ -297,6 +377,7 @@ import { redactEventPayload, redactSensitiveText } from "../redaction.js"; import { createRunSecretRedactionRegistry } from "./run-secret-redaction.js"; import { hasSessionCompactionThresholds, + resolvePaperclipRunnerPermissionMode, resolveSessionCompactionPolicy, type RuntimeStatusUpdate, type SessionCompactionPolicy, @@ -338,7 +419,10 @@ import { assertLowTrustRuntimeServicesAllowed, assertLowTrustWorkspaceIsolation, } from "./low-trust-runtime-containment.js"; -import { resolveCoreTrustPreset, type TrustPresetResolution } from "./trust-preset-resolver.js"; +import { + resolveCoreTrustPreset, + type TrustPresetResolution, +} from "./trust-preset-resolver.js"; import { createEffectiveRunConfigFingerprints, createEffectiveRunConfigSubcategoryFingerprints, @@ -348,30 +432,20 @@ import { } from "./effective-run-config-fingerprints.js"; import type { PluginWorkerManager } from "./plugin-worker-manager.js"; import { serverVersion } from "../version.js"; -import { executeNativeCodexRunner } from "./native-runtime/native-codex-runner.js"; -import { prepareNativeHeartbeatRun } from "./native-runtime/prepare-native-run.js"; -import { - NativeRunnerSelectionError, - resolveHeartbeatRuntimeMode, -} from "./native-runtime/runtime-mode.js"; const MAX_LIVE_LOG_CHUNK_BYTES = 8 * 1024; const MAX_PERSISTED_LOG_CHUNK_CHARS = 64 * 1024; const MAX_RUN_EVENT_PAYLOAD_STRING_CHARS = 16 * 1024; const MAX_RUN_EVENT_PAYLOAD_ARRAY_ITEMS = 50; -function nativeRunnerErrorCode(error: unknown): string | null { - if (error instanceof NativeRunnerSelectionError) return error.code; - const message = error instanceof Error ? error.message : String(error); - return message.match(/^(paperclip_runner_[a-z0-9_]+)/)?.[1] ?? null; -} - export function redactDetectedSuccessfulRunProgressSummaryForBoard( summary: string, currentUserRedactionOptions?: CurrentUserRedactionOptions, ) { const normalized = summary.replace(/\s+/g, " ").trim(); - const redacted = redactSensitiveText(redactCurrentUserText(normalized, currentUserRedactionOptions)); + const redacted = redactSensitiveText( + redactCurrentUserText(normalized, currentUserRedactionOptions), + ); return redacted.length <= 280 ? redacted : `${redacted.slice(0, 277)}...`; } @@ -380,7 +454,9 @@ export function redactSuccessfulRunHandoffEvidence( currentUserRedactionOptions?: CurrentUserRedactionOptions, ) { if (!value) return null; - return redactSensitiveText(redactCurrentUserText(value, currentUserRedactionOptions)); + return redactSensitiveText( + redactCurrentUserText(value, currentUserRedactionOptions), + ); } const MAX_RUN_EVENT_PAYLOAD_OBJECT_KEYS = 100; @@ -395,9 +471,15 @@ const LIVENESS_BOOKKEEPING_ACTIVITY_ACTIONS = [ const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext"; const WAKE_COMMENT_IDS_KEY = "wakeCommentIds"; const PAPERCLIP_WAKE_PAYLOAD_KEY = "paperclipWake"; +const ACCEPTED_PLAN_CONVERSION_SKILL_KEY = + "paperclipai/paperclip/paperclip-converting-plans-to-tasks"; const PAPERCLIP_AGENT_MESSAGE_KEY = "paperclipAgentMessage"; const PAPERCLIP_HARNESS_CHECKOUT_KEY = "paperclipHarnessCheckedOut"; const DETACHED_PROCESS_ERROR_CODE = "process_detached"; +const NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE = + "native_execution_ownership_unverified"; +const NATIVE_OWNERSHIP_UNVERIFIED_MESSAGE = + "Native execution ownership could not be verified; automatic recovery is blocked"; // The reaper sweeps at most this many pending_cleanup leases per tick. const PENDING_CLEANUP_SWEEP_PAGE_SIZE = 20; // The reaper stops retrying a pending_cleanup lease after this many attempts. @@ -489,15 +571,30 @@ export const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS = [ const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_JITTER_RATIO = 0.25; const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON = "transient_failure"; const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON = "transient_failure_retry"; -const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS = BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS.length; -export const INTERACTION_CONTINUATION_INFRA_RETRY_REASON = "interaction_continuation_infra_retry"; -export const INTERACTION_CONTINUATION_INFRA_WAKE_REASON = "interaction_continuation_infra_retry"; +const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS = + BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS.length; +export const INTERACTION_CONTINUATION_INFRA_RETRY_REASON = + "interaction_continuation_infra_retry"; +export const INTERACTION_CONTINUATION_INFRA_WAKE_REASON = + "interaction_continuation_infra_retry"; const INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS = 3; -const RESOLVED_INTERACTION_CONTINUATION_STATUSES = new Set(["accepted", "answered", "rejected"]); +const RESOLVED_INTERACTION_CONTINUATION_STATUSES = new Set([ + "accepted", + "answered", + "rejected", +]); const WORKSPACE_VALIDATION_FAILURE_CODE = "workspace_validation_failed"; const WORKSPACE_VALIDATION_RECOVERY_CAUSE = "workspace_validation_failed"; const CONFIGURATION_INCOMPLETE_FAILURE_CODE = "configuration_incomplete"; const CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE = "configuration_incomplete"; +const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON = + "execution_review_participant_recovery"; +const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON = + "execution_review_participant_recovery"; +const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE = + "execution_review_participant_recovery"; +const GITHUB_PR_WORKFLOW_SKILL_KEY = + "paperclipai/bundled/software-development/github-pr-workflow"; // Error codes that mark a pre-dispatch setup failure. The adapter process never // started, so no agent could post an issue comment. The setup catch writes one // of these codes when a failure happens before `adapter.execute`. @@ -506,10 +603,6 @@ const PRE_ADAPTER_SETUP_FAILURE_CODES = new Set([ CONFIGURATION_INCOMPLETE_FAILURE_CODE, WORKSPACE_VALIDATION_FAILURE_CODE, ]); -const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON = "execution_review_participant_recovery"; -const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON = "execution_review_participant_recovery"; -const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE = "execution_review_participant_recovery"; -const GITHUB_PR_WORKFLOW_SKILL_KEY = "paperclipai/bundled/software-development/github-pr-workflow"; const GITHUB_PR_WORKFLOW_SKILL_SLUG = "github-pr-workflow"; const PUSH_CAPABILITY_ENV_KEYS = ["GH_TOKEN", "GITHUB_TOKEN"] as const; // Keep this in sync with local adapters that require a git workspace before launch. @@ -530,7 +623,11 @@ const MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS = 2; const MAX_TURN_CONTINUATION_MAX_ATTEMPTS_CAP = 10; const MAX_TURN_CONTINUATION_DEFAULT_DELAY_MS = 1_000; const MAX_TURN_CONTINUATION_MAX_DELAY_MS = 5 * 60 * 1000; -const MAX_TURN_CONTINUATION_LIVE_RUN_STATUSES = ["scheduled_retry", "queued", "running"] as const; +const MAX_TURN_CONTINUATION_LIVE_RUN_STATUSES = [ + "scheduled_retry", + "queued", + "running", +] as const; export const WORKSPACE_BUSY_RETRY_REASON = "workspace_busy"; export const WORKSPACE_BUSY_RETRY_WAKE_REASON = "workspace_busy_retry"; export const WORKSPACE_BUSY_ERROR_CODE = "workspace_busy"; @@ -544,14 +641,19 @@ export const WORKSPACE_BUSY_RETRY_JITTER_MS = 60 * 1000; // contrast, never gets overtaken — a deferred run keeps rescheduling until the // workspace frees, because dispatching alongside a live holder is exactly the // concurrent-mutation failure this gate exists to prevent. -export const WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS = RECOVERY_ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS; +export const WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS = + RECOVERY_ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS; // Issue-level executionWorkspaceSettings.mode values that unambiguously opt an // issue's runs out of the shared project workspace, and therefore out of // shared-workspace serialization ("isolated" is the legacy alias // parseIssueExecutionWorkspaceSettings normalizes to isolated_workspace). Any // other value — including agent_default and an absent mode — may still resolve // to the shared workspace and counts as a holder. -const ISOLATED_EXECUTION_WORKSPACE_MODES = ["isolated_workspace", "operator_branch", "isolated"] as const; +const ISOLATED_EXECUTION_WORKSPACE_MODES = [ + "isolated_workspace", + "operator_branch", + "isolated", +] as const; type CodexTransientFallbackMode = | "same_session" | "safer_invocation" @@ -655,13 +757,20 @@ export class WorkspaceBusyDeferral extends Error { } } -function isWorkspaceBusyDeferral(error: unknown): error is WorkspaceBusyDeferral { +function isWorkspaceBusyDeferral( + error: unknown, +): error is WorkspaceBusyDeferral { return error instanceof WorkspaceBusyDeferral; } -export function computeWorkspaceBusyRetryDelayMs(random: () => number = Math.random) { +export function computeWorkspaceBusyRetryDelayMs( + random: () => number = Math.random, +) { const jitter = Math.min(Math.max(random(), 0), 1); - return WORKSPACE_BUSY_RETRY_BASE_DELAY_MS + Math.floor(jitter * WORKSPACE_BUSY_RETRY_JITTER_MS); + return ( + WORKSPACE_BUSY_RETRY_BASE_DELAY_MS + + Math.floor(jitter * WORKSPACE_BUSY_RETRY_JITTER_MS) + ); } // True for the retry of a workspace-busy deferral whose original run did NOT @@ -680,7 +789,9 @@ export function isNonAssigneeWorkspaceBusyRetry( ); } -function resolveCodexTransientFallbackMode(attempt: number): CodexTransientFallbackMode { +function resolveCodexTransientFallbackMode( + attempt: number, +): CodexTransientFallbackMode { if (attempt <= 1) return "same_session"; if (attempt === 2) return "safer_invocation"; if (attempt === 3) return "fresh_session"; @@ -713,14 +824,20 @@ function isMaxTurnExhaustionRun( const resultJson = parseObject(run.resultJson); return Boolean( normalizeMaxTurnStopReason(resultJson.stopReason) ?? - normalizeMaxTurnStopReason(run.errorCode), + normalizeMaxTurnStopReason(run.errorCode), ); } -function readTransientRetryNotBeforeFromRun(run: Pick) { +function readTransientRetryNotBeforeFromRun( + run: Pick, +) { const resultJson = parseObject(run.resultJson); const value = resultJson.retryNotBefore ?? resultJson.transientRetryNotBefore; - if (!(typeof value === "string" || typeof value === "number" || value instanceof Date)) { + if (!( + typeof value === "string" || + typeof value === "number" || + value instanceof Date + )) { return null; } const parsed = new Date(value); @@ -731,7 +848,8 @@ function readTransientRecoveryContractFromRun( run: Pick, ) { const errorFamily = readHeartbeatRunErrorFamily(run); - return errorFamily === "transient_upstream" || errorFamily === "provider_quota" + return errorFamily === "transient_upstream" || + errorFamily === "provider_quota" ? { errorFamily, retryNotBefore: readTransientRetryNotBeforeFromRun(run), @@ -739,12 +857,15 @@ function readTransientRecoveryContractFromRun( : null; } -function isResolvedInteractionContinuationWakeContext(contextSnapshot: unknown) { +function isResolvedInteractionContinuationWakeContext( + contextSnapshot: unknown, +) { const context = parseObject(contextSnapshot); const interactionId = readNonEmptyString(context.interactionId); const interactionStatus = readNonEmptyString(context.interactionStatus); if (!interactionId || !interactionStatus) return false; - if (!RESOLVED_INTERACTION_CONTINUATION_STATUSES.has(interactionStatus)) return false; + if (!RESOLVED_INTERACTION_CONTINUATION_STATUSES.has(interactionStatus)) + return false; const mutation = readNonEmptyString(context.mutation); const wakeReason = readNonEmptyString(context.wakeReason); @@ -779,17 +900,26 @@ function isSpawnLikeFailureMessage(value: unknown) { // describes a terminal condition that must not be retried. function isSandboxProviderWorkerUnavailableFailureMessage(value: unknown) { if (typeof value !== "string") return false; - return /sandbox provider .* is installed via plugin .* but its worker is not running/i.test(value); + return /sandbox provider .* is installed via plugin .* but its worker is not running/i.test( + value, + ); } function isRetryableInteractionContinuationInfrastructureFailure( - run: Pick, + run: Pick< + typeof heartbeatRuns.$inferSelect, + "error" | "errorCode" | "resultJson" + >, ) { - if (run.errorCode === WORKSPACE_VALIDATION_FAILURE_CODE || run.errorCode === "process_lost") { + if ( + run.errorCode === WORKSPACE_VALIDATION_FAILURE_CODE || + run.errorCode === "process_lost" + ) { return true; } - if (run.errorCode !== "adapter_failed" && run.errorCode !== "setup_failed") return false; + if (run.errorCode !== "adapter_failed" && run.errorCode !== "setup_failed") + return false; const resultJson = parseObject(run.resultJson); return ( @@ -809,7 +939,8 @@ function mergeAdapterRecoveryMetadata(input: { }) { const errorFamily = readNonEmptyString(input.errorFamily); const retryNotBefore = readNonEmptyString(input.retryNotBefore); - if (!input.resultJson && !errorFamily && !retryNotBefore) return input.resultJson ?? null; + if (!input.resultJson && !errorFamily && !retryNotBefore) + return input.resultJson ?? null; return { ...(input.resultJson ?? {}), @@ -818,7 +949,9 @@ function mergeAdapterRecoveryMetadata(input: { ? { retryNotBefore, transientRetryNotBefore: retryNotBefore, - ...(errorFamily === "provider_quota" ? { providerQuotaRetryNotBefore: retryNotBefore } : {}), + ...(errorFamily === "provider_quota" + ? { providerQuotaRetryNotBefore: retryNotBefore } + : {}), } : {}), }; @@ -874,6 +1007,10 @@ const activeRunExecutionPromises = new Set>(); // can await a wake that is still before run registration. A caller that tears // down a shared database (a test afterEach) then cannot race a late wake. const activeWakeupPromises = new Set>(); +const nativeSessionResumeDispatchTimers = new Map< + string, + ReturnType +>(); // Task drain: an operator-controlled hold on new run admission, so a caller // can wait for active work to finish before it stops the process. The state // lives in process memory only — a process restart clears it — and it sits at @@ -950,20 +1087,23 @@ type RuntimeConfigSecretResolver = Pick< | "collectMissingAdapterConfigRuntimeBindings" >; -function formatMissingBindingForOperator(missing: MissingRuntimeBinding): string { +function formatMissingBindingForOperator( + missing: MissingRuntimeBinding, +): string { if (missing.bindingType === "user_secret_ref") { - const definitionLabel = - missing.userSecretDefinitionName - ? `"${missing.userSecretDefinitionName}"` - : missing.userSecretDefinitionKey - ? `"${missing.userSecretDefinitionKey}"` - : "declared user secret"; - const ownerLabel = missing.responsibleUserId ? ` for responsible user ${missing.responsibleUserId}` : ""; + const definitionLabel = missing.userSecretDefinitionName + ? `"${missing.userSecretDefinitionName}"` + : missing.userSecretDefinitionKey + ? `"${missing.userSecretDefinitionKey}"` + : "declared user secret"; + const ownerLabel = missing.responsibleUserId + ? ` for responsible user ${missing.responsibleUserId}` + : ""; return `user secret ${definitionLabel}${ownerLabel} not available at ${missing.consumerType} ${missing.configPath}`; } const secretLabel = missing.secretName ? `"${missing.secretName}"` - : missing.secretId ?? "unknown"; + : (missing.secretId ?? "unknown"); return `secret ${secretLabel} not bound at ${missing.consumerType} ${missing.configPath}`; } @@ -979,9 +1119,11 @@ function isConfiguredEnvBindingValue(binding: unknown) { function hasGithubPrWorkflowSkill(desiredSkills: string[]) { return desiredSkills.some((skill) => { const normalized = skill.trim(); - return normalized === GITHUB_PR_WORKFLOW_SKILL_KEY - || normalized === GITHUB_PR_WORKFLOW_SKILL_SLUG - || normalized.endsWith(`/${GITHUB_PR_WORKFLOW_SKILL_SLUG}`); + return ( + normalized === GITHUB_PR_WORKFLOW_SKILL_KEY || + normalized === GITHUB_PR_WORKFLOW_SKILL_SLUG || + normalized.endsWith(`/${GITHUB_PR_WORKFLOW_SKILL_SLUG}`) + ); }); } @@ -990,9 +1132,11 @@ export function requiresPushCapabilityPreflight(input: { issueId: string | null | undefined; explicitRunScopedSkillKeys: string[]; }) { - return Boolean(input.issueId) - && GIT_SENSITIVE_LOCAL_ADAPTER_TYPES.has(input.adapterType) - && hasGithubPrWorkflowSkill(input.explicitRunScopedSkillKeys); + return ( + Boolean(input.issueId) && + GIT_SENSITIVE_LOCAL_ADAPTER_TYPES.has(input.adapterType) && + hasGithubPrWorkflowSkill(input.explicitRunScopedSkillKeys) + ); } const LOW_TRUST_SENSITIVE_ENV_KEY_RE = @@ -1008,15 +1152,21 @@ const LOW_TRUST_SENSITIVE_ENV_KEY_RE = // the run env like any non-prefixed binding. const FORBIDDEN_ENV_BINDING_KEYS = new Set(["PAPERCLIP_API_KEY"]); -function stripForbiddenEnvBindings(envValue: unknown): Record | null { +function stripForbiddenEnvBindings( + envValue: unknown, +): Record | null { const record = parseObject(envValue); const filtered = Object.fromEntries( - Object.entries(record).filter(([key]) => !FORBIDDEN_ENV_BINDING_KEYS.has(key)), + Object.entries(record).filter( + ([key]) => !FORBIDDEN_ENV_BINDING_KEYS.has(key), + ), ); return Object.keys(filtered).length > 0 ? filtered : null; } -function stripForbiddenEnvFromAdapterConfig(config: Record): Record { +function stripForbiddenEnvFromAdapterConfig( + config: Record, +): Record { if (!Object.prototype.hasOwnProperty.call(config, "env")) return config; return { ...config, @@ -1033,11 +1183,17 @@ function assertLowTrustEnvConfigAllowed(envValue: unknown, source: string) { const binding = parsed.data; const isPlainBinding = typeof binding === "string" || - (typeof binding === "object" && binding !== null && binding.type === "plain"); + (typeof binding === "object" && + binding !== null && + binding.type === "plain"); if (isPlainBinding && LOW_TRUST_SENSITIVE_ENV_KEY_RE.test(key)) { - throw new HttpError(422, `Low-trust execution cannot use inline sensitive env value ${source}.${key}`, { - code: "low_trust_inline_sensitive_env_denied", - }); + throw new HttpError( + 422, + `Low-trust execution cannot use inline sensitive env value ${source}.${key}`, + { + code: "low_trust_inline_sensitive_env_denied", + }, + ); } } } @@ -1066,14 +1222,17 @@ export async function resolveExecutionRunAdapterConfig(input: { remediation: string; }; }) { - const executionRunConfig = stripForbiddenEnvFromAdapterConfig(input.executionRunConfig); + const executionRunConfig = stripForbiddenEnvFromAdapterConfig( + input.executionRunConfig, + ); const environmentEnv = stripForbiddenEnvBindings(input.environmentEnv); const projectEnv = stripForbiddenEnvBindings(input.projectEnv); const routineEnv = stripForbiddenEnvBindings(input.routineEnv); const agentEnv = parseObject(executionRunConfig.env); - const lowTrustAllowedBindingIds = input.trustPreset?.kind === "low_trust_review" - ? input.trustPreset.boundary.allowedSecretBindingIds ?? [] - : undefined; + const lowTrustAllowedBindingIds = + input.trustPreset?.kind === "low_trust_review" + ? (input.trustPreset.boundary.allowedSecretBindingIds ?? []) + : undefined; if (input.trustPreset?.kind === "low_trust_review") { assertLowTrustEnvConfigAllowed(environmentEnv, "environment.env"); assertLowTrustEnvConfigAllowed(executionRunConfig.env, "agent.env"); @@ -1082,28 +1241,31 @@ export async function resolveExecutionRunAdapterConfig(input: { } const requiredScopedEnvBinding = input.requiredScopedEnvBinding ?? null; const requiredScopedBindingsConfigured = requiredScopedEnvBinding - ? requiredScopedEnvBinding.keys.some((key) => ( - requiredScopedEnvBinding.consumerScopes.includes("agent") - && isConfiguredEnvBindingValue(agentEnv[key]) - ) || ( - requiredScopedEnvBinding.consumerScopes.includes("project") - && isConfiguredEnvBindingValue(projectEnv?.[key]) - )) + ? requiredScopedEnvBinding.keys.some( + (key) => + (requiredScopedEnvBinding.consumerScopes.includes("agent") && + isConfiguredEnvBindingValue(agentEnv[key])) || + (requiredScopedEnvBinding.consumerScopes.includes("project") && + isConfiguredEnvBindingValue(projectEnv?.[key])), + ) : false; if (requiredScopedEnvBinding && !requiredScopedBindingsConfigured) { - throw new ConfigurationIncompleteFailure(`configuration incomplete: ${requiredScopedEnvBinding.remediation}`, { - configurationIncomplete: { - reason: requiredScopedEnvBinding.reason, - companyId: input.companyId, - agentId: input.agentId ?? null, - issueId: input.issueId ?? null, - projectId: input.projectId ?? null, - routineId: input.routineId ?? null, - requiredEnvKeys: requiredScopedEnvBinding.keys, - requiredScopes: requiredScopedEnvBinding.consumerScopes, - missingBindings: [], + throw new ConfigurationIncompleteFailure( + `configuration incomplete: ${requiredScopedEnvBinding.remediation}`, + { + configurationIncomplete: { + reason: requiredScopedEnvBinding.reason, + companyId: input.companyId, + agentId: input.agentId ?? null, + issueId: input.issueId ?? null, + projectId: input.projectId ?? null, + routineId: input.routineId ?? null, + requiredEnvKeys: requiredScopedEnvBinding.keys, + requiredScopes: requiredScopedEnvBinding.consumerScopes, + missingBindings: [], + }, }, - }); + ); } // Pre-dispatch binding-validation gate: detect declared secret refs that have // no binding before resolving any secret value. Missing bindings short-circuit @@ -1136,7 +1298,10 @@ export async function resolveExecutionRunAdapterConfig(input: { }, )), ); - if (typeof input.secretsSvc.collectMissingAdapterConfigRuntimeBindings === "function") { + if ( + typeof input.secretsSvc.collectMissingAdapterConfigRuntimeBindings === + "function" + ) { missingBindings.push( ...(await input.secretsSvc.collectMissingAdapterConfigRuntimeBindings( input.companyId, @@ -1180,12 +1345,15 @@ export async function resolveExecutionRunAdapterConfig(input: { if (requiredScopedEnvBinding) { const requiredEnvKeys = new Set(requiredScopedEnvBinding.keys); const requiredScopes = new Set(requiredScopedEnvBinding.consumerScopes); - const requiredMissingBindings = missingBindings.filter((binding) => - requiredScopes.has(binding.consumerType as "agent" | "project") - && requiredEnvKeys.has(binding.envKey), + const requiredMissingBindings = missingBindings.filter( + (binding) => + requiredScopes.has(binding.consumerType as "agent" | "project") && + requiredEnvKeys.has(binding.envKey), ); if (requiredMissingBindings.length > 0) { - const detail = requiredMissingBindings.map(formatMissingBindingForOperator).join("; "); + const detail = requiredMissingBindings + .map(formatMissingBindingForOperator) + .join("; "); throw new ConfigurationIncompleteFailure( `configuration incomplete: ${requiredScopedEnvBinding.remediation}; ${detail}`, { @@ -1205,18 +1373,23 @@ export async function resolveExecutionRunAdapterConfig(input: { } } if (missingBindings.length > 0) { - const detail = missingBindings.map(formatMissingBindingForOperator).join("; "); - throw new ConfigurationIncompleteFailure(`configuration incomplete: ${detail}`, { - configurationIncomplete: { - reason: "secret_binding_missing", - companyId: input.companyId, - agentId: input.agentId ?? null, - issueId: input.issueId ?? null, - projectId: input.projectId ?? null, - routineId: input.routineId ?? null, - missingBindings, + const detail = missingBindings + .map(formatMissingBindingForOperator) + .join("; "); + throw new ConfigurationIncompleteFailure( + `configuration incomplete: ${detail}`, + { + configurationIncomplete: { + reason: "secret_binding_missing", + companyId: input.companyId, + agentId: input.agentId ?? null, + issueId: input.issueId ?? null, + projectId: input.projectId ?? null, + routineId: input.routineId ?? null, + missingBindings, + }, }, - }); + ); } } const environmentEnvResolution = environmentEnv @@ -1232,12 +1405,18 @@ export async function resolveExecutionRunAdapterConfig(input: { responsibleUserId: input.responsibleUserId ?? null, issueId: input.issueId ?? null, heartbeatRunId: input.heartbeatRunId ?? null, - ...(lowTrustAllowedBindingIds !== undefined ? { allowedBindingIds: lowTrustAllowedBindingIds } : {}), + ...(lowTrustAllowedBindingIds !== undefined + ? { allowedBindingIds: lowTrustAllowedBindingIds } + : {}), } : undefined, ) : { env: {}, secretKeys: new Set(), manifest: [] }; - const { config: resolvedConfig, secretKeys, manifest } = await input.secretsSvc.resolveAdapterConfigForRuntime( + const { + config: resolvedConfig, + secretKeys, + manifest, + } = await input.secretsSvc.resolveAdapterConfigForRuntime( input.companyId, executionRunConfig, input.agentId @@ -1249,7 +1428,9 @@ export async function resolveExecutionRunAdapterConfig(input: { responsibleUserId: input.responsibleUserId ?? null, issueId: input.issueId ?? null, heartbeatRunId: input.heartbeatRunId ?? null, - ...(lowTrustAllowedBindingIds !== undefined ? { allowedBindingIds: lowTrustAllowedBindingIds } : {}), + ...(lowTrustAllowedBindingIds !== undefined + ? { allowedBindingIds: lowTrustAllowedBindingIds } + : {}), } : undefined, { adapterType: input.adapterType ?? null }, @@ -1276,7 +1457,9 @@ export async function resolveExecutionRunAdapterConfig(input: { responsibleUserId: input.responsibleUserId ?? null, issueId: input.issueId ?? null, heartbeatRunId: input.heartbeatRunId ?? null, - ...(lowTrustAllowedBindingIds !== undefined ? { allowedBindingIds: lowTrustAllowedBindingIds } : {}), + ...(lowTrustAllowedBindingIds !== undefined + ? { allowedBindingIds: lowTrustAllowedBindingIds } + : {}), } : undefined, ) @@ -1303,7 +1486,9 @@ export async function resolveExecutionRunAdapterConfig(input: { responsibleUserId: input.responsibleUserId ?? null, issueId: input.issueId ?? null, heartbeatRunId: input.heartbeatRunId ?? null, - ...(lowTrustAllowedBindingIds !== undefined ? { allowedBindingIds: lowTrustAllowedBindingIds } : {}), + ...(lowTrustAllowedBindingIds !== undefined + ? { allowedBindingIds: lowTrustAllowedBindingIds } + : {}), } : undefined, ) @@ -1332,7 +1517,10 @@ export async function resolveExecutionRunAdapterConfig(input: { // adapter can probe once the sandbox is up — and on managed cloud hosts a // host-side login never exists at all. The adapter's execute-time gate // remains the authority there; it probes the sandbox before failing. - if ((input.adapterType ?? null) === "codex_local" && (input.environmentDriver ?? null) !== "sandbox") { + if ( + (input.adapterType ?? null) === "codex_local" && + (input.environmentDriver ?? null) !== "sandbox" + ) { const resolvedEnv = parseObject(resolvedConfig.env); const readiness = await evaluateCodexCredentialReadiness({ env: process.env, @@ -1393,11 +1581,7 @@ export function applyRunScopedMentionedSkillKeys( skillKeys: string[], ): Record { const normalizedSkillKeys = Array.from( - new Set( - skillKeys - .map((value) => value.trim()) - .filter(Boolean), - ), + new Set(skillKeys.map((value) => value.trim()).filter(Boolean)), ); if (normalizedSkillKeys.length === 0) return config; @@ -1417,7 +1601,8 @@ export function computeBoundedTransientHeartbeatRetrySchedule( const baseDelayMs = BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS[attempt - 1]; if (typeof baseDelayMs !== "number") return null; const sample = Math.min(1, Math.max(0, random())); - const jitterMultiplier = 1 + (((sample * 2) - 1) * BOUNDED_TRANSIENT_HEARTBEAT_RETRY_JITTER_RATIO); + const jitterMultiplier = + 1 + (sample * 2 - 1) * BOUNDED_TRANSIENT_HEARTBEAT_RETRY_JITTER_RATIO; const delayMs = Math.max(1_000, Math.round(baseDelayMs * jitterMultiplier)); return { attempt, @@ -1441,7 +1626,9 @@ async function resolveRunScopedMentionedSkillKeys(input: { description: issues.description, }) .from(issues) - .where(and(eq(issues.id, input.issueId), eq(issues.companyId, input.companyId))) + .where( + and(eq(issues.id, input.issueId), eq(issues.companyId, input.companyId)), + ) .then((rows) => rows[0] ?? null); if (!issue) return []; @@ -1498,7 +1685,9 @@ export function applyPersistedExecutionWorkspaceConfig(input: { if (input.workspaceConfig?.workspaceRuntime === null) { delete nextConfig.workspaceRuntime; } else if (input.workspaceConfig?.workspaceRuntime) { - nextConfig.workspaceRuntime = { ...input.workspaceConfig.workspaceRuntime }; + nextConfig.workspaceRuntime = { + ...input.workspaceConfig.workspaceRuntime, + }; } if (input.workspaceConfig?.desiredState === null) { delete nextConfig.desiredState; @@ -1514,11 +1703,16 @@ export function applyPersistedExecutionWorkspaceConfig(input: { if (input.workspaceConfig && input.mode === "isolated_workspace") { const nextStrategy = parseObject(nextConfig.workspaceStrategy); - if (input.workspaceConfig.provisionCommand === null) delete nextStrategy.provisionCommand; + if (input.workspaceConfig.provisionCommand === null) + delete nextStrategy.provisionCommand; else nextStrategy.provisionCommand = input.workspaceConfig.provisionCommand; - if (input.workspaceConfig.runtimeProvisionCommand === null) delete nextStrategy.runtimeProvisionCommand; - else nextStrategy.runtimeProvisionCommand = input.workspaceConfig.runtimeProvisionCommand; - if (input.workspaceConfig.teardownCommand === null) delete nextStrategy.teardownCommand; + if (input.workspaceConfig.runtimeProvisionCommand === null) + delete nextStrategy.runtimeProvisionCommand; + else + nextStrategy.runtimeProvisionCommand = + input.workspaceConfig.runtimeProvisionCommand; + if (input.workspaceConfig.teardownCommand === null) + delete nextStrategy.teardownCommand; else nextStrategy.teardownCommand = input.workspaceConfig.teardownCommand; nextConfig.workspaceStrategy = nextStrategy; } @@ -1550,10 +1744,7 @@ export function mergeExecutionWorkspaceMetadataForPersistence(input: { } const existingSnapshot = parseObject(base.baseRefSnapshot); - if ( - typeof existingSnapshot.resolvedSha !== "string" - && input.baseRefSha - ) { + if (typeof existingSnapshot.resolvedSha !== "string" && input.baseRefSha) { base.baseRefSnapshot = { baseRef: input.baseRef ?? null, resolvedSha: input.baseRefSha, @@ -1570,7 +1761,10 @@ export function mergeExecutionWorkspaceMetadataForPersistence(input: { }; } - if ((input.shouldReuseExisting && !input.shouldRefreshConfigSnapshot) || !input.configSnapshot) { + if ( + (input.shouldReuseExisting && !input.shouldRefreshConfigSnapshot) || + !input.configSnapshot + ) { return base; } @@ -1578,12 +1772,17 @@ export function mergeExecutionWorkspaceMetadataForPersistence(input: { } export function resolveExecutionWorkspaceBranchOwnership( - executionWorkspace: Pick, + executionWorkspace: Pick< + RealizedExecutionWorkspace, + "created" | "branchCreatedByRuntime" + >, ) { return executionWorkspace.branchCreatedByRuntime; } -export function stripWorkspaceRuntimeFromExecutionRunConfig(config: Record) { +export function stripWorkspaceRuntimeFromExecutionRunConfig( + config: Record, +) { const nextConfig = { ...config }; delete nextConfig.workspaceRuntime; return nextConfig; @@ -1604,38 +1803,54 @@ function buildExecutionWorkspaceConfigSnapshot( } if ("workspaceStrategy" in config) { - snapshot.provisionCommand = typeof strategy.provisionCommand === "string" ? strategy.provisionCommand : null; + snapshot.provisionCommand = + typeof strategy.provisionCommand === "string" + ? strategy.provisionCommand + : null; snapshot.runtimeProvisionCommand = - typeof strategy.runtimeProvisionCommand === "string" ? strategy.runtimeProvisionCommand : null; - snapshot.teardownCommand = typeof strategy.teardownCommand === "string" ? strategy.teardownCommand : null; + typeof strategy.runtimeProvisionCommand === "string" + ? strategy.runtimeProvisionCommand + : null; + snapshot.teardownCommand = + typeof strategy.teardownCommand === "string" + ? strategy.teardownCommand + : null; } if ("workspaceRuntime" in config) { const workspaceRuntime = parseObject(config.workspaceRuntime); - snapshot.workspaceRuntime = Object.keys(workspaceRuntime).length > 0 ? workspaceRuntime : null; + snapshot.workspaceRuntime = + Object.keys(workspaceRuntime).length > 0 ? workspaceRuntime : null; } if ("desiredState" in config) { snapshot.desiredState = - config.desiredState === "running" || config.desiredState === "stopped" || config.desiredState === "manual" + config.desiredState === "running" || + config.desiredState === "stopped" || + config.desiredState === "manual" ? config.desiredState : null; } if ("serviceStates" in config) { const serviceStates = parseObject(config.serviceStates); - snapshot.serviceStates = Object.keys(serviceStates).length > 0 - ? Object.fromEntries( - Object.entries(serviceStates).filter(([, state]) => - state === "running" || state === "stopped" || state === "manual" - ), - ) as ExecutionWorkspaceConfig["serviceStates"] - : null; + snapshot.serviceStates = + Object.keys(serviceStates).length > 0 + ? (Object.fromEntries( + Object.entries(serviceStates).filter( + ([, state]) => + state === "running" || + state === "stopped" || + state === "manual", + ), + ) as ExecutionWorkspaceConfig["serviceStates"]) + : null; } - const hasSnapshot = Object.values(snapshot).some((value) => { - if (value === null) return false; - if (typeof value === "object") return Object.keys(value).length > 0; - return true; - }) || hasExplicitEnvironmentSelection; + const hasSnapshot = + Object.values(snapshot).some((value) => { + if (value === null) return false; + if (typeof value === "object") return Object.keys(value).length > 0; + return true; + }) || hasExplicitEnvironmentSelection; return hasSnapshot ? snapshot : null; } @@ -1649,9 +1864,10 @@ export function stripHostWorkspaceProvisionForLowTrustSandbox(input: { const workspaceStrategy = parseObject(input.config.workspaceStrategy); if ( - typeof workspaceStrategy.provisionCommand !== "string" - && typeof workspaceStrategy.runtimeProvisionCommand !== "string" - ) return input.config; + typeof workspaceStrategy.provisionCommand !== "string" && + typeof workspaceStrategy.runtimeProvisionCommand !== "string" + ) + return input.config; const nextWorkspaceStrategy = { ...workspaceStrategy }; delete nextWorkspaceStrategy.provisionCommand; @@ -1668,10 +1884,17 @@ export async function preflightLowTrustWorkspaceIsolation(input: { trustPreset: TrustPresetResolution; isolatedWorkspacesEnabled: boolean; effectiveExecutionWorkspaceMode: string | null | undefined; - issue: { companyId: string; id?: string | null; projectId?: string | null } | null; + issue: { + companyId: string; + id?: string | null; + projectId?: string | null; + } | null; resolveSelectedEnvironmentDriver: () => Promise; }): Promise { - if (input.trustPreset.kind !== "denied" && input.trustPreset.kind !== "low_trust_review") { + if ( + input.trustPreset.kind !== "denied" && + input.trustPreset.kind !== "low_trust_review" + ) { return null; } @@ -1692,15 +1915,24 @@ export async function preflightLowTrustWorkspaceIsolation(input: { return selectedEnvironmentDriver ?? null; } -export async function resolveWorkspaceAfterLowTrustPreflight(input: { +export async function resolveWorkspaceAfterLowTrustPreflight< + TWorkspace, +>(input: { db?: Db; trustPreset: TrustPresetResolution; isolatedWorkspacesEnabled: boolean; effectiveExecutionWorkspaceMode: string | null | undefined; - issue: { companyId: string; id?: string | null; projectId?: string | null } | null; + issue: { + companyId: string; + id?: string | null; + projectId?: string | null; + } | null; resolveSelectedEnvironmentDriver: () => Promise; resolveWorkspace: () => Promise; -}): Promise<{ selectedEnvironmentDriver: string | null; workspace: TWorkspace }> { +}): Promise<{ + selectedEnvironmentDriver: string | null; + workspace: TWorkspace; +}> { const selectedEnvironmentDriver = await preflightLowTrustWorkspaceIsolation({ db: input.db, trustPreset: input.trustPreset, @@ -1722,7 +1954,12 @@ function deriveRepoNameFromRepoUrl(repoUrl: string | null): string | null { try { const parsed = new URL(trimmed); const cleanedPath = parsed.pathname.replace(/\/+$/, ""); - const repoName = cleanedPath.split("/").filter(Boolean).pop()?.replace(/\.git$/i, "") ?? ""; + const repoName = + cleanedPath + .split("/") + .filter(Boolean) + .pop() + ?.replace(/\.git$/i, "") ?? ""; return repoName || null; } catch { return null; @@ -1735,7 +1972,10 @@ function deriveRepoNameFromRepoUrl(repoUrl: string | null): string | null { * clone target — the loser saw "destination path already exists" and its failure cleanup * deleted the winner's in-progress clone, so both runs failed every round. */ -const managedCheckoutMaterializations = new Map>(); +const managedCheckoutMaterializations = new Map< + string, + Promise<{ cwd: string; warning: string | null }> +>(); export async function ensureManagedProjectWorkspace(input: { companyId: string; @@ -1799,44 +2039,59 @@ async function materializeManagedProjectWorkspace( // is never created in a partial state and never removed on failure, so a concurrent // materialization (another process, or a run racing this one) can neither adopt a broken // checkout nor lose its own completed one. - const auth = input.resolveGitAuth ? await input.resolveGitAuth(input.repoUrl) : null; + const auth = input.resolveGitAuth + ? await input.resolveGitAuth(input.repoUrl) + : null; const cloneTmpDir = await fs.mkdtemp(`${cwd}.clone-`); try { - await execFile("git", [...(auth?.configArgs ?? []), "clone", input.repoUrl, cloneTmpDir], { - env: { - // Spread order matters: the sanitizer strips PAPERCLIP_*, which would remove the - // credential-helper token env if it came first. GIT_TERMINAL_PROMPT=0 fails a - // credential-less private clone immediately instead of hanging on a prompt until - // the clone timeout. - ...sanitizeRuntimeServiceBaseEnv(process.env), - GIT_TERMINAL_PROMPT: "0", - ...(auth?.env ?? {}), + await execFile( + "git", + [...(auth?.configArgs ?? []), "clone", input.repoUrl, cloneTmpDir], + { + env: { + // Spread order matters: the sanitizer strips PAPERCLIP_*, which would remove the + // credential-helper token env if it came first. GIT_TERMINAL_PROMPT=0 fails a + // credential-less private clone immediately instead of hanging on a prompt until + // the clone timeout. + ...sanitizeRuntimeServiceBaseEnv(process.env), + GIT_TERMINAL_PROMPT: "0", + ...(auth?.env ?? {}), + }, + timeout: MANAGED_WORKSPACE_GIT_CLONE_TIMEOUT_MS, }, - timeout: MANAGED_WORKSPACE_GIT_CLONE_TIMEOUT_MS, - }); + ); } catch (error) { - await fs.rm(cloneTmpDir, { recursive: true, force: true }).catch(() => undefined); + await fs + .rm(cloneTmpDir, { recursive: true, force: true }) + .catch(() => undefined); const reason = error instanceof Error ? error.message : String(error); const authNote = describeGitAuthFailure({ error: reason, used: auth ? { source: auth.source, secretName: auth.secretName } : null, }); - throw new Error(scrubGitCredentialText( - `Failed to prepare managed checkout for "${input.repoUrl}" at "${cwd}": ${reason}${authNote ? ` ${authNote}` : ""}`, - )); + throw new Error( + scrubGitCredentialText( + `Failed to prepare managed checkout for "${input.repoUrl}" at "${cwd}": ${reason}${authNote ? ` ${authNote}` : ""}`, + ), + ); } try { await fs.rename(cloneTmpDir, cwd); } catch (renameError) { - await fs.rm(cloneTmpDir, { recursive: true, force: true }).catch(() => undefined); + await fs + .rm(cloneTmpDir, { recursive: true, force: true }) + .catch(() => undefined); // The target appearing between the emptiness check and the rename means another // materialization won the race; adopt its checkout instead of failing the run. if (await hasAdoptableGitDir()) { return { cwd, warning: null }; } - const reason = renameError instanceof Error ? renameError.message : String(renameError); - throw new Error(`Failed to move managed checkout into place at "${cwd}": ${reason}`); + const reason = + renameError instanceof Error ? renameError.message : String(renameError); + throw new Error( + `Failed to move managed checkout into place at "${cwd}": ${reason}`, + ); } return { cwd, warning: null }; } @@ -1882,23 +2137,34 @@ export interface ResolveAdditionalProjectWorkspaceDeps { } /** Build the real dependencies for {@link resolveAdditionalProjectWorkspace}. */ -function defaultAdditionalProjectWorkspaceDeps(db: Db): ResolveAdditionalProjectWorkspaceDeps { +function defaultAdditionalProjectWorkspaceDeps( + db: Db, +): ResolveAdditionalProjectWorkspaceDeps { return { loadProjectWorkspaceRows: (companyId, projectId) => db .select() .from(projectWorkspaces) - .where(and(eq(projectWorkspaces.companyId, companyId), eq(projectWorkspaces.projectId, projectId))) + .where( + and( + eq(projectWorkspaces.companyId, companyId), + eq(projectWorkspaces.projectId, projectId), + ), + ) .orderBy(asc(projectWorkspaces.createdAt), asc(projectWorkspaces.id)), resolveConfiguredOrManagedProjectCwd: (input) => resolveConfiguredOrManagedProjectCwd({ ...input, - resolveGitAuth: input.resolveGitAuth ?? createGitRemoteAuthProvider(db, input.companyId), + resolveGitAuth: + input.resolveGitAuth ?? + createGitRemoteAuthProvider(db, input.companyId), }), ensureManagedProjectWorkspace: (input) => ensureManagedProjectWorkspace({ ...input, - resolveGitAuth: input.resolveGitAuth ?? createGitRemoteAuthProvider(db, input.companyId), + resolveGitAuth: + input.resolveGitAuth ?? + createGitRemoteAuthProvider(db, input.companyId), }), // A realized workspace must hold real content. An empty directory gives the agent an empty // referenced workspace, so treat an empty directory the same as a missing one. @@ -1935,12 +2201,16 @@ export async function resolveAdditionalProjectWorkspace( ): Promise { const { companyId } = input; const projectId = input.project.projectId; - const workspaceRows = await deps.loadProjectWorkspaceRows(companyId, projectId); + const workspaceRows = await deps.loadProjectWorkspaceRows( + companyId, + projectId, + ); for (const workspace of workspaceRows) { // A row realizes real content only through a configured checkout directory or a repository URL // to clone. A row with neither can produce only an empty managed directory, so skip it here. const configuredCwd = readNonEmptyString(workspace.cwd); - const hasConfiguredCwd = Boolean(configuredCwd) && configuredCwd !== REPO_ONLY_CWD_SENTINEL; + const hasConfiguredCwd = + Boolean(configuredCwd) && configuredCwd !== REPO_ONLY_CWD_SENTINEL; if (!hasConfiguredCwd && !readNonEmptyString(workspace.repoUrl)) { continue; } @@ -1966,8 +2236,11 @@ export async function resolveAdditionalProjectWorkspace( // real source: the first workspace row that supplies a repository URL. Without a real source, do // not fabricate an empty managed directory and report success. Throw instead, so the caller drops // only this referenced project and adds a clear warning. - const fallbackRow = workspaceRows.find((row) => readNonEmptyString(row.repoUrl)) ?? null; - const fallbackRepoUrl = fallbackRow ? readNonEmptyString(fallbackRow.repoUrl) : null; + const fallbackRow = + workspaceRows.find((row) => readNonEmptyString(row.repoUrl)) ?? null; + const fallbackRepoUrl = fallbackRow + ? readNonEmptyString(fallbackRow.repoUrl) + : null; if (!fallbackRow || !fallbackRepoUrl) { throw new Error( `Referenced project ${projectId} has no workspace checkout or repository URL to realize.`, @@ -1987,20 +2260,24 @@ export async function resolveAdditionalProjectWorkspace( }; } -type WorkspaceValidationFailureLike = WorkspaceValidationFailure | { - code: typeof WORKSPACE_VALIDATION_FAILURE_CODE; - resultJson: Record; -}; +type WorkspaceValidationFailureLike = + | WorkspaceValidationFailure + | { + code: typeof WORKSPACE_VALIDATION_FAILURE_CODE; + resultJson: Record; + }; -function isWorkspaceValidationFailure(error: unknown): error is WorkspaceValidationFailureLike { +function isWorkspaceValidationFailure( + error: unknown, +): error is WorkspaceValidationFailureLike { if (error instanceof WorkspaceValidationFailure) return true; const maybe = error as { code?: unknown; resultJson?: unknown } | null; return Boolean( maybe && - maybe.code === WORKSPACE_VALIDATION_FAILURE_CODE && - maybe.resultJson && - typeof maybe.resultJson === "object" && - !Array.isArray(maybe.resultJson), + maybe.code === WORKSPACE_VALIDATION_FAILURE_CODE && + maybe.resultJson && + typeof maybe.resultJson === "object" && + !Array.isArray(maybe.resultJson), ); } @@ -2022,7 +2299,13 @@ function stableStringifyForFingerprint(value: unknown): string { } if (value && typeof value === "object") { const rec = value as Record; - return `{${Object.keys(rec).sort().map((key) => `${JSON.stringify(key)}:${stableStringifyForFingerprint(rec[key])}`).join(",")}}`; + return `{${Object.keys(rec) + .sort() + .map( + (key) => + `${JSON.stringify(key)}:${stableStringifyForFingerprint(rec[key])}`, + ) + .join(",")}}`; } return JSON.stringify(value); } @@ -2033,29 +2316,40 @@ function fingerprintFinalizeWorkspaceBranchValidation(input: { inspection: ReturnType; }) { const digest = createHash("sha256") - .update(stableStringifyForFingerprint({ - version: 1, - reason: "git_worktree_branch_incoherence", - issueId: input.issueId, - executionWorkspaceId: input.executionWorkspaceId, - worktreePath: input.inspection.worktreePath ? path.resolve(input.inspection.worktreePath) : null, - repoRoot: input.inspection.repoRoot ? path.resolve(input.inspection.repoRoot) : null, - expectedBranchName: input.inspection.expectedBranchName, - actualBranchName: input.inspection.actualBranchName, - reasonCode: input.inspection.reasonCode, - })) + .update( + stableStringifyForFingerprint({ + version: 1, + reason: "git_worktree_branch_incoherence", + issueId: input.issueId, + executionWorkspaceId: input.executionWorkspaceId, + worktreePath: input.inspection.worktreePath + ? path.resolve(input.inspection.worktreePath) + : null, + repoRoot: input.inspection.repoRoot + ? path.resolve(input.inspection.repoRoot) + : null, + expectedBranchName: input.inspection.expectedBranchName, + actualBranchName: input.inspection.actualBranchName, + reasonCode: input.inspection.reasonCode, + }), + ) .digest("hex"); return `workspace_finalize_branch_mismatch:v1:sha256:${digest}`; } -function isConfigurationIncompleteFailure(error: unknown): error is ConfigurationIncompleteFailure { +function isConfigurationIncompleteFailure( + error: unknown, +): error is ConfigurationIncompleteFailure { return error instanceof ConfigurationIncompleteFailure; } export function isConfigurationIncompleteFailedRun( run: Pick | null | undefined, ) { - return run?.errorCode === CONFIGURATION_INCOMPLETE_FAILURE_CODE || run?.errorCode === "model_not_found"; + return ( + run?.errorCode === CONFIGURATION_INCOMPLETE_FAILURE_CODE || + run?.errorCode === "model_not_found" + ); } async function hasGitMetadata(cwd: string | null | undefined) { @@ -2075,7 +2369,10 @@ async function isGitCheckout(cwd: string | null | undefined) { .catch(() => false); } -function sameResolvedPath(left: string | null | undefined, right: string | null | undefined) { +function sameResolvedPath( + left: string | null | undefined, + right: string | null | undefined, +) { const leftPath = readNonEmptyString(left); const rightPath = readNonEmptyString(right); if (!leftPath || !rightPath) return false; @@ -2095,7 +2392,11 @@ async function hasGitPushRemote(cwd: string | null | undefined) { .catch(() => []); for (const remoteName of remoteNames) { - const pushUrl = await execFile("git", ["remote", "get-url", "--push", remoteName], { cwd: normalized }) + const pushUrl = await execFile( + "git", + ["remote", "get-url", "--push", remoteName], + { cwd: normalized }, + ) .then((result) => readNonEmptyString(result.stdout)) .catch(() => null); if (pushUrl) return true; @@ -2104,7 +2405,9 @@ async function hasGitPushRemote(cwd: string | null | undefined) { } export async function assertGitWorktreeBaseWorkspaceReady(input: { - requestedExecutionWorkspaceMode: ReturnType; + requestedExecutionWorkspaceMode: ReturnType< + typeof resolveExecutionWorkspaceMode + >; config: Record; issue: { id: string; @@ -2140,8 +2443,13 @@ export async function assertGitWorktreeBaseWorkspaceReady(input: { if (strategyType !== "git_worktree") return; const issueLabel = input.issue.identifier ?? input.issue.id; - const remediation = "This task needs a project / project workspace or a reusable execution workspace before it can run."; - const fail = (reason: string, message: string, extra: Record = {}) => { + const remediation = + "This task needs a project / project workspace or a reusable execution workspace before it can run."; + const fail = ( + reason: string, + message: string, + extra: Record = {}, + ) => { throw new WorkspaceValidationFailure(message, { workspaceValidation: { reason, @@ -2150,7 +2458,8 @@ export async function assertGitWorktreeBaseWorkspaceReady(input: { issueProjectId: input.issue!.projectId, issueProjectWorkspaceId: input.issue!.projectWorkspaceId, issueExecutionWorkspaceId: input.issue!.executionWorkspaceId ?? null, - issueExecutionWorkspacePreference: input.issue!.executionWorkspacePreference ?? null, + issueExecutionWorkspacePreference: + input.issue!.executionWorkspacePreference ?? null, requestedExecutionWorkspaceMode: input.requestedExecutionWorkspaceMode, workspaceStrategyType: strategyType, resolvedWorkspaceSource: input.base.source, @@ -2185,7 +2494,7 @@ export async function assertGitWorktreeBaseWorkspaceReady(input: { ); } - if (!await isGitCheckout(input.base.baseCwd)) { + if (!(await isGitCheckout(input.base.baseCwd))) { fail( "git_worktree_base_not_git_checkout", `Issue ${issueLabel} requested ${input.requestedExecutionWorkspaceMode} with git_worktree, but base workspace "${input.base.baseCwd}" is not a git checkout. ${remediation}`, @@ -2269,13 +2578,17 @@ export async function assertGitSensitiveAdapterWorkspaceValid(input: { }) { if (!GIT_SENSITIVE_LOCAL_ADAPTER_TYPES.has(input.adapterType)) return; - const executionTargetKind = readNonEmptyString((input.executionTarget as { kind?: unknown } | null)?.kind) ?? "local"; + const executionTargetKind = + readNonEmptyString( + (input.executionTarget as { kind?: unknown } | null)?.kind, + ) ?? "local"; if (executionTargetKind !== "local") return; const issue = input.issue; if (!issue) return; - const environmentDriver = readNonEmptyString(input.environmentDriver) ?? "local"; + const environmentDriver = + readNonEmptyString(input.environmentDriver) ?? "local"; const leaseMetadata = parseObject(input.leaseMetadata); const leaseProviderMetadata = parseObject(leaseMetadata.providerMetadata); const leaseRemoteCwd = @@ -2283,14 +2596,20 @@ export async function assertGitSensitiveAdapterWorkspaceValid(input: { readNonEmptyString(leaseProviderMetadata.remoteCwd); const effectiveCwd = readNonEmptyString(input.executionWorkspace.cwd); - const persistedCwd = readNonEmptyString(input.persistedExecutionWorkspace?.cwd); + const persistedCwd = readNonEmptyString( + input.persistedExecutionWorkspace?.cwd, + ); const agentFallbackCwd = resolveDefaultAgentWorkspaceDir(input.agentId); const workspaceExpectation = Boolean(issue.projectWorkspaceId) || Boolean(input.resolvedWorkspace.workspaceId) || input.executionWorkspace.strategy === "git_worktree"; - const fail = (reason: string, message: string, extra: Record = {}) => { + const fail = ( + reason: string, + message: string, + extra: Record = {}, + ) => { throw new WorkspaceValidationFailure(message, { workspaceValidation: { reason, @@ -2306,13 +2625,19 @@ export async function assertGitSensitiveAdapterWorkspaceValid(input: { executionWorkspaceCwd: effectiveCwd, executionWorkspaceStrategy: input.executionWorkspace.strategy, executionWorkspaceProjectId: input.executionWorkspace.projectId, - executionWorkspaceProjectWorkspaceId: input.executionWorkspace.workspaceId, - persistedExecutionWorkspaceId: input.persistedExecutionWorkspace?.id ?? null, + executionWorkspaceProjectWorkspaceId: + input.executionWorkspace.workspaceId, + persistedExecutionWorkspaceId: + input.persistedExecutionWorkspace?.id ?? null, persistedWorkspaceCwd: persistedCwd, - persistedWorkspaceStrategy: input.persistedExecutionWorkspace?.strategyType ?? null, - persistedProjectId: input.persistedExecutionWorkspace?.projectId ?? null, - persistedProjectWorkspaceId: input.persistedExecutionWorkspace?.projectWorkspaceId ?? null, - persistedProviderRef: input.persistedExecutionWorkspace?.providerRef ?? null, + persistedWorkspaceStrategy: + input.persistedExecutionWorkspace?.strategyType ?? null, + persistedProjectId: + input.persistedExecutionWorkspace?.projectId ?? null, + persistedProjectWorkspaceId: + input.persistedExecutionWorkspace?.projectWorkspaceId ?? null, + persistedProviderRef: + input.persistedExecutionWorkspace?.providerRef ?? null, ...extra, }, }); @@ -2325,7 +2650,8 @@ export async function assertGitSensitiveAdapterWorkspaceValid(input: { ); } - if (!input.executionTarget && environmentDriver !== "local" && leaseRemoteCwd) return; + if (!input.executionTarget && environmentDriver !== "local" && leaseRemoteCwd) + return; if (workspaceExpectation && !input.persistedExecutionWorkspace) { fail( @@ -2353,7 +2679,8 @@ export async function assertGitSensitiveAdapterWorkspaceValid(input: { ); } - const expectedProjectWorkspaceId = issue.projectWorkspaceId ?? input.resolvedWorkspace.workspaceId ?? null; + const expectedProjectWorkspaceId = + issue.projectWorkspaceId ?? input.resolvedWorkspace.workspaceId ?? null; if ( expectedProjectWorkspaceId && input.persistedExecutionWorkspace && @@ -2368,7 +2695,8 @@ export async function assertGitSensitiveAdapterWorkspaceValid(input: { if ( expectedProjectWorkspaceId && input.persistedExecutionWorkspace?.projectWorkspaceId && - input.persistedExecutionWorkspace.projectWorkspaceId !== expectedProjectWorkspaceId + input.persistedExecutionWorkspace.projectWorkspaceId !== + expectedProjectWorkspaceId ) { fail( "project_workspace_mismatch", @@ -2376,7 +2704,11 @@ export async function assertGitSensitiveAdapterWorkspaceValid(input: { ); } - if (workspaceExpectation && effectiveCwd && sameResolvedPath(effectiveCwd, agentFallbackCwd)) { + if ( + workspaceExpectation && + effectiveCwd && + sameResolvedPath(effectiveCwd, agentFallbackCwd) + ) { fail( "fallback_agent_home_cwd", `Issue ${issue.identifier ?? issue.id} expected a project workspace, but ${input.adapterType} would launch from agent fallback cwd "${effectiveCwd}".`, @@ -2387,7 +2719,10 @@ export async function assertGitSensitiveAdapterWorkspaceValid(input: { input.persistedExecutionWorkspace?.strategyType === "git_worktree" && input.persistedExecutionWorkspace.providerRef && effectiveCwd && - !sameResolvedPath(effectiveCwd, input.persistedExecutionWorkspace.providerRef) + !sameResolvedPath( + effectiveCwd, + input.persistedExecutionWorkspace.providerRef, + ) ) { fail( "git_worktree_provider_ref_mismatch", @@ -2395,7 +2730,11 @@ export async function assertGitSensitiveAdapterWorkspaceValid(input: { ); } - if (workspaceExpectation && effectiveCwd && !await hasGitMetadata(effectiveCwd)) { + if ( + workspaceExpectation && + effectiveCwd && + !(await hasGitMetadata(effectiveCwd)) + ) { fail( "missing_git_metadata", `Issue ${issue.identifier ?? issue.id} expected a git workspace for ${input.adapterType}, but "${effectiveCwd}" has no .git metadata.`, @@ -2418,7 +2757,10 @@ export async function assertGitSensitiveAdapterWorkspaceValid(input: { fail( "git_worktree_branch_mismatch", `Issue ${issue.identifier ?? issue.id} expected git worktree branch "${expectedManagedBranchName}" at "${effectiveCwd}", but ${inspection.reason ?? "the checked-out branch could not be verified"}.`, - { managedGitWorktreeBranch: formatManagedGitWorktreeBranchInspection(inspection) }, + { + managedGitWorktreeBranch: + formatManagedGitWorktreeBranchInspection(inspection), + }, ); } } @@ -2488,24 +2830,66 @@ const heartbeatRunSummaryListColumns = { } as const; const heartbeatRunListContextColumns = { - contextIssueId: sql`${heartbeatRuns.contextSnapshot} ->> 'issueId'`.as("contextIssueId"), - contextTaskId: sql`${heartbeatRuns.contextSnapshot} ->> 'taskId'`.as("contextTaskId"), - contextTaskKey: sql`${heartbeatRuns.contextSnapshot} ->> 'taskKey'`.as("contextTaskKey"), - contextCommentId: sql`${heartbeatRuns.contextSnapshot} ->> 'commentId'`.as("contextCommentId"), - contextWakeCommentId: sql`${heartbeatRuns.contextSnapshot} ->> 'wakeCommentId'`.as("contextWakeCommentId"), - contextWakeReason: sql`${heartbeatRuns.contextSnapshot} ->> 'wakeReason'`.as("contextWakeReason"), - contextWakeSource: sql`${heartbeatRuns.contextSnapshot} ->> 'wakeSource'`.as("contextWakeSource"), - contextWakeTriggerDetail: sql`${heartbeatRuns.contextSnapshot} ->> 'wakeTriggerDetail'`.as("contextWakeTriggerDetail"), + contextIssueId: sql< + string | null + >`${heartbeatRuns.contextSnapshot} ->> 'issueId'`.as("contextIssueId"), + contextTaskId: sql< + string | null + >`${heartbeatRuns.contextSnapshot} ->> 'taskId'`.as("contextTaskId"), + contextTaskKey: sql< + string | null + >`${heartbeatRuns.contextSnapshot} ->> 'taskKey'`.as("contextTaskKey"), + contextCommentId: sql< + string | null + >`${heartbeatRuns.contextSnapshot} ->> 'commentId'`.as("contextCommentId"), + contextWakeCommentId: sql< + string | null + >`${heartbeatRuns.contextSnapshot} ->> 'wakeCommentId'`.as( + "contextWakeCommentId", + ), + contextWakeReason: sql< + string | null + >`${heartbeatRuns.contextSnapshot} ->> 'wakeReason'`.as("contextWakeReason"), + contextWakeSource: sql< + string | null + >`${heartbeatRuns.contextSnapshot} ->> 'wakeSource'`.as("contextWakeSource"), + contextWakeTriggerDetail: sql< + string | null + >`${heartbeatRuns.contextSnapshot} ->> 'wakeTriggerDetail'`.as( + "contextWakeTriggerDetail", + ), } as const; const heartbeatRunListResultColumns = { - resultSummary: sql`left(${heartbeatRuns.resultJson} ->> 'summary', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS})`.as("resultSummary"), - resultResult: sql`left(${heartbeatRuns.resultJson} ->> 'result', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS})`.as("resultResult"), - resultMessage: sql`left(${heartbeatRuns.resultJson} ->> 'message', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS})`.as("resultMessage"), - resultError: sql`left(${heartbeatRuns.resultJson} ->> 'error', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS})`.as("resultError"), - resultTotalCostUsd: sql`${heartbeatRuns.resultJson} ->> 'total_cost_usd'`.as("resultTotalCostUsd"), - resultCostUsd: sql`${heartbeatRuns.resultJson} ->> 'cost_usd'`.as("resultCostUsd"), - resultCostUsdCamel: sql`${heartbeatRuns.resultJson} ->> 'costUsd'`.as("resultCostUsdCamel"), + resultSummary: sql< + string | null + >`left(${heartbeatRuns.resultJson} ->> 'summary', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS})`.as( + "resultSummary", + ), + resultResult: sql< + string | null + >`left(${heartbeatRuns.resultJson} ->> 'result', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS})`.as( + "resultResult", + ), + resultMessage: sql< + string | null + >`left(${heartbeatRuns.resultJson} ->> 'message', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS})`.as( + "resultMessage", + ), + resultError: sql< + string | null + >`left(${heartbeatRuns.resultJson} ->> 'error', ${HEARTBEAT_RUN_RESULT_SUMMARY_MAX_CHARS})`.as( + "resultError", + ), + resultTotalCostUsd: sql< + string | null + >`${heartbeatRuns.resultJson} ->> 'total_cost_usd'`.as("resultTotalCostUsd"), + resultCostUsd: sql< + string | null + >`${heartbeatRuns.resultJson} ->> 'cost_usd'`.as("resultCostUsd"), + resultCostUsdCamel: sql< + string | null + >`${heartbeatRuns.resultJson} ->> 'costUsd'`.as("resultCostUsdCamel"), } as const; const heartbeatRunSafeResultJsonColumn = sql | null>` @@ -2582,8 +2966,14 @@ const heartbeatRunIssueSummaryColumns = { status: heartbeatRuns.status, invocationSource: heartbeatRuns.invocationSource, triggerDetail: heartbeatRuns.triggerDetail, - contextCommentId: sql`${heartbeatRuns.contextSnapshot} ->> 'commentId'`.as("contextCommentId"), - contextWakeCommentId: sql`${heartbeatRuns.contextSnapshot} ->> 'wakeCommentId'`.as("contextWakeCommentId"), + contextCommentId: sql< + string | null + >`${heartbeatRuns.contextSnapshot} ->> 'commentId'`.as("contextCommentId"), + contextWakeCommentId: sql< + string | null + >`${heartbeatRuns.contextSnapshot} ->> 'wakeCommentId'`.as( + "contextWakeCommentId", + ), startedAt: heartbeatRuns.startedAt, finishedAt: heartbeatRuns.finishedAt, createdAt: heartbeatRuns.createdAt, @@ -2599,7 +2989,9 @@ const heartbeatRunIssueSummaryColumns = { lastOutputSeq: heartbeatRuns.lastOutputSeq, lastOutputStream: heartbeatRuns.lastOutputStream, lastOutputBytes: heartbeatRuns.lastOutputBytes, - issueId: sql`${heartbeatRuns.contextSnapshot} ->> 'issueId'`.as("issueId"), + issueId: sql< + string | null + >`${heartbeatRuns.contextSnapshot} ->> 'issueId'`.as("issueId"), } as const; function appendExcerpt(prev: string, chunk: string) { @@ -2612,14 +3004,18 @@ function truncateRunEventString(value: string) { return `${value.slice(0, MAX_RUN_EVENT_PAYLOAD_STRING_CHARS)}\n[truncated ${omittedChars} chars]`; } -function boundRunEventValue(value: unknown, depth: number, seen: WeakSet): unknown { +function boundRunEventValue( + value: unknown, + depth: number, + seen: WeakSet, +): unknown { if (typeof value === "string") { return truncateRunEventString(value); } if ( - value === null - || typeof value === "number" - || typeof value === "boolean" + value === null || + typeof value === "number" || + typeof value === "boolean" ) { return value; } @@ -2664,7 +3060,10 @@ function boundRunEventValue(value: unknown, depth: number, seen: WeakSet } const out: Record = {}; - for (const [key, entryValue] of entries.slice(0, MAX_RUN_EVENT_PAYLOAD_OBJECT_KEYS)) { + for (const [key, entryValue] of entries.slice( + 0, + MAX_RUN_EVENT_PAYLOAD_OBJECT_KEYS, + )) { out[key] = boundRunEventValue(entryValue, depth + 1, seen); } if (entries.length > MAX_RUN_EVENT_PAYLOAD_OBJECT_KEYS) { @@ -2675,18 +3074,25 @@ function boundRunEventValue(value: unknown, depth: number, seen: WeakSet return out; } -export function boundHeartbeatRunEventPayloadForStorage(payload: Record): Record { +export function boundHeartbeatRunEventPayloadForStorage( + payload: Record, +): Record { const bounded = boundRunEventValue(payload, 0, new WeakSet()); return parseObject(bounded) ?? { _truncated: true }; } function redactInlineBase64ImageData(chunk: string) { - return chunk.replace(INLINE_BASE64_IMAGE_DATA_RE, (_match, prefix: string, data: string, suffix: string) => - `${prefix}[omitted base64 image data: ${data.length} chars]${suffix}`, + return chunk.replace( + INLINE_BASE64_IMAGE_DATA_RE, + (_match, prefix: string, data: string, suffix: string) => + `${prefix}[omitted base64 image data: ${data.length} chars]${suffix}`, ); } -export function compactRunLogChunk(chunk: string, maxChars = MAX_PERSISTED_LOG_CHUNK_CHARS) { +export function compactRunLogChunk( + chunk: string, + maxChars = MAX_PERSISTED_LOG_CHUNK_CHARS, +) { const normalized = redactSensitiveText(redactInlineBase64ImageData(chunk)); if (normalized.length <= maxChars) return normalized; @@ -2698,9 +3104,14 @@ export function compactRunLogChunk(chunk: string, maxChars = MAX_PERSISTED_LOG_C } function normalizeMaxConcurrentRuns(value: unknown) { - const parsed = Math.floor(asNumber(value, HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT)); + const parsed = Math.floor( + asNumber(value, HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT), + ); if (!Number.isFinite(parsed)) return HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT; - return Math.max(HEARTBEAT_MAX_CONCURRENT_RUNS_MIN, Math.min(HEARTBEAT_MAX_CONCURRENT_RUNS_MAX, parsed)); + return Math.max( + HEARTBEAT_MAX_CONCURRENT_RUNS_MIN, + Math.min(HEARTBEAT_MAX_CONCURRENT_RUNS_MAX, parsed), + ); } interface WakeupOptions { @@ -2852,7 +3263,10 @@ export function buildAnchorFallbackWorkspaceNotes(input: { ? `Project workspace path "${firstMissing}" and ${extraMissingCount} other configured path(s) are not available yet. Using fallback workspace "${input.fallbackCwd}" for this run.` : `Project workspace path "${firstMissing}" is not available yet. Using fallback workspace "${input.fallbackCwd}" for this run.`, ); - } else if (input.materializationFailures.length === 0 && !input.hasConfiguredProjectCwd) { + } else if ( + input.materializationFailures.length === 0 && + !input.hasConfiguredProjectCwd + ) { warnings.push( `Project workspace has no local cwd configured. Using fallback workspace "${input.fallbackCwd}" for this run.`, ); @@ -2873,7 +3287,10 @@ export function buildAnchorFallbackWorkspaceNotes(input: { * unchanged in the production default. */ export function buildRunWorkspaceHints( - resolved: Pick, + resolved: Pick< + ResolvedWorkspaceForRun, + "workspaceHints" | "additionalWorkspaces" + >, ): Array> { return [ ...resolved.workspaceHints, @@ -2891,14 +3308,19 @@ type ProjectWorkspaceCandidate = { id: string; }; -export function prioritizeProjectWorkspaceCandidatesForRun( - rows: T[], - preferredWorkspaceId: string | null | undefined, -): T[] { +export function prioritizeProjectWorkspaceCandidatesForRun< + T extends ProjectWorkspaceCandidate, +>(rows: T[], preferredWorkspaceId: string | null | undefined): T[] { if (!preferredWorkspaceId) return rows; - const preferredIndex = rows.findIndex((row) => row.id === preferredWorkspaceId); + const preferredIndex = rows.findIndex( + (row) => row.id === preferredWorkspaceId, + ); if (preferredIndex <= 0) return rows; - return [rows[preferredIndex]!, ...rows.slice(0, preferredIndex), ...rows.slice(preferredIndex + 1)]; + return [ + rows[preferredIndex]!, + ...rows.slice(0, preferredIndex), + ...rows.slice(preferredIndex + 1), + ]; } /** @@ -2909,7 +3331,8 @@ export function prioritizeProjectWorkspaceCandidatesForRun [project.id, project])); @@ -3151,7 +3592,9 @@ export async function resolveRunReferencedProjects( for (const projectId of batchCandidateIds) { const project = byId.get(projectId); if (!project) { - warnings.push(`Referenced project ${projectId} was skipped because it is not available in this company.`); + warnings.push( + `Referenced project ${projectId} was skipped because it is not available in this company.`, + ); failures.push({ projectId, reason: "resolution" }); continue; } @@ -3162,12 +3605,15 @@ export async function resolveRunReferencedProjects( // Hydrate the anchor on its own if the candidate loop never ran (no mentions to co-hydrate it with). if (!anchorHydrated && anchorProjectId) { const hydrated = await projects.listByIds(companyId, [anchorProjectId]); - anchorRecord = hydrated.find((project) => project.id === anchorProjectId) ?? null; + anchorRecord = + hydrated.find((project) => project.id === anchorProjectId) ?? null; anchorHydrated = true; } const anchor: RunReferencedProject | null = - anchorRecord && anchorProjectId ? { projectId: anchorProjectId, project: anchorRecord } : null; + anchorRecord && anchorProjectId + ? { projectId: anchorProjectId, project: anchorRecord } + : null; // The loop already bounds `availableCandidates` to at most `evaluationCap` entries. Any mentions left // un-hydrated past the window (the fan-out cap dropped them before hydration/authorization) are @@ -3205,7 +3651,9 @@ export async function resolveRunReferencedProjects( } if (!allowed) { - warnings.push(`Referenced project ${projectId} was skipped because it is not authorized for this run.`); + warnings.push( + `Referenced project ${projectId} was skipped because it is not authorized for this run.`, + ); failures.push({ projectId, reason: "authorization" }); continue; } @@ -3217,7 +3665,8 @@ export async function resolveRunReferencedProjects( // skipped count includes both the still-unconsidered evaluated candidates and any available // candidates that were dropped before evaluation by the fan-out cap above. if (capReachedAtIndex !== null) { - const skipped = candidates.length - capReachedAtIndex + unevaluatedCandidateCount; + const skipped = + candidates.length - capReachedAtIndex + unevaluatedCandidateCount; warnings.push( `Only the first ${cap} referenced project(s) will be synced for this run; ${skipped} additional referenced project(s) were skipped.`, ); @@ -3235,7 +3684,10 @@ export async function resolveRunReferencedProjects( // dropped before hydration carry their id from the ordered mention set. if (capReachedAtIndex !== null) { for (let index = capReachedAtIndex; index < candidates.length; index++) { - failures.push({ projectId: candidates[index]!.projectId, reason: "resolution" }); + failures.push({ + projectId: candidates[index]!.projectId, + reason: "resolution", + }); } } for (const projectId of allCandidateIds.slice(hydrationCursor)) { @@ -3255,7 +3707,9 @@ export interface ResolveAdditionalRunWorkspacesOptions { projects: Pick, "listByIds">; access: Pick, "decide">; /** Resolve one authorized referenced project to its own workspace cwd (injectable for tests). */ - resolveProjectWorkspace: (project: RunReferencedProject) => Promise; + resolveProjectWorkspace: ( + project: RunReferencedProject, + ) => Promise; maxAdditionalProjects?: number; maxCandidateEvaluations?: number; /** @@ -3314,7 +3768,8 @@ export async function resolveAdditionalRunWorkspaces( // exposes an inaccessible referenced path to the agent. if (opts.executionTargetIsRemote) { const remoteReferencedSyncOpen = - (opts.remoteReferencedSyncEnabled ?? false) && (opts.targetStagesConfined ?? false); + (opts.remoteReferencedSyncEnabled ?? false) && + (opts.targetStagesConfined ?? false); if (!remoteReferencedSyncOpen) { const mentionedIds = await opts.issues.findMentionedProjectIds(issueId, { includeCommentBodies: true, @@ -3326,7 +3781,9 @@ export async function resolveAdditionalRunWorkspaces( // emits its structured sync log. Warn only when the issue actually mentions a project, so a // remote run without any referenced mention stays silent. const droppedProjectIds = [ - ...new Set(mentionedIds.filter((projectId) => projectId !== anchorProjectId)), + ...new Set( + mentionedIds.filter((projectId) => projectId !== anchorProjectId), + ), ]; return { additionalWorkspaces: [], @@ -3336,7 +3793,10 @@ export async function resolveAdditionalRunWorkspaces( "Referenced-project workspaces are available only on a local execution target or a confined sandbox target. This run uses a different remote execution target, so no referenced-project workspace was attached.", ] : [], - failures: droppedProjectIds.map((projectId) => ({ projectId, reason: "staging" as const })), + failures: droppedProjectIds.map((projectId) => ({ + projectId, + reason: "staging" as const, + })), }; } // Fall through: a confined sandbox target with the remote flag on resolves and authorizes the @@ -3345,15 +3805,19 @@ export async function resolveAdditionalRunWorkspaces( // directory. The per-project `project:read` check below still runs against the run actor. } - const referenced = await resolveRunReferencedProjects(issueId, anchorProjectId, { - companyId: opts.companyId, - actor: opts.actor, - issues: opts.issues, - projects: opts.projects, - access: opts.access, - maxAdditionalProjects: opts.maxAdditionalProjects, - maxCandidateEvaluations: opts.maxCandidateEvaluations, - }); + const referenced = await resolveRunReferencedProjects( + issueId, + anchorProjectId, + { + companyId: opts.companyId, + actor: opts.actor, + issues: opts.issues, + projects: opts.projects, + access: opts.access, + maxAdditionalProjects: opts.maxAdditionalProjects, + maxCandidateEvaluations: opts.maxCandidateEvaluations, + }, + ); const additionalWorkspaces: ResolvedAdditionalWorkspace[] = []; const warnings = [...referenced.warnings]; @@ -3399,7 +3863,8 @@ export function buildReferencedProjectRunObservability(input: { failures: readonly ReferencedProjectFailure[]; }): ReferencedProjectRunObservability { return { - referenced_projects_requested: input.syncedProjectIds.length + input.failures.length, + referenced_projects_requested: + input.syncedProjectIds.length + input.failures.length, referenced_projects_synced: input.syncedProjectIds.length, referenced_project_failures: input.failures.map((failure) => ({ project_id: failure.projectId, @@ -3418,7 +3883,10 @@ function readNonEmptyString(value: unknown): string | null { function sanitizeAgentSessionMessageText(value: unknown): string | null { const text = readNonEmptyString(value); if (!text) return null; - const redacted = redactSensitiveText(text).slice(0, MAX_AGENT_SESSION_MESSAGE_CHARS); + const redacted = redactSensitiveText(text).slice( + 0, + MAX_AGENT_SESSION_MESSAGE_CHARS, + ); return redacted.trim().length > 0 ? redacted : null; } @@ -3444,7 +3912,9 @@ function configuredPaperclipApiBaseUrl(): string | null { function paperclipApiBaseUrl(): string { const configured = configuredPaperclipApiBaseUrl(); if (!configured) { - throw new Error("PAPERCLIP_API_URL is required to deliver managed runtime MCP servers"); + throw new Error( + "PAPERCLIP_API_URL is required to deliver managed runtime MCP servers", + ); } return configured; } @@ -3458,18 +3928,21 @@ export async function revokeHeartbeatRunGatewayTokens(input: { await input.db .update(toolMcpGatewayTokens) .set({ revokedAt: now, updatedAt: now }) - .where(and( - eq(toolMcpGatewayTokens.companyId, input.companyId), - eq(toolMcpGatewayTokens.subjectType, "heartbeat_run"), - eq(toolMcpGatewayTokens.subjectId, input.runId), - isNull(toolMcpGatewayTokens.revokedAt), - )); + .where( + and( + eq(toolMcpGatewayTokens.companyId, input.companyId), + eq(toolMcpGatewayTokens.subjectType, "heartbeat_run"), + eq(toolMcpGatewayTokens.subjectId, input.runId), + isNull(toolMcpGatewayTokens.revokedAt), + ), + ); } export async function buildPaperclipRuntimeMcpServers(input: { db: Db; agent: Pick; runId: string; + failOnUnavailableAssignedConnection?: boolean; }): Promise { const access = toolAccessService(input.db); const effective = await access.getEffectiveProfilesForAgent( @@ -3487,16 +3960,12 @@ export async function buildPaperclipRuntimeMcpServers(input: { ); const permittedConnections = permittedConnectionIds.size > 0 ? await input.db - .select({ - id: toolConnections.id, - name: toolConnections.name, - transport: toolConnections.transport, - }) - .from(toolConnections) - .where(and( - eq(toolConnections.companyId, input.agent.companyId), - inArray(toolConnections.id, [...permittedConnectionIds]), - )) + .select({ id: toolConnections.id, name: toolConnections.name, transport: toolConnections.transport }) + .from(toolConnections) + .where(and( + eq(toolConnections.companyId, input.agent.companyId), + inArray(toolConnections.id, [...permittedConnectionIds]), + )) : []; const permittedNotInstalledConnections = permittedConnections .filter((connection) => @@ -3512,6 +3981,16 @@ export async function buildPaperclipRuntimeMcpServers(input: { && !["degraded", "failed", "error", "missing_secret"].includes(connection.healthStatus) && (connection.transport === "mcp_remote" || connection.transport === "local_stdio") ); + const unhealthyConnections = effective.installedConnections.filter((connection) => + permittedConnectionIds.has(connection.id) + && (connection.transport === "mcp_remote" || connection.transport === "local_stdio") + && (!connection.enabled || connection.status !== "active" || ["degraded", "failed", "error", "missing_secret"].includes(connection.healthStatus)), + ); + if (input.failOnUnavailableAssignedConnection && unhealthyConnections.length) { + throw new Error( + `assigned native MCP connection is unavailable: ${unhealthyConnections.map((connection) => connection.id).join(", ")}`, + ); + } const service = createToolGatewayService(input.db); if (assignedConnections.length === 0) { await service.recordRuntimeMcpDeliveryDiagnostic({ @@ -3522,7 +4001,6 @@ export async function buildPaperclipRuntimeMcpServers(input: { }); return []; } - const assignment = { version: 1, agentId: input.agent.id, @@ -3732,9 +4210,24 @@ function gatewayAppliesToRun(input: { if (gateway.agentId && gateway.agentId !== agentId) return false; if (gateway.projectId && gateway.projectId !== projectId) return false; if (gateway.issueId && gateway.issueId !== issueId) return false; - if (gateway.contextScopeType === "agent" && gateway.contextScopeId && gateway.contextScopeId !== agentId) return false; - if (gateway.contextScopeType === "project" && gateway.contextScopeId && gateway.contextScopeId !== projectId) return false; - if (gateway.contextScopeType === "issue" && gateway.contextScopeId && gateway.contextScopeId !== issueId) return false; + if ( + gateway.contextScopeType === "agent" && + gateway.contextScopeId && + gateway.contextScopeId !== agentId + ) + return false; + if ( + gateway.contextScopeType === "project" && + gateway.contextScopeId && + gateway.contextScopeId !== projectId + ) + return false; + if ( + gateway.contextScopeType === "issue" && + gateway.contextScopeId && + gateway.contextScopeId !== issueId + ) + return false; return true; } @@ -3801,7 +4294,10 @@ async function gatewayConnectionIds(input: { export async function createManagedMcpRunConfig(input: { db: Db; - agent: Pick; + agent: Pick< + typeof agents.$inferSelect, + "id" | "companyId" | "name" | "adapterType" + >; runId: string; config: Record; projectId: string | null; @@ -3813,21 +4309,43 @@ export async function createManagedMcpRunConfig(input: { const rows = await input.db .select() .from(toolMcpGateways) - .where(and( - eq(toolMcpGateways.companyId, input.agent.companyId), - eq(toolMcpGateways.status, "active"), - isNull(toolMcpGateways.archivedAt), - )) + .where( + and( + eq(toolMcpGateways.companyId, input.agent.companyId), + eq(toolMcpGateways.status, "active"), + isNull(toolMcpGateways.archivedAt), + ), + ) .orderBy(asc(toolMcpGateways.name)); const installRows = await input.db - .select({ connectionId: toolConnectionInstalls.connectionId }) + .select({ + connectionId: toolConnectionInstalls.connectionId, + enabled: toolConnections.enabled, + status: toolConnections.status, + healthStatus: toolConnections.healthStatus, + }) .from(toolConnectionInstalls) + .innerJoin( + toolConnections, + and( + eq(toolConnections.id, toolConnectionInstalls.connectionId), + eq(toolConnections.companyId, toolConnectionInstalls.companyId), + ), + ) .where(and( eq(toolConnectionInstalls.companyId, input.agent.companyId), sql`((${toolConnectionInstalls.targetType} = 'company' and ${toolConnectionInstalls.targetId} = ${input.agent.companyId}) or (${toolConnectionInstalls.targetType} = 'agent' and ${toolConnectionInstalls.targetId} = ${input.agent.id}))`, )); - const installedConnectionIds = new Set(installRows.map((install) => install.connectionId)); + const availableInstalledConnectionIds = new Set( + installRows + .filter((install) => + install.enabled + && install.status === "active" + && !["degraded", "failed", "error", "missing_secret"].includes(install.healthStatus) + ) + .map((install) => install.connectionId), + ); const applicableGateways = rows.filter((gateway) => gatewayAppliesToRun({ gateway, @@ -3845,7 +4363,7 @@ export async function createManagedMcpRunConfig(input: { })))) .filter(({ connectionIds }) => connectionIds.size > 0 - && [...connectionIds].every((connectionId) => installedConnectionIds.has(connectionId))) + && [...connectionIds].every((connectionId) => availableInstalledConnectionIds.has(connectionId))) .map(({ gateway }) => gateway); if (gateways.length === 0) return null; @@ -3901,8 +4419,13 @@ export function normalizeModelProfileWakeContext(input: { contextSnapshot: Record; payload: Record | null | undefined; }): Record { - const modelProfileFromPayload = readModelProfileKey(input.payload?.modelProfile); - if (!readContextModelProfile(input.contextSnapshot) && modelProfileFromPayload) { + const modelProfileFromPayload = readModelProfileKey( + input.payload?.modelProfile, + ); + if ( + !readContextModelProfile(input.contextSnapshot) && + modelProfileFromPayload + ) { input.contextSnapshot.modelProfile = modelProfileFromPayload; } return input.contextSnapshot; @@ -3911,7 +4434,11 @@ export function normalizeModelProfileWakeContext(input: { function readAgentRuntimeModelProfile( runtimeConfig: unknown, key: ModelProfileKey, -): { enabled: boolean; adapterConfig: Record; configured: boolean } { +): { + enabled: boolean; + adapterConfig: Record; + configured: boolean; +} { const modelProfiles = parseObject(parseObject(runtimeConfig).modelProfiles); const profile = parseObject(modelProfiles[key]); if (Object.keys(profile).length === 0) { @@ -3952,19 +4479,26 @@ export function resolveModelProfileApplication(input: { }; } - const adapterProfile = input.adapterModelProfiles.find((profile) => profile.key === requested) ?? null; + const adapterProfile = + input.adapterModelProfiles.find((profile) => profile.key === requested) ?? + null; if (!adapterProfile) { return { requested, requestedBy, applied: null, configSource: null, - fallbackReason: input.profileResolutionFallbackReason ?? "adapter_profile_not_supported", + fallbackReason: + input.profileResolutionFallbackReason ?? + "adapter_profile_not_supported", adapterConfig: null, }; } - const runtimeProfile = readAgentRuntimeModelProfile(input.agentRuntimeConfig, requested); + const runtimeProfile = readAgentRuntimeModelProfile( + input.agentRuntimeConfig, + requested, + ); if (!runtimeProfile.enabled) { return { requested, @@ -3980,7 +4514,9 @@ export function resolveModelProfileApplication(input: { requested, requestedBy, applied: requested, - configSource: runtimeProfile.configured ? "agent_runtime" : "adapter_default", + configSource: runtimeProfile.configured + ? "agent_runtime" + : "adapter_default", fallbackReason: null, adapterConfig: { ...parseObject(adapterProfile.adapterConfig), @@ -4085,7 +4621,10 @@ export function summarizeHeartbeatRunListResultJson(input: { } function didAutomaticRecoveryFail( - latestRun: Pick | null, + latestRun: Pick< + typeof heartbeatRuns.$inferSelect, + "status" | "contextSnapshot" + > | null, expectedRetryReason: | "assignment_recovery" | "issue_continuation_needed" @@ -4108,7 +4647,10 @@ function isExecutionReviewParticipantRecoveryRun( ) { if (!run) return false; const context = parseObject(run.contextSnapshot); - return readNonEmptyString(context.retryReason) === EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON; + return ( + readNonEmptyString(context.retryReason) === + EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON + ); } function isExecutionReviewParticipantRecoveryEligibleRun( @@ -4145,10 +4687,17 @@ function normalizeLedgerBillingType(value: unknown): BillingType { } function resolveLedgerBiller(result: AdapterExecutionResult): string { - return readNonEmptyString(result.biller) ?? readNonEmptyString(result.provider) ?? "unknown"; + return ( + readNonEmptyString(result.biller) ?? + readNonEmptyString(result.provider) ?? + "unknown" + ); } -function normalizeBilledCostCents(costUsd: number | null | undefined, billingType: BillingType): number { +function normalizeBilledCostCents( + costUsd: number | null | undefined, + billingType: BillingType, +): number { if (billingType === "subscription_included") return 0; if (typeof costUsd !== "number" || !Number.isFinite(costUsd)) return 0; return Math.max(0, Math.round(costUsd * 100)); @@ -4160,7 +4709,10 @@ export function resolveLedgerCostStatus(input: { cachedInputTokens: number; outputTokens: number; }): CostStatus { - const hasTokenUsage = input.inputTokens > 0 || input.cachedInputTokens > 0 || input.outputTokens > 0; + const hasTokenUsage = + input.inputTokens > 0 || + input.cachedInputTokens > 0 || + input.outputTokens > 0; return input.costUsd == null && hasTokenUsage ? "unpriced" : "reported"; } @@ -4169,11 +4721,19 @@ export function resolveCacheAdjustedCostUsd(input: { cacheAdjustedCostUsd?: number | null; }) { const explicit = input.cacheAdjustedCostUsd; - if (typeof explicit === "number" && Number.isFinite(explicit) && explicit >= 0) { + if ( + typeof explicit === "number" && + Number.isFinite(explicit) && + explicit >= 0 + ) { return explicit; } const reported = input.costUsd; - if (typeof reported === "number" && Number.isFinite(reported) && reported >= 0) { + if ( + typeof reported === "number" && + Number.isFinite(reported) && + reported >= 0 + ) { return reported; } return null; @@ -4228,50 +4788,66 @@ export function buildExplicitResumeSessionOverride(input: { taskSession: ResumeSessionRow | null; sessionCodec: AdapterSessionCodec; }) { - const resumeRunSessionIdAfter = truncateDisplayId(input.resumeRunSessionIdAfter); - const resumeRunSessionIdBefore = truncateDisplayId(input.resumeRunSessionIdBefore); + const resumeRunSessionIdAfter = truncateDisplayId( + input.resumeRunSessionIdAfter, + ); + const resumeRunSessionIdBefore = truncateDisplayId( + input.resumeRunSessionIdBefore, + ); const desiredDisplayId = requiresCanonicalSessionIds(input.adapterType) ? isCanonicalSessionIdForAdapter(input.adapterType, resumeRunSessionIdAfter) ? resumeRunSessionIdAfter - : isCanonicalSessionIdForAdapter(input.adapterType, resumeRunSessionIdBefore) + : isCanonicalSessionIdForAdapter( + input.adapterType, + resumeRunSessionIdBefore, + ) ? resumeRunSessionIdBefore : null - : resumeRunSessionIdAfter ?? resumeRunSessionIdBefore; + : (resumeRunSessionIdAfter ?? resumeRunSessionIdBefore); const runSessionParams = requiresCanonicalSessionIds(input.adapterType) ? normalizeResumeParamsForAdapter( input.adapterType, input.sessionCodec.deserialize(input.resumeRunSessionParams ?? null), ) : null; - const runSessionDisplayId = truncateDisplayId(readNonEmptyString(runSessionParams?.sessionId)); + const runSessionDisplayId = truncateDisplayId( + readNonEmptyString(runSessionParams?.sessionId), + ); const taskSessionParams = normalizeResumeParamsForAdapter( input.adapterType, - input.sessionCodec.deserialize(input.taskSession?.sessionParamsJson ?? null), + input.sessionCodec.deserialize( + input.taskSession?.sessionParamsJson ?? null, + ), ); const taskSessionRawDisplayId = input.taskSession?.sessionDisplayId ?? null; const taskSessionDisplayId = truncateDisplayId( requiresCanonicalSessionIds(input.adapterType) - ? readNonEmptyString(taskSessionParams?.sessionId) ?? - (isCanonicalSessionIdForAdapter(input.adapterType, taskSessionRawDisplayId) ? taskSessionRawDisplayId : null) - : taskSessionRawDisplayId ?? - (input.sessionCodec.getDisplayId ? input.sessionCodec.getDisplayId(taskSessionParams) : null) ?? - readNonEmptyString(taskSessionParams?.sessionId), + ? (readNonEmptyString(taskSessionParams?.sessionId) ?? + (isCanonicalSessionIdForAdapter( + input.adapterType, + taskSessionRawDisplayId, + ) + ? taskSessionRawDisplayId + : null)) + : (taskSessionRawDisplayId ?? + (input.sessionCodec.getDisplayId + ? input.sessionCodec.getDisplayId(taskSessionParams) + : null) ?? + readNonEmptyString(taskSessionParams?.sessionId)), ); const canReuseTaskSessionParams = input.taskSession != null && - (!requiresCanonicalSessionIds(input.adapterType) || taskSessionParams != null) && - ( - input.taskSession.lastRunId === input.resumeFromRunId || - (!!desiredDisplayId && taskSessionDisplayId === desiredDisplayId) - ); - const sessionParams = - canReuseTaskSessionParams - ? taskSessionParams - : runSessionParams - ? runSessionParams - : desiredDisplayId - ? { sessionId: desiredDisplayId } - : null; + (!requiresCanonicalSessionIds(input.adapterType) || + taskSessionParams != null) && + (input.taskSession.lastRunId === input.resumeFromRunId || + (!!desiredDisplayId && taskSessionDisplayId === desiredDisplayId)); + const sessionParams = canReuseTaskSessionParams + ? taskSessionParams + : runSessionParams + ? runSessionParams + : desiredDisplayId + ? { sessionId: desiredDisplayId } + : null; const sessionDisplayId = canReuseTaskSessionParams ? taskSessionDisplayId : runSessionParams @@ -4285,11 +4861,16 @@ export function buildExplicitResumeSessionOverride(input: { }; } -function normalizeUsageTotals(usage: UsageSummary | null | undefined): UsageTotals | null { +function normalizeUsageTotals( + usage: UsageSummary | null | undefined, +): UsageTotals | null { if (!usage) return null; return { inputTokens: Math.max(0, Math.floor(asNumber(usage.inputTokens, 0))), - cachedInputTokens: Math.max(0, Math.floor(asNumber(usage.cachedInputTokens, 0))), + cachedInputTokens: Math.max( + 0, + Math.floor(asNumber(usage.cachedInputTokens, 0)), + ), outputTokens: Math.max(0, Math.floor(asNumber(usage.outputTokens, 0))), }; } @@ -4300,15 +4881,24 @@ function readRawUsageTotals(usageJson: unknown): UsageTotals | null { const inputTokens = Math.max( 0, - Math.floor(asNumber(parsed.rawInputTokens, asNumber(parsed.inputTokens, 0))), + Math.floor( + asNumber(parsed.rawInputTokens, asNumber(parsed.inputTokens, 0)), + ), ); const cachedInputTokens = Math.max( 0, - Math.floor(asNumber(parsed.rawCachedInputTokens, asNumber(parsed.cachedInputTokens, 0))), + Math.floor( + asNumber( + parsed.rawCachedInputTokens, + asNumber(parsed.cachedInputTokens, 0), + ), + ), ); const outputTokens = Math.max( 0, - Math.floor(asNumber(parsed.rawOutputTokens, asNumber(parsed.outputTokens, 0))), + Math.floor( + asNumber(parsed.rawOutputTokens, asNumber(parsed.outputTokens, 0)), + ), ); if (inputTokens <= 0 && cachedInputTokens <= 0 && outputTokens <= 0) { @@ -4322,19 +4912,25 @@ function readRawUsageTotals(usageJson: unknown): UsageTotals | null { }; } -function deriveNormalizedUsageDelta(current: UsageTotals | null, previous: UsageTotals | null): UsageTotals | null { +function deriveNormalizedUsageDelta( + current: UsageTotals | null, + previous: UsageTotals | null, +): UsageTotals | null { if (!current) return null; if (!previous) return { ...current }; - const inputTokens = current.inputTokens >= previous.inputTokens - ? current.inputTokens - previous.inputTokens - : current.inputTokens; - const cachedInputTokens = current.cachedInputTokens >= previous.cachedInputTokens - ? current.cachedInputTokens - previous.cachedInputTokens - : current.cachedInputTokens; - const outputTokens = current.outputTokens >= previous.outputTokens - ? current.outputTokens - previous.outputTokens - : current.outputTokens; + const inputTokens = + current.inputTokens >= previous.inputTokens + ? current.inputTokens - previous.inputTokens + : current.inputTokens; + const cachedInputTokens = + current.cachedInputTokens >= previous.cachedInputTokens + ? current.cachedInputTokens - previous.cachedInputTokens + : current.cachedInputTokens; + const outputTokens = + current.outputTokens >= previous.outputTokens + ? current.outputTokens - previous.outputTokens + : current.outputTokens; return { inputTokens: Math.max(0, inputTokens), @@ -4348,8 +4944,11 @@ function formatCount(value: number | null | undefined) { return value.toLocaleString("en-US"); } -export function parseSessionCompactionPolicy(agent: typeof agents.$inferSelect): SessionCompactionPolicy { - return resolveSessionCompactionPolicy(agent.adapterType, agent.runtimeConfig).policy; +export function parseSessionCompactionPolicy( + agent: typeof agents.$inferSelect, +): SessionCompactionPolicy { + return resolveSessionCompactionPolicy(agent.adapterType, agent.runtimeConfig) + .policy; } export function resolveRuntimeSessionParamsForWorkspace(input: { @@ -4358,7 +4957,9 @@ export function resolveRuntimeSessionParamsForWorkspace(input: { resolvedWorkspace: ResolvedWorkspaceForRun; }) { const { agentId, previousSessionParams, resolvedWorkspace } = input; - const previousSessionId = readNonEmptyString(previousSessionParams?.sessionId); + const previousSessionId = readNonEmptyString( + previousSessionParams?.sessionId, + ); const previousCwd = readNonEmptyString(previousSessionParams?.cwd); if (!previousSessionId || !previousCwd) { return { @@ -4392,7 +4993,9 @@ export function resolveRuntimeSessionParamsForWorkspace(input: { warning: null as string | null, }; } - const previousWorkspaceId = readNonEmptyString(previousSessionParams?.workspaceId); + const previousWorkspaceId = readNonEmptyString( + previousSessionParams?.workspaceId, + ); if ( previousWorkspaceId && resolvedWorkspace.workspaceId && @@ -4408,9 +5011,12 @@ export function resolveRuntimeSessionParamsForWorkspace(input: { ...(previousSessionParams ?? {}), cwd: projectCwd, }; - if (resolvedWorkspace.workspaceId) migratedSessionParams.workspaceId = resolvedWorkspace.workspaceId; - if (resolvedWorkspace.repoUrl) migratedSessionParams.repoUrl = resolvedWorkspace.repoUrl; - if (resolvedWorkspace.repoRef) migratedSessionParams.repoRef = resolvedWorkspace.repoRef; + if (resolvedWorkspace.workspaceId) + migratedSessionParams.workspaceId = resolvedWorkspace.workspaceId; + if (resolvedWorkspace.repoUrl) + migratedSessionParams.repoUrl = resolvedWorkspace.repoUrl; + if (resolvedWorkspace.repoRef) + migratedSessionParams.repoRef = resolvedWorkspace.repoRef; return { sessionParams: migratedSessionParams, @@ -4424,8 +5030,10 @@ function parseIssueAssigneeAdapterOverrides( raw: unknown, ): ParsedIssueAssigneeAdapterOverrides | null { const parsed = parseObject(raw); - const modelProfile = MODEL_PROFILE_KEYS.includes(parsed.modelProfile as ModelProfileKey) - ? parsed.modelProfile as ModelProfileKey + const modelProfile = MODEL_PROFILE_KEYS.includes( + parsed.modelProfile as ModelProfileKey, + ) + ? (parsed.modelProfile as ModelProfileKey) : null; const parsedAdapterConfig = parseObject(parsed.adapterConfig); const adapterConfig = @@ -4434,7 +5042,8 @@ function parseIssueAssigneeAdapterOverrides( typeof parsed.useProjectWorkspace === "boolean" ? parsed.useProjectWorkspace : null; - if (!modelProfile && !adapterConfig && useProjectWorkspace === null) return null; + if (!modelProfile && !adapterConfig && useProjectWorkspace === null) + return null; return { modelProfile, adapterConfig, @@ -4529,7 +5138,11 @@ function allowsIssueInteractionWake( contextSnapshot: Record | null | undefined, ) { const wakeReason = readNonEmptyString(contextSnapshot?.wakeReason); - if (!wakeReason || !ISSUE_TREE_CONTROL_INTERACTION_WAKE_REASONS.has(wakeReason)) return false; + if ( + !wakeReason || + !ISSUE_TREE_CONTROL_INTERACTION_WAKE_REASONS.has(wakeReason) + ) + return false; return Boolean(deriveCommentId(contextSnapshot, null)); } @@ -4595,27 +5208,29 @@ export function isZombieRun( */ export function filterZombieCoalesceTarget< T extends { status: string; id: string }, ->( - target: T | null, - tracked: { has(id: string): boolean }, -): T | null { +>(target: T | null, tracked: { has(id: string): boolean }): T | null { return target && isZombieRun(target, tracked) ? null : target; } export function describeSessionResetReason( contextSnapshot: Record | null | undefined, ) { - if (contextSnapshot?.forceFreshSession === true) return "forceFreshSession was requested"; + if (contextSnapshot?.forceFreshSession === true) + return "forceFreshSession was requested"; const wakeReason = readNonEmptyString(contextSnapshot?.wakeReason); if (wakeReason === "issue_assigned") return "wake reason is issue_assigned"; if (wakeReason === EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON) { return `wake reason is ${EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON}`; } - if (wakeReason === "execution_approval_requested") return "wake reason is execution_approval_requested"; + if (wakeReason === "execution_approval_requested") + return "wake reason is execution_approval_requested"; // PF-4: paired with shouldResetTaskSessionForWake — keep the reason wording // explicit so run logs make session reuse/reset behavior legible. - if (wakeReason === "heartbeat_timer" && !deriveTaskKey(contextSnapshot, null)) { + if ( + wakeReason === "heartbeat_timer" && + !deriveTaskKey(contextSnapshot, null) + ) { return "wake reason is heartbeat_timer (unscoped timer wake starts fresh)"; } return null; @@ -4636,9 +5251,13 @@ const WORKSPACE_SYNC_CONFLICT_SIGNATURES = [ "lacks these prerequisite commits", ]; -export function isWorkspaceSyncConflictFailure(message: string | null | undefined): boolean { +export function isWorkspaceSyncConflictFailure( + message: string | null | undefined, +): boolean { if (!message) return false; - return WORKSPACE_SYNC_CONFLICT_SIGNATURES.some((signature) => message.includes(signature)); + return WORKSPACE_SYNC_CONFLICT_SIGNATURES.some((signature) => + message.includes(signature), + ); } export function shouldDeferFollowupWakeForSameIssue(input: { @@ -4657,9 +5276,11 @@ export function shouldDeferFollowupWakeForSameIssue(input: { const SESSION_CONFIGURED_MODEL_KEY = "__paperclipConfiguredModel"; const SESSION_CONFIG_FINGERPRINT_KEY = "__paperclipConfigFingerprint"; -const SESSION_CONFIG_FINGERPRINT_VERSION_KEY = "__paperclipConfigFingerprintVersion"; +const SESSION_CONFIG_FINGERPRINT_VERSION_KEY = + "__paperclipConfigFingerprintVersion"; const SESSION_CONFIG_CATEGORIES_KEY = "__paperclipConfigCategories"; -const SESSION_CONFIG_CATEGORY_FINGERPRINTS_KEY = "__paperclipConfigCategoryFingerprints"; +const SESSION_CONFIG_CATEGORY_FINGERPRINTS_KEY = + "__paperclipConfigCategoryFingerprints"; const PAPERCLIP_SESSION_METADATA_KEYS = new Set([ SESSION_CONFIGURED_MODEL_KEY, SESSION_CONFIG_FINGERPRINT_KEY, @@ -4692,8 +5313,10 @@ const EFFECTIVE_RUN_WORKSPACE_CONFIG_CATEGORIES = [ "realization", ] as const; -type EffectiveRunSessionConfigCategory = (typeof EFFECTIVE_RUN_SESSION_CONFIG_CATEGORIES)[number]; -type EffectiveRunWorkspaceConfigCategory = (typeof EFFECTIVE_RUN_WORKSPACE_CONFIG_CATEGORIES)[number]; +type EffectiveRunSessionConfigCategory = + (typeof EFFECTIVE_RUN_SESSION_CONFIG_CATEGORIES)[number]; +type EffectiveRunWorkspaceConfigCategory = + (typeof EFFECTIVE_RUN_WORKSPACE_CONFIG_CATEGORIES)[number]; type EffectiveRunSessionConfigMetadata = { version: typeof EFFECTIVE_RUN_CONFIG_FINGERPRINT_VERSION; @@ -4720,7 +5343,8 @@ export type EffectiveRunWorkspaceConfigMetadata = { evaluatedAt: string; }; -type WorkspaceConfigFreshnessDecisionAction = "create" | "reuse" | "refresh" | "replace"; +type WorkspaceConfigFreshnessDecisionAction = + "create" | "reuse" | "refresh" | "replace"; type ExecutionWorkspaceConfigFreshnessDecision = { action: WorkspaceConfigFreshnessDecisionAction; @@ -4750,10 +5374,13 @@ type ExecutionWorkspaceReuseProvisioningPolicy = { shouldPersistLatestWorkspaceConfigMetadata: boolean; }; -type WorkspaceReuseIssueRef = { - id?: string | null; - identifier?: string | null; -} | null | undefined; +type WorkspaceReuseIssueRef = + | { + id?: string | null; + identifier?: string | null; + } + | null + | undefined; export type ExecutionWorkspaceReuseRequestForIssue = { requestedExecutionWorkspaceId: string | null; @@ -4781,15 +5408,19 @@ export function resolveExecutionWorkspaceReuseRequestForIssue(input: { requestedExistingBranch?: string | null; existingExecutionWorkspaceBranchName?: string | null; }): ExecutionWorkspaceReuseRequestForIssue { - const requestedExecutionWorkspaceId = readNonEmptyString(input.issueExecutionWorkspaceId); + const requestedExecutionWorkspaceId = readNonEmptyString( + input.issueExecutionWorkspaceId, + ); // An explicitly pinned existing branch outranks an inherited reuse_existing // binding: a persisted workspace on any other branch (or with no recorded - // branch) is stale for this issue, so dispatch realizes the pinned branch - // instead of restoring the mismatched workspace. - const requestedExistingBranch = readNonEmptyString(input.requestedExistingBranch); + // branch) is stale for this issue, so dispatch realizes the pinned branch. + const requestedExistingBranch = readNonEmptyString( + input.requestedExistingBranch, + ); const existingWorkspaceMatchesRequestedBranch = requestedExistingBranch === null || - readNonEmptyString(input.existingExecutionWorkspaceBranchName) === requestedExistingBranch; + readNonEmptyString(input.existingExecutionWorkspaceBranchName) === + requestedExistingBranch; const requestedShouldReuseExisting = input.issueExecutionWorkspacePreference === "reuse_existing" && requestedExecutionWorkspaceId !== null && @@ -4812,7 +5443,8 @@ export function resolveExecutionWorkspaceReuseProvisioningPolicy(input: { }): ExecutionWorkspaceReuseProvisioningPolicy { const shouldRestoreExistingWorkspace = input.requestedShouldReuseExisting; const replacementClassDrift = - input.requestedShouldReuseExisting && input.workspaceConfigFreshness.action === "replace"; + input.requestedShouldReuseExisting && + input.workspaceConfigFreshness.action === "replace"; return { shouldRestoreExistingWorkspace, @@ -4825,23 +5457,28 @@ export function resolveExecutionWorkspaceReuseProvisioningPolicy(input: { } function formatInheritedExecutionWorkspaceReuseFailure(input: { - reason: "inherited_workspace_reuse_failed" | "inherited_workspace_reuse_unavailable"; + reason: + | "inherited_workspace_reuse_failed" + | "inherited_workspace_reuse_unavailable"; issueRef: WorkspaceReuseIssueRef; runId: string; executionWorkspaceId: string | null | undefined; workspaceConfigFreshness: ExecutionWorkspaceConfigFreshnessDecision; cause?: unknown; }) { - const issueLabel = input.issueRef?.identifier ?? input.issueRef?.id ?? input.runId; + const issueLabel = + input.issueRef?.identifier ?? input.issueRef?.id ?? input.runId; const workspaceLabel = input.executionWorkspaceId ?? "unknown workspace"; - const causeMessage = input.cause instanceof Error - ? input.cause.message - : input.cause != null - ? String(input.cause) - : null; - const remediation = input.reason === "inherited_workspace_reuse_failed" - ? "Inspect the referenced execution workspace restore/provision logs, repair or unarchive the workspace, or intentionally clear the issue's reuse_existing workspace binding before retrying." - : "Repair or unarchive the referenced execution workspace, or intentionally clear the issue's reuse_existing workspace binding before retrying."; + const causeMessage = + input.cause instanceof Error + ? input.cause.message + : input.cause != null + ? String(input.cause) + : null; + const remediation = + input.reason === "inherited_workspace_reuse_failed" + ? "Inspect the referenced execution workspace restore/provision logs, repair or unarchive the workspace, or intentionally clear the issue's reuse_existing workspace binding before retrying." + : "Repair or unarchive the referenced execution workspace, or intentionally clear the issue's reuse_existing workspace binding before retrying."; const message = causeMessage ? `Issue ${issueLabel} requested inherited execution workspace reuse for ${workspaceLabel}, but the workspace could not be restored because ${causeMessage}.` : `Issue ${issueLabel} requested inherited execution workspace reuse for ${workspaceLabel}, but the workspace could not be restored.`; @@ -4849,7 +5486,9 @@ function formatInheritedExecutionWorkspaceReuseFailure(input: { return `${message} ${remediation}`; } -export async function provisionExecutionWorkspaceForFreshnessDecision(input: { +export async function provisionExecutionWorkspaceForFreshnessDecision< + T extends { warnings?: string[] }, +>(input: { requestedShouldReuseExisting: boolean; existingExecutionWorkspaceId?: string | null; issueRef: WorkspaceReuseIssueRef; @@ -4895,18 +5534,22 @@ export async function provisionExecutionWorkspaceForFreshnessDecision = { +const EFFECTIVE_RUN_SESSION_CONFIG_CATEGORY_LABELS: Record< + EffectiveRunSessionConfigCategory, + string +> = { adapter: "adapter", adapterConfig: "adapter config", agentRuntimeConfig: "agent runtime config", @@ -4929,7 +5575,10 @@ const EFFECTIVE_RUN_SESSION_CONFIG_CATEGORY_LABELS: Record = { +const EFFECTIVE_RUN_WORKSPACE_CONFIG_CATEGORY_LABELS: Record< + EffectiveRunWorkspaceConfigCategory, + string +> = { mode: "workspace mode", projectWorkspace: "project workspace", strategy: "workspace strategy", @@ -4939,14 +5588,15 @@ const EFFECTIVE_RUN_WORKSPACE_CONFIG_CATEGORY_LABELS: Record([ - "mode", - "projectWorkspace", - "strategy", - "repo", - "environment", - "realization", -]); +const WORKSPACE_REPLACEMENT_CONFIG_CATEGORIES = + new Set([ + "mode", + "projectWorkspace", + "strategy", + "repo", + "environment", + "realization", + ]); function parseStoredConfigCategoryFingerprints(value: unknown) { const parsed = parseObject(value); @@ -4961,13 +5611,17 @@ function parseStoredConfigCategoryFingerprints(value: unknown) { function readConfigCategoriesFromSessionParams( sessionParams: Record | null | undefined, ) { - const rawCategories = Array.isArray(sessionParams?.[SESSION_CONFIG_CATEGORIES_KEY]) + const rawCategories = Array.isArray( + sessionParams?.[SESSION_CONFIG_CATEGORIES_KEY], + ) ? sessionParams?.[SESSION_CONFIG_CATEGORIES_KEY] : []; return rawCategories.filter( (category): category is EffectiveRunSessionConfigCategory => typeof category === "string" && - (EFFECTIVE_RUN_SESSION_CONFIG_CATEGORIES as readonly string[]).includes(category), + (EFFECTIVE_RUN_SESSION_CONFIG_CATEGORIES as readonly string[]).includes( + category, + ), ); } @@ -4975,8 +5629,13 @@ function readConfigFingerprintFromSessionParams( sessionParams: Record | null | undefined, ) { if (!sessionParams) return null; - const fingerprint = readNonEmptyString(sessionParams[SESSION_CONFIG_FINGERPRINT_KEY]); - const version = asNumber(sessionParams[SESSION_CONFIG_FINGERPRINT_VERSION_KEY], 0); + const fingerprint = readNonEmptyString( + sessionParams[SESSION_CONFIG_FINGERPRINT_KEY], + ); + const version = asNumber( + sessionParams[SESSION_CONFIG_FINGERPRINT_VERSION_KEY], + 0, + ); if (!fingerprint || version <= 0) return null; return { fingerprint, @@ -4988,8 +5647,12 @@ function readConfigFingerprintFromSessionParams( }; } -function describeEffectiveRunConfigCategories(categories: readonly EffectiveRunSessionConfigCategory[]) { - return categories.map((category) => EFFECTIVE_RUN_SESSION_CONFIG_CATEGORY_LABELS[category]).join(", "); +function describeEffectiveRunConfigCategories( + categories: readonly EffectiveRunSessionConfigCategory[], +) { + return categories + .map((category) => EFFECTIVE_RUN_SESSION_CONFIG_CATEGORY_LABELS[category]) + .join(", "); } function changedEffectiveRunSessionConfigCategories(input: { @@ -4999,7 +5662,9 @@ function changedEffectiveRunSessionConfigCategories(input: { const changed = EFFECTIVE_RUN_SESSION_CONFIG_CATEGORIES.filter( (category) => input.previous[category] !== input.next[category], ); - return changed.length > 0 ? changed : [...EFFECTIVE_RUN_SESSION_CONFIG_CATEGORIES]; + return changed.length > 0 + ? changed + : [...EFFECTIVE_RUN_SESSION_CONFIG_CATEGORIES]; } function parseStoredWorkspaceConfigCategoryFingerprints(value: unknown) { @@ -5017,29 +5682,39 @@ function readWorkspaceConfigCategoriesFromMetadata(value: unknown) { return rawCategories.filter( (category): category is EffectiveRunWorkspaceConfigCategory => typeof category === "string" && - (EFFECTIVE_RUN_WORKSPACE_CONFIG_CATEGORIES as readonly string[]).includes(category), + (EFFECTIVE_RUN_WORKSPACE_CONFIG_CATEGORIES as readonly string[]).includes( + category, + ), ); } function readWorkspaceConfigFingerprintFromMetadata( metadata: Record | null | undefined, ) { - const raw = parseObject(metadata?.[WORKSPACE_CONFIG_FINGERPRINT_METADATA_KEY]); - const fingerprint = readNonEmptyString(raw.workspaceHash) ?? readNonEmptyString(raw.fingerprint); + const raw = parseObject( + metadata?.[WORKSPACE_CONFIG_FINGERPRINT_METADATA_KEY], + ); + const fingerprint = + readNonEmptyString(raw.workspaceHash) ?? + readNonEmptyString(raw.fingerprint); const version = asNumber(raw.version, 0); if (!fingerprint || version <= 0) return null; return { fingerprint, version, categories: readWorkspaceConfigCategoriesFromMetadata(raw.categories), - categoryFingerprints: parseStoredWorkspaceConfigCategoryFingerprints(raw.categoryFingerprints), + categoryFingerprints: parseStoredWorkspaceConfigCategoryFingerprints( + raw.categoryFingerprints, + ), }; } function describeEffectiveRunWorkspaceConfigCategories( categories: readonly EffectiveRunWorkspaceConfigCategory[], ) { - return categories.map((category) => EFFECTIVE_RUN_WORKSPACE_CONFIG_CATEGORY_LABELS[category]).join(", "); + return categories + .map((category) => EFFECTIVE_RUN_WORKSPACE_CONFIG_CATEGORY_LABELS[category]) + .join(", "); } function changedEffectiveRunWorkspaceConfigCategories(input: { @@ -5049,10 +5724,14 @@ function changedEffectiveRunWorkspaceConfigCategories(input: { const changed = EFFECTIVE_RUN_WORKSPACE_CONFIG_CATEGORIES.filter( (category) => input.previous[category] !== input.next[category], ); - return changed.length > 0 ? changed : [...EFFECTIVE_RUN_WORKSPACE_CONFIG_CATEGORIES]; + return changed.length > 0 + ? changed + : [...EFFECTIVE_RUN_WORKSPACE_CONFIG_CATEGORIES]; } -function workspaceConfigFreshnessActionLabel(action: WorkspaceConfigFreshnessDecisionAction) { +function workspaceConfigFreshnessActionLabel( + action: WorkspaceConfigFreshnessDecisionAction, +) { switch (action) { case "refresh": return "refreshed execution workspace config"; @@ -5065,8 +5744,14 @@ function workspaceConfigFreshnessActionLabel(action: WorkspaceConfigFreshnessDec } } -export function buildWorkspaceConfigFreshnessOperation(input: WorkspaceConfigFreshnessOperationInput) { - if (!input.reuseRequested || !input.hasExistingWorkspace || input.decision.reasons.length === 0) { +export function buildWorkspaceConfigFreshnessOperation( + input: WorkspaceConfigFreshnessOperationInput, +) { + if ( + !input.reuseRequested || + !input.hasExistingWorkspace || + input.decision.reasons.length === 0 + ) { return null; } @@ -5074,7 +5759,9 @@ export function buildWorkspaceConfigFreshnessOperation(input: WorkspaceConfigFre (category) => EFFECTIVE_RUN_WORKSPACE_CONFIG_CATEGORY_LABELS[category], ); const categorySummary = - changedCategoryLabels.length > 0 ? ` (${changedCategoryLabels.join(", ")})` : ""; + changedCategoryLabels.length > 0 + ? ` (${changedCategoryLabels.join(", ")})` + : ""; const reasonSummary = input.decision.reasons.join("; "); return { @@ -5091,15 +5778,16 @@ export function buildWorkspaceConfigFreshnessOperation(input: WorkspaceConfigFre previousWorkspaceId: input.previousWorkspaceId, activeWorkspaceId: input.activeWorkspaceId, }, - system: - `[paperclip] ${workspaceConfigFreshnessActionLabel(input.decision.action)} after config freshness check${categorySummary}: ${reasonSummary}\n`, + system: `[paperclip] ${workspaceConfigFreshnessActionLabel(input.decision.action)} after config freshness check${categorySummary}: ${reasonSummary}\n`, }; } -async function recordWorkspaceConfigFreshnessOperation(input: WorkspaceConfigFreshnessOperationInput & { - recorder: WorkspaceOperationRecorder; - runId: string; -}) { +async function recordWorkspaceConfigFreshnessOperation( + input: WorkspaceConfigFreshnessOperationInput & { + recorder: WorkspaceOperationRecorder; + runId: string; + }, +) { const operation = buildWorkspaceConfigFreshnessOperation(input); if (!operation) return; @@ -5136,12 +5824,16 @@ function sanitizeSecretManifestForConfigFingerprint( envKey: readNonEmptyString(record.envKey), secretId: readNonEmptyString(record.secretId) ?? "", bindingId: readNonEmptyString(record.bindingId), - version: typeof record.version === "number" && Number.isFinite(record.version) - ? record.version - : readNonEmptyString(record.version), + version: + typeof record.version === "number" && Number.isFinite(record.version) + ? record.version + : readNonEmptyString(record.version), provider: readNonEmptyString(record.provider), providerVersionRef: readNonEmptyString(record.providerVersionRef), - outcome: record.outcome === "success" || record.outcome === "failure" ? record.outcome : null, + outcome: + record.outcome === "success" || record.outcome === "failure" + ? record.outcome + : null, }; }); } @@ -5153,10 +5845,11 @@ async function hashFileContentsForConfigFingerprint(filePath: string) { function isPathInsideRoot(input: { rootPath: string; filePath: string }) { const relative = path.relative(input.rootPath, input.filePath); - return relative === "" || ( - relative.length > 0 - && !relative.startsWith("..") - && !path.isAbsolute(relative) + return ( + relative === "" || + (relative.length > 0 && + !relative.startsWith("..") && + !path.isAbsolute(relative)) ); } @@ -5164,17 +5857,26 @@ function resolveRootBoundInstructionsFingerprintPath(input: { instructionsFilePath: string | null; instructionsRootPath: string | null; instructionsEntryFile: string | null; -}): { filePath: string; skippedReason: null } | { filePath: null; skippedReason: string | null } { - if (!input.instructionsRootPath || !path.isAbsolute(input.instructionsRootPath)) { +}): + | { filePath: string; skippedReason: null } + | { filePath: null; skippedReason: string | null } { + if ( + !input.instructionsRootPath || + !path.isAbsolute(input.instructionsRootPath) + ) { return { filePath: null, - skippedReason: input.instructionsFilePath ? "missing_absolute_root" : null, + skippedReason: input.instructionsFilePath + ? "missing_absolute_root" + : null, }; } const rootPath = path.resolve(input.instructionsRootPath); - const candidatePath = input.instructionsEntryFile ?? input.instructionsFilePath; - if (!candidatePath) return { filePath: null, skippedReason: "missing_entry_file" }; + const candidatePath = + input.instructionsEntryFile ?? input.instructionsFilePath; + if (!candidatePath) + return { filePath: null, skippedReason: "missing_entry_file" }; const resolvedPath = path.isAbsolute(candidatePath) ? path.resolve(candidatePath) @@ -5187,33 +5889,46 @@ function resolveRootBoundInstructionsFingerprintPath(input: { return { filePath: resolvedPath, skippedReason: null }; } -async function resolveInstructionsConfigFingerprintMetadata(config: Record) { +async function resolveInstructionsConfigFingerprintMetadata( + config: Record, +) { const instructionsFilePath = readNonEmptyString(config.instructionsFilePath); const instructionsRootPath = readNonEmptyString(config.instructionsRootPath); - const instructionsEntryFile = readNonEmptyString(config.instructionsEntryFile); + const instructionsEntryFile = readNonEmptyString( + config.instructionsEntryFile, + ); const resolved = resolveRootBoundInstructionsFingerprintPath({ instructionsFilePath, instructionsRootPath, instructionsEntryFile, }); - const configuredPath = resolved.filePath ?? instructionsFilePath ?? ( - instructionsRootPath && instructionsEntryFile + const configuredPath = + resolved.filePath ?? + instructionsFilePath ?? + (instructionsRootPath && instructionsEntryFile ? path.resolve(instructionsRootPath, instructionsEntryFile) - : null - ); - if (!configuredPath && !instructionsRootPath && !instructionsEntryFile) return null; + : null); + if (!configuredPath && !instructionsRootPath && !instructionsEntryFile) + return null; const metadata: Record = { configured: true, bundleMode: readNonEmptyString(config.instructionsBundleMode), entryFile: instructionsEntryFile, - pathKind: configuredPath ? (path.isAbsolute(configuredPath) ? "absolute" : "relative") : null, + pathKind: configuredPath + ? path.isAbsolute(configuredPath) + ? "absolute" + : "relative" + : null, readPolicy: "root_bound", }; - if (resolved.skippedReason) metadata.readSkippedReason = resolved.skippedReason; + if (resolved.skippedReason) + metadata.readSkippedReason = resolved.skippedReason; if (resolved.filePath) { try { - metadata.contentHash = await hashFileContentsForConfigFingerprint(resolved.filePath); + metadata.contentHash = await hashFileContentsForConfigFingerprint( + resolved.filePath, + ); metadata.readable = true; } catch { metadata.readable = false; @@ -5238,7 +5953,9 @@ function buildSessionConfigCategoryValues(input: { runtimeSkills: unknown; agentConfigRevision: unknown; }) { - const sanitizedSecretManifest = sanitizeSecretManifestForConfigFingerprint(input.secretManifest); + const sanitizedSecretManifest = sanitizeSecretManifestForConfigFingerprint( + input.secretManifest, + ); const workspaceConfig = { ...parseObject(input.workspaceConfig) }; // issues.updatedAt also advances for comments and status changes. Those are // wake deltas, not execution-workspace configuration changes, so including @@ -5283,7 +6000,9 @@ export async function buildEffectiveRunSessionConfigMetadata(input: { agentConfigRevision?: unknown; }): Promise { const secretManifest = input.secretManifest ?? []; - const instructions = await resolveInstructionsConfigFingerprintMetadata(input.effectiveAdapterConfig); + const instructions = await resolveInstructionsConfigFingerprintMetadata( + input.effectiveAdapterConfig, + ); const categoryValues = buildSessionConfigCategoryValues({ adapterType: input.adapterType, effectiveAdapterConfig: input.effectiveAdapterConfig, @@ -5405,9 +6124,10 @@ export function buildEffectiveRunWorkspaceConfigMetadata(input: { subcategories: EFFECTIVE_RUN_WORKSPACE_CONFIG_CATEGORIES, secretManifest, }); - const evaluatedAt = input.evaluatedAt instanceof Date - ? input.evaluatedAt.toISOString() - : readNonEmptyString(input.evaluatedAt) ?? new Date().toISOString(); + const evaluatedAt = + input.evaluatedAt instanceof Date + ? input.evaluatedAt.toISOString() + : (readNonEmptyString(input.evaluatedAt) ?? new Date().toISOString()); return { version: EFFECTIVE_RUN_CONFIG_FINGERPRINT_VERSION, fingerprint: fingerprints.workspaceFingerprint.fingerprint, @@ -5438,7 +6158,9 @@ export function resolveExecutionWorkspaceConfigFreshness(input: { }; } - const stored = readWorkspaceConfigFingerprintFromMetadata(input.existingWorkspaceMetadata); + const stored = readWorkspaceConfigFingerprintFromMetadata( + input.existingWorkspaceMetadata, + ); const previous = stored ? { version: stored.version, @@ -5461,7 +6183,9 @@ export function resolveExecutionWorkspaceConfigFreshness(input: { reasons: [], changedCategories: [], storedFingerprint: stored?.fingerprint ?? null, - inferredFingerprint: stored ? null : input.inferredMetadata?.fingerprint ?? null, + inferredFingerprint: stored + ? null + : (input.inferredMetadata?.fingerprint ?? null), nextFingerprint: null, storedFingerprintPresent: Boolean(stored), }; @@ -5472,7 +6196,9 @@ export function resolveExecutionWorkspaceConfigFreshness(input: { action: "replace", shouldReuseExisting: false, shouldRefreshConfigSnapshot: false, - reasons: ["execution workspace configuration fingerprint metadata is missing"], + reasons: [ + "execution workspace configuration fingerprint metadata is missing", + ], changedCategories: [...input.nextMetadata.categories], storedFingerprint: null, inferredFingerprint: null, @@ -5491,7 +6217,9 @@ export function resolveExecutionWorkspaceConfigFreshness(input: { ], changedCategories: [...input.nextMetadata.categories], storedFingerprint: stored?.fingerprint ?? null, - inferredFingerprint: stored ? null : input.inferredMetadata?.fingerprint ?? null, + inferredFingerprint: stored + ? null + : (input.inferredMetadata?.fingerprint ?? null), nextFingerprint: input.nextMetadata.fingerprint, storedFingerprintPresent: Boolean(stored), }; @@ -5502,10 +6230,14 @@ export function resolveExecutionWorkspaceConfigFreshness(input: { action: "reuse", shouldReuseExisting: true, shouldRefreshConfigSnapshot: !stored, - reasons: stored ? [] : ["execution workspace configuration fingerprint metadata is missing"], + reasons: stored + ? [] + : ["execution workspace configuration fingerprint metadata is missing"], changedCategories: [], storedFingerprint: stored?.fingerprint ?? null, - inferredFingerprint: stored ? null : input.inferredMetadata?.fingerprint ?? null, + inferredFingerprint: stored + ? null + : (input.inferredMetadata?.fingerprint ?? null), nextFingerprint: input.nextMetadata.fingerprint, storedFingerprintPresent: Boolean(stored), }; @@ -5516,9 +6248,11 @@ export function resolveExecutionWorkspaceConfigFreshness(input: { next: input.nextMetadata.categoryFingerprints, }); const replacementRequired = changedCategories.some((category) => - WORKSPACE_REPLACEMENT_CONFIG_CATEGORIES.has(category) + WORKSPACE_REPLACEMENT_CONFIG_CATEGORIES.has(category), ); - const action: WorkspaceConfigFreshnessDecisionAction = replacementRequired ? "replace" : "refresh"; + const action: WorkspaceConfigFreshnessDecisionAction = replacementRequired + ? "replace" + : "refresh"; return { action, shouldReuseExisting: action !== "replace", @@ -5528,7 +6262,9 @@ export function resolveExecutionWorkspaceConfigFreshness(input: { ], changedCategories, storedFingerprint: stored?.fingerprint ?? null, - inferredFingerprint: stored ? null : input.inferredMetadata?.fingerprint ?? null, + inferredFingerprint: stored + ? null + : (input.inferredMetadata?.fingerprint ?? null), nextFingerprint: input.nextMetadata.fingerprint, storedFingerprintPresent: Boolean(stored), }; @@ -5552,7 +6288,8 @@ function attachPaperclipSessionMetadataToSessionParams( next[SESSION_CONFIG_FINGERPRINT_KEY] = configMetadata.fingerprint; next[SESSION_CONFIG_FINGERPRINT_VERSION_KEY] = configMetadata.version; next[SESSION_CONFIG_CATEGORIES_KEY] = configMetadata.categories; - next[SESSION_CONFIG_CATEGORY_FINGERPRINTS_KEY] = configMetadata.categoryFingerprints; + next[SESSION_CONFIG_CATEGORY_FINGERPRINTS_KEY] = + configMetadata.categoryFingerprints; } return next; } @@ -5612,27 +6349,41 @@ export function resolveTaskSessionConfigFreshness(input: { } const reasons: string[] = []; - const storedConfig = readConfigFingerprintFromSessionParams(input.taskSessionParams); - const taskSessionConfiguredModel = readConfiguredModelFromSessionParams(input.taskSessionParams); + const storedConfig = readConfigFingerprintFromSessionParams( + input.taskSessionParams, + ); + const taskSessionConfiguredModel = readConfiguredModelFromSessionParams( + input.taskSessionParams, + ); const modelChangedSinceTaskSession = shouldResetTaskSessionForModelChange({ configuredModel: input.configuredModel, taskSessionParams: input.taskSessionParams, }); if (modelChangedSinceTaskSession && taskSessionConfiguredModel) { - reasons.push(`configured model changed from "${taskSessionConfiguredModel}" to "${input.configuredModel}"`); + reasons.push( + `configured model changed from "${taskSessionConfiguredModel}" to "${input.configuredModel}"`, + ); } let changedCategories: EffectiveRunSessionConfigCategory[] = []; if (input.configMetadata) { if (!storedConfig && !input.preserveLegacySessionWithoutConfigMetadata) { changedCategories = [...input.configMetadata.categories]; - reasons.push("effective run configuration fingerprint metadata is missing"); - } else if (storedConfig && storedConfig.version !== input.configMetadata.version) { + reasons.push( + "effective run configuration fingerprint metadata is missing", + ); + } else if ( + storedConfig && + storedConfig.version !== input.configMetadata.version + ) { changedCategories = [...input.configMetadata.categories]; reasons.push( `effective run configuration fingerprint version changed from ${storedConfig.version} to ${input.configMetadata.version}`, ); - } else if (storedConfig && storedConfig.fingerprint !== input.configMetadata.fingerprint) { + } else if ( + storedConfig && + storedConfig.fingerprint !== input.configMetadata.fingerprint + ) { changedCategories = changedEffectiveRunSessionConfigCategories({ previous: storedConfig.categoryFingerprints, next: input.configMetadata.categoryFingerprints, @@ -5701,11 +6452,17 @@ export function shouldQueueFollowupForRunningIssueWake(input: { return true; } const wakeReason = readNonEmptyString(input.contextSnapshot?.wakeReason); - return Boolean(wakeReason && RUNNING_ISSUE_WAKE_REASONS_REQUIRING_FOLLOWUP.has(wakeReason)); + return Boolean( + wakeReason && RUNNING_ISSUE_WAKE_REASONS_REQUIRING_FOLLOWUP.has(wakeReason), + ); } function isCheckoutConflictError(error: unknown): boolean { - return error instanceof HttpError && error.status === 409 && error.message === "Issue checkout conflict"; + return ( + error instanceof HttpError && + error.status === 409 && + error.message === "Issue checkout conflict" + ); } function deriveCommentId( @@ -5774,11 +6531,16 @@ function enrichWakeContextSnapshot(input: { payload: Record | null; }) { const { contextSnapshot, reason, source, triggerDetail, payload } = input; - const issueIdFromPayload = readNonEmptyString(payload?.["issueId"]) ?? readNonEmptyString(payload?.["taskId"]); + const issueIdFromPayload = + readNonEmptyString(payload?.["issueId"]) ?? + readNonEmptyString(payload?.["taskId"]); const commentIdFromPayload = readNonEmptyString(payload?.["commentId"]); const taskKey = deriveTaskKey(contextSnapshot, payload); const wakeCommentId = deriveCommentId(contextSnapshot, payload); - const wakeCommentIds = mergeWakeCommentIds(contextSnapshot, commentIdFromPayload); + const wakeCommentIds = mergeWakeCommentIds( + contextSnapshot, + commentIdFromPayload, + ); if (!readNonEmptyString(contextSnapshot["wakeReason"]) && reason) { contextSnapshot.wakeReason = reason; @@ -5792,7 +6554,10 @@ function enrichWakeContextSnapshot(input: { if (!readNonEmptyString(contextSnapshot["taskKey"]) && taskKey) { contextSnapshot.taskKey = taskKey; } - if (!readNonEmptyString(contextSnapshot["commentId"]) && commentIdFromPayload) { + if ( + !readNonEmptyString(contextSnapshot["commentId"]) && + commentIdFromPayload + ) { contextSnapshot.commentId = commentIdFromPayload; } if (wakeCommentIds.length > 0) { @@ -5803,13 +6568,19 @@ function enrichWakeContextSnapshot(input: { // Once comment ids are normalized into the snapshot, rebuild the structured // wake payload from those ids later instead of carrying forward stale data. delete contextSnapshot[PAPERCLIP_WAKE_PAYLOAD_KEY]; - } else if (!readNonEmptyString(contextSnapshot["wakeCommentId"]) && wakeCommentId) { + } else if ( + !readNonEmptyString(contextSnapshot["wakeCommentId"]) && + wakeCommentId + ) { contextSnapshot.wakeCommentId = wakeCommentId; } if (!readNonEmptyString(contextSnapshot["wakeSource"]) && source) { contextSnapshot.wakeSource = source; } - if (!readNonEmptyString(contextSnapshot["wakeTriggerDetail"]) && triggerDetail) { + if ( + !readNonEmptyString(contextSnapshot["wakeTriggerDetail"]) && + triggerDetail + ) { contextSnapshot.wakeTriggerDetail = triggerDetail; } normalizeModelProfileWakeContext({ contextSnapshot, payload }); @@ -5834,18 +6605,26 @@ const INTERACTION_CONTINUATION_CONTEXT_KEYS = [ "newlyResolvedItemIds", ] as const; -function isInteractionResolutionWakePayload(payload: Record | null | undefined) { +function isInteractionResolutionWakePayload( + payload: Record | null | undefined, +) { return readNonEmptyString(payload?.mutation) === "interaction"; } -function clearInteractionContinuationWakeContext(contextSnapshot: Record) { +function clearInteractionContinuationWakeContext( + contextSnapshot: Record, +) { for (const key of INTERACTION_CONTINUATION_CONTEXT_KEYS) { delete contextSnapshot[key]; } } -function hasInteractionContinuationWakeContext(contextSnapshot: Record) { - return INTERACTION_CONTINUATION_CONTEXT_KEYS.some((key) => readNonEmptyString(contextSnapshot[key])); +function hasInteractionContinuationWakeContext( + contextSnapshot: Record, +) { + return INTERACTION_CONTINUATION_CONTEXT_KEYS.some((key) => + readNonEmptyString(contextSnapshot[key]), + ); } function normalizeInteractionContinuationWakeContext( @@ -5883,21 +6662,28 @@ async function resolveAcceptedPlanWakeRoutingDecision(args: { }) .from(issuePlanDecompositions) .innerJoin(issues, eq(issues.id, issuePlanDecompositions.sourceIssueId)) - .where(and( - eq(issuePlanDecompositions.companyId, args.companyId), - eq(issuePlanDecompositions.ownerAgentId, args.agentId), - eq(issuePlanDecompositions.status, "in_flight"), - )) - .orderBy(desc(issuePlanDecompositions.updatedAt), asc(issuePlanDecompositions.createdAt)); + .where( + and( + eq(issuePlanDecompositions.companyId, args.companyId), + eq(issuePlanDecompositions.ownerAgentId, args.agentId), + eq(issuePlanDecompositions.status, "in_flight"), + ), + ) + .orderBy( + desc(issuePlanDecompositions.updatedAt), + asc(issuePlanDecompositions.createdAt), + ); if (activeClaims.length === 0) return null; - if (activeClaims.some((claim) => claim.sourceIssueId === args.issueId)) return null; + if (activeClaims.some((claim) => claim.sourceIssueId === args.issueId)) + return null; const otherActiveClaim = activeClaims[0]; if (!otherActiveClaim) return null; const hasAcceptedContinuationWake = - readNonEmptyString(args.contextSnapshot.interactionKind) === "request_confirmation" && + readNonEmptyString(args.contextSnapshot.interactionKind) === + "request_confirmation" && readNonEmptyString(args.contextSnapshot.interactionStatus) === "accepted"; return { @@ -5919,7 +6705,10 @@ export function mergeCoalescedContextSnapshot( ...existing, ...incoming, }; - if (existing.forceFreshSession === true || incoming.forceFreshSession === true) { + if ( + existing.forceFreshSession === true || + incoming.forceFreshSession === true + ) { merged.forceFreshSession = true; } const mergedCommentIds = mergeWakeCommentIds(existing, incoming); @@ -5948,28 +6737,24 @@ export async function buildPaperclipWakePayload(input: { db: Db; companyId: string; contextSnapshot: Record; - continuationSummary?: - | { - key: string; - title: string | null; - body: string; - sourceTrust?: SourceTrustMetadata | null; - updatedAt: Date; - } - | null; - issueSummary?: - | { - id: string; - identifier: string | null; - title: string; - description: string | null; - status: string; - priority: string; - workMode: string; - projectId?: string | null; - executionPolicy?: unknown; - } - | null; + continuationSummary?: { + key: string; + title: string | null; + body: string; + sourceTrust?: SourceTrustMetadata | null; + updatedAt: Date; + } | null; + issueSummary?: { + id: string; + identifier: string | null; + title: string; + description: string | null; + status: string; + priority: string; + workMode: string; + projectId?: string | null; + executionPolicy?: unknown; + } | null; exposeLowTrustRaw?: boolean; // Experimental: agents write user-interaction content in ASD-STE100 // Simplified Technical English (rendered as a prompt directive downstream). @@ -5977,10 +6762,14 @@ export async function buildPaperclipWakePayload(input: { }) { const executionStage = parseObject(input.contextSnapshot.executionStage); const commentIds = extractWakeCommentIds(input.contextSnapshot); - const annotationCommentId = readNonEmptyString(input.contextSnapshot.annotationCommentId); + const annotationCommentId = readNonEmptyString( + input.contextSnapshot.annotationCommentId, + ); const issueId = readNonEmptyString(input.contextSnapshot.issueId); const continuationSummary = input.continuationSummary ?? null; - const agentMessage = parseObject(input.contextSnapshot[PAPERCLIP_AGENT_MESSAGE_KEY]); + const agentMessage = parseObject( + input.contextSnapshot[PAPERCLIP_AGENT_MESSAGE_KEY], + ); const agentMessageText = sanitizeAgentSessionMessageText(agentMessage.text); const issueSummary = input.issueSummary ?? @@ -5996,15 +6785,18 @@ export async function buildPaperclipWakePayload(input: { workMode: issues.workMode, }) .from(issues) - .where(and(eq(issues.id, issueId), eq(issues.companyId, input.companyId))) + .where( + and(eq(issues.id, issueId), eq(issues.companyId, input.companyId)), + ) .then((rows) => rows[0] ?? null) : null); if ( - commentIds.length === 0 - && Object.keys(executionStage).length === 0 - && !issueSummary - && !agentMessageText - ) return null; + commentIds.length === 0 && + Object.keys(executionStage).length === 0 && + !issueSummary && + !agentMessageText + ) + return null; const commentRows = commentIds.length === 0 @@ -6035,10 +6827,13 @@ export async function buildPaperclipWakePayload(input: { ), ); - const commentsById = new Map(commentRows.map((comment) => [comment.id, comment])); + const commentsById = new Map( + commentRows.map((comment) => [comment.id, comment]), + ); const issueDescription = issueSummary?.description ?? null; const issueDescriptionTruncated = - issueDescription !== null && issueDescription.length > MAX_INLINE_WAKE_ISSUE_DESCRIPTION_CHARS; + issueDescription !== null && + issueDescription.length > MAX_INLINE_WAKE_ISSUE_DESCRIPTION_CHARS; const inlineIssueDescription = issueDescriptionTruncated ? issueDescription.slice(0, MAX_INLINE_WAKE_ISSUE_DESCRIPTION_CHARS) : issueDescription; @@ -6064,15 +6859,24 @@ export async function buildPaperclipWakePayload(input: { } const deletedAt = row.deletedAt ?? null; - const safeRow = deletedAt || input.exposeLowTrustRaw ? row : sanitizeQuarantinedCommentForHigherTrust(row); + const safeRow = + deletedAt || input.exposeLowTrustRaw + ? row + : sanitizeQuarantinedCommentForHigherTrust(row); const fullBody = deletedAt ? "" : safeRow.body; - const allowedBodyChars = Math.min(MAX_INLINE_WAKE_COMMENT_BODY_CHARS, remainingBodyChars); + const allowedBodyChars = Math.min( + MAX_INLINE_WAKE_COMMENT_BODY_CHARS, + remainingBodyChars, + ); if (allowedBodyChars <= 0) { truncated = true; break; } - const body = fullBody.length > allowedBodyChars ? fullBody.slice(0, allowedBodyChars) : fullBody; + const body = + fullBody.length > allowedBodyChars + ? fullBody.slice(0, allowedBodyChars) + : fullBody; const bodyTruncated = body.length < fullBody.length; if (bodyTruncated) truncated = true; remainingBodyChars -= body.length; @@ -6080,16 +6884,18 @@ export async function buildPaperclipWakePayload(input: { comments.push({ id: row.id, issueId: row.issueId, - authorType: row.authorType ?? (row.authorAgentId ? "agent" : row.authorUserId ? "user" : "system"), + authorType: + row.authorType ?? + (row.authorAgentId ? "agent" : row.authorUserId ? "user" : "system"), body, bodyTruncated, - presentation: deletedAt ? null : safeRow.presentation ?? null, - metadata: deletedAt ? null : safeRow.metadata ?? null, + presentation: deletedAt ? null : (safeRow.presentation ?? null), + metadata: deletedAt ? null : (safeRow.metadata ?? null), deletedAt: deletedAt ? deletedAt.toISOString() : null, - deletedByType: deletedAt ? row.deletedByType ?? null : null, - deletedByAgentId: deletedAt ? row.deletedByAgentId ?? null : null, - deletedByUserId: deletedAt ? row.deletedByUserId ?? null : null, - deletedByRunId: deletedAt ? row.deletedByRunId ?? null : null, + deletedByType: deletedAt ? (row.deletedByType ?? null) : null, + deletedByAgentId: deletedAt ? (row.deletedByAgentId ?? null) : null, + deletedByUserId: deletedAt ? (row.deletedByUserId ?? null) : null, + deletedByRunId: deletedAt ? (row.deletedByRunId ?? null) : null, sourceTrust: row.sourceTrust ?? null, createdAt: row.createdAt.toISOString(), author: row.authorAgentId @@ -6100,123 +6906,162 @@ export async function buildPaperclipWakePayload(input: { }); } - const annotationDeltas = annotationCommentId && issueId - ? await input.db - .select({ - id: documentAnnotationComments.id, - issueId: documentAnnotationComments.issueId, - threadId: documentAnnotationComments.threadId, - body: documentAnnotationComments.body, - authorType: documentAnnotationComments.authorType, - authorAgentId: documentAnnotationComments.authorAgentId, - authorUserId: documentAnnotationComments.authorUserId, - createdAt: documentAnnotationComments.createdAt, - documentKey: documentAnnotationThreads.documentKey, - status: documentAnnotationThreads.status, - anchorState: documentAnnotationThreads.anchorState, - anchorConfidence: documentAnnotationThreads.anchorConfidence, - currentRevisionNumber: documentAnnotationThreads.currentRevisionNumber, - selectedText: documentAnnotationThreads.selectedText, - prefixText: documentAnnotationThreads.prefixText, - suffixText: documentAnnotationThreads.suffixText, - }) - .from(documentAnnotationComments) - .innerJoin(documentAnnotationThreads, eq(documentAnnotationComments.threadId, documentAnnotationThreads.id)) - .where(and( - eq(documentAnnotationComments.companyId, input.companyId), - eq(documentAnnotationComments.issueId, issueId), - eq(documentAnnotationComments.id, annotationCommentId), - eq(documentAnnotationThreads.companyId, input.companyId), - eq(documentAnnotationThreads.issueId, issueId), - )) - .then((rows) => rows.map((row) => ({ - id: row.id, - issueId: row.issueId, - threadId: row.threadId, - documentKey: row.documentKey, - revisionNumber: row.currentRevisionNumber, - quote: row.selectedText, - prefix: row.prefixText, - suffix: row.suffixText, - threadStatus: row.status, - anchorState: row.anchorState, - anchorConfidence: row.anchorConfidence, - body: row.body.length > MAX_INLINE_WAKE_COMMENT_BODY_CHARS - ? row.body.slice(0, MAX_INLINE_WAKE_COMMENT_BODY_CHARS) - : row.body, - bodyTruncated: row.body.length > MAX_INLINE_WAKE_COMMENT_BODY_CHARS, - createdAt: row.createdAt.toISOString(), - author: row.authorAgentId - ? { type: "agent", id: row.authorAgentId } - : row.authorUserId - ? { type: "user", id: row.authorUserId } - : { type: row.authorType, id: null }, - }))) - : []; + const annotationDeltas = + annotationCommentId && issueId + ? await input.db + .select({ + id: documentAnnotationComments.id, + issueId: documentAnnotationComments.issueId, + threadId: documentAnnotationComments.threadId, + body: documentAnnotationComments.body, + authorType: documentAnnotationComments.authorType, + authorAgentId: documentAnnotationComments.authorAgentId, + authorUserId: documentAnnotationComments.authorUserId, + createdAt: documentAnnotationComments.createdAt, + documentKey: documentAnnotationThreads.documentKey, + status: documentAnnotationThreads.status, + anchorState: documentAnnotationThreads.anchorState, + anchorConfidence: documentAnnotationThreads.anchorConfidence, + currentRevisionNumber: + documentAnnotationThreads.currentRevisionNumber, + selectedText: documentAnnotationThreads.selectedText, + prefixText: documentAnnotationThreads.prefixText, + suffixText: documentAnnotationThreads.suffixText, + }) + .from(documentAnnotationComments) + .innerJoin( + documentAnnotationThreads, + eq( + documentAnnotationComments.threadId, + documentAnnotationThreads.id, + ), + ) + .where( + and( + eq(documentAnnotationComments.companyId, input.companyId), + eq(documentAnnotationComments.issueId, issueId), + eq(documentAnnotationComments.id, annotationCommentId), + eq(documentAnnotationThreads.companyId, input.companyId), + eq(documentAnnotationThreads.issueId, issueId), + ), + ) + .then((rows) => + rows.map((row) => ({ + id: row.id, + issueId: row.issueId, + threadId: row.threadId, + documentKey: row.documentKey, + revisionNumber: row.currentRevisionNumber, + quote: row.selectedText, + prefix: row.prefixText, + suffix: row.suffixText, + threadStatus: row.status, + anchorState: row.anchorState, + anchorConfidence: row.anchorConfidence, + body: + row.body.length > MAX_INLINE_WAKE_COMMENT_BODY_CHARS + ? row.body.slice(0, MAX_INLINE_WAKE_COMMENT_BODY_CHARS) + : row.body, + bodyTruncated: + row.body.length > MAX_INLINE_WAKE_COMMENT_BODY_CHARS, + createdAt: row.createdAt.toISOString(), + author: row.authorAgentId + ? { type: "agent", id: row.authorAgentId } + : row.authorUserId + ? { type: "user", id: row.authorUserId } + : { type: row.authorType, id: null }, + })), + ) + : []; const interactionId = readNonEmptyString(input.contextSnapshot.interactionId); - const interactionKind = readNonEmptyString(input.contextSnapshot.interactionKind); - const interactionStatus = readNonEmptyString(input.contextSnapshot.interactionStatus); - const checkboxSelection = parseObject(input.contextSnapshot.checkboxSelection); + const interactionKind = readNonEmptyString( + input.contextSnapshot.interactionKind, + ); + const interactionStatus = readNonEmptyString( + input.contextSnapshot.interactionStatus, + ); + const checkboxSelection = parseObject( + input.contextSnapshot.checkboxSelection, + ); const planReviewContext = issueId ? await buildPlanReviewContext({ - db: input.db, - companyId: input.companyId, - issueId, - issueWorkMode: issueSummary?.workMode ?? null, - includeForIssueComment: commentIds.length > 0, - includeForAnnotationDelta: annotationDeltas.length > 0, - interactionId, - }) + db: input.db, + companyId: input.companyId, + issueId, + issueWorkMode: issueSummary?.workMode ?? null, + includeForIssueComment: commentIds.length > 0, + includeForAnnotationDelta: annotationDeltas.length > 0, + interactionId, + }) : null; const documentReviewContext = issueId ? await buildDocumentReviewContext({ - db: input.db, - companyId: input.companyId, - issueId, - includeForIssueComment: commentIds.length > 0, - includeForAnnotationDelta: annotationDeltas.length > 0, - }) + db: input.db, + companyId: input.companyId, + issueId, + includeForIssueComment: commentIds.length > 0, + includeForAnnotationDelta: annotationDeltas.length > 0, + }) : null; - const payloadTruncated = truncated || issueDescriptionTruncated || planReviewContext?.truncated === true || documentReviewContext?.truncated === true; - const recoveryActionId = readNonEmptyString(input.contextSnapshot.recoveryActionId); + const payloadTruncated = + truncated || + issueDescriptionTruncated || + planReviewContext?.truncated === true || + documentReviewContext?.truncated === true; + const recoveryActionId = readNonEmptyString( + input.contextSnapshot.recoveryActionId, + ); const recoveryCause = readNonEmptyString(input.contextSnapshot.recoveryCause); const recoveryAction = recoveryActionId ? await input.db - .select() - .from(issueRecoveryActions) - .where(and( - eq(issueRecoveryActions.id, recoveryActionId), - eq(issueRecoveryActions.companyId, input.companyId), - )) - .then((rows) => rows[0] ?? null) + .select() + .from(issueRecoveryActions) + .where( + and( + eq(issueRecoveryActions.id, recoveryActionId), + eq(issueRecoveryActions.companyId, input.companyId), + ), + ) + .then((rows) => rows[0] ?? null) : null; const recoveryEvidence = parseObject(recoveryAction?.evidence); - const originalAssigneeId = recoveryAction?.returnOwnerAgentId ?? recoveryAction?.previousOwnerAgentId ?? null; + const originalAssigneeId = + recoveryAction?.returnOwnerAgentId ?? + recoveryAction?.previousOwnerAgentId ?? + null; const originalAssignee = originalAssigneeId ? await input.db - .select({ id: agents.id, name: agents.name }) - .from(agents) - .where(and(eq(agents.id, originalAssigneeId), eq(agents.companyId, input.companyId))) - .then((rows) => rows[0] ?? null) + .select({ id: agents.id, name: agents.name }) + .from(agents) + .where( + and( + eq(agents.id, originalAssigneeId), + eq(agents.companyId, input.companyId), + ), + ) + .then((rows) => rows[0] ?? null) : null; const payload = { reason: readNonEmptyString(input.contextSnapshot.wakeReason), - recovery: recoveryAction || recoveryCause - ? { - cause: recoveryAction?.cause ?? recoveryCause, - failureSummary: readNonEmptyString(recoveryEvidence.failureSummary), - originalAssignee: originalAssignee - ? { id: originalAssignee.id, name: originalAssignee.name } - : originalAssigneeId - ? { id: originalAssigneeId, name: null } - : null, - attemptCount: recoveryAction?.attemptCount ?? null, - maxAttempts: recoveryAction?.maxAttempts ?? null, - nextAction: recoveryAction?.nextAction ?? null, - routingFallbackReason: readNonEmptyString(recoveryEvidence.routingFallbackReason), - } - : null, + recovery: + recoveryAction || recoveryCause + ? { + cause: recoveryAction?.cause ?? recoveryCause, + failureSummary: readNonEmptyString(recoveryEvidence.failureSummary), + originalAssignee: originalAssignee + ? { id: originalAssignee.id, name: originalAssignee.name } + : originalAssigneeId + ? { id: originalAssigneeId, name: null } + : null, + attemptCount: recoveryAction?.attemptCount ?? null, + maxAttempts: recoveryAction?.maxAttempts ?? null, + nextAction: recoveryAction?.nextAction ?? null, + routingFallbackReason: readNonEmptyString( + recoveryEvidence.routingFallbackReason, + ), + } + : null, issue: issueSummary ? { id: issueSummary.id, @@ -6237,38 +7082,65 @@ export async function buildPaperclipWakePayload(input: { sessionId: readNonEmptyString(agentMessage.sessionId), } : null, - childIssueSummaries: Array.isArray(input.contextSnapshot.childIssueSummaries) + childIssueSummaries: Array.isArray( + input.contextSnapshot.childIssueSummaries, + ) ? input.contextSnapshot.childIssueSummaries : [], - childIssueSummaryTruncated: input.contextSnapshot.childIssueSummaryTruncated === true, - livenessContinuation: readNonEmptyString(input.contextSnapshot.livenessContinuationState) || - readNonEmptyString(input.contextSnapshot.livenessContinuationInstruction) || - readNonEmptyString(input.contextSnapshot.livenessContinuationSourceRunId) || + childIssueSummaryTruncated: + input.contextSnapshot.childIssueSummaryTruncated === true, + livenessContinuation: + readNonEmptyString(input.contextSnapshot.livenessContinuationState) || + readNonEmptyString( + input.contextSnapshot.livenessContinuationInstruction, + ) || + readNonEmptyString( + input.contextSnapshot.livenessContinuationSourceRunId, + ) || typeof input.contextSnapshot.livenessContinuationAttempt === "number" - ? { - attempt: input.contextSnapshot.livenessContinuationAttempt, - maxAttempts: input.contextSnapshot.livenessContinuationMaxAttempts, - sourceRunId: readNonEmptyString(input.contextSnapshot.livenessContinuationSourceRunId), - state: readNonEmptyString(input.contextSnapshot.livenessContinuationState), - reason: readNonEmptyString(input.contextSnapshot.livenessContinuationReason), - instruction: readNonEmptyString(input.contextSnapshot.livenessContinuationInstruction), - } - : null, + ? { + attempt: input.contextSnapshot.livenessContinuationAttempt, + maxAttempts: input.contextSnapshot.livenessContinuationMaxAttempts, + sourceRunId: readNonEmptyString( + input.contextSnapshot.livenessContinuationSourceRunId, + ), + state: readNonEmptyString( + input.contextSnapshot.livenessContinuationState, + ), + reason: readNonEmptyString( + input.contextSnapshot.livenessContinuationReason, + ), + instruction: readNonEmptyString( + input.contextSnapshot.livenessContinuationInstruction, + ), + } + : null, interactionKind, interactionStatus, - checkboxSelection: Object.keys(checkboxSelection).length > 0 ? checkboxSelection : null, - checkedOutByHarness: input.contextSnapshot[PAPERCLIP_HARNESS_CHECKOUT_KEY] === true, + checkboxSelection: + Object.keys(checkboxSelection).length > 0 ? checkboxSelection : null, + checkedOutByHarness: + input.contextSnapshot[PAPERCLIP_HARNESS_CHECKOUT_KEY] === true, simplifiedEnglishInteractions: input.simplifiedEnglishInteractions === true, - dependencyBlockedInteraction: input.contextSnapshot.dependencyBlockedInteraction === true, + dependencyBlockedInteraction: + input.contextSnapshot.dependencyBlockedInteraction === true, treeHoldInteraction: input.contextSnapshot.treeHoldInteraction === true, activeTreeHold: parseObject(input.contextSnapshot.activeTreeHold), - unresolvedBlockerIssueIds: Array.isArray(input.contextSnapshot.unresolvedBlockerIssueIds) - ? input.contextSnapshot.unresolvedBlockerIssueIds.filter((value): value is string => typeof value === "string" && value.length > 0) + unresolvedBlockerIssueIds: Array.isArray( + input.contextSnapshot.unresolvedBlockerIssueIds, + ) + ? input.contextSnapshot.unresolvedBlockerIssueIds.filter( + (value): value is string => + typeof value === "string" && value.length > 0, + ) : [], - unresolvedBlockerSummaries: Array.isArray(input.contextSnapshot.unresolvedBlockerSummaries) + unresolvedBlockerSummaries: Array.isArray( + input.contextSnapshot.unresolvedBlockerSummaries, + ) ? input.contextSnapshot.unresolvedBlockerSummaries : [], - executionStage: Object.keys(executionStage).length > 0 ? executionStage : null, + executionStage: + Object.keys(executionStage).length > 0 ? executionStage : null, taskWatchdog: (input.contextSnapshot.taskWatchdog ?? null) as unknown, skillTest: (input.contextSnapshot.paperclipSkillTest ?? null) as unknown, continuationSummary: safeContinuationSummary @@ -6299,12 +7171,19 @@ export async function buildPaperclipWakePayload(input: { fallbackFetchNeeded: payloadTruncated || missingCommentCount > 0, }; return issueId - ? createRunSecretRedactionRegistry(input.db).redactForIssue(input.companyId, issueId, payload) + ? createRunSecretRedactionRegistry(input.db).redactForIssue( + input.companyId, + issueId, + payload, + ) : payload; } function runTaskKey(run: typeof heartbeatRuns.$inferSelect) { - return deriveTaskKey(run.contextSnapshot as Record | null, null); + return deriveTaskKey( + run.contextSnapshot as Record | null, + null, + ); } function isSameTaskScope(left: string | null, right: string | null) { @@ -6354,7 +7233,9 @@ export function buildHeartbeatRunStatusLiveEventPayload( }; } -function isHeartbeatRunRuntimeStatusActive(status: string | null | undefined): boolean { +function isHeartbeatRunRuntimeStatusActive( + status: string | null | undefined, +): boolean { return status === "queued" || status === "running"; } @@ -6377,7 +7258,9 @@ function readRuntimeStatusIssueIdCandidate( return undefined; } -function decorateHeartbeatRunRuntimeStatus( +function decorateHeartbeatRunRuntimeStatus< + T extends HeartbeatRunRuntimeStatusRunLike, +>( run: T, expected: { companyId?: string | null; @@ -6398,7 +7281,9 @@ function decorateHeartbeatRunRuntimeStatus, + run: Pick< + typeof heartbeatRuns.$inferSelect, + "id" | "companyId" | "agentId" | "status" | "contextSnapshot" + >, update: RuntimeStatusUpdate, issueId: string | null, ) { @@ -6461,7 +7349,8 @@ function recordHeartbeatRunRuntimeProgress( phase: update.phase as HeartbeatRunStatusPhase, message: update.message, currentToolName: readNonEmptyString(update.currentToolName) ?? null, - lastAssistantSnippet: readNonEmptyString(update.lastAssistantSnippet) ?? null, + lastAssistantSnippet: + readNonEmptyString(update.lastAssistantSnippet) ?? null, lastEventAt: update.lastEventAt ? new Date(update.lastEventAt) : new Date(), }); if (!status) return null; @@ -6470,7 +7359,10 @@ function recordHeartbeatRunRuntimeProgress( return status; } -function sanitizeLiveRunProgressText(value: string, maxChars: number): string | null { +function sanitizeLiveRunProgressText( + value: string, + maxChars: number, +): string | null { const normalized = value.replace(/\s+/g, " ").trim(); if (!normalized) return null; const redacted = redactSensitiveText(normalized); @@ -6478,11 +7370,19 @@ function sanitizeLiveRunProgressText(value: string, maxChars: number): string | return `${redacted.slice(0, maxChars - 3)}...`; } -function readLiveRunProgressString(value: unknown, maxChars: number): string | null { - return typeof value === "string" ? sanitizeLiveRunProgressText(value, maxChars) : null; +function readLiveRunProgressString( + value: unknown, + maxChars: number, +): string | null { + return typeof value === "string" + ? sanitizeLiveRunProgressText(value, maxChars) + : null; } -function readFirstLiveRunProgressString(maxChars: number, values: unknown[]): string | null { +function readFirstLiveRunProgressString( + maxChars: number, + values: unknown[], +): string | null { for (const value of values) { const text = readLiveRunProgressString(value, maxChars); if (text) return text; @@ -6490,25 +7390,35 @@ function readFirstLiveRunProgressString(maxChars: number, values: unknown[]): st return null; } -function readLiveRunToolName(payload: Record | null, eventType: string): string | null { - const toolCall = parseObject(payload?.tool_call) ?? parseObject(payload?.toolCall); +function readLiveRunToolName( + payload: Record | null, + eventType: string, +): string | null { + const toolCall = + parseObject(payload?.tool_call) ?? parseObject(payload?.toolCall); const message = parseObject(payload?.message); - const direct = readFirstLiveRunProgressString(MAX_HEARTBEAT_RUN_RUNTIME_TOOL_NAME_CHARS, [ - payload?.toolName, - payload?.tool_name, - payload?.tool, - payload?.name, - payload?.title, - toolCall?.name, - toolCall?.toolName, - message?.name, - message?.toolName, - ]); + const direct = readFirstLiveRunProgressString( + MAX_HEARTBEAT_RUN_RUNTIME_TOOL_NAME_CHARS, + [ + payload?.toolName, + payload?.tool_name, + payload?.tool, + payload?.name, + payload?.title, + toolCall?.name, + toolCall?.toolName, + message?.name, + message?.toolName, + ], + ); if (direct) return direct; const normalizedEventType = eventType.toLowerCase(); if (!normalizedEventType.includes("tool")) return null; - return readLiveRunProgressString(eventType.replace(/[._-]+/g, " "), MAX_HEARTBEAT_RUN_RUNTIME_TOOL_NAME_CHARS); + return readLiveRunProgressString( + eventType.replace(/[._-]+/g, " "), + MAX_HEARTBEAT_RUN_RUNTIME_TOOL_NAME_CHARS, + ); } function readLiveRunAssistantSnippet( @@ -6518,15 +7428,18 @@ function readLiveRunAssistantSnippet( ): string | null { const normalizedEventType = eventType.toLowerCase(); const messagePayload = parseObject(payload?.message); - const direct = readFirstLiveRunProgressString(MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS, [ - payload?.text, - payload?.delta, - payload?.text_delta, - payload?.content, - payload?.summary, - messagePayload?.text, - messagePayload?.content, - ]); + const direct = readFirstLiveRunProgressString( + MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS, + [ + payload?.text, + payload?.delta, + payload?.text_delta, + payload?.content, + payload?.summary, + messagePayload?.text, + messagePayload?.content, + ], + ); if (direct) return direct; if ( @@ -6535,7 +7448,12 @@ function readLiveRunAssistantSnippet( normalizedEventType.includes("message.delta") || normalizedEventType.includes("message_delta") ) { - return message ? sanitizeLiveRunProgressText(message, MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS) : null; + return message + ? sanitizeLiveRunProgressText( + message, + MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS, + ) + : null; } return null; @@ -6548,22 +7466,31 @@ function buildRunEventRuntimeProgress(input: { at: Date; }) { const normalizedEventType = input.eventType.toLowerCase(); - if (normalizedEventType === "lifecycle" || normalizedEventType === "adapter.invoke") { + if ( + normalizedEventType === "lifecycle" || + normalizedEventType === "adapter.invoke" + ) { return null; } const currentToolName = readLiveRunToolName(input.payload, input.eventType); - const lastAssistantSnippet = readLiveRunAssistantSnippet(input.payload, input.eventType, input.message); + const lastAssistantSnippet = readLiveRunAssistantSnippet( + input.payload, + input.eventType, + input.message, + ); const fallbackMessage = - readLiveRunProgressString(input.message, MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS) ?? + readLiveRunProgressString( + input.message, + MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS, + ) ?? sanitizeLiveRunProgressText( input.eventType.replace(/[._-]+/g, " "), MAX_HEARTBEAT_RUN_RUNTIME_ASSISTANT_SNIPPET_CHARS, ); - const message = - currentToolName - ? `Using ${currentToolName}` - : lastAssistantSnippet ?? fallbackMessage; + const message = currentToolName + ? `Using ${currentToolName}` + : (lastAssistantSnippet ?? fallbackMessage); if (!message) return null; return { @@ -6598,6 +7525,11 @@ export function buildPaperclipTaskMarkdown(input: { kind?: string | null; status?: string | null; } | null; + acceptedPlan?: { + documentId?: string | null; + revisionId?: string | null; + revisionNumber?: number | null; + } | null; acceptedPlanContinuation?: boolean; // false builds the compact variant used for resume deltas, where the session // already received the description with the assignment. @@ -6617,11 +7549,10 @@ export function buildPaperclipTaskMarkdown(input: { const wakeComment = input.wakeComment ?? null; const acceptedPlanContinuation = !wakeComment && - (input.acceptedPlanContinuation || ( - input.interaction?.kind === "request_confirmation" && - input.interaction.status === "accepted" && - issue?.workMode === "planning" - )); + (input.acceptedPlanContinuation || + (input.interaction?.kind === "request_confirmation" && + input.interaction.status === "accepted" && + issue?.workMode === "planning")); if (!issue && !wakeComment) return null; const lines = [ @@ -6641,12 +7572,15 @@ export function buildPaperclipTaskMarkdown(input: { "Answer the question directly in the issue thread. Do not write implementation code, and do not produce an implementation plan. Use tools only for investigation or temporary scratch work when needed; the deliverable is the answer.", ); } else if (issue.workMode === "planning") { - let directive = "Make the plan only. Do not write code or perform implementation work."; + let directive = + "Make the plan only. Do not write code or perform implementation work."; if (wakeComment) { - directive = "Update the plan only. Do not write code or perform implementation work."; + directive = + "Update the plan only. Do not write code or perform implementation work."; } if (acceptedPlanContinuation) { - directive = "Create child issues from the approved plan only. Do not write code or perform implementation work on the planning issue."; + directive = + "Implement the accepted plan on this issue when the work is small and cohesive. Use the paperclip-converting-plans-to-tasks skill to decide whether decomposition is justified. Create the minimum child issue graph only for qualifying ownership, parallelism, dependency, review, or lifecycle boundaries. Do not create a child merely because a plan was accepted."; } lines.push( `- Work mode: ${quoteTaskScalar("planning")}`, @@ -6665,10 +7599,22 @@ export function buildPaperclipTaskMarkdown(input: { lines.push( "", "Accepted plan directive:", - "Create child issues from the approved plan only. Do not write code or perform implementation work on the source issue.", + "Implement the accepted plan on this issue when the work is small and cohesive. Use the paperclip-converting-plans-to-tasks skill to decide whether decomposition is justified. Create the minimum child issue graph only for qualifying ownership, parallelism, dependency, review, or lifecycle boundaries. Do not create a child merely because a plan was accepted.", ); } - const description = input.includeDescription === false ? "" : issue.description?.trim(); + if (acceptedPlanContinuation && input.acceptedPlan?.revisionId) { + const revisionNumber = input.acceptedPlan.revisionNumber + ? ` revision ${input.acceptedPlan.revisionNumber}` + : " revision"; + const documentId = input.acceptedPlan.documentId + ? ` of document ${input.acceptedPlan.documentId}` + : ""; + lines.push( + `- Approved plan:${revisionNumber} ${input.acceptedPlan.revisionId}${documentId}. Follow this exact revision, not a later draft.`, + ); + } + const description = + input.includeDescription === false ? "" : issue.description?.trim(); if (description) { lines.push("", "Issue description:", fenceTaskText(description)); } @@ -6680,10 +7626,14 @@ export function buildPaperclipTaskMarkdown(input: { const status = ancestor.status ? ` (${ancestor.status})` : ""; const priority = ancestor.priority ? ` [${ancestor.priority}]` : ""; const title = ancestor.title ? ` ${ancestor.title}` : ""; - lines.push(`- ${index === 0 ? "Parent" : `Ancestor ${index + 1}`}: ${label}${title}${status}${priority}`); + lines.push( + `- ${index === 0 ? "Parent" : `Ancestor ${index + 1}`}: ${label}${title}${status}${priority}`, + ); } if ((input.ancestors ?? []).length > ancestors.length) { - lines.push(`- [ancestor context truncated after ${ancestors.length} entries]`); + lines.push( + `- [ancestor context truncated after ${ancestors.length} entries]`, + ); } } if (wakeComment?.body.trim()) { @@ -6704,7 +7654,8 @@ export function buildPaperclipTaskMarkdown(input: { // On Linux, PIDs can be recycled, so this is a best-effort signal rather // than proof that the original child is still alive. function isProcessAlive(pid: number | null | undefined) { - if (typeof pid !== "number" || !Number.isInteger(pid) || pid <= 0) return false; + if (typeof pid !== "number" || !Number.isInteger(pid) || pid <= 0) + return false; try { process.kill(pid, 0); return true; @@ -6716,6 +7667,27 @@ function isProcessAlive(pid: number | null | undefined) { } } +export async function persistHeartbeatRunProcessMetadata( + db: Db, + runId: string, + meta: { pid: number; processGroupId: number | null; startedAt: string }, +) { + const startedAt = new Date(meta.startedAt); + return db + .update(heartbeatRuns) + .set({ + processPid: meta.pid, + processGroupId: meta.processGroupId, + processStartedAt: Number.isNaN(startedAt.getTime()) + ? new Date() + : startedAt, + updatedAt: new Date(), + }) + .where(eq(heartbeatRuns.id, runId)) + .returning() + .then((rows) => rows[0] ?? null); +} + async function terminateHeartbeatRunProcess(input: { pid: number | null | undefined; processGroupId: number | null | undefined; @@ -6732,7 +7704,9 @@ async function terminateHeartbeatRunProcess(input: { ? pid : (processGroupId ?? 0), processGroupId: - typeof processGroupId === "number" && Number.isInteger(processGroupId) && processGroupId > 0 + typeof processGroupId === "number" && + Number.isInteger(processGroupId) && + processGroupId > 0 ? processGroupId : null, }, @@ -6740,10 +7714,13 @@ async function terminateHeartbeatRunProcess(input: { ); } -function buildProcessLossMessage(run: { - processPid: number | null; - processGroupId: number | null; -}, options?: { descendantOnly?: boolean }) { +function buildProcessLossMessage( + run: { + processPid: number | null; + processGroupId: number | null; + }, + options?: { descendantOnly?: boolean }, +) { if (options?.descendantOnly && run.processGroupId) { return `Process lost -- parent pid ${run.processPid ?? "unknown"} exited, but descendant process group ${run.processGroupId} was still alive and was terminated`; } @@ -6756,10 +7733,13 @@ function buildProcessLossMessage(run: { return "Process lost -- server may have restarted"; } -function readHotRestartAdoptionMetadata(resultJson: Record | null | undefined) { +function readHotRestartAdoptionMetadata( + resultJson: Record | null | undefined, +) { const result = parseObject(resultJson); const hotRestart = parseObject(result.hotRestart); - if (hotRestart.adopted !== true || typeof hotRestart.adoptedAt !== "string") return null; + if (hotRestart.adopted !== true || typeof hotRestart.adoptedAt !== "string") + return null; return hotRestart; } @@ -6808,7 +7788,9 @@ const defaultSessionCodec: AdapterSessionCodec = { deserialize(raw: unknown) { const asObj = parseObject(raw); if (Object.keys(asObj).length > 0) return asObj; - const sessionId = readNonEmptyString((raw as Record | null)?.sessionId); + const sessionId = readNonEmptyString( + (raw as Record | null)?.sessionId, + ); if (sessionId) return { sessionId }; return null; }, @@ -6826,12 +7808,15 @@ function getAdapterSessionCodec(adapterType: string) { return adapter.sessionCodec ?? defaultSessionCodec; } -export function normalizeSessionParams(params: Record | null | undefined) { +export function normalizeSessionParams( + params: Record | null | undefined, +) { if (!params) return null; return Object.keys(params).length > 0 ? params : null; } -type RunSessionOutcome = "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out"; +type RunSessionOutcome = + "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out"; type SkillTestHeartbeatCompletion = { outcome: "failed" | "cancelled"; @@ -6868,7 +7853,8 @@ export function resolveSkillTestRunCompletionForHeartbeatOutcome( } const HERMES_ADAPTER_TYPE = "hermes_local"; -const HERMES_SESSION_ID_REGEX = /^(?:\d{8}_\d{6}_[A-Za-z0-9_-]{4,}|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})$/; +const HERMES_SESSION_ID_REGEX = + /^(?:\d{8}_\d{6}_[A-Za-z0-9_-]{4,}|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})$/; function requiresCanonicalSessionIds(adapterType: string | null | undefined) { return adapterType === HERMES_ADAPTER_TYPE; @@ -6891,7 +7877,9 @@ function normalizeResumeParamsForAdapter( if (!normalized) return null; if (!requiresCanonicalSessionIds(adapterType)) return normalized; const sessionId = readNonEmptyString(normalized.sessionId); - return isCanonicalSessionIdForAdapter(adapterType, sessionId) ? normalized : null; + return isCanonicalSessionIdForAdapter(adapterType, sessionId) + ? normalized + : null; } export function resolveNextSessionState(input: { @@ -6903,7 +7891,14 @@ export function resolveNextSessionState(input: { previousDisplayId: string | null; previousLegacySessionId: string | null; }) { - const { adapterType, codec, adapterResult, previousParams, previousDisplayId, previousLegacySessionId } = input; + const { + adapterType, + codec, + adapterResult, + previousParams, + previousDisplayId, + previousLegacySessionId, + } = input; if (adapterResult.clearSession) { return { @@ -6919,17 +7914,23 @@ export function resolveNextSessionState(input: { const hasExplicitSessionId = adapterResult.sessionId !== undefined; const explicitSessionId = readNonEmptyString(adapterResult.sessionId); const hasExplicitDisplay = adapterResult.sessionDisplayId !== undefined; - const explicitDisplayId = readNonEmptyString(adapterResult.sessionDisplayId); - const shouldUsePrevious = !hasExplicitParams && !hasExplicitSessionId && !hasExplicitDisplay; + const explicitDisplayId = readNonEmptyString( + adapterResult.sessionDisplayId, + ); + const shouldUsePrevious = + !hasExplicitParams && !hasExplicitSessionId && !hasExplicitDisplay; - const candidateParams = - hasExplicitParams - ? explicitParams - : hasExplicitSessionId - ? (explicitSessionId ? { sessionId: explicitSessionId } : null) - : previousParams; + const candidateParams = hasExplicitParams + ? explicitParams + : hasExplicitSessionId + ? explicitSessionId + ? { sessionId: explicitSessionId } + : null + : previousParams; - const serialized = normalizeSessionParams(codec.serialize(normalizeSessionParams(candidateParams) ?? null)); + const serialized = normalizeSessionParams( + codec.serialize(normalizeSessionParams(candidateParams) ?? null), + ); const deserialized = normalizeSessionParams(codec.deserialize(serialized)); const displayId = truncateDisplayId( @@ -6956,12 +7957,20 @@ export function resolveNextSessionState(input: { const previousSerializedParams = normalizeResumeParamsForAdapter( adapterType, - codec.serialize(normalizeResumeParamsForAdapter(adapterType, previousParams)), + codec.serialize( + normalizeResumeParamsForAdapter(adapterType, previousParams), + ), ); - const validPreviousDisplayId = isCanonicalSessionIdForAdapter(adapterType, previousDisplayId) + const validPreviousDisplayId = isCanonicalSessionIdForAdapter( + adapterType, + previousDisplayId, + ) ? previousDisplayId : null; - const validPreviousLegacySessionId = isCanonicalSessionIdForAdapter(adapterType, previousLegacySessionId) + const validPreviousLegacySessionId = isCanonicalSessionIdForAdapter( + adapterType, + previousLegacySessionId, + ) ? previousLegacySessionId : null; const previousState = () => { @@ -6973,7 +7982,10 @@ export function resolveNextSessionState(input: { return { params: previousSerializedParams, displayId, - legacySessionId: readNonEmptyString(previousSerializedParams?.sessionId) ?? displayId ?? validPreviousLegacySessionId, + legacySessionId: + readNonEmptyString(previousSerializedParams?.sessionId) ?? + displayId ?? + validPreviousLegacySessionId, }; }; @@ -6984,11 +7996,17 @@ export function resolveNextSessionState(input: { const explicitParams = adapterResult.sessionParams; const hasExplicitParams = adapterResult.sessionParams !== undefined; const explicitSessionId = readNonEmptyString(adapterResult.sessionId); - const validExplicitSessionId = isCanonicalSessionIdForAdapter(adapterType, explicitSessionId) + const validExplicitSessionId = isCanonicalSessionIdForAdapter( + adapterType, + explicitSessionId, + ) ? explicitSessionId : null; const explicitDisplayId = readNonEmptyString(adapterResult.sessionDisplayId); - const validExplicitDisplayId = isCanonicalSessionIdForAdapter(adapterType, explicitDisplayId) + const validExplicitDisplayId = isCanonicalSessionIdForAdapter( + adapterType, + explicitDisplayId, + ) ? explicitDisplayId : null; const explicitSerializedParams = hasExplicitParams @@ -7015,7 +8033,8 @@ export function resolveNextSessionState(input: { (codec.getDisplayId ? codec.getDisplayId(serialized) : null) ?? explicitCanonicalSessionId, ); - const legacySessionId = readNonEmptyString(serialized?.sessionId) ?? explicitCanonicalSessionId; + const legacySessionId = + readNonEmptyString(serialized?.sessionId) ?? explicitCanonicalSessionId; return { params: serialized, @@ -7024,12 +8043,22 @@ export function resolveNextSessionState(input: { }; } -export type HeartbeatEnvironmentRuntime = ReturnType; +export type HeartbeatEnvironmentRuntime = ReturnType< + typeof environmentRuntimeService +>; export interface HeartbeatServiceOptions { pluginWorkerManager?: PluginWorkerManager; environmentRuntime?: HeartbeatEnvironmentRuntime; runtimeEnv?: Record; + /** + * Provider-boundary seam for persisted native-run recovery tests. Keeping + * the seam here exercises the production reaper, claim, execution, package + * session loop, persistence port, and finalizer without spawning a provider. + */ + nativeSessionBackendFactory?: ( + execution: NativeExecutionInput, + ) => NativeSessionBackend; /** Test seam for changing a continuation issue at the final pre-dispatch boundary. */ beforeResolvedInteractionContinuationDispatchCheck?: (input: { runId: string; @@ -7042,6 +8071,42 @@ export interface HeartbeatServiceOptions { }) => Promise; } +export async function cancelHeartbeatNativeRun(input: { + db: Db; + runId: string; + reason: string; + runtimeMode: string | null; + cancel?: ( + runId: string, + reason: string, + options: { db: Db; scope: "run" }, + ) => Promise<{ decision: unknown | null; auditId: string | null }>; +}) { + if (input.runtimeMode !== "native") { + return { decision: null, auditId: null }; + } + const cancellation = input.cancel + ? await input.cancel(input.runId, input.reason, { + db: input.db, + scope: "run", + }) + : await cancelNativeSession(input.runId, input.reason, { + db: input.db, + scope: "run", + }); + if (!cancellation.decision || !cancellation.auditId) { + throw new Error("native_cancellation_outcome_not_audited"); + } + return cancellation; +} + +class NativeSessionResumeScheduledError extends Error { + constructor(readonly original: unknown) { + super("Native session recovery has been scheduled for the same run."); + this.name = "NativeSessionResumeScheduledError"; + } +} + type WorkspaceReadyCommentWriter = { addComment: ( issueId: string, @@ -7100,20 +8165,29 @@ export function resolveHeartbeatSchedulingSuppression( return { suppressed: false, reason: null }; } -export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) { +export function heartbeatService( + db: Db, + options: HeartbeatServiceOptions = {}, +) { const instanceSettings = instanceSettingsService(db); const getCurrentUserRedactionOptions = async () => ({ enabled: (await instanceSettings.getGeneral()).censorUsernameInLogs, }); const runtimeEnv = options.runtimeEnv ?? process.env; - const inWorktreeRuntime = isTruthyRuntimeEnvValue(runtimeEnv.PAPERCLIP_IN_WORKTREE); + const inWorktreeRuntime = isTruthyRuntimeEnvValue( + runtimeEnv.PAPERCLIP_IN_WORKTREE, + ); // Preview worktree instances suppress the run engine by default. Users can lift // that per-worktree via the `enableWorktreeRunExecution` experimental setting // (worktree instances have their own isolated DB, so it can't affect the parent). // Only worktree runtimes ever read the setting; a short TTL keeps the hot-path // suppression checks off the DB, and a read failure falls back to prior/default // (fail closed to suppression). - let cachedWorktreeRunExecutionOverride: { allowed: boolean; cutoff: Date | null; at: number } = { + let cachedWorktreeRunExecutionOverride: { + allowed: boolean; + cutoff: Date | null; + at: number; + } = { allowed: false, cutoff: null, at: 0, @@ -7122,7 +8196,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const resolveWorktreeRunExecutionOverride = async () => { if (!inWorktreeRuntime) return { allowed: false, cutoff: null }; const now = Date.now(); - if (now - cachedWorktreeRunExecutionOverride.at < WORKTREE_RUN_EXECUTION_OVERRIDE_TTL_MS) { + if ( + now - cachedWorktreeRunExecutionOverride.at < + WORKTREE_RUN_EXECUTION_OVERRIDE_TTL_MS + ) { return cachedWorktreeRunExecutionOverride; } try { @@ -7132,7 +8209,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) ); const cutoff = activation.armed ? new Date(activation.cutoff) : null; cachedWorktreeRunExecutionOverride = { - allowed: Boolean(activation.armed && cutoff && !Number.isNaN(cutoff.getTime())), + allowed: Boolean( + activation.armed && cutoff && !Number.isNaN(cutoff.getTime()), + ), cutoff: cutoff && !Number.isNaN(cutoff.getTime()) ? cutoff : null, at: now, }; @@ -7154,15 +8233,18 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) }; const runLogStore = getRunLogStore(); + const traceStore = providerTraceStore(db); const secretsSvc = secretService(db); const companySkills = companySkillService(db); const issuesSvc = issueService(db); const treeControlSvc = issueTreeControlService(db); const executionWorkspacesSvc = executionWorkspaceService(db); const environmentsSvc = environmentService(db); - const environmentRuntime = options.environmentRuntime ?? environmentRuntimeService(db, { - pluginWorkerManager: options.pluginWorkerManager, - }); + const environmentRuntime = + options.environmentRuntime ?? + environmentRuntimeService(db, { + pluginWorkerManager: options.pluginWorkerManager, + }); const envOrchestrator = environmentRunOrchestrator(db, { pluginWorkerManager: options.pluginWorkerManager, environmentRuntime, @@ -7181,8 +8263,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) function isPlanApprovalConfirmationPayload(payload: unknown) { const target = parseObject(parseObject(payload).target); - return readNonEmptyString(target.type) === "issue_document" && - readNonEmptyString(target.key) === "plan"; + return ( + readNonEmptyString(target.type) === "issue_document" && + readNonEmptyString(target.key) === "plan" + ); } async function getAcceptedPlanApprovalInteractionForRun( @@ -7212,11 +8296,19 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .then((rows) => rows[0] ?? null); if (!interaction) return null; - if (interaction.kind !== "request_confirmation" || interaction.status !== "accepted") return null; - return isPlanApprovalConfirmationPayload(interaction.payload) ? interaction : null; + if ( + interaction.kind !== "request_confirmation" || + interaction.status !== "accepted" + ) + return null; + return isPlanApprovalConfirmationPayload(interaction.payload) + ? interaction + : null; } - function planApprovalResumeFailureErrorCode(run: typeof heartbeatRuns.$inferSelect) { + function planApprovalResumeFailureErrorCode( + run: typeof heartbeatRuns.$inferSelect, + ) { return readNonEmptyString(run.errorCode) ?? "unknown_error"; } @@ -7254,7 +8346,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } async function updatePlanApprovalInteractionResumeFailure(input: { - interaction: NonNullable>>; + interaction: NonNullable< + Awaited> + >; failure: NonNullable; }) { const result = parseObject(input.interaction.result); @@ -7295,10 +8389,18 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .limit(1) .then((rows) => rows[0] ?? null); if (existing) return null; - return issuesSvc.addComment(input.issueId, input.body, { runId: input.run.id }, { authorType: "system" }); + return issuesSvc.addComment( + input.issueId, + input.body, + { runId: input.run.id }, + { authorType: "system" }, + ); } - async function getActiveRecoveryActionId(companyId: string, sourceIssueId: string) { + async function getActiveRecoveryActionId( + companyId: string, + sourceIssueId: string, + ) { return db .select({ id: issueRecoveryActions.id }) .from(issueRecoveryActions) @@ -7321,7 +8423,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) attempt: number; maxAttempts: number; }) { - const interaction = await getAcceptedPlanApprovalInteractionForRun(input.run, input.issueId); + const interaction = await getAcceptedPlanApprovalInteractionForRun( + input.run, + input.issueId, + ); if (!interaction || !input.issueId) return null; const body = buildPlanApprovalResumeFailureComment({ @@ -7354,16 +8459,29 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) attempt: number; maxAttempts: number; }) { - const interaction = await getAcceptedPlanApprovalInteractionForRun(input.run, input.issueId); + const interaction = await getAcceptedPlanApprovalInteractionForRun( + input.run, + input.issueId, + ); if (!interaction || !input.issueId) return null; const issue = await db .select() .from(issues) - .where(and(eq(issues.companyId, input.run.companyId), eq(issues.id, input.issueId))) + .where( + and( + eq(issues.companyId, input.run.companyId), + eq(issues.id, input.issueId), + ), + ) .then((rows) => rows[0] ?? null); if (!issue) return null; - if (issue.status !== "todo" && issue.status !== "in_progress" && issue.status !== "in_review") return null; + if ( + issue.status !== "todo" && + issue.status !== "in_progress" && + issue.status !== "in_review" + ) + return null; const body = buildPlanApprovalResumeFailureComment({ run: input.run, @@ -7383,7 +8501,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) body, }); - const recoveryActionId = await getActiveRecoveryActionId(issue.companyId, issue.id); + const recoveryActionId = await getActiveRecoveryActionId( + issue.companyId, + issue.id, + ); await updatePlanApprovalInteractionResumeFailure({ interaction, failure: buildPlanApprovalResumeFailureResult({ @@ -7408,7 +8529,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) outcome: RunSessionOutcome; error: string | null; }) { - const completion = resolveSkillTestRunCompletionForHeartbeatOutcome(input.outcome, input.error); + const completion = resolveSkillTestRunCompletionForHeartbeatOutcome( + input.outcome, + input.error, + ); if (!completion || !input.issueId) return null; let isSkillTestIssue = input.issueWorkMode === "skill_test"; @@ -7419,9 +8543,16 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) harnessKind: issues.harnessKind, }) .from(issues) - .where(and(eq(issues.companyId, input.run.companyId), eq(issues.id, input.issueId))) + .where( + and( + eq(issues.companyId, input.run.companyId), + eq(issues.id, input.issueId), + ), + ) .then((rows) => rows[0] ?? null); - isSkillTestIssue = issueRow?.workMode === "skill_test" || issueRow?.harnessKind === "skill_test"; + isSkillTestIssue = + issueRow?.workMode === "skill_test" || + issueRow?.harnessKind === "skill_test"; } if (!isSkillTestIssue) return null; @@ -7431,12 +8562,18 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) status: companySkillTestRuns.status, }) .from(companySkillTestRuns) - .where(and( - eq(companySkillTestRuns.companyId, input.run.companyId), - eq(companySkillTestRuns.issueId, input.issueId), - )) + .where( + and( + eq(companySkillTestRuns.companyId, input.run.companyId), + eq(companySkillTestRuns.issueId, input.issueId), + ), + ) .then((rows) => rows[0] ?? null); - if (!existingRun || ["succeeded", "failed", "cancelled"].includes(existingRun.status)) return null; + if ( + !existingRun || + ["succeeded", "failed", "cancelled"].includes(existingRun.status) + ) + return null; const completedRun = await companySkills.completeTestRunForIssue({ companyId: input.run.companyId, @@ -7475,19 +8612,28 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) status: string | null | undefined; failureReason?: string | null; }) { - const releaseResult = await envOrchestrator.releaseForRun({ - heartbeatRunId: input.runId, - companyId: input.companyId, - agentId: input.agentId, - status: leaseReleaseStatusForRunStatus(input.status), - failureReason: input.failureReason ?? undefined, - }).catch((err) => { - logger.warn({ err, runId: input.runId }, "failed to release environment leases for heartbeat run"); - return null; - }); + const releaseResult = await envOrchestrator + .releaseForRun({ + heartbeatRunId: input.runId, + companyId: input.companyId, + agentId: input.agentId, + status: leaseReleaseStatusForRunStatus(input.status), + failureReason: input.failureReason ?? undefined, + }) + .catch((err) => { + logger.warn( + { err, runId: input.runId }, + "failed to release environment leases for heartbeat run", + ); + return null; + }); for (const releaseError of releaseResult?.errors ?? []) { logger.warn( - { err: releaseError.error, leaseId: releaseError.leaseId, runId: input.runId }, + { + err: releaseError.error, + leaseId: releaseError.leaseId, + runId: input.runId, + }, "failed to release environment lease for heartbeat run", ); } @@ -7496,16 +8642,25 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) async function hasUnsafeTextProjectionDatabase() { if (!unsafeTextProjectionPromise) { unsafeTextProjectionPromise = db - .execute(sql`select current_setting('server_encoding') as server_encoding`) + .execute( + sql`select current_setting('server_encoding') as server_encoding`, + ) .then((rows) => { const first = Array.isArray(rows) ? rows[0] : null; - const serverEncoding = typeof first === "object" && first !== null - ? (first as Record).server_encoding - : null; - return typeof serverEncoding === "string" && serverEncoding.toUpperCase() === "SQL_ASCII"; + const serverEncoding = + typeof first === "object" && first !== null + ? (first as Record).server_encoding + : null; + return ( + typeof serverEncoding === "string" && + serverEncoding.toUpperCase() === "SQL_ASCII" + ); }) .catch((err) => { - logger.warn({ err }, "failed to inspect database server encoding; using conservative heartbeat result projection"); + logger.warn( + { err }, + "failed to inspect database server encoding; using conservative heartbeat result projection", + ); return true; }); } @@ -7520,11 +8675,18 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .then((rows) => rows[0] ?? null); } - async function getAgentInvokability(agent: typeof agents.$inferSelect | null | undefined) { + async function getAgentInvokability( + agent: typeof agents.$inferSelect | null | undefined, + ) { return evaluateAgentInvokabilityFromDb(db, agent); } - function toAgentOrgRow(agent: Pick): AgentOrgRow { + function toAgentOrgRow( + agent: Pick< + typeof agents.$inferSelect, + "id" | "companyId" | "name" | "reportsTo" | "status" + >, + ): AgentOrgRow { return { id: agent.id, companyId: agent.companyId, @@ -7534,7 +8696,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) }; } - async function listCompanyAgentOrgRows(companyId: string): Promise { + async function listCompanyAgentOrgRows( + companyId: string, + ): Promise { return db .select({ id: agents.id, @@ -7560,8 +8724,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) return byCompany; } - async function getRun(runId: string, opts?: { unsafeFullResultJson?: boolean }) { - const safeForLegacyEncoding = !opts?.unsafeFullResultJson && await hasUnsafeTextProjectionDatabase(); + async function getRun( + runId: string, + opts?: { unsafeFullResultJson?: boolean }, + ) { + const safeForLegacyEncoding = + !opts?.unsafeFullResultJson && (await hasUnsafeTextProjectionDatabase()); return db .select( opts?.unsafeFullResultJson @@ -7576,7 +8744,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } async function recordCurrentHeartbeatRunRuntimeProgress( - run: Pick, + run: Pick< + typeof heartbeatRuns.$inferSelect, + "id" | "companyId" | "agentId" | "status" | "contextSnapshot" + >, update: RuntimeStatusUpdate, issueId: string | null, ) { @@ -7655,21 +8826,29 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) eq(companySkillVersions.companyId, companySkillTestRuns.companyId), ), ) - .where(and(eq(companySkillTestRuns.companyId, companyId), eq(companySkillTestRuns.issueId, issueId))) + .where( + and( + eq(companySkillTestRuns.companyId, companyId), + eq(companySkillTestRuns.issueId, issueId), + ), + ) .then((rows) => rows[0] ?? null); if (!row) return null; const fileInventory = Array.isArray(row.fileInventory) ? row.fileInventory.flatMap((entry) => { - if (!entry || typeof entry !== "object" || Array.isArray(entry)) return []; - const record = entry as unknown as Record; - const path = typeof record.path === "string" ? record.path : ""; - if (!path) return []; - return [{ - path, - kind: typeof record.kind === "string" ? record.kind : "other", - content: typeof record.content === "string" ? record.content : "", - }]; - }) + if (!entry || typeof entry !== "object" || Array.isArray(entry)) + return []; + const record = entry as unknown as Record; + const path = typeof record.path === "string" ? record.path : ""; + if (!path) return []; + return [ + { + path, + kind: typeof record.kind === "string" ? record.kind : "other", + content: typeof record.content === "string" ? record.content : "", + }, + ]; + }) : []; return { testRunId: row.testRunId, @@ -7687,7 +8866,11 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) companyId: string, issueContext: Awaited> | null, ) { - if (!issueContext || issueContext.originKind !== "routine_execution" || !issueContext.originId) { + if ( + !issueContext || + issueContext.originKind !== "routine_execution" || + !issueContext.originId + ) { return { routineId: null, env: null, responsibleUserId: null }; } @@ -7723,25 +8906,38 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) ), ) .then((rows) => rows[0] ?? null); - const snapshot = revision?.snapshot as RoutineRevisionSnapshotV1 | undefined; + const snapshot = revision?.snapshot as + RoutineRevisionSnapshotV1 | undefined; if (snapshot?.version === 1) { return { routineId: issueContext.originId, env: snapshot.routine.env ?? null, - responsibleUserId: revision?.responsibleUserId ?? snapshot.routine.responsibleUserId ?? null, + responsibleUserId: + revision?.responsibleUserId ?? + snapshot.routine.responsibleUserId ?? + null, }; } } const routine = await db - .select({ env: routines.env, responsibleUserId: routines.responsibleUserId }) + .select({ + env: routines.env, + responsibleUserId: routines.responsibleUserId, + }) .from(routines) - .where(and(eq(routines.id, issueContext.originId), eq(routines.companyId, companyId))) + .where( + and( + eq(routines.id, issueContext.originId), + eq(routines.companyId, companyId), + ), + ) .then((rows) => rows[0] ?? null); return { routineId: issueContext.originId, env: routine?.env ?? null, - responsibleUserId: routineRun?.responsibleUserId ?? routine?.responsibleUserId ?? null, + responsibleUserId: + routineRun?.responsibleUserId ?? routine?.responsibleUserId ?? null, }; } @@ -7751,7 +8947,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .from(companies) .where(eq(companies.id, companyId)) .then((rows) => rows[0] ?? null); - const explicitDefault = readNonEmptyString(company?.defaultResponsibleUserId); + const explicitDefault = readNonEmptyString( + company?.defaultResponsibleUserId, + ); if (explicitDefault) return explicitDefault; const owner = await db @@ -7786,7 +8984,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) return firstUser?.userId ?? null; } - async function resolveParentIssueResponsibleUserId(companyId: string, parentId: string | null | undefined) { + async function resolveParentIssueResponsibleUserId( + companyId: string, + parentId: string | null | undefined, + ) { if (!parentId) return null; const parent = await db .select({ @@ -7807,7 +9008,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) }) { if (input.requestedByActorType !== "user") return false; const wakeReason = readNonEmptyString(input.contextSnapshot.wakeReason); - if (wakeReason && ISSUE_RESPONSIBLE_USER_WAKE_REASONS.has(wakeReason)) return false; + if (wakeReason && ISSUE_RESPONSIBLE_USER_WAKE_REASONS.has(wakeReason)) + return false; return input.source === "on_demand" || input.triggerDetail === "manual"; } @@ -7815,25 +9017,37 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) companyId: string; contextSnapshot: Record; issueContext: Awaited> | null; - routineEnvContext: Awaited>; + routineEnvContext: Awaited< + ReturnType + >; requestedByActorType?: "user" | "agent" | "system" | null; requestedByActorId?: string | null; source?: WakeupOptions["source"] | null; triggerDetail?: WakeupOptions["triggerDetail"] | null; existingRunResponsibleUserId?: string | null; }) { - const contextResponsibleUserId = readNonEmptyString(input.contextSnapshot.responsibleUserId); - const requestedUserId = input.requestedByActorType === "user" - ? readNonEmptyString(input.requestedByActorId) - : null; + const contextResponsibleUserId = readNonEmptyString( + input.contextSnapshot.responsibleUserId, + ); + const requestedUserId = + input.requestedByActorType === "user" + ? readNonEmptyString(input.requestedByActorId) + : null; if (contextResponsibleUserId) return contextResponsibleUserId; - if (input.existingRunResponsibleUserId) return input.existingRunResponsibleUserId; - if (input.routineEnvContext.responsibleUserId) return input.routineEnvContext.responsibleUserId; + if (input.existingRunResponsibleUserId) + return input.existingRunResponsibleUserId; + if (input.routineEnvContext.responsibleUserId) + return input.routineEnvContext.responsibleUserId; if (isManualUserRun(input) && requestedUserId) return requestedUserId; - if (input.issueContext?.responsibleUserId) return input.issueContext.responsibleUserId; - const parentResponsibleUserId = await resolveParentIssueResponsibleUserId(input.companyId, input.issueContext?.parentId); + if (input.issueContext?.responsibleUserId) + return input.issueContext.responsibleUserId; + const parentResponsibleUserId = await resolveParentIssueResponsibleUserId( + input.companyId, + input.issueContext?.parentId, + ); if (parentResponsibleUserId) return parentResponsibleUserId; - if (input.issueContext) return resolveCompanyDefaultResponsibleUserId(input.companyId); + if (input.issueContext) + return resolveCompanyDefaultResponsibleUserId(input.companyId); if (requestedUserId) return requestedUserId; return resolveCompanyDefaultResponsibleUserId(input.companyId); } @@ -7842,7 +9056,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) run: typeof heartbeatRuns.$inferSelect; contextSnapshot: Record; issueContext: Awaited> | null; - routineEnvContext: Awaited>; + routineEnvContext: Awaited< + ReturnType + >; }) { const responsibleUserId = await resolveResponsibleUserIdForRunSeed({ companyId: input.run.companyId, @@ -7854,16 +9070,20 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) triggerDetail: input.run.triggerDetail as WakeupOptions["triggerDetail"], }); if (!responsibleUserId) { - throw new HttpError(422, "Unable to resolve responsible user for heartbeat run dispatch", { - code: "responsible_user_unresolved", - runId: input.run.id, - agentId: input.run.agentId, - companyId: input.run.companyId, - issueId: input.issueContext?.id ?? null, - invocationSource: input.run.invocationSource, - triggerDetail: input.run.triggerDetail, - wakeReason: readNonEmptyString(input.contextSnapshot.wakeReason), - }); + throw new HttpError( + 422, + "Unable to resolve responsible user for heartbeat run dispatch", + { + code: "responsible_user_unresolved", + runId: input.run.id, + agentId: input.run.agentId, + companyId: input.run.companyId, + issueId: input.issueContext?.id ?? null, + invocationSource: input.run.invocationSource, + triggerDetail: input.run.triggerDetail, + wakeReason: readNonEmptyString(input.contextSnapshot.wakeReason), + }, + ); } return responsibleUserId; } @@ -7872,13 +9092,20 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) run: typeof heartbeatRuns.$inferSelect, contextSnapshot: Record, ) { - const issueId = readNonEmptyString(contextSnapshot.issueId) ?? readNonEmptyString(contextSnapshot.taskId); - const issueContext = issueId ? await getIssueExecutionContext(run.companyId, issueId) : null; + const issueId = + readNonEmptyString(contextSnapshot.issueId) ?? + readNonEmptyString(contextSnapshot.taskId); + const issueContext = issueId + ? await getIssueExecutionContext(run.companyId, issueId) + : null; return resolveResponsibleUserIdForRun({ run, contextSnapshot, issueContext, - routineEnvContext: await getRoutineEnvForExecutionIssue(run.companyId, issueContext), + routineEnvContext: await getRoutineEnvForExecutionIssue( + run.companyId, + issueContext, + ), }); } @@ -7890,7 +9117,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .then((rows) => rows[0] ?? null); } - async function getLatestAgentConfigRevision(companyId: string, agentId: string) { + async function getLatestAgentConfigRevision( + companyId: string, + agentId: string, + ) { return db .select({ id: agentConfigRevisions.id, @@ -7898,8 +9128,16 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) createdAt: agentConfigRevisions.createdAt, }) .from(agentConfigRevisions) - .where(and(eq(agentConfigRevisions.companyId, companyId), eq(agentConfigRevisions.agentId, agentId))) - .orderBy(desc(agentConfigRevisions.createdAt), desc(agentConfigRevisions.id)) + .where( + and( + eq(agentConfigRevisions.companyId, companyId), + eq(agentConfigRevisions.agentId, agentId), + ), + ) + .orderBy( + desc(agentConfigRevisions.createdAt), + desc(agentConfigRevisions.id), + ) .limit(1) .then((rows) => rows[0] ?? null); } @@ -8043,7 +9281,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) }; } - function formatIssueIdentifierLink(identifier: string | null, fallback: string) { + function formatIssueIdentifierLink( + identifier: string | null, + fallback: string, + ) { if (!identifier) return fallback; const prefix = identifier.split("-")[0]; if (!prefix || !/^[A-Z][A-Z0-9]*-\d+$/.test(identifier)) return identifier; @@ -8056,7 +9297,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) recoveryPolicy: IssueExecutionMonitorRecoveryPolicy; nextAttemptCount: number; }) { - const label = formatIssueIdentifierLink(input.issue.identifier, input.issue.id); + const label = formatIssueIdentifierLink( + input.issue.identifier, + input.issue.id, + ); const reason = input.clearReason === "timeout_exceeded" ? "its timeout was reached" @@ -8071,7 +9315,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) ].join("\n"); } - async function findOpenIssueMonitorRecoveryIssue(claimed: IssueMonitorDispatchRow) { + async function findOpenIssueMonitorRecoveryIssue( + claimed: IssueMonitorDispatchRow, + ) { return db .select() .from(issues) @@ -8102,15 +9348,17 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) runId: string | null; activitySource: "manual" | "scheduled"; }) { - const reviewPathLost = input.claimed.status === "in_review" - && (await issuesSvc + const reviewPathLost = + input.claimed.status === "in_review" && + (await issuesSvc .listReviewAttention(input.claimed.companyId, [input.claimed]) - .then((attention) => attention.get(input.claimed.id)?.state === "stalled")); + .then( + (attention) => attention.get(input.claimed.id)?.state === "stalled", + )); const reviewPathContext = reviewPathLost ? { reviewPathLost: true, - reviewPathConsumedRef: - `monitor:${input.claimed.id}:${input.clearReason}:${input.scheduledAtIso}`, + reviewPathConsumedRef: `monitor:${input.claimed.id}:${input.clearReason}:${input.scheduledAtIso}`, reviewPathInstruction: REVIEW_PATH_RECOVERY_INSTRUCTION, } : null; @@ -8125,7 +9373,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) }); if (input.recoveryPolicy === "create_recovery_issue") { - let recoveryIssue = await findOpenIssueMonitorRecoveryIssue(input.claimed); + let recoveryIssue = await findOpenIssueMonitorRecoveryIssue( + input.claimed, + ); if (!recoveryIssue) { recoveryIssue = await issuesSvc.create(input.claimed.companyId, { title: `Recover external-service monitor for ${input.claimed.identifier ?? input.claimed.title}`, @@ -8141,7 +9391,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) projectId: input.claimed.projectId, goalId: input.claimed.goalId, assigneeAgentId: input.claimed.assigneeAgentId, - assigneeAdapterOverrides: recoveryAssigneeAdapterOverrides("status_only"), + assigneeAdapterOverrides: + recoveryAssigneeAdapterOverrides("status_only"), originKind: RECOVERY_ORIGIN_KINDS.strandedIssueRecovery, originId: input.claimed.id, originFingerprint: `issue_monitor:${input.clearReason}`, @@ -8155,15 +9406,21 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) triggerDetail: "system", reason: "issue_monitor_recovery_issue", idempotencyKey: `issue-monitor-recovery-issue:${input.claimed.id}:${input.clearReason}:${input.scheduledAtIso}`, - payload: withRecoveryModelProfileHint({ issueId: recoveryIssue.id, sourceIssueId: input.claimed.id }, "status_only"), + payload: withRecoveryModelProfileHint( + { issueId: recoveryIssue.id, sourceIssueId: input.claimed.id }, + "status_only", + ), requestedByActorType: input.actorType, requestedByActorId: input.actorId, - contextSnapshot: withRecoveryModelProfileHint({ - issueId: recoveryIssue.id, - sourceIssueId: input.claimed.id, - source: "issue.monitor.recovery_issue", - wakeReason: "issue_monitor_recovery_issue", - }, "status_only"), + contextSnapshot: withRecoveryModelProfileHint( + { + issueId: recoveryIssue.id, + sourceIssueId: input.claimed.id, + source: "issue.monitor.recovery_issue", + wakeReason: "issue_monitor_recovery_issue", + }, + "status_only", + ), }); } @@ -8216,30 +9473,36 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) triggerDetail: "system", reason: "issue_monitor_recovery", idempotencyKey: `issue-monitor-recovery:${input.claimed.id}:${input.clearReason}:${input.scheduledAtIso}`, - payload: withRecoveryModelProfileHint({ - issueId: input.claimed.id, - monitorAttemptCount: input.nextAttemptCount, - monitorNotes: input.claimed.monitorNotes ?? null, - clearReason: input.clearReason, - serviceName: input.monitor?.serviceName ?? null, - timeoutAt: input.monitor?.timeoutAt ?? null, - maxAttempts: input.monitor?.maxAttempts ?? null, - ...(reviewPathContext ?? {}), - }, "status_only"), + payload: withRecoveryModelProfileHint( + { + issueId: input.claimed.id, + monitorAttemptCount: input.nextAttemptCount, + monitorNotes: input.claimed.monitorNotes ?? null, + clearReason: input.clearReason, + serviceName: input.monitor?.serviceName ?? null, + timeoutAt: input.monitor?.timeoutAt ?? null, + maxAttempts: input.monitor?.maxAttempts ?? null, + ...(reviewPathContext ?? {}), + }, + "status_only", + ), requestedByActorType: input.actorType, requestedByActorId: input.actorId, - contextSnapshot: withRecoveryModelProfileHint({ - issueId: input.claimed.id, - source: "issue.monitor.recovery", - wakeReason: "issue_monitor_recovery", - monitorAttemptCount: input.nextAttemptCount, - monitorNotes: input.claimed.monitorNotes ?? null, - clearReason: input.clearReason, - serviceName: input.monitor?.serviceName ?? null, - timeoutAt: input.monitor?.timeoutAt ?? null, - maxAttempts: input.monitor?.maxAttempts ?? null, - ...(reviewPathContext ?? {}), - }, "status_only"), + contextSnapshot: withRecoveryModelProfileHint( + { + issueId: input.claimed.id, + source: "issue.monitor.recovery", + wakeReason: "issue_monitor_recovery", + monitorAttemptCount: input.nextAttemptCount, + monitorNotes: input.claimed.monitorNotes ?? null, + clearReason: input.clearReason, + serviceName: input.monitor?.serviceName ?? null, + timeoutAt: input.monitor?.timeoutAt ?? null, + maxAttempts: input.monitor?.maxAttempts ?? null, + ...(reviewPathContext ?? {}), + }, + "status_only", + ), }); await logActivity(db, { @@ -8341,9 +9604,15 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const scheduledAtIso = claimed.monitorNextCheckAt.toISOString(); const nextAttemptCount = (claimed.monitorAttemptCount ?? 0) + 1; - const policy = normalizeIssueExecutionPolicy(claimed.executionPolicy ?? null); + const policy = normalizeIssueExecutionPolicy( + claimed.executionPolicy ?? null, + ); const monitor = policy?.monitor ?? null; - const clearReason = issueMonitorLimitClearReason({ monitor, nextAttemptCount, now: input.now }); + const clearReason = issueMonitorLimitClearReason({ + monitor, + nextAttemptCount, + now: input.now, + }); const recoveryPolicy = monitorRecoveryPolicy(monitor); const monitorMetadata = { serviceName: monitor?.serviceName ?? null, @@ -8351,16 +9620,18 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) maxAttempts: monitor?.maxAttempts ?? null, recoveryPolicy: monitor?.recoveryPolicy ?? null, }; - const executionState = claimed.status === "in_review" - ? parseIssueExecutionState(claimed.executionState) - : null; - const currentParticipant = executionState?.status === "pending" - ? executionState.currentParticipant - : null; - const reviewParticipantAgentId = currentParticipant?.type === "agent" - ? currentParticipant.agentId - : null; - const isProviderQuotaReviewMonitor = monitor?.serviceName === PROVIDER_QUOTA_MONITOR_SERVICE_NAME && + const executionState = + claimed.status === "in_review" + ? parseIssueExecutionState(claimed.executionState) + : null; + const currentParticipant = + executionState?.status === "pending" + ? executionState.currentParticipant + : null; + const reviewParticipantAgentId = + currentParticipant?.type === "agent" ? currentParticipant.agentId : null; + const isProviderQuotaReviewMonitor = + monitor?.serviceName === PROVIDER_QUOTA_MONITOR_SERVICE_NAME && Boolean(reviewParticipantAgentId); const targetAgentId = isProviderQuotaReviewMonitor ? reviewParticipantAgentId @@ -8418,7 +9689,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) requestedByActorId: input.actorId, contextSnapshot: { issueId: claimed.id, - source: isProviderQuotaReviewMonitor ? "issue.execution_review_recovery" : "issue.monitor", + source: isProviderQuotaReviewMonitor + ? "issue.execution_review_recovery" + : "issue.monitor", wakeReason, nextCheckAt: scheduledAtIso, monitorAttemptCount: nextAttemptCount, @@ -8521,17 +9794,21 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } } - async function triggerIssueMonitor(issueId: string, input?: { - now?: Date; - actorType?: "user" | "agent" | "system"; - actorId?: string | null; - agentId?: string | null; - runId?: string | null; - wakeReason?: string; - }) { + async function triggerIssueMonitor( + issueId: string, + input?: { + now?: Date; + actorType?: "user" | "agent" | "system"; + actorId?: string | null; + agentId?: string | null; + runId?: string | null; + wakeReason?: string; + }, + ) { const now = input?.now ?? new Date(); const actorType = input?.actorType ?? "system"; - const actorId = input?.actorId ?? (actorType === "system" ? "heartbeat_scheduler" : null); + const actorId = + input?.actorId ?? (actorType === "system" ? "heartbeat_scheduler" : null); if (!actorId) { throw conflict("Issue monitor trigger requires an actor"); } @@ -8552,7 +9829,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) throw conflict("Issue monitor requires an agent assignee"); } if (!["in_progress", "in_review"].includes(issue.status)) { - throw conflict("Issue monitor can only run while the issue is in progress or in review"); + throw conflict( + "Issue monitor can only run while the issue is in progress or in review", + ); } const staleClaimThreshold = new Date(now.getTime() - 5 * 60 * 1000); @@ -8686,7 +9965,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) createdAt: heartbeatRuns.createdAt, }) .from(heartbeatRuns) - .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.sessionIdAfter, sessionId))) + .where( + and( + eq(heartbeatRuns.agentId, agentId), + eq(heartbeatRuns.sessionIdAfter, sessionId), + ), + ) .orderBy(asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id)) .limit(1) .then((rows) => rows[0] ?? null); @@ -8711,7 +9995,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) }; } - const previousRun = await getLatestRunForSession(agentId, sessionId, { excludeRunId: runId }); + const previousRun = await getLatestRunForSession(agentId, sessionId, { + excludeRunId: runId, + }); const previousRawUsage = readRawUsageTotals(previousRun?.usageJson); return { normalizedUsage: deriveNormalizedUsageDelta(rawUsage, previousRawUsage), @@ -8746,7 +10032,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) }; } - const fetchLimit = Math.max(policy.maxSessionRuns > 0 ? policy.maxSessionRuns + 1 : 0, 4); + const fetchLimit = Math.max( + policy.maxSessionRuns > 0 ? policy.maxSessionRuns + 1 : 0, + 4, + ); const runs = await db .select({ id: heartbeatRuns.id, @@ -8756,7 +10045,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) ...heartbeatRunListResultColumns, }) .from(heartbeatRuns) - .where(and(eq(heartbeatRuns.agentId, agent.id), eq(heartbeatRuns.sessionIdAfter, sessionId))) + .where( + and( + eq(heartbeatRuns.agentId, agent.id), + eq(heartbeatRuns.sessionIdAfter, sessionId), + ), + ) .orderBy(desc(heartbeatRuns.createdAt)) .limit(fetchLimit); @@ -8773,13 +10067,15 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const oldestRun = policy.maxSessionAgeHours > 0 ? await getOldestRunForSession(agent.id, sessionId) - : runs[runs.length - 1] ?? latestRun; + : (runs[runs.length - 1] ?? latestRun); const latestRawUsage = readRawUsageTotals(latestRun?.usageJson); const sessionAgeHours = latestRun && oldestRun ? Math.max( 0, - (new Date(latestRun.createdAt).getTime() - new Date(oldestRun.createdAt).getTime()) / (1000 * 60 * 60), + (new Date(latestRun.createdAt).getTime() - + new Date(oldestRun.createdAt).getTime()) / + (1000 * 60 * 60), ) : 0; @@ -8794,7 +10090,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) reason = `session raw input reached ${formatCount(latestRawUsage.inputTokens)} tokens ` + `(threshold ${formatCount(policy.maxRawInputTokens)})`; - } else if (policy.maxSessionAgeHours > 0 && sessionAgeHours >= policy.maxSessionAgeHours) { + } else if ( + policy.maxSessionAgeHours > 0 && + sessionAgeHours >= policy.maxSessionAgeHours + ) { reason = `session age reached ${Math.floor(sessionAgeHours)} hours`; } @@ -8870,7 +10169,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) return runtimeForRun?.sessionId ?? null; } - async function hasResolvableSessionWorkspaceCwd(sessionParams: Record | null | undefined) { + async function hasResolvableSessionWorkspaceCwd( + sessionParams: Record | null | undefined, + ) { const cwd = readNonEmptyString(sessionParams?.cwd); if (!cwd || isUnsafeSessionWorkspaceCwd(cwd)) return false; return fs @@ -8883,9 +10184,16 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) agent: typeof agents.$inferSelect; contextSnapshot: Record; taskKey: string | null; - explicitResumeSession: Awaited> | null; + explicitResumeSession: Awaited< + ReturnType + > | null; }) { - if (await hasResolvableSessionWorkspaceCwd(input.explicitResumeSession?.sessionParams)) return true; + if ( + await hasResolvableSessionWorkspaceCwd( + input.explicitResumeSession?.sessionParams, + ) + ) + return true; if (shouldResetTaskSessionForWake(input.contextSnapshot)) return false; if (!input.taskKey) return false; @@ -8933,19 +10241,27 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const resumeContext = parseObject(resumeRun.contextSnapshot); const resumeTaskKey = deriveTaskKey(resumeContext, null) ?? taskKey; const resumeTaskSession = resumeTaskKey - ? await getTaskSession(agent.companyId, agent.id, agent.adapterType, resumeTaskKey) + ? await getTaskSession( + agent.companyId, + agent.id, + agent.adapterType, + resumeTaskKey, + ) : null; const sessionCodec = getAdapterSessionCodec(agent.adapterType); const resumeRunResult = parseObject(resumeRun.resultJson); const resumeRunSessionId = requiresCanonicalSessionIds(agent.adapterType) - ? readNonEmptyString(resumeRunResult.sessionId) ?? readNonEmptyString(resumeRunResult.session_id) + ? (readNonEmptyString(resumeRunResult.sessionId) ?? + readNonEmptyString(resumeRunResult.session_id)) : null; const sessionOverride = buildExplicitResumeSessionOverride({ adapterType: agent.adapterType, resumeFromRunId, resumeRunSessionIdBefore: resumeRun.sessionIdBefore, resumeRunSessionIdAfter: resumeRun.sessionIdAfter, - resumeRunSessionParams: resumeRunSessionId ? { sessionId: resumeRunSessionId } : null, + resumeRunSessionParams: resumeRunSessionId + ? { sessionId: resumeRunSessionId } + : null, taskSession: resumeTaskSession, sessionCodec, }); @@ -8955,7 +10271,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) resumeFromRunId, taskKey: resumeTaskKey, issueId: readNonEmptyString(resumeContext.issueId), - taskId: readNonEmptyString(resumeContext.taskId) ?? readNonEmptyString(resumeContext.issueId), + taskId: + readNonEmptyString(resumeContext.taskId) ?? + readNonEmptyString(resumeContext.issueId), sessionDisplayId: sessionOverride.sessionDisplayId, sessionParams: sessionOverride.sessionParams, }; @@ -8967,9 +10285,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) previousSessionParams: Record | null, opts?: { useProjectWorkspace?: boolean | null }, ): Promise { - const issueId = readNonEmptyString(context.issueId) ?? readNonEmptyString(context.taskId); + const issueId = + readNonEmptyString(context.issueId) ?? readNonEmptyString(context.taskId); const contextProjectId = readNonEmptyString(context.projectId); - const contextProjectWorkspaceId = readNonEmptyString(context.projectWorkspaceId); + const contextProjectWorkspaceId = readNonEmptyString( + context.projectWorkspaceId, + ); const issueProjectRef = issueId ? await db .select({ @@ -8977,7 +10298,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) projectWorkspaceId: issues.projectWorkspaceId, }) .from(issues) - .where(and(eq(issues.id, issueId), eq(issues.companyId, agent.companyId))) + .where( + and(eq(issues.id, issueId), eq(issues.companyId, agent.companyId)), + ) .then((rows) => rows[0] ?? null) : null; const issueProjectId = issueProjectRef?.projectId ?? null; @@ -9013,24 +10336,28 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) if (projectWorkspaceRows.length > 0) { const preferredWorkspace = preferredProjectWorkspaceId - ? projectWorkspaceRows.find((workspace) => workspace.id === preferredProjectWorkspaceId) ?? null + ? (projectWorkspaceRows.find( + (workspace) => workspace.id === preferredProjectWorkspaceId, + ) ?? null) : null; const missingProjectCwds: string[] = []; const materializationFailures: WorkspaceMaterializationFailure[] = []; let hasConfiguredProjectCwd = false; let preferredWorkspaceWarning: string | null = null; if (preferredProjectWorkspaceId && !preferredWorkspace) { - preferredWorkspaceWarning = - `Selected project workspace "${preferredProjectWorkspaceId}" is not available on this project.`; + preferredWorkspaceWarning = `Selected project workspace "${preferredProjectWorkspaceId}" is not available on this project.`; } - const resolveGitAuth = createGitRemoteAuthProvider(db, agent.companyId, { issueId }); + const resolveGitAuth = createGitRemoteAuthProvider(db, agent.companyId, { + issueId, + }); for (const workspace of projectWorkspaceRows) { let projectCwd: string; let managedWorkspaceWarning: string | null = null; try { const resolvedCwd = await resolveConfiguredOrManagedProjectCwd({ companyId: agent.companyId, - projectId: workspaceProjectId ?? resolvedProjectId ?? workspace.projectId, + projectId: + workspaceProjectId ?? resolvedProjectId ?? workspace.projectId, cwd: workspace.cwd, repoUrl: workspace.repoUrl, resolveGitAuth, @@ -9044,7 +10371,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const workspaceRepoUrl = readNonEmptyString(workspace.repoUrl); materializationFailures.push({ projectWorkspaceId: workspace.id, - repoUrl: workspaceRepoUrl ? scrubGitCredentialText(workspaceRepoUrl) : null, + repoUrl: workspaceRepoUrl + ? scrubGitCredentialText(workspaceRepoUrl) + : null, error: scrubbedError, }); if (preferredWorkspace?.id === workspace.id) { @@ -9066,16 +10395,16 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) repoUrl: workspace.repoUrl, repoRef: workspace.repoRef, workspaceHints, - warnings: [preferredWorkspaceWarning, managedWorkspaceWarning].filter( - (value): value is string => Boolean(value), - ), + warnings: [ + preferredWorkspaceWarning, + managedWorkspaceWarning, + ].filter((value): value is string => Boolean(value)), baseCwdFallback: false, materializationFailures, }; } if (preferredWorkspace?.id === workspace.id) { - preferredWorkspaceWarning = - `Selected project workspace path "${projectCwd}" is not available yet.`; + preferredWorkspaceWarning = `Selected project workspace path "${projectCwd}" is not available yet.`; } missingProjectCwds.push(projectCwd); } @@ -9189,11 +10518,23 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) agent: typeof agents.$inferSelect, context: Record, previousSessionParams: Record | null, - opts?: { useProjectWorkspace?: boolean | null; executionEnvironmentDriver?: string | null }, + opts?: { + useProjectWorkspace?: boolean | null; + executionEnvironmentDriver?: string | null; + }, ): Promise { - const anchor = await resolveAnchorWorkspaceForRun(agent, context, previousSessionParams, opts); + const anchor = await resolveAnchorWorkspaceForRun( + agent, + context, + previousSessionParams, + opts, + ); if (!isMultiProjectWorkspaceSyncEnabled()) { - return { ...anchor, additionalWorkspaces: [], referencedProjectFailures: [] }; + return { + ...anchor, + additionalWorkspaces: [], + referencedProjectFailures: [], + }; } // Derive the remote-transport facts from the selected environment driver. `executionTargetIsRemote` @@ -9201,14 +10542,17 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) // whether that remote target confines each staged tree (only the sandbox driver does). The remote // flag is the targeted kill switch; with it off, a remote run fails closed. const executionEnvironmentDriver = opts?.executionEnvironmentDriver ?? null; - const issueId = readNonEmptyString(context.issueId) ?? readNonEmptyString(context.taskId); - const { additionalWorkspaces, warnings, failures } = await resolveAdditionalRunWorkspaces( - issueId, - anchor.projectId, - { + const issueId = + readNonEmptyString(context.issueId) ?? readNonEmptyString(context.taskId); + const { additionalWorkspaces, warnings, failures } = + await resolveAdditionalRunWorkspaces(issueId, anchor.projectId, { enabled: true, - executionTargetIsRemote: isRemoteExecutionEnvironmentDriver(executionEnvironmentDriver), - targetStagesConfined: isConfinedRemoteStagingDriver(executionEnvironmentDriver), + executionTargetIsRemote: isRemoteExecutionEnvironmentDriver( + executionEnvironmentDriver, + ), + targetStagesConfined: isConfinedRemoteStagingDriver( + executionEnvironmentDriver, + ), remoteReferencedSyncEnabled: isMultiProjectWorkspaceSyncRemoteEnabled(), companyId: agent.companyId, actor: { @@ -9225,14 +10569,16 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) { companyId: agent.companyId, project }, defaultAdditionalProjectWorkspaceDeps(db), ), - }, - ); + }); return { ...anchor, additionalWorkspaces, referencedProjectFailures: failures, - warnings: warnings.length > 0 ? [...anchor.warnings, ...warnings] : anchor.warnings, + warnings: + warnings.length > 0 + ? [...anchor.warnings, ...warnings] + : anchor.warnings, }; } @@ -9381,7 +10727,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const updated = await db .update(heartbeatRuns) .set({ status, ...patch, updatedAt: new Date() }) - .where(and(eq(heartbeatRuns.id, runId), inArray(heartbeatRuns.status, fromStatuses))) + .where( + and( + eq(heartbeatRuns.id, runId), + inArray(heartbeatRuns.status, fromStatuses), + ), + ) .returning() .then((rows) => rows[0] ?? null); @@ -9426,8 +10777,11 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) // already reached a terminal status, the run reached its goal, so use the // matching terminal run status. Otherwise the teardown cut the run short, // so use "interrupted". - const issueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId); - let terminalStatus: "succeeded" | "cancelled" | "interrupted" = "interrupted"; + const issueId = readNonEmptyString( + parseObject(run.contextSnapshot).issueId, + ); + let terminalStatus: "succeeded" | "cancelled" | "interrupted" = + "interrupted"; if (issueId) { const issueStatus = await db .select({ status: issues.status }) @@ -9438,16 +10792,24 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) else if (issueStatus === "cancelled") terminalStatus = "cancelled"; } - const message = - `run terminalized on environment lease release: heartbeat_runs.status was still ${run.status} at teardown`; + const message = `run terminalized on environment lease release: heartbeat_runs.status was still ${run.status} at teardown`; // Match both "running" and "queued". A queued run has released its lease but // never reached "running", so a running-only update would miss it and leave // a phantom live run behind. - const write = await setRunStatusFromLive(run.id, terminalStatus, ["running", "queued"], { - finishedAt: run.finishedAt ?? new Date(), - error: run.error ?? (terminalStatus === "interrupted" ? message : null), - errorCode: run.errorCode ?? (terminalStatus === "interrupted" ? "lease_released_before_terminal" : null), - }); + const write = await setRunStatusFromLive( + run.id, + terminalStatus, + ["running", "queued"], + { + finishedAt: run.finishedAt ?? new Date(), + error: run.error ?? (terminalStatus === "interrupted" ? message : null), + errorCode: + run.errorCode ?? + (terminalStatus === "interrupted" + ? "lease_released_before_terminal" + : null), + }, + ); if (!write.updated) { // Another path already finalized the run. Keep that terminal outcome. return write.run ?? run; @@ -9476,7 +10838,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) return terminalRun ?? run; } - function publishRunLifecyclePluginEvent(run: typeof heartbeatRuns.$inferSelect) { + function publishRunLifecyclePluginEvent( + run: typeof heartbeatRuns.$inferSelect, + ) { const eventType = run.status === "running" ? "agent.run.started" @@ -9505,11 +10869,15 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) triggerDetail: run.triggerDetail, error: run.error ?? null, errorCode: run.errorCode ?? null, - issueId: typeof run.contextSnapshot === "object" && run.contextSnapshot !== null - ? (run.contextSnapshot as Record).issueId ?? null - : null, + issueId: + typeof run.contextSnapshot === "object" && + run.contextSnapshot !== null + ? ((run.contextSnapshot as Record).issueId ?? null) + : null, startedAt: run.startedAt ? new Date(run.startedAt).toISOString() : null, - finishedAt: run.finishedAt ? new Date(run.finishedAt).toISOString() : null, + finishedAt: run.finishedAt + ? new Date(run.finishedAt).toISOString() + : null, }, }); } @@ -9551,9 +10919,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) }); } - async function handleRunLivenessContinuation(run: typeof heartbeatRuns.$inferSelect) { + async function handleRunLivenessContinuation( + run: typeof heartbeatRuns.$inferSelect, + ) { const livenessState = run.livenessState as RunLivenessState | null; - if (livenessState !== "plan_only" && livenessState !== "empty_response") return; + if (livenessState !== "plan_only" && livenessState !== "empty_response") + return; const context = parseObject(run.contextSnapshot); const issueId = readNonEmptyString(context.issueId); @@ -9588,20 +10959,20 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const budgetBlock = issue && agent ? await budgets.getInvocationBlock(issue.companyId, agent.id, { - issueId: issue.id, - projectId: issue.projectId, - }) + issueId: issue.id, + projectId: issue.projectId, + }) : null; if (issue) { - const productivityHold = await productivityReviews.isProductivityReviewContinuationHoldActive({ - companyId: issue.companyId, - issueId: issue.id, - agentId: run.agentId, - }); + const productivityHold = + await productivityReviews.isProductivityReviewContinuationHoldActive({ + companyId: issue.companyId, + issueId: issue.id, + agentId: run.agentId, + }); if (productivityHold.held) { await setRunStatus(run.id, run.status, { - livenessReason: - `${run.livenessReason ?? "Run ended without concrete progress"}; continuation held by productivity review ${productivityHold.reviewIdentifier ?? productivityHold.reviewIssueId}`, + livenessReason: `${run.livenessReason ?? "Run ended without concrete progress"}; continuation held by productivity review ${productivityHold.reviewIdentifier ?? productivityHold.reviewIssueId}`, }); await productivityReviews.recordContinuationHold({ companyId: issue.companyId, @@ -9619,17 +10990,17 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const nextAttempt = readContinuationAttempt(run.continuationAttempt) + 1; const idempotencyKey = issue ? buildRunLivenessContinuationIdempotencyKey({ - issueId: issue.id, - sourceRunId: run.id, - livenessState, - nextAttempt, - }) + issueId: issue.id, + sourceRunId: run.id, + livenessState, + nextAttempt, + }) : null; const existingWake = idempotencyKey ? await findExistingRunLivenessContinuationWake(db, { - companyId: run.companyId, - idempotencyKey, - }) + companyId: run.companyId, + idempotencyKey, + }) : null; const decision = decideRunLivenessContinuation({ @@ -9679,20 +11050,28 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } } - function issueUiLink(issue: Pick) { + function issueUiLink( + issue: Pick, + ) { const label = issue.identifier ?? issue.id; const prefix = issue.identifier?.split("-")[0] || "PAP"; return `[${label}](/${prefix}/issues/${label})`; } - function hasUnmanagedBackgroundTaskEvidence(resultJson: Record | null | undefined) { + function hasUnmanagedBackgroundTaskEvidence( + resultJson: Record | null | undefined, + ) { const evidence = parseObject(resultJson?.unmanagedBackgroundTask); - return evidence.stopped === true && + return ( + evidence.stopped === true && (evidence.stopReason === UNMANAGED_BACKGROUND_TASK_STOP_REASON || - evidence.reason === UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON); + evidence.reason === UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON) + ); } - function withUnmanagedBackgroundTaskStopReason(resultJson: Record | null | undefined) { + function withUnmanagedBackgroundTaskStopReason( + resultJson: Record | null | undefined, + ) { return { ...(resultJson ?? {}), stopReason: UNMANAGED_BACKGROUND_TASK_STOP_REASON, @@ -9705,8 +11084,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) ) { const resultJson = parseObject(run.resultJson); const candidates = [ - hasUnmanagedBackgroundTaskEvidence(resultJson) ? UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON : null, - readNonEmptyString(run.nextAction) ? `Next action noted: ${readNonEmptyString(run.nextAction)}` : null, + hasUnmanagedBackgroundTaskEvidence(resultJson) + ? UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON + : null, + readNonEmptyString(run.nextAction) + ? `Next action noted: ${readNonEmptyString(run.nextAction)}` + : null, readNonEmptyString(run.livenessReason), readNonEmptyString(resultJson.summary), readNonEmptyString(resultJson.result), @@ -9721,7 +11104,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } async function addSuccessfulRunHandoffCommentOnce(input: { - issue: Pick; + issue: Pick< + typeof issues.$inferSelect, + "id" | "identifier" | "title" | "status" + >; run: typeof heartbeatRuns.$inferSelect; agent: Pick; detectedProgressSummary: string; @@ -9753,10 +11139,14 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) ); } - async function handleSuccessfulRunHandoff(run: typeof heartbeatRuns.$inferSelect, agent: typeof agents.$inferSelect) { + async function handleSuccessfulRunHandoff( + run: typeof heartbeatRuns.$inferSelect, + agent: typeof agents.$inferSelect, + ) { if (run.status !== "succeeded") return; const context = parseObject(run.contextSnapshot); - const issueId = readNonEmptyString(context.issueId) ?? readNonEmptyString(context.taskId); + const issueId = + readNonEmptyString(context.issueId) ?? readNonEmptyString(context.taskId); if (!issueId) return; const issue = await db @@ -9779,9 +11169,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .then((rows) => rows[0] ?? null); const idempotencyKey = issue ? buildFinishSuccessfulRunHandoffIdempotencyKey({ - issueId: issue.id, - sourceRunId: run.id, - }) + issueId: issue.id, + sourceRunId: run.id, + }) : null; const taskKey = deriveTaskKeyWithHeartbeatFallback(context, null); const currentUserRedactionOptions = await getCurrentUserRedactionOptions(); @@ -9817,82 +11207,88 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) ] = await Promise.all([ issue ? db - .select({ id: heartbeatRuns.id }) - .from(heartbeatRuns) - .where( - and( - eq(heartbeatRuns.companyId, issue.companyId), - eq(heartbeatRuns.agentId, run.agentId), - inArray(heartbeatRuns.status, [...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES]), - sql`( + .select({ id: heartbeatRuns.id }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, issue.companyId), + eq(heartbeatRuns.agentId, run.agentId), + inArray(heartbeatRuns.status, [ + ...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES, + ]), + sql`( ${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issue.id} or ${heartbeatRuns.contextSnapshot} ->> 'taskId' = ${issue.id} )`, - sql`${heartbeatRuns.id} <> ${run.id}`, - ), - ) - .limit(1) - .then((rows) => rows[0] ?? null) + sql`${heartbeatRuns.id} <> ${run.id}`, + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null) : Promise.resolve(null), issue ? db - .select({ id: agentWakeupRequests.id }) - .from(agentWakeupRequests) - .where( - and( - eq(agentWakeupRequests.companyId, issue.companyId), - eq(agentWakeupRequests.agentId, run.agentId), - inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution", "claimed"]), - sql`( + .select({ id: agentWakeupRequests.id }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, issue.companyId), + eq(agentWakeupRequests.agentId, run.agentId), + inArray(agentWakeupRequests.status, [ + "queued", + "deferred_issue_execution", + "claimed", + ]), + sql`( ${agentWakeupRequests.payload} ->> 'issueId' = ${issue.id} or ${agentWakeupRequests.payload} ->> 'taskId' = ${issue.id} or ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'issueId' = ${issue.id} or ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'taskId' = ${issue.id} )`, - ), - ) - .limit(1) - .then((rows) => rows[0] ?? null) + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null) : Promise.resolve(null), issue ? db - .select({ id: issueThreadInteractions.id }) - .from(issueThreadInteractions) - .where( - and( - eq(issueThreadInteractions.companyId, issue.companyId), - eq(issueThreadInteractions.issueId, issue.id), - eq(issueThreadInteractions.status, "pending"), - ), - ) - .limit(1) - .then((rows) => rows[0] ?? null) + .select({ id: issueThreadInteractions.id }) + .from(issueThreadInteractions) + .where( + and( + eq(issueThreadInteractions.companyId, issue.companyId), + eq(issueThreadInteractions.issueId, issue.id), + eq(issueThreadInteractions.status, "pending"), + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null) : Promise.resolve(null), issue ? db - .select({ id: issueApprovals.approvalId }) - .from(issueApprovals) - .innerJoin(approvals, eq(issueApprovals.approvalId, approvals.id)) - .where( - and( - eq(issueApprovals.companyId, issue.companyId), - eq(issueApprovals.issueId, issue.id), - inArray(approvals.status, ["pending", "revision_requested"]), - ), - ) - .limit(1) - .then((rows) => rows[0] ?? null) + .select({ id: issueApprovals.approvalId }) + .from(issueApprovals) + .innerJoin(approvals, eq(issueApprovals.approvalId, approvals.id)) + .where( + and( + eq(issueApprovals.companyId, issue.companyId), + eq(issueApprovals.issueId, issue.id), + inArray(approvals.status, ["pending", "revision_requested"]), + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null) : Promise.resolve(null), issue ? db - .select({ id: issueRelations.issueId }) - .from(issueRelations) - .where( - and( - eq(issueRelations.companyId, issue.companyId), - eq(issueRelations.relatedIssueId, issue.id), - eq(issueRelations.type, "blocks"), - sql`exists ( + .select({ id: issueRelations.issueId }) + .from(issueRelations) + .where( + and( + eq(issueRelations.companyId, issue.companyId), + eq(issueRelations.relatedIssueId, issue.id), + eq(issueRelations.type, "blocks"), + sql`exists ( select 1 from issues blocker where blocker.id = ${issueRelations.issueId} @@ -9900,58 +11296,58 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) and blocker.status not in ('done', 'cancelled') and blocker.hidden_at is null )`, - ), - ) - .limit(1) - .then((rows) => rows[0] ?? null) + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null) : Promise.resolve(null), issue ? db - .select({ id: issues.id }) - .from(issues) - .where( - and( - eq(issues.companyId, issue.companyId), - inArray(issues.originKind, [ - RECOVERY_ORIGIN_KINDS.strandedIssueRecovery, - RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation, - ]), - eq(issues.originId, issue.id), - visibleIssueCondition(), - notInArray(issues.status, ["done", "cancelled"]), - ), - ) - .limit(1) - .then((rows) => rows[0] ?? null) + .select({ id: issues.id }) + .from(issues) + .where( + and( + eq(issues.companyId, issue.companyId), + inArray(issues.originKind, [ + RECOVERY_ORIGIN_KINDS.strandedIssueRecovery, + RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation, + ]), + eq(issues.originId, issue.id), + visibleIssueCondition(), + notInArray(issues.status, ["done", "cancelled"]), + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null) : Promise.resolve(null), idempotencyKey ? findExistingFinishSuccessfulRunHandoffWake(db, { - companyId: run.companyId, - idempotencyKey, - }) + companyId: run.companyId, + idempotencyKey, + }) : Promise.resolve(null), issue ? budgets.getInvocationBlock(issue.companyId, run.agentId, { - issueId: issue.id, - projectId: issue.projectId, - }) + issueId: issue.id, + projectId: issue.projectId, + }) : Promise.resolve(null), issue ? treeControlSvc.getActivePauseHoldGate(issue.companyId, issue.id) : Promise.resolve(null), issue ? db - .select({ id: routines.id }) - .from(routines) - .where( - and( - eq(routines.companyId, issue.companyId), - eq(routines.parentIssueId, issue.id), - eq(routines.status, "active"), - ), - ) - .limit(1) - .then((rows) => rows[0] ?? null) + .select({ id: routines.id }) + .from(routines) + .where( + and( + eq(routines.companyId, issue.companyId), + eq(routines.parentIssueId, issue.id), + eq(routines.status, "active"), + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null) : Promise.resolve(null), ]); @@ -9966,7 +11362,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) taskKey, hasActiveExecutionPath: Boolean(activeExecutionPath), hasQueuedWake: Boolean(queuedWake), - hasPendingInteractionOrApproval: Boolean(pendingInteraction || pendingApproval), + hasPendingInteractionOrApproval: Boolean( + pendingInteraction || pendingApproval, + ), hasPersistedMonitor: Boolean(issue?.monitorNextCheckAt), hasExplicitBlockerPath: Boolean(explicitBlocker), hasOpenRecoveryIssue: Boolean(openRecoveryIssue), @@ -9994,7 +11392,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .update(heartbeatRuns) .set({ livenessReason: UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON, - resultJson: withUnmanagedBackgroundTaskStopReason(parseObject(run.resultJson)), + resultJson: withUnmanagedBackgroundTaskStopReason( + parseObject(run.resultJson), + ), updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, run.id)); @@ -10016,7 +11416,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) issue, run, agent, - detectedProgressSummary: detectedProgressSummary ?? "The run reported progress, but did not choose a next step.", + detectedProgressSummary: + detectedProgressSummary ?? + "The run reported progress, but did not choose a next step.", }); await logActivity(db, { companyId: issue.companyId, @@ -10039,9 +11441,13 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) }); } - async function handleIssueReviewPathDisposition(run: typeof heartbeatRuns.$inferSelect) { + async function handleIssueReviewPathDisposition( + run: typeof heartbeatRuns.$inferSelect, + ) { const contextSnapshot = parseObject(run.contextSnapshot); - const issueId = readNonEmptyString(contextSnapshot.issueId) ?? readNonEmptyString(contextSnapshot.taskId); + const issueId = + readNonEmptyString(contextSnapshot.issueId) ?? + readNonEmptyString(contextSnapshot.taskId); if (!issueId) return; const issue = await db @@ -10055,11 +11461,19 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .from(issues) .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId))) .then((rows) => rows[0] ?? null); - if (!issue || issue.status !== "in_review" || !issue.assigneeAgentId) return; + if (!issue || issue.status !== "in_review" || !issue.assigneeAgentId) + return; const reviewAttention = await issuesSvc .listReviewAttention(issue.companyId, [issue]) - .then((map) => map.get(issue.id) ?? { state: "none" as const, paths: [], reason: null }); + .then( + (map) => + map.get(issue.id) ?? { + state: "none" as const, + paths: [], + reason: null, + }, + ); if (reviewAttention.state !== "stalled") return; const consumedPathRef = reviewPathConsumedRefFromRun({ @@ -10074,11 +11488,13 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const existingWake = await db .select({ id: agentWakeupRequests.id }) .from(agentWakeupRequests) - .where(and( - eq(agentWakeupRequests.companyId, issue.companyId), - eq(agentWakeupRequests.idempotencyKey, idempotencyKey), - notInArray(agentWakeupRequests.status, ["skipped"]), - )) + .where( + and( + eq(agentWakeupRequests.companyId, issue.companyId), + eq(agentWakeupRequests.idempotencyKey, idempotencyKey), + notInArray(agentWakeupRequests.status, ["skipped"]), + ), + ) .limit(1) .then((rows) => rows[0] ?? null); @@ -10145,9 +11561,14 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const boundedPayload = event.payload ? boundHeartbeatRunEventPayloadForStorage(event.payload) : event.payload; - const secretSanitizedPayload = boundedPayload ? redactEventPayload(boundedPayload) : boundedPayload; + const secretSanitizedPayload = boundedPayload + ? redactEventPayload(boundedPayload) + : boundedPayload; const sanitizedPayload = secretSanitizedPayload - ? redactCurrentUserValue(secretSanitizedPayload, currentUserRedactionOptions) + ? redactCurrentUserValue( + secretSanitizedPayload, + currentUserRedactionOptions, + ) : secretSanitizedPayload; const issueId = readRuntimeStatusIssueIdCandidate(run) ?? null; const progress = buildRunEventRuntimeProgress({ @@ -10156,13 +11577,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) payload: sanitizedPayload ?? null, at: eventAt, }); - const seq = await allocateHeartbeatRunEventSeq(db, run.id); - - await db.insert(heartbeatRunEvents).values({ + const persistedEvent = await appendHeartbeatRunEvent(db, { companyId: run.companyId, runId: run.id, agentId: run.agentId, - seq, eventType: event.eventType, stream: event.stream, level: event.level, @@ -10170,6 +11588,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) message: sanitizedMessage, payload: sanitizedPayload, }); + const seq = persistedEvent.row.seq; publishLiveEvent({ companyId: run.companyId, @@ -10211,18 +11630,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) runId: string, meta: { pid: number; processGroupId: number | null; startedAt: string }, ) { - const startedAt = new Date(meta.startedAt); - return db - .update(heartbeatRuns) - .set({ - processPid: meta.pid, - processGroupId: meta.processGroupId, - processStartedAt: Number.isNaN(startedAt.getTime()) ? new Date() : startedAt, - updatedAt: new Date(), - }) - .where(eq(heartbeatRuns.id, runId)) - .returning() - .then((rows) => rows[0] ?? null); + return persistHeartbeatRunProcessMetadata(db, runId, meta); } async function clearDetachedRunWarning(runId: string) { @@ -10233,7 +11641,13 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) errorCode: null, updatedAt: new Date(), }) - .where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.status, "running"), eq(heartbeatRuns.errorCode, DETACHED_PROCESS_ERROR_CODE))) + .where( + and( + eq(heartbeatRuns.id, runId), + eq(heartbeatRuns.status, "running"), + eq(heartbeatRuns.errorCode, DETACHED_PROCESS_ERROR_CODE), + ), + ) .returning() .then((rows) => rows[0] ?? null); if (!updated) return null; @@ -10242,14 +11656,22 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) eventType: "lifecycle", stream: "system", level: "info", - message: "Detached child process reported activity; cleared detached warning", + message: + "Detached child process reported activity; cleared detached warning", }); return updated; } async function patchRunIssueCommentStatus( runId: string, - patch: Partial>, + patch: Partial< + Pick< + typeof heartbeatRuns.$inferInsert, + | "issueCommentStatus" + | "issueCommentSatisfiedByCommentId" + | "issueCommentRetryQueuedAt" + > + >, ) { return db .update(heartbeatRuns) @@ -10259,10 +11681,16 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .then((rows) => rows[0] ?? null); } - async function findRunIssueComment(runId: string, companyId: string, issueId: string) { - return db + async function findRunIssueComment( + runId: string, + companyId: string, + issueId: string, + resultJson?: Record | null, + ) { + const comments = await db .select({ id: issueComments.id, + body: issueComments.body, }) .from(issueComments) .where( @@ -10272,9 +11700,73 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) eq(issueComments.createdByRunId, runId), ), ) - .orderBy(desc(issueComments.createdAt), desc(issueComments.id)) - .limit(1) - .then((rows) => rows[0] ?? null); + .orderBy(desc(issueComments.createdAt), desc(issueComments.id)); + return findHeartbeatRunCompletionComment(comments, resultJson); + } + + async function findLatestCompletedFinalAgentMessage( + runId: string, + companyId: string, + ) { + const rows = await db + .select({ + seq: heartbeatRunEvents.seq, + payload: heartbeatRunEvents.payload, + }) + .from(heartbeatRunEvents) + .where( + and( + eq(heartbeatRunEvents.companyId, companyId), + eq(heartbeatRunEvents.runId, runId), + eq(heartbeatRunEvents.eventType, "item.completed"), + ), + ) + .orderBy(desc(heartbeatRunEvents.seq)) + .limit(200); + const candidates: Array<{ + seq: number; + text: string; + sourceEventId: string | null; + }> = []; + for (const row of rows) { + const prpEvent = parseObject(parseObject(row.payload).prpEvent); + const payload = parseObject(prpEvent.payload); + if (payload.kind !== "agentMessage" || payload.channel !== "final") + continue; + const text = readNonEmptyString(payload.text); + if (!text) continue; + candidates.push({ + seq: row.seq, + text, + sourceEventId: readNonEmptyString(prpEvent.sourceEventId) ?? null, + }); + } + const recoveryBoundary = await db + .select({ + seq: heartbeatRunEvents.seq, + payload: heartbeatRunEvents.payload, + }) + .from(heartbeatRunEvents) + .where( + and( + eq(heartbeatRunEvents.companyId, companyId), + eq(heartbeatRunEvents.runId, runId), + eq(heartbeatRunEvents.eventType, "lifecycle"), + ), + ) + .orderBy(heartbeatRunEvents.seq) + .limit(200) + .then((lifecycleRows) => + lifecycleRows.find( + (row) => + parseObject(row.payload).retryReasonCode === + "semantic_result_missing", + )?.seq ?? null, + ); + return selectHeartbeatRunFinalAgentMessage({ + candidates, + semanticResultRecoveryAfterSeq: recoveryBoundary, + }); } async function refreshContinuationSummaryForRun( @@ -10329,7 +11821,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) eventType: "lifecycle", stream: "system", level: "warn", - message: "Missing-comment retry suppressed because the agent is not invokable", + message: + "Missing-comment retry suppressed because the agent is not invokable", payload: { reason: invokability.reason, invalidOrgChain: invokability.invalidOrgChain, @@ -10342,14 +11835,20 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const contextSnapshot = parseObject(run.contextSnapshot); const taskKey = deriveTaskKeyWithHeartbeatFallback(contextSnapshot, null); const sessionBefore = await resolveSessionBeforeForWakeup(agent, taskKey); - const retryContextSnapshot = withRecoveryModelProfileHint({ - ...contextSnapshot, - retryOfRunId: run.id, - wakeReason: "missing_issue_comment", - retryReason: "missing_issue_comment", - missingIssueCommentForRunId: run.id, - }, "status_only"); - const responsibleUserId = await resolveResponsibleUserIdForRunContext(run, retryContextSnapshot); + const retryContextSnapshot = withRecoveryModelProfileHint( + { + ...contextSnapshot, + retryOfRunId: run.id, + wakeReason: "missing_issue_comment", + retryReason: "missing_issue_comment", + missingIssueCommentForRunId: run.id, + }, + "status_only", + ); + const responsibleUserId = await resolveResponsibleUserIdForRunContext( + run, + retryContextSnapshot, + ); const now = new Date(); const retryRun = await db.transaction(async (tx) => { @@ -10360,7 +11859,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const issue = await tx .select({ id: issues.id }) .from(issues) - .where(and(eq(issues.companyId, run.companyId), eq(issues.executionRunId, run.id))) + .where( + and( + eq(issues.companyId, run.companyId), + eq(issues.executionRunId, run.id), + ), + ) .then((rows) => rows[0] ?? null); if (!issue) return null; @@ -10372,11 +11876,14 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) source: "automation", triggerDetail: "system", reason: "missing_issue_comment", - payload: withRecoveryModelProfileHint({ - issueId, - retryOfRunId: run.id, - retryReason: "missing_issue_comment", - }, "status_only"), + payload: withRecoveryModelProfileHint( + { + issueId, + retryOfRunId: run.id, + retryReason: "missing_issue_comment", + }, + "status_only", + ), status: "queued", requestedByActorType: "system", requestedByActorId: null, @@ -10451,7 +11958,11 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) return retryRun; } - async function hasDeferredIssueCommentWake(companyId: string, issueId: string, agentId: string) { + async function hasDeferredIssueCommentWake( + companyId: string, + issueId: string, + agentId: string, + ) { const deferredPayloads = await db .select({ payload: agentWakeupRequests.payload }) .from(agentWakeupRequests) @@ -10466,7 +11977,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) return deferredPayloads.some(({ payload }) => { const parsedPayload = parseObject(payload); - const deferredContext = parseObject(parsedPayload[DEFERRED_WAKE_CONTEXT_KEY]); + const deferredContext = parseObject( + parsedPayload[DEFERRED_WAKE_CONTEXT_KEY], + ); return Boolean(deriveCommentId(deferredContext, parsedPayload)); }); } @@ -10474,6 +11987,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) async function finalizeIssueCommentPolicy( run: typeof heartbeatRuns.$inferSelect, agent: typeof agents.$inferSelect, + presentationDecision?: RunPresentationDecision | null, ) { const contextSnapshot = parseObject(run.contextSnapshot); const issueId = readNonEmptyString(contextSnapshot.issueId); @@ -10508,6 +12022,24 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) return { outcome: "not_applicable" as const, queuedRun: null }; } + // A settled run may legitimately have no user-facing prose. The response + // resolver owns that decision; do not wake the agent again merely to force + // an artificial comment into the issue thread. + if ( + presentationDecision?.chosenSource === "none" && + (hasAcceptedSemanticResult(parseObject(run.resultJson)) || + presentationDecision.reasonCodes.includes( + "legacy_adapter_summary_ambiguous", + )) + ) { + await patchRunIssueCommentStatus(run.id, { + issueCommentStatus: "not_applicable", + issueCommentSatisfiedByCommentId: null, + issueCommentRetryQueuedAt: null, + }); + return { outcome: "not_applicable" as const, queuedRun: null }; + } + // A pre-dispatch setup failure means the adapter process never started (for // example an unresolved workspace base ref). No agent could run, so no agent // could post an issue comment. A missing-comment retry cannot help and would @@ -10524,7 +12056,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) return { outcome: "not_applicable" as const, queuedRun: null }; } - const postedComment = await findRunIssueComment(run.id, run.companyId, issueId); + const postedComment = await findRunIssueComment( + run.id, + run.companyId, + issueId, + parseObject(run.resultJson), + ); if (postedComment) { await patchRunIssueCommentStatus(run.id, { issueCommentStatus: "satisfied", @@ -10534,7 +12071,25 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) return { outcome: "satisfied" as const, queuedRun: null }; } - if (readNonEmptyString(contextSnapshot.retryReason) === "missing_issue_comment") { + // Missing-comment recovery is a legacy compatibility path for otherwise + // successful runs. A failed, timed-out, or cancelled run is already owned + // by lifecycle recovery and its terminal system presentation. Queuing a + // prose-only retry here can seize the issue execution lock before the + // authoritative continuation is materialized, replacing real recovery + // with a cheap status-only turn. + if (run.status !== "succeeded") { + await patchRunIssueCommentStatus(run.id, { + issueCommentStatus: "not_applicable", + issueCommentSatisfiedByCommentId: null, + issueCommentRetryQueuedAt: null, + }); + return { outcome: "not_applicable" as const, queuedRun: null }; + } + + if ( + readNonEmptyString(contextSnapshot.retryReason) === + "missing_issue_comment" + ) { await patchRunIssueCommentStatus(run.id, { issueCommentStatus: "retry_exhausted", issueCommentSatisfiedByCommentId: null, @@ -10543,7 +12098,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) eventType: "lifecycle", stream: "system", level: "warn", - message: "Run ended without an issue comment after one retry; no further comment wake will be queued", + message: + "Run ended without an issue comment after one retry; no further comment wake will be queued", }); return { outcome: "retry_exhausted" as const, queuedRun: null }; } @@ -10559,7 +12115,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) return { outcome: "not_applicable" as const, queuedRun: null }; } - if (await hasDeferredIssueCommentWake(run.companyId, issueId, run.agentId)) { + if ( + await hasDeferredIssueCommentWake(run.companyId, issueId, run.agentId) + ) { await patchRunIssueCommentStatus(run.id, { issueCommentStatus: "not_applicable", issueCommentSatisfiedByCommentId: null, @@ -10569,18 +12127,24 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) eventType: "lifecycle", stream: "system", level: "info", - message: "Run ended without an issue comment; a deferred comment wake already exists for this issue", + message: + "Run ended without an issue comment; a deferred comment wake already exists for this issue", }); return { outcome: "not_applicable" as const, queuedRun: null }; } - const queuedRun = await enqueueMissingIssueCommentRetry(run, agent, issueId); + const queuedRun = await enqueueMissingIssueCommentRetry( + run, + agent, + issueId, + ); if (queuedRun) { await appendRunEvent(run, { eventType: "lifecycle", stream: "system", level: "warn", - message: "Run ended without an issue comment; queued one follow-up wake to require a comment", + message: + "Run ended without an issue comment; queued one follow-up wake to require a comment", }); return { outcome: "retry_queued" as const, queuedRun }; } @@ -10600,7 +12164,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const existingRetry = await db .select() .from(heartbeatRuns) - .where(and(eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.retryOfRunId, run.id))) + .where( + and( + eq(heartbeatRuns.companyId, run.companyId), + eq(heartbeatRuns.retryOfRunId, run.id), + ), + ) .orderBy(asc(heartbeatRuns.createdAt)) .limit(1) .then((rows) => rows[0] ?? null); @@ -10609,7 +12178,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) eventType: "lifecycle", stream: "system", level: "warn", - message: "Process-loss retry already exists; skipping duplicate retry enqueue", + message: + "Process-loss retry already exists; skipping duplicate retry enqueue", payload: { retryRunId: existingRetry.id, retryRunStatus: existingRetry.status, @@ -10624,7 +12194,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) eventType: "lifecycle", stream: "system", level: "warn", - message: "Process-loss retry suppressed because the agent is not invokable", + message: + "Process-loss retry suppressed because the agent is not invokable", payload: { reason: invokability.reason, invalidOrgChain: invokability.invalidOrgChain, @@ -10637,18 +12208,25 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const contextSnapshot = parseObject(run.contextSnapshot); const issueId = readNonEmptyString(contextSnapshot.issueId); - const retryReason = readNonEmptyString(contextSnapshot.wakeReason) === "issue_monitor_due" - ? "issue_continuation_needed" - : "process_lost"; + const retryReason = + readNonEmptyString(contextSnapshot.wakeReason) === "issue_monitor_due" + ? "issue_continuation_needed" + : "process_lost"; const taskKey = deriveTaskKeyWithHeartbeatFallback(contextSnapshot, null); const sessionBefore = await resolveSessionBeforeForWakeup(agent, taskKey); - const retryContextSnapshot = withRecoveryModelProfileHint({ - ...contextSnapshot, - retryOfRunId: run.id, - wakeReason: "process_lost_retry", - retryReason, - }, "normal_model"); - const responsibleUserId = await resolveResponsibleUserIdForRunContext(run, retryContextSnapshot); + const retryContextSnapshot = withRecoveryModelProfileHint( + { + ...contextSnapshot, + retryOfRunId: run.id, + wakeReason: "process_lost_retry", + retryReason, + }, + "normal_model", + ); + const responsibleUserId = await resolveResponsibleUserIdForRunContext( + run, + retryContextSnapshot, + ); const queued = await db.transaction(async (tx) => { const wakeupRequest = await tx @@ -10659,10 +12237,13 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) source: "automation", triggerDetail: "system", reason: "process_lost_retry", - payload: withRecoveryModelProfileHint({ - ...(issueId ? { issueId } : {}), - retryOfRunId: run.id, - }, "normal_model"), + payload: withRecoveryModelProfileHint( + { + ...(issueId ? { issueId } : {}), + retryOfRunId: run.id, + }, + "normal_model", + ), status: "queued", requestedByActorType: "system", requestedByActorId: null, @@ -10708,7 +12289,13 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) executionLockedAt: now, updatedAt: now, }) - .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId), eq(issues.executionRunId, run.id))); + .where( + and( + eq(issues.id, issueId), + eq(issues.companyId, run.companyId), + eq(issues.executionRunId, run.id), + ), + ); } return retryRun; @@ -10730,7 +12317,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) eventType: "lifecycle", stream: "system", level: "warn", - message: "Queued automatic retry after orphaned child process was confirmed dead", + message: + "Queued automatic retry after orphaned child process was confirmed dead", payload: { retryOfRunId: run.id, }, @@ -10762,35 +12350,71 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) adapterConfig: unknown; }) { const context = parseObject(input.run.contextSnapshot); - if (context.processTopology === "server_stdio" || context.executionEngine === "acp") { + if ( + context.processTopology === "server_stdio" || + context.executionEngine === "acp" + ) { return true; } - if (context.processTopology === "detached" || context.executionEngine === "cli") { + if ( + context.processTopology === "detached" || + context.executionEngine === "cli" + ) { return false; } - if (!["claude_local", "codex_local", "gemini_local"].includes(input.adapterType)) { + if ( + !["claude_local", "codex_local", "gemini_local"].includes( + input.adapterType, + ) + ) { return false; } - return readNonEmptyString(parseObject(input.adapterConfig).engine) !== "cli"; + return ( + readNonEmptyString(parseObject(input.adapterConfig).engine) !== "cli" + ); } - async function prepareHotRestartShutdown(signal: "SIGINT" | "SIGTERM", now = new Date()) { + async function prepareHotRestartShutdown( + signal: "SIGINT" | "SIGTERM", + now = new Date(), + ) { let intent: Awaited>; try { intent = await readHotRestartIntent(); } catch (err) { - logger.warn({ err }, "failed to read hot-restart intent; falling back to normal shutdown drain"); - return { mode: "read_error" as const, skipDrain: false as const, activeRunIds: [] as string[] }; + logger.warn( + { err }, + "failed to read hot-restart intent; falling back to normal shutdown drain", + ); + return { + mode: "read_error" as const, + skipDrain: false as const, + activeRunIds: [] as string[], + }; } - if (!intent) return { mode: "not_requested" as const, skipDrain: false as const, activeRunIds: [] as string[] }; - if (intent.drainRequired) return { mode: "drain_required" as const, skipDrain: false as const, activeRunIds: [] as string[] }; + if (!intent) + return { + mode: "not_requested" as const, + skipDrain: false as const, + activeRunIds: [] as string[], + }; + if (intent.drainRequired) + return { + mode: "drain_required" as const, + skipDrain: false as const, + activeRunIds: [] as string[], + }; if (!shouldHonorHotRestartIntentForProcess(intent)) { logger.warn( { expectedPid: intent.previousServerPid, currentPid: process.pid }, "hot-restart intent targets a different server pid; falling back to normal shutdown drain", ); - return { mode: "pid_mismatch" as const, skipDrain: false as const, activeRunIds: [] as string[] }; + return { + mode: "pid_mismatch" as const, + skipDrain: false as const, + activeRunIds: [] as string[], + }; } const activeRuns = await db @@ -10853,7 +12477,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) eventType: "lifecycle", stream: "system", level: "info", - message: "Hot restart requested; leaving child process alive for startup adoption", + message: + "Hot restart requested; leaving child process alive for startup adoption", payload: { signal, previousServerPid: intent.previousServerPid, @@ -10865,7 +12490,11 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } logger.info( - { signal, previousServerPid: intent.previousServerPid, activeRunIds: snapshotRuns.map((run) => run.runId) }, + { + signal, + previousServerPid: intent.previousServerPid, + activeRunIds: snapshotRuns.map((run) => run.runId), + }, "hot-restart shutdown snapshot captured; skipping graceful run drain", ); @@ -10881,7 +12510,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) try { intent = await readHotRestartIntent(); } catch (err) { - logger.warn({ err }, "failed to read hot-restart intent on startup; skipping adoption"); + logger.warn( + { err }, + "failed to read hot-restart intent on startup; skipping adoption", + ); return { mode: "read_error" as const, adoptedRunIds: [] as string[], @@ -10901,7 +12533,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } if (!intent.shutdownSnapshot) { - const log = intent.drainRequired ? logger.info.bind(logger) : logger.warn.bind(logger); + const log = intent.drainRequired + ? logger.info.bind(logger) + : logger.warn.bind(logger); log( { previousServerPid: intent.previousServerPid, @@ -10916,18 +12550,22 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const candidates = intent.shutdownSnapshot?.activeRuns ?? []; const missingSnapshotRunIds = findMissingHotRestartSnapshotRunIds(intent); const reconciliationRunIds = [ - ...new Set([...candidates.map((run) => run.runId), ...missingSnapshotRunIds]), + ...new Set([ + ...candidates.map((run) => run.runId), + ...missingSnapshotRunIds, + ]), ]; - const currentRows = reconciliationRunIds.length > 0 - ? await db - .select({ - run: heartbeatRuns, - adapterType: agents.adapterType, - }) - .from(heartbeatRuns) - .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id)) - .where(inArray(heartbeatRuns.id, reconciliationRunIds)) - : []; + const currentRows = + reconciliationRunIds.length > 0 + ? await db + .select({ + run: heartbeatRuns, + adapterType: agents.adapterType, + }) + .from(heartbeatRuns) + .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id)) + .where(inArray(heartbeatRuns.id, reconciliationRunIds)) + : []; const currentByRunId = new Map(currentRows.map((row) => [row.run.id, row])); const reportRuns: HotRestartReportRun[] = []; @@ -10942,10 +12580,16 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) reason: string, patch?: Partial, ) => { - const run = { ...candidate, ...patch, classification, reason } satisfies HotRestartReportRun; + const run = { + ...candidate, + ...patch, + classification, + reason, + } satisfies HotRestartReportRun; reportRuns.push(run); if (classification === "adopted") adoptedRunIds.push(candidate.runId); - else if (classification === "finalized_while_down") finalizedWhileDownRunIds.push(candidate.runId); + else if (classification === "finalized_while_down") + finalizedWhileDownRunIds.push(candidate.runId); else if (classification === "lost") lostRunIds.push(candidate.runId); else skippedRunIds.push(candidate.runId); }; @@ -10959,7 +12603,11 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const candidate = toHotRestartIntentRun(current); if (current.run.status !== "running") { - classify(candidate, "finalized_while_down", `run_status_${current.run.status}`); + classify( + candidate, + "finalized_while_down", + `run_status_${current.run.status}`, + ); } else { classify(candidate, "lost", "missing_shutdown_snapshot"); } @@ -10988,13 +12636,22 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) }; if (run.status !== "running") { - classify(candidate, "finalized_while_down", `run_status_${run.status}`, patch); + classify( + candidate, + "finalized_while_down", + `run_status_${run.status}`, + patch, + ); continue; } - const hasSelectiveAcpDrain = intent.drainReason === "active_acp_run" - && (intent.drainRunIds?.length ?? 0) > 0; - if (hasSelectiveAcpDrain && intent.drainRunIds?.includes(candidate.runId)) { + const hasSelectiveAcpDrain = + intent.drainReason === "active_acp_run" && + (intent.drainRunIds?.length ?? 0) > 0; + if ( + hasSelectiveAcpDrain && + intent.drainRunIds?.includes(candidate.runId) + ) { // A selective ACP drain is expected to persist a terminal row before // the new server starts. If the process was terminated but that write // failed, surface the run as lost instead of hiding it as an expected @@ -11002,16 +12659,18 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) classify(candidate, "lost", "selective_drain_not_finalized", patch); continue; } - if ( - intent.drainRequired - && !hasSelectiveAcpDrain - ) { + if (intent.drainRequired && !hasSelectiveAcpDrain) { classify(candidate, "skipped", "drain_required", patch); continue; } if (!isTrackedLocalChildProcessAdapter(adapterType)) { - classify(candidate, "skipped", "adapter_not_local_child_process", patch); + classify( + candidate, + "skipped", + "adapter_not_local_child_process", + patch, + ); continue; } @@ -11028,24 +12687,36 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) continue; } - const resultJson = mergeHotRestartAdoptionResultJson(parseObject(run.resultJson), { - adoptedAt: now, - previousServerPid: intent.previousServerPid, - newServerPid: process.pid, - previousServerVersion: intent.previousServerVersion, - newServerVersion: serverVersion, - processPid, - processGroupId, - }); + const resultJson = mergeHotRestartAdoptionResultJson( + parseObject(run.resultJson), + { + adoptedAt: now, + previousServerPid: intent.previousServerPid, + newServerPid: process.pid, + previousServerVersion: intent.previousServerVersion, + newServerVersion: serverVersion, + processPid, + processGroupId, + }, + ); const updated = await db .update(heartbeatRuns) .set({ resultJson, - error: run.errorCode === DETACHED_PROCESS_ERROR_CODE ? null : run.error, - errorCode: run.errorCode === DETACHED_PROCESS_ERROR_CODE ? null : run.errorCode, + error: + run.errorCode === DETACHED_PROCESS_ERROR_CODE ? null : run.error, + errorCode: + run.errorCode === DETACHED_PROCESS_ERROR_CODE + ? null + : run.errorCode, updatedAt: now, }) - .where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "running"))) + .where( + and( + eq(heartbeatRuns.id, run.id), + eq(heartbeatRuns.status, "running"), + ), + ) .returning() .then((rows) => rows[0] ?? null); @@ -11056,7 +12727,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .where(eq(heartbeatRuns.id, run.id)) .then((rows) => rows[0] ?? null); if (latest && latest.status !== "running") { - classify(candidate, "finalized_while_down", `run_status_${latest.status}`, patch); + classify( + candidate, + "finalized_while_down", + `run_status_${latest.status}`, + patch, + ); } else { classify(candidate, "lost", "adoption_update_not_applied", patch); } @@ -11077,7 +12753,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) processGroupId, }, }); - classify(candidate, "adopted", processPidAlive ? "process_pid_alive" : "process_group_alive", patch); + classify( + candidate, + "adopted", + processPidAlive ? "process_pid_alive" : "process_group_alive", + patch, + ); } const report = await writeHotRestartReport({ @@ -11085,7 +12766,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) requestedAt: intent.requestedAt, completedAt: now.toISOString(), drainRequired: intent.drainRequired, - drainReason: intent.drainReason ?? (intent.drainRequired ? "requested" : null), + drainReason: + intent.drainReason ?? (intent.drainRequired ? "requested" : null), previousServerPid: intent.previousServerPid, newServerPid: process.pid, previousServerVersion: intent.previousServerVersion, @@ -11139,9 +12821,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .where( selectedRunIds ? and( - eq(heartbeatRuns.status, "running"), - inArray(heartbeatRuns.id, selectedRunIds), - ) + eq(heartbeatRuns.status, "running"), + inArray(heartbeatRuns.id, selectedRunIds), + ) : eq(heartbeatRuns.status, "running"), ); @@ -11149,8 +12831,17 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const retryRunIds: string[] = []; for (const { run, agent } of activeRuns) { + const message = `Interrupted by graceful server shutdown (${signal}); retry queued for restart recovery`; const running = runningProcesses.get(run.id); try { + if (run.runtimeMode === "native") { + await cancelHeartbeatNativeRun({ + db, + runId: run.id, + reason: message, + runtimeMode: run.runtimeMode, + }); + } if (running) { await terminateHeartbeatRunProcess({ pid: running.child.pid, @@ -11162,25 +12853,39 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) runningProcesses.delete(run.id); } - const message = `Interrupted by graceful server shutdown (${signal}); retry queued for restart recovery`; - const interruptedStatus = await setRunStatusIfRunning(run.id, "interrupted", { - finishedAt: now, - error: message, - errorCode: "server_shutdown_interrupted", - signal, - resultJson: mergeRunStopMetadataForAgent(agent, "interrupted", { - resultJson: parseObject(run.resultJson), + const persistedCancellationResult = + run.runtimeMode === "native" + ? await getRun(run.id).then((current) => + parseObject(current?.resultJson), + ) + : parseObject(run.resultJson); + + const interruptedStatus = await setRunStatusIfRunning( + run.id, + "interrupted", + { + finishedAt: now, + error: message, errorCode: "server_shutdown_interrupted", - errorMessage: message, - }), - }); + signal, + resultJson: mergeRunStopMetadataForAgent(agent, "interrupted", { + resultJson: persistedCancellationResult, + errorCode: "server_shutdown_interrupted", + errorMessage: message, + }), + }, + ); if (!interruptedStatus.updated || !interruptedStatus.run) continue; let interrupted = interruptedStatus.run; await setWakeupStatus(run.wakeupRequestId, "cancelled", { finishedAt: now, error: null, }); - interrupted = await classifyAndPersistRunLiveness(interrupted, parseObject(interrupted.resultJson)) ?? interrupted; + interrupted = + (await classifyAndPersistRunLiveness( + interrupted, + parseObject(interrupted.resultJson), + )) ?? interrupted; await releaseEnvironmentLeasesForRun({ runId: interrupted.id, @@ -11218,7 +12923,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) if (interruptedRunIds.length > 0) { logger.warn( - { signal, interrupted: interruptedRunIds.length, interruptedRunIds, retryRunIds }, + { + signal, + interrupted: interruptedRunIds.length, + interruptedRunIds, + retryRunIds, + }, "interrupted running heartbeat runs for graceful shutdown", ); } @@ -11252,7 +12962,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) issueId: string | null; details: Record; }; - type BlockedScheduledRetryGate = Extract; + type BlockedScheduledRetryGate = Extract< + ScheduledRetryGate, + { allowed: false } + >; async function evaluateScheduledRetryGate(input: { run: typeof heartbeatRuns.$inferSelect; @@ -11263,14 +12976,21 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) }): Promise { const { run, agent, contextSnapshot } = input; const retryReason = - input.retryReason ?? readNonEmptyString(contextSnapshot.retryReason) ?? run.scheduledRetryReason ?? null; + input.retryReason ?? + readNonEmptyString(contextSnapshot.retryReason) ?? + run.scheduledRetryReason ?? + null; const issueId = readNonEmptyString(contextSnapshot.issueId); const projectId = readNonEmptyString(contextSnapshot.projectId); - const budgetBlock = await budgets.getInvocationBlock(run.companyId, run.agentId, { - issueId, - projectId, - }); + const budgetBlock = await budgets.getInvocationBlock( + run.companyId, + run.agentId, + { + issueId, + projectId, + }, + ); if (budgetBlock) { return { allowed: false, @@ -11329,7 +13049,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) if (!issue) { return { allowed: false, - reason: "Scheduled retry suppressed because the target issue no longer exists", + reason: + "Scheduled retry suppressed because the target issue no longer exists", errorCode: "issue_not_found", issueId, details: { issueId }, @@ -11385,19 +13106,29 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) return { allowed: false, reason: `Scheduled retry suppressed because issue reached terminal status (${issue.status})`, - errorCode: issue.status === "cancelled" ? "issue_cancelled" : "issue_terminal_status", + errorCode: + issue.status === "cancelled" + ? "issue_cancelled" + : "issue_terminal_status", issueId, details: { issueId, currentStatus: issue.status }, }; } - if (retryReason === MAX_TURN_CONTINUATION_RETRY_REASON && issue.status !== "in_progress") { + if ( + retryReason === MAX_TURN_CONTINUATION_RETRY_REASON && + issue.status !== "in_progress" + ) { return { allowed: false, reason: `Scheduled max-turn continuation suppressed because issue is no longer in_progress (current status: ${issue.status})`, errorCode: "issue_not_in_progress", issueId, - details: { issueId, currentStatus: issue.status, requiredStatus: "in_progress" }, + details: { + issueId, + currentStatus: issue.status, + requiredStatus: "in_progress", + }, }; } @@ -11408,7 +13139,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) ) { return { allowed: false, - reason: "Scheduled max-turn continuation suppressed because the issue execution lock belongs to a different run", + reason: + "Scheduled max-turn continuation suppressed because the issue execution lock belongs to a different run", errorCode: "issue_execution_lock_changed", issueId, details: { @@ -11424,11 +13156,13 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const currentParticipant = executionState?.currentParticipant ?? null; if (currentParticipant) { const participantMatches = - currentParticipant.type === "agent" && currentParticipant.agentId === run.agentId; + currentParticipant.type === "agent" && + currentParticipant.agentId === run.agentId; if (!participantMatches) { return { allowed: false, - reason: "Scheduled retry suppressed because the issue is waiting on another review participant", + reason: + "Scheduled retry suppressed because the issue is waiting on another review participant", errorCode: "issue_review_participant_changed", issueId, details: { @@ -11441,11 +13175,15 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } } - const activePauseHold = await treeControlSvc.getActivePauseHoldGate(run.companyId, issueId); + const activePauseHold = await treeControlSvc.getActivePauseHoldGate( + run.companyId, + issueId, + ); if (activePauseHold) { return { allowed: false, - reason: "Scheduled retry suppressed because the issue is held by an active subtree pause hold", + reason: + "Scheduled retry suppressed because the issue is held by an active subtree pause hold", errorCode: "issue_paused", issueId, details: { @@ -11456,12 +13194,16 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) }; } - const dependencyReadiness = await issuesSvc.listDependencyReadiness(run.companyId, [issueId]); + const dependencyReadiness = await issuesSvc.listDependencyReadiness( + run.companyId, + [issueId], + ); const readiness = dependencyReadiness.get(issueId); if (readiness && !readiness.isDependencyReady) { return { allowed: false, - reason: "Scheduled retry suppressed because issue dependencies are still blocked", + reason: + "Scheduled retry suppressed because issue dependencies are still blocked", errorCode: "issue_dependencies_blocked", issueId, details: { @@ -11539,7 +13281,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) payload: { ...gate.details, scheduledRetryAttempt: cancelled.scheduledRetryAttempt, - scheduledRetryAt: cancelled.scheduledRetryAt ? new Date(cancelled.scheduledRetryAt).toISOString() : null, + scheduledRetryAt: cancelled.scheduledRetryAt + ? new Date(cancelled.scheduledRetryAt).toISOString() + : null, scheduledRetryReason: cancelled.scheduledRetryReason, }, }); @@ -11566,7 +13310,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) allowed: false as const, reason: "Scheduled retry suppressed because the agent no longer exists", errorCode: "agent_not_invokable" as const, - issueId: readNonEmptyString(parseObject(dueRun.contextSnapshot).issueId), + issueId: readNonEmptyString( + parseObject(dueRun.contextSnapshot).issueId, + ), details: { agentId: dueRun.agentId }, }; const cancelled = await cancelScheduledRetryForGate(dueRun, gate, now); @@ -11586,7 +13332,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) agent, contextSnapshot, retryReason: dueRun.scheduledRetryReason, - enforceIssueExecutionLock: dueRun.scheduledRetryReason === MAX_TURN_CONTINUATION_RETRY_REASON, + enforceIssueExecutionLock: + dueRun.scheduledRetryReason === MAX_TURN_CONTINUATION_RETRY_REASON, }); if (!gate.allowed) { if ( @@ -11629,10 +13376,13 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) eventType: "lifecycle", stream: "system", level: "info", - message: "Scheduled retry became due and was promoted to the queued run pool", + message: + "Scheduled retry became due and was promoted to the queued run pool", payload: { scheduledRetryAttempt: promoted.scheduledRetryAttempt, - scheduledRetryAt: promoted.scheduledRetryAt ? new Date(promoted.scheduledRetryAt).toISOString() : null, + scheduledRetryAt: promoted.scheduledRetryAt + ? new Date(promoted.scheduledRetryAt).toISOString() + : null, scheduledRetryReason: promoted.scheduledRetryReason, }, }); @@ -11665,30 +13415,47 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) }, ) { const now = opts?.now ?? new Date(); - const retryReason = opts?.retryReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON; - const wakeReason = opts?.wakeReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON; - const maxAttempts = Math.max(0, Math.floor(opts?.maxAttempts ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS)); + const retryReason = + opts?.retryReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON; + const wakeReason = + opts?.wakeReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON; + const maxAttempts = Math.max( + 0, + Math.floor( + opts?.maxAttempts ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS, + ), + ); const nextAttempt = (run.scheduledRetryAttempt ?? 0) + 1; - const computedBaseSchedule = opts?.delayMs != null - ? nextAttempt <= maxAttempts - ? { - attempt: nextAttempt, - baseDelayMs: Math.max(0, Math.floor(opts.delayMs)), - delayMs: Math.max(0, Math.floor(opts.delayMs)), - dueAt: new Date(now.getTime() + Math.max(0, Math.floor(opts.delayMs))), - maxAttempts, - } - : null - : nextAttempt <= maxAttempts - ? computeBoundedTransientHeartbeatRetrySchedule(nextAttempt, now, opts?.random) - : null; - const baseSchedule = computedBaseSchedule ? { ...computedBaseSchedule, maxAttempts } : null; + const computedBaseSchedule = + opts?.delayMs != null + ? nextAttempt <= maxAttempts + ? { + attempt: nextAttempt, + baseDelayMs: Math.max(0, Math.floor(opts.delayMs)), + delayMs: Math.max(0, Math.floor(opts.delayMs)), + dueAt: new Date( + now.getTime() + Math.max(0, Math.floor(opts.delayMs)), + ), + maxAttempts, + } + : null + : nextAttempt <= maxAttempts + ? computeBoundedTransientHeartbeatRetrySchedule( + nextAttempt, + now, + opts?.random, + ) + : null; + const baseSchedule = computedBaseSchedule + ? { ...computedBaseSchedule, maxAttempts } + : null; const transientRecovery = retryReason === BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON ? readTransientRecoveryContractFromRun(run) : null; const codexTransientFallbackMode = - agent.adapterType === "codex_local" && transientRecovery?.errorFamily === "transient_upstream" + agent.adapterType === "codex_local" && + transientRecovery?.errorFamily === "transient_upstream" ? resolveCodexTransientFallbackMode(nextAttempt) : null; const transientRetryNotBefore = transientRecovery?.retryNotBefore ?? null; @@ -11711,7 +13478,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) await escalatePlanApprovalResumeFailureNeedsAttention({ run, issueId, - attempt: Math.min(run.scheduledRetryAttempt ?? maxAttempts, maxAttempts), + attempt: Math.min( + run.scheduledRetryAttempt ?? maxAttempts, + maxAttempts, + ), maxAttempts, }).catch((error) => { logger.warn( @@ -11734,7 +13504,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) eventType: "lifecycle", stream: "system", level: "warn", - message: "Scheduled retry suppressed because the agent is not invokable", + message: + "Scheduled retry suppressed because the agent is not invokable", payload: { retryReason, scheduledRetryAttempt: nextAttempt, @@ -11746,7 +13517,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) }); return { outcome: "not_scheduled" as const, - reason: "Scheduled retry suppressed because the agent is not invokable", + reason: + "Scheduled retry suppressed because the agent is not invokable", errorCode: "agent_not_invokable" as const, issueId, }; @@ -11754,11 +13526,15 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } const schedule = - transientRetryNotBefore && transientRetryNotBefore.getTime() > baseSchedule.dueAt.getTime() + transientRetryNotBefore && + transientRetryNotBefore.getTime() > baseSchedule.dueAt.getTime() ? { ...baseSchedule, dueAt: transientRetryNotBefore, - delayMs: Math.max(0, transientRetryNotBefore.getTime() - now.getTime()), + delayMs: Math.max( + 0, + transientRetryNotBefore.getTime() - now.getTime(), + ), } : baseSchedule; @@ -11771,7 +13547,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) agent, contextSnapshot, retryReason, - enforceIssueExecutionLock: retryReason === MAX_TURN_CONTINUATION_RETRY_REASON, + enforceIssueExecutionLock: + retryReason === MAX_TURN_CONTINUATION_RETRY_REASON, }); if (!gate.allowed) { await appendRunEvent(run, { @@ -11796,53 +13573,84 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } const taskKey = deriveTaskKeyWithHeartbeatFallback(contextSnapshot, null); const sessionBefore = await resolveSessionBeforeForWakeup(agent, taskKey); - const interactionContinuationPayload = retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON - ? { - mutation: "interaction", - interactionId: readNonEmptyString(contextSnapshot.interactionId), - interactionKind: readNonEmptyString(contextSnapshot.interactionKind), - interactionStatus: readNonEmptyString(contextSnapshot.interactionStatus), - continuationPolicy: readNonEmptyString(contextSnapshot.continuationPolicy), - } - : {}; + const interactionContinuationPayload = + retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON + ? { + mutation: "interaction", + interactionId: readNonEmptyString(contextSnapshot.interactionId), + interactionKind: readNonEmptyString( + contextSnapshot.interactionKind, + ), + interactionStatus: readNonEmptyString( + contextSnapshot.interactionStatus, + ), + continuationPolicy: readNonEmptyString( + contextSnapshot.continuationPolicy, + ), + } + : {}; const workspaceValidationRetryPayload = - retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON && isWorkspaceValidationFailedRun(run) + retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON && + isWorkspaceValidationFailedRun(run) ? readWorkspaceValidationPayloadFromRun(run) : null; const shouldQuarantineWorkspaceForRetry = workspaceValidationRetryPayload !== null && Object.keys(workspaceValidationRetryPayload).length > 0; - const retryContextSnapshot: Record = withRecoveryModelProfileHint({ - ...contextSnapshot, - retryOfRunId: run.id, - wakeReason, - retryReason, - ...(shouldQuarantineWorkspaceForRetry - ? { - workspaceValidationRecovery: { - strategy: "quarantine_failed_workspace_and_retry_clean", - sourceRunId: run.id, - reason: readNonEmptyString(workspaceValidationRetryPayload?.reason) ?? WORKSPACE_VALIDATION_FAILURE_CODE, - fingerprint: readNonEmptyString(workspaceValidationRetryPayload?.fingerprint), - failedExecutionWorkspaceId: readNonEmptyString(workspaceValidationRetryPayload?.executionWorkspaceId), - }, - } - : {}), - ...(transientRecovery ? { errorFamily: transientRecovery.errorFamily } : {}), - scheduledRetryAttempt: schedule.attempt, - scheduledRetryAt: schedule.dueAt.toISOString(), - ...(transientRetryNotBefore ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } : {}), - ...(transientRecovery?.errorFamily === "provider_quota" && transientRetryNotBefore - ? { providerQuotaRetryNotBefore: transientRetryNotBefore.toISOString() } - : {}), - ...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}), - }, "normal_model"); - const responsibleUserId = await resolveResponsibleUserIdForRunContext(run, retryContextSnapshot); - const continuationRetryIdempotencyKey = retryReason === MAX_TURN_CONTINUATION_RETRY_REASON - ? `max-turn-continuation:${run.companyId}:${issueId ?? "no-issue"}:${run.id}:${schedule.attempt}` - : retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON - ? `interaction-continuation:${run.companyId}:${issueId ?? "no-issue"}:${run.id}:${schedule.attempt}` - : null; + const retryContextSnapshot: Record = + withRecoveryModelProfileHint( + { + ...contextSnapshot, + retryOfRunId: run.id, + wakeReason, + retryReason, + ...(shouldQuarantineWorkspaceForRetry + ? { + workspaceValidationRecovery: { + strategy: "quarantine_failed_workspace_and_retry_clean", + sourceRunId: run.id, + reason: + readNonEmptyString( + workspaceValidationRetryPayload?.reason, + ) ?? WORKSPACE_VALIDATION_FAILURE_CODE, + fingerprint: readNonEmptyString( + workspaceValidationRetryPayload?.fingerprint, + ), + failedExecutionWorkspaceId: readNonEmptyString( + workspaceValidationRetryPayload?.executionWorkspaceId, + ), + }, + } + : {}), + ...(transientRecovery + ? { errorFamily: transientRecovery.errorFamily } + : {}), + scheduledRetryAttempt: schedule.attempt, + scheduledRetryAt: schedule.dueAt.toISOString(), + ...(transientRetryNotBefore + ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } + : {}), + ...(transientRecovery?.errorFamily === "provider_quota" && + transientRetryNotBefore + ? { + providerQuotaRetryNotBefore: + transientRetryNotBefore.toISOString(), + } + : {}), + ...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}), + }, + "normal_model", + ); + const responsibleUserId = await resolveResponsibleUserIdForRunContext( + run, + retryContextSnapshot, + ); + const continuationRetryIdempotencyKey = + retryReason === MAX_TURN_CONTINUATION_RETRY_REASON + ? `max-turn-continuation:${run.companyId}:${issueId ?? "no-issue"}:${run.id}:${schedule.attempt}` + : retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON + ? `interaction-continuation:${run.companyId}:${issueId ?? "no-issue"}:${run.id}:${schedule.attempt}` + : null; type ScheduledRetryTransactionResult = | { @@ -11864,362 +13672,437 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) details: Record; }; - const scheduleResult = await db.transaction(async (tx): Promise => { - if (retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON) { - if (issueId) { - await tx.execute( - sql`select id from issues where company_id = ${run.companyId} and id = ${issueId} for update`, - ); - } else { - await tx.execute( - sql`select id from heartbeat_runs where company_id = ${run.companyId} and id = ${run.id} for update`, - ); - } - - const existingContinuation = await tx - .select() - .from(heartbeatRuns) - .where( - and( - eq(heartbeatRuns.companyId, run.companyId), - eq(heartbeatRuns.retryOfRunId, run.id), - eq(heartbeatRuns.scheduledRetryReason, retryReason), - eq(heartbeatRuns.scheduledRetryAttempt, schedule.attempt), - inArray(heartbeatRuns.status, [...MAX_TURN_CONTINUATION_LIVE_RUN_STATUSES]), - issueId - ? sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}` - : sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' is null`, - ), - ) - .orderBy(asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id)) - .limit(1) - .then((rows) => rows[0] ?? null); - - if (existingContinuation) { - if (existingContinuation.wakeupRequestId) { - const existingWakeup = await tx - .select({ coalescedCount: agentWakeupRequests.coalescedCount }) - .from(agentWakeupRequests) - .where(eq(agentWakeupRequests.id, existingContinuation.wakeupRequestId)) - .then((rows) => rows[0] ?? null); - - await tx - .update(agentWakeupRequests) - .set({ - coalescedCount: (existingWakeup?.coalescedCount ?? 0) + 1, - updatedAt: now, - }) - .where(eq(agentWakeupRequests.id, existingContinuation.wakeupRequestId)); + const scheduleResult = await db.transaction( + async (tx): Promise => { + if (retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON) { + if (issueId) { + await tx.execute( + sql`select id from issues where company_id = ${run.companyId} and id = ${issueId} for update`, + ); + } else { + await tx.execute( + sql`select id from heartbeat_runs where company_id = ${run.companyId} and id = ${run.id} for update`, + ); } - return { - outcome: "scheduled", - run: existingContinuation, - reusedExisting: true, - }; - } - } + const existingContinuation = await tx + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, run.companyId), + eq(heartbeatRuns.retryOfRunId, run.id), + eq(heartbeatRuns.scheduledRetryReason, retryReason), + eq(heartbeatRuns.scheduledRetryAttempt, schedule.attempt), + inArray(heartbeatRuns.status, [ + ...MAX_TURN_CONTINUATION_LIVE_RUN_STATUSES, + ]), + issueId + ? sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}` + : sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' is null`, + ), + ) + .orderBy(asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id)) + .limit(1) + .then((rows) => rows[0] ?? null); - if (retryReason === MAX_TURN_CONTINUATION_RETRY_REASON) { - if (issueId) { - await tx.execute( - sql`select id from issues where company_id = ${run.companyId} and id = ${issueId} for update`, - ); - } else { - await tx.execute( - sql`select id from heartbeat_runs where company_id = ${run.companyId} and id = ${run.id} for update`, - ); + if (existingContinuation) { + if (existingContinuation.wakeupRequestId) { + const existingWakeup = await tx + .select({ coalescedCount: agentWakeupRequests.coalescedCount }) + .from(agentWakeupRequests) + .where( + eq( + agentWakeupRequests.id, + existingContinuation.wakeupRequestId, + ), + ) + .then((rows) => rows[0] ?? null); + + await tx + .update(agentWakeupRequests) + .set({ + coalescedCount: (existingWakeup?.coalescedCount ?? 0) + 1, + updatedAt: now, + }) + .where( + eq( + agentWakeupRequests.id, + existingContinuation.wakeupRequestId, + ), + ); + } + + return { + outcome: "scheduled", + run: existingContinuation, + reusedExisting: true, + }; + } } - const existingContinuation = await tx - .select() - .from(heartbeatRuns) - .where( - and( - eq(heartbeatRuns.companyId, run.companyId), - eq(heartbeatRuns.retryOfRunId, run.id), - eq(heartbeatRuns.scheduledRetryReason, retryReason), - eq(heartbeatRuns.scheduledRetryAttempt, schedule.attempt), - inArray(heartbeatRuns.status, [...MAX_TURN_CONTINUATION_LIVE_RUN_STATUSES]), - issueId - ? sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}` - : sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' is null`, - ), - ) - .orderBy(asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id)) - .limit(1) - .then((rows) => rows[0] ?? null); - - if (existingContinuation) { - if (existingContinuation.wakeupRequestId) { - const existingWakeup = await tx - .select({ coalescedCount: agentWakeupRequests.coalescedCount }) - .from(agentWakeupRequests) - .where(eq(agentWakeupRequests.id, existingContinuation.wakeupRequestId)) - .then((rows) => rows[0] ?? null); - - await tx - .update(agentWakeupRequests) - .set({ - coalescedCount: (existingWakeup?.coalescedCount ?? 0) + 1, - updatedAt: now, - }) - .where(eq(agentWakeupRequests.id, existingContinuation.wakeupRequestId)); + if (retryReason === MAX_TURN_CONTINUATION_RETRY_REASON) { + if (issueId) { + await tx.execute( + sql`select id from issues where company_id = ${run.companyId} and id = ${issueId} for update`, + ); + } else { + await tx.execute( + sql`select id from heartbeat_runs where company_id = ${run.companyId} and id = ${run.id} for update`, + ); } - return { - outcome: "scheduled", - run: existingContinuation, - reusedExisting: true, - }; + const existingContinuation = await tx + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, run.companyId), + eq(heartbeatRuns.retryOfRunId, run.id), + eq(heartbeatRuns.scheduledRetryReason, retryReason), + eq(heartbeatRuns.scheduledRetryAttempt, schedule.attempt), + inArray(heartbeatRuns.status, [ + ...MAX_TURN_CONTINUATION_LIVE_RUN_STATUSES, + ]), + issueId + ? sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}` + : sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' is null`, + ), + ) + .orderBy(asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id)) + .limit(1) + .then((rows) => rows[0] ?? null); + + if (existingContinuation) { + if (existingContinuation.wakeupRequestId) { + const existingWakeup = await tx + .select({ coalescedCount: agentWakeupRequests.coalescedCount }) + .from(agentWakeupRequests) + .where( + eq( + agentWakeupRequests.id, + existingContinuation.wakeupRequestId, + ), + ) + .then((rows) => rows[0] ?? null); + + await tx + .update(agentWakeupRequests) + .set({ + coalescedCount: (existingWakeup?.coalescedCount ?? 0) + 1, + updatedAt: now, + }) + .where( + eq( + agentWakeupRequests.id, + existingContinuation.wakeupRequestId, + ), + ); + } + + return { + outcome: "scheduled", + run: existingContinuation, + reusedExisting: true, + }; + } + + if (issueId) { + const lockedIssue = await tx + .select({ + id: issues.id, + status: issues.status, + assigneeAgentId: issues.assigneeAgentId, + executionRunId: issues.executionRunId, + }) + .from(issues) + .where( + and( + eq(issues.id, issueId), + eq(issues.companyId, run.companyId), + ), + ) + .then((rows) => rows[0] ?? null); + + if (!lockedIssue) { + return { + outcome: "not_scheduled", + reason: + "Scheduled max-turn continuation suppressed because the target issue no longer exists", + errorCode: "issue_not_found", + issueId, + details: { issueId }, + }; + } + + if (lockedIssue.assigneeAgentId !== run.agentId) { + return { + outcome: "not_scheduled", + reason: + "Scheduled max-turn continuation suppressed because issue ownership changed", + errorCode: "issue_reassigned", + issueId, + details: { + issueId, + previousAssigneeAgentId: run.agentId, + currentAssigneeAgentId: lockedIssue.assigneeAgentId, + }, + }; + } + + if ( + lockedIssue.status === "cancelled" || + lockedIssue.status === "done" + ) { + return { + outcome: "not_scheduled", + reason: `Scheduled max-turn continuation suppressed because issue reached terminal status (${lockedIssue.status})`, + errorCode: + lockedIssue.status === "cancelled" + ? "issue_cancelled" + : "issue_terminal_status", + issueId, + details: { issueId, currentStatus: lockedIssue.status }, + }; + } + + if (lockedIssue.status !== "in_progress") { + return { + outcome: "not_scheduled", + reason: `Scheduled max-turn continuation suppressed because issue is no longer in_progress (current status: ${lockedIssue.status})`, + errorCode: "issue_not_in_progress", + issueId, + details: { + issueId, + currentStatus: lockedIssue.status, + requiredStatus: "in_progress", + }, + }; + } + + if (lockedIssue.executionRunId !== run.id) { + return { + outcome: "not_scheduled", + reason: + "Scheduled max-turn continuation suppressed because the issue execution lock belongs to a different run", + errorCode: "issue_execution_lock_changed", + issueId, + details: { + issueId, + expectedExecutionRunId: run.id, + currentExecutionRunId: lockedIssue.executionRunId, + }, + }; + } + } } - if (issueId) { - const lockedIssue = await tx + const wakeupRequest = await tx + .insert(agentWakeupRequests) + .values({ + companyId: run.companyId, + agentId: run.agentId, + source: "automation", + triggerDetail: "system", + reason: wakeReason, + payload: withRecoveryModelProfileHint( + { + ...(issueId ? { issueId } : {}), + retryOfRunId: run.id, + ...interactionContinuationPayload, + retryReason, + ...(transientRecovery + ? { errorFamily: transientRecovery.errorFamily } + : {}), + scheduledRetryAttempt: schedule.attempt, + scheduledRetryAt: schedule.dueAt.toISOString(), + ...(transientRetryNotBefore + ? { + transientRetryNotBefore: + transientRetryNotBefore.toISOString(), + } + : {}), + ...(transientRecovery?.errorFamily === "provider_quota" && + transientRetryNotBefore + ? { + providerQuotaRetryNotBefore: + transientRetryNotBefore.toISOString(), + } + : {}), + ...(codexTransientFallbackMode + ? { codexTransientFallbackMode } + : {}), + }, + "normal_model", + ), + status: "queued", + requestedByActorType: "system", + requestedByActorId: null, + idempotencyKey: continuationRetryIdempotencyKey, + updatedAt: now, + }) + .returning() + .then((rows) => rows[0]); + + const scheduledRun = await tx + .insert(heartbeatRuns) + .values({ + companyId: run.companyId, + agentId: run.agentId, + invocationSource: "automation", + triggerDetail: "system", + status: "scheduled_retry", + wakeupRequestId: wakeupRequest.id, + contextSnapshot: retryContextSnapshot, + responsibleUserId, + sessionIdBefore: sessionBefore, + retryOfRunId: run.id, + scheduledRetryAt: schedule.dueAt, + scheduledRetryAttempt: schedule.attempt, + scheduledRetryReason: retryReason, + continuationAttempt: readContinuationAttempt( + retryContextSnapshot.livenessContinuationAttempt, + ), + updatedAt: now, + }) + .returning() + .then((rows) => rows[0]); + + await tx + .update(agentWakeupRequests) + .set({ + runId: scheduledRun.id, + updatedAt: now, + }) + .where(eq(agentWakeupRequests.id, wakeupRequest.id)); + + let detachWorkspaceFromIssue = false; + if (issueId && shouldQuarantineWorkspaceForRetry) { + const issueWorkspace = await tx .select({ id: issues.id, - status: issues.status, - assigneeAgentId: issues.assigneeAgentId, - executionRunId: issues.executionRunId, + companyId: issues.companyId, + executionWorkspaceId: issues.executionWorkspaceId, }) .from(issues) - .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId))) - .then((rows) => rows[0] ?? null); - - if (!lockedIssue) { - return { - outcome: "not_scheduled", - reason: "Scheduled max-turn continuation suppressed because the target issue no longer exists", - errorCode: "issue_not_found", - issueId, - details: { issueId }, - }; - } - - if (lockedIssue.assigneeAgentId !== run.agentId) { - return { - outcome: "not_scheduled", - reason: "Scheduled max-turn continuation suppressed because issue ownership changed", - errorCode: "issue_reassigned", - issueId, - details: { - issueId, - previousAssigneeAgentId: run.agentId, - currentAssigneeAgentId: lockedIssue.assigneeAgentId, - }, - }; - } - - if (lockedIssue.status === "cancelled" || lockedIssue.status === "done") { - return { - outcome: "not_scheduled", - reason: `Scheduled max-turn continuation suppressed because issue reached terminal status (${lockedIssue.status})`, - errorCode: lockedIssue.status === "cancelled" ? "issue_cancelled" : "issue_terminal_status", - issueId, - details: { issueId, currentStatus: lockedIssue.status }, - }; - } - - if (lockedIssue.status !== "in_progress") { - return { - outcome: "not_scheduled", - reason: `Scheduled max-turn continuation suppressed because issue is no longer in_progress (current status: ${lockedIssue.status})`, - errorCode: "issue_not_in_progress", - issueId, - details: { issueId, currentStatus: lockedIssue.status, requiredStatus: "in_progress" }, - }; - } - - if (lockedIssue.executionRunId !== run.id) { - return { - outcome: "not_scheduled", - reason: - "Scheduled max-turn continuation suppressed because the issue execution lock belongs to a different run", - errorCode: "issue_execution_lock_changed", - issueId, - details: { - issueId, - expectedExecutionRunId: run.id, - currentExecutionRunId: lockedIssue.executionRunId, - }, - }; - } - } - } - - const wakeupRequest = await tx - .insert(agentWakeupRequests) - .values({ - companyId: run.companyId, - agentId: run.agentId, - source: "automation", - triggerDetail: "system", - reason: wakeReason, - payload: withRecoveryModelProfileHint({ - ...(issueId ? { issueId } : {}), - retryOfRunId: run.id, - ...interactionContinuationPayload, - retryReason, - ...(transientRecovery ? { errorFamily: transientRecovery.errorFamily } : {}), - scheduledRetryAttempt: schedule.attempt, - scheduledRetryAt: schedule.dueAt.toISOString(), - ...(transientRetryNotBefore ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } : {}), - ...(transientRecovery?.errorFamily === "provider_quota" && transientRetryNotBefore - ? { providerQuotaRetryNotBefore: transientRetryNotBefore.toISOString() } - : {}), - ...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}), - }, "normal_model"), - status: "queued", - requestedByActorType: "system", - requestedByActorId: null, - idempotencyKey: continuationRetryIdempotencyKey, - updatedAt: now, - }) - .returning() - .then((rows) => rows[0]); - - const scheduledRun = await tx - .insert(heartbeatRuns) - .values({ - companyId: run.companyId, - agentId: run.agentId, - invocationSource: "automation", - triggerDetail: "system", - status: "scheduled_retry", - wakeupRequestId: wakeupRequest.id, - contextSnapshot: retryContextSnapshot, - responsibleUserId, - sessionIdBefore: sessionBefore, - retryOfRunId: run.id, - scheduledRetryAt: schedule.dueAt, - scheduledRetryAttempt: schedule.attempt, - scheduledRetryReason: retryReason, - continuationAttempt: readContinuationAttempt(retryContextSnapshot.livenessContinuationAttempt), - updatedAt: now, - }) - .returning() - .then((rows) => rows[0]); - - await tx - .update(agentWakeupRequests) - .set({ - runId: scheduledRun.id, - updatedAt: now, - }) - .where(eq(agentWakeupRequests.id, wakeupRequest.id)); - - let detachWorkspaceFromIssue = false; - if (issueId && shouldQuarantineWorkspaceForRetry) { - const issueWorkspace = await tx - .select({ - id: issues.id, - companyId: issues.companyId, - executionWorkspaceId: issues.executionWorkspaceId, - }) - .from(issues) - .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId))) - .for("update") - .then((rows) => rows[0] ?? null); - const failedExecutionWorkspaceId = - readNonEmptyString(workspaceValidationRetryPayload?.executionWorkspaceId) ?? - readNonEmptyString(issueWorkspace?.executionWorkspaceId); - - if (issueWorkspace && failedExecutionWorkspaceId) { - const failedWorkspace = await tx - .select({ - id: executionWorkspaces.id, - companyId: executionWorkspaces.companyId, - sourceIssueId: executionWorkspaces.sourceIssueId, - status: executionWorkspaces.status, - metadata: executionWorkspaces.metadata, - }) - .from(executionWorkspaces) - .where(and( - eq(executionWorkspaces.id, failedExecutionWorkspaceId), - eq(executionWorkspaces.companyId, run.companyId), - )) + .where( + and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)), + ) .for("update") .then((rows) => rows[0] ?? null); + const failedExecutionWorkspaceId = + readNonEmptyString( + workspaceValidationRetryPayload?.executionWorkspaceId, + ) ?? readNonEmptyString(issueWorkspace?.executionWorkspaceId); - const workspaceBelongsToIssue = - failedWorkspace + if (issueWorkspace && failedExecutionWorkspaceId) { + const failedWorkspace = await tx + .select({ + id: executionWorkspaces.id, + companyId: executionWorkspaces.companyId, + sourceIssueId: executionWorkspaces.sourceIssueId, + status: executionWorkspaces.status, + metadata: executionWorkspaces.metadata, + }) + .from(executionWorkspaces) + .where( + and( + eq(executionWorkspaces.id, failedExecutionWorkspaceId), + eq(executionWorkspaces.companyId, run.companyId), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null); + + const workspaceBelongsToIssue = failedWorkspace ? failedWorkspace.sourceIssueId === issueId : false; - if ( - failedWorkspace && - workspaceBelongsToIssue && - issueWorkspace.executionWorkspaceId === failedExecutionWorkspaceId - ) { - const existingMetadata = parseObject(failedWorkspace.metadata); - const quarantine = { - reason: WORKSPACE_VALIDATION_FAILURE_CODE, - retryReason, - sourceRunId: run.id, - retryRunId: scheduledRun.id, - issueId, - sourceIssueId: failedWorkspace.sourceIssueId ?? null, - quarantinedAt: now.toISOString(), - workspaceValidation: workspaceValidationRetryPayload ?? {}, - }; - await tx - .update(executionWorkspaces) - .set({ - status: "archived", - closedAt: now, - cleanupEligibleAt: null, - cleanupReason: WORKSPACE_VALIDATION_FAILURE_CODE, - metadata: { - ...existingMetadata, - workspaceValidationQuarantine: quarantine, - }, - updatedAt: now, - }) - .where(and( - eq(executionWorkspaces.id, failedWorkspace.id), - eq(executionWorkspaces.companyId, run.companyId), - )); + if ( + failedWorkspace && + workspaceBelongsToIssue && + issueWorkspace.executionWorkspaceId === failedExecutionWorkspaceId + ) { + const existingMetadata = parseObject(failedWorkspace.metadata); + const quarantine = { + reason: WORKSPACE_VALIDATION_FAILURE_CODE, + retryReason, + sourceRunId: run.id, + retryRunId: scheduledRun.id, + issueId, + sourceIssueId: failedWorkspace.sourceIssueId ?? null, + quarantinedAt: now.toISOString(), + workspaceValidation: workspaceValidationRetryPayload ?? {}, + }; + await tx + .update(executionWorkspaces) + .set({ + status: "archived", + closedAt: now, + cleanupEligibleAt: null, + cleanupReason: WORKSPACE_VALIDATION_FAILURE_CODE, + metadata: { + ...existingMetadata, + workspaceValidationQuarantine: quarantine, + }, + updatedAt: now, + }) + .where( + and( + eq(executionWorkspaces.id, failedWorkspace.id), + eq(executionWorkspaces.companyId, run.companyId), + ), + ); - await logActivity(tx as unknown as Db, { - companyId: run.companyId, - actorType: "system", - actorId: "heartbeat", - agentId: run.agentId, - runId: run.id, - action: "execution_workspace.workspace_validation_quarantined", - entityType: "execution_workspace", - entityId: failedWorkspace.id, - details: quarantine, - }); - detachWorkspaceFromIssue = issueWorkspace.executionWorkspaceId === failedExecutionWorkspaceId; + await logActivity(tx as unknown as Db, { + companyId: run.companyId, + actorType: "system", + actorId: "heartbeat", + agentId: run.agentId, + runId: run.id, + action: "execution_workspace.workspace_validation_quarantined", + entityType: "execution_workspace", + entityId: failedWorkspace.id, + details: quarantine, + }); + detachWorkspaceFromIssue = + issueWorkspace.executionWorkspaceId === + failedExecutionWorkspaceId; + } } } - } - if (issueId) { - await tx - .update(issues) - .set({ - executionRunId: scheduledRun.id, - executionAgentNameKey: normalizeAgentNameKey(agent.name), - executionLockedAt: now, - ...(detachWorkspaceFromIssue - ? { - executionWorkspaceId: null, - executionWorkspacePreference: null, - } - : {}), - updatedAt: now, - }) - .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId), eq(issues.executionRunId, run.id))); - } + if (issueId) { + await tx + .update(issues) + .set({ + executionRunId: scheduledRun.id, + executionAgentNameKey: normalizeAgentNameKey(agent.name), + executionLockedAt: now, + ...(detachWorkspaceFromIssue + ? { + executionWorkspaceId: null, + executionWorkspacePreference: null, + } + : {}), + updatedAt: now, + }) + .where( + and( + eq(issues.id, issueId), + eq(issues.companyId, run.companyId), + eq(issues.executionRunId, run.id), + ), + ); + } - return { - outcome: "scheduled", - run: scheduledRun, - reusedExisting: false, - }; - }); + return { + outcome: "scheduled", + run: scheduledRun, + reusedExisting: false, + }; + }, + ); if (scheduleResult.outcome === "not_scheduled") { await appendRunEvent(run, { @@ -12243,7 +14126,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } const retryRun = scheduleResult.run; - const dueAt = retryRun.scheduledRetryAt ? new Date(retryRun.scheduledRetryAt) : schedule.dueAt; + const dueAt = retryRun.scheduledRetryAt + ? new Date(retryRun.scheduledRetryAt) + : schedule.dueAt; if (scheduleResult.reusedExisting) { await appendRunEvent(run, { @@ -12278,14 +14163,22 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) payload: { retryRunId: retryRun.id, retryReason, - ...(transientRecovery ? { errorFamily: transientRecovery.errorFamily } : {}), + ...(transientRecovery + ? { errorFamily: transientRecovery.errorFamily } + : {}), scheduledRetryAttempt: schedule.attempt, scheduledRetryAt: schedule.dueAt.toISOString(), baseDelayMs: schedule.baseDelayMs, delayMs: schedule.delayMs, - ...(transientRetryNotBefore ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } : {}), - ...(transientRecovery?.errorFamily === "provider_quota" && transientRetryNotBefore - ? { providerQuotaRetryNotBefore: transientRetryNotBefore.toISOString() } + ...(transientRetryNotBefore + ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } + : {}), + ...(transientRecovery?.errorFamily === "provider_quota" && + transientRetryNotBefore + ? { + providerQuotaRetryNotBefore: + transientRetryNotBefore.toISOString(), + } : {}), ...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}), }, @@ -12335,7 +14228,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) now?: Date; }): Promise { const staleCutoff = new Date( - (input.now ?? new Date()).getTime() - WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS, + (input.now ?? new Date()).getTime() - + WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS, ); return await db .select({ @@ -12429,19 +14323,24 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) error: deferral.message, }).catch(() => undefined); - const cancelledRun = cancelWrite.run ?? (await getRun(run.id).catch(() => null)); + const cancelledRun = + cancelWrite.run ?? (await getRun(run.id).catch(() => null)); const agentRow = await getAgent(run.agentId).catch(() => null); let scheduleOutcome: string | null = null; if (cancelledRun && agentRow) { - const scheduleResult = await scheduleBoundedRetryForRun(cancelledRun, agentRow, { - now, - retryReason: WORKSPACE_BUSY_RETRY_REASON, - wakeReason: WORKSPACE_BUSY_RETRY_WAKE_REASON, - // Always admit the next attempt: workspace-busy deferral is bounded by - // holder liveness, not by an attempt counter. - maxAttempts: (cancelledRun.scheduledRetryAttempt ?? 0) + 1, - delayMs: computeWorkspaceBusyRetryDelayMs(), - }).catch((scheduleErr) => { + const scheduleResult = await scheduleBoundedRetryForRun( + cancelledRun, + agentRow, + { + now, + retryReason: WORKSPACE_BUSY_RETRY_REASON, + wakeReason: WORKSPACE_BUSY_RETRY_WAKE_REASON, + // Always admit the next attempt: workspace-busy deferral is bounded by + // holder liveness, not by an attempt counter. + maxAttempts: (cancelledRun.scheduledRetryAttempt ?? 0) + 1, + delayMs: computeWorkspaceBusyRetryDelayMs(), + }, + ).catch((scheduleErr) => { logger.error( { err: scheduleErr, runId: run.id }, "failed to schedule workspace-busy retry after deferral", @@ -12471,12 +14370,14 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } if (cancelledRun && scheduleOutcome !== "scheduled") { - await releaseIssueExecutionAndPromote(cancelledRun).catch((releaseErr) => { - logger.error( - { err: releaseErr, runId: run.id }, - "failed to release issue execution after workspace-busy deferral", - ); - }); + await releaseIssueExecutionAndPromote(cancelledRun).catch( + (releaseErr) => { + logger.error( + { err: releaseErr, runId: run.id }, + "failed to release issue execution after workspace-busy deferral", + ); + }, + ); } await finalizeAgentStatus(run.agentId, "cancelled", null, { @@ -12489,14 +14390,19 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) agent: typeof agents.$inferSelect, ) { if (!run.wakeupRequestId) return null; - if (!isResolvedInteractionContinuationWakeContext(run.contextSnapshot)) return null; + if (!isResolvedInteractionContinuationWakeContext(run.contextSnapshot)) + return null; if (!isRetryableInteractionContinuationInfrastructureFailure(run)) { const context = parseObject(run.contextSnapshot); const issueId = readNonEmptyString(context.issueId); await escalatePlanApprovalResumeFailureNeedsAttention({ run, issueId, - attempt: Math.min(run.scheduledRetryAttempt ?? INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS, INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS), + attempt: Math.min( + run.scheduledRetryAttempt ?? + INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS, + INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS, + ), maxAttempts: INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS, }).catch((error) => { logger.warn( @@ -12526,7 +14432,11 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined, ), ) - .orderBy(asc(heartbeatRuns.scheduledRetryAt), asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id)) + .orderBy( + asc(heartbeatRuns.scheduledRetryAt), + asc(heartbeatRuns.createdAt), + asc(heartbeatRuns.id), + ) .limit(50); const promotedRunIds: string[] = []; @@ -12565,17 +14475,23 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) sql`${heartbeatRuns.retryOfRunId} is not null`, ), ) - .orderBy(desc(heartbeatRuns.updatedAt), desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)) + .orderBy( + desc(heartbeatRuns.updatedAt), + desc(heartbeatRuns.createdAt), + desc(heartbeatRuns.id), + ) .limit(1) .then((rows) => rows[0] ?? null); } - function summarizeIssueScheduledRetryRun( - row: { run: typeof heartbeatRuns.$inferSelect; agentName: string | null }, - ) { + function summarizeIssueScheduledRetryRun(row: { + run: typeof heartbeatRuns.$inferSelect; + agentName: string | null; + }) { return { runId: row.run.id, - status: row.run.status as "scheduled_retry" | "queued" | "running" | "cancelled", + status: row.run.status as + "scheduled_retry" | "queued" | "running" | "cancelled", agentId: row.run.agentId, agentName: row.agentName, retryOfRunId: row.run.retryOfRunId, @@ -12589,7 +14505,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) async function retryScheduledRetryNow(input: { issueId: string; - actor?: { actorType?: "user" | "agent" | "system"; actorId?: string | null }; + actor?: { + actorType?: "user" | "agent" | "system"; + actorId?: string | null; + }; now?: Date; }) { const now = input.now ?? new Date(); @@ -12600,9 +14519,15 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .then((rows) => rows[0] ?? null); if (!issue) throw notFound("Issue not found"); - const scheduled = await getIssueRetryRun(issue.companyId, issue.id, ["scheduled_retry"]); + const scheduled = await getIssueRetryRun(issue.companyId, issue.id, [ + "scheduled_retry", + ]); if (!scheduled) { - const alreadyPromoted = await getIssueRetryRun(issue.companyId, issue.id, ["queued", "running"]); + const alreadyPromoted = await getIssueRetryRun( + issue.companyId, + issue.id, + ["queued", "running"], + ); if (alreadyPromoted) { return { outcome: "already_promoted" as const, @@ -12633,20 +14558,25 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) contextSnapshot, updatedAt: now, }) - .where(and(eq(heartbeatRuns.id, scheduled.run.id), eq(heartbeatRuns.status, "scheduled_retry"))) + .where( + and( + eq(heartbeatRuns.id, scheduled.run.id), + eq(heartbeatRuns.status, "scheduled_retry"), + ), + ) .returning() .then((rows) => rows[0] ?? null); if (!row) return null; if (row.wakeupRequestId) { const wakeupPayload = { - ...(parseObject( + ...parseObject( await tx .select({ payload: agentWakeupRequests.payload }) .from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, row.wakeupRequestId)) .then((rows) => rows[0]?.payload ?? null), - )), + ), scheduledRetryAt: now.toISOString(), retryNowRequestedAt: now.toISOString(), }; @@ -12663,7 +14593,11 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) }); if (!updated) { - const alreadyPromoted = await getIssueRetryRun(issue.companyId, issue.id, ["queued", "running"]); + const alreadyPromoted = await getIssueRetryRun( + issue.companyId, + issue.id, + ["queued", "running"], + ); if (alreadyPromoted) { return { outcome: "already_promoted" as const, @@ -12686,7 +14620,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) payload: { issueId: issue.id, scheduledRetryAttempt: updated.scheduledRetryAttempt, - scheduledRetryAt: updated.scheduledRetryAt ? new Date(updated.scheduledRetryAt).toISOString() : null, + scheduledRetryAt: updated.scheduledRetryAt + ? new Date(updated.scheduledRetryAt).toISOString() + : null, scheduledRetryReason: updated.scheduledRetryReason, requestedByActorType: input.actor?.actorType ?? null, requestedByActorId: input.actor?.actorId ?? null, @@ -12694,10 +14630,17 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) }); const promotion = await promoteScheduledRetryRun(updated, now); - const promotedRow = await getIssueRetryRun(issue.companyId, issue.id, ["queued", "running", "cancelled"]); + const promotedRow = await getIssueRetryRun(issue.companyId, issue.id, [ + "queued", + "running", + "cancelled", + ]); const scheduledRetry = promotedRow ? summarizeIssueScheduledRetryRun(promotedRow) - : summarizeIssueScheduledRetryRun({ run: promotion.run ?? updated, agentName: scheduled.agentName }); + : summarizeIssueScheduledRetryRun({ + run: promotion.run ?? updated, + agentName: scheduled.agentName, + }); if (promotion.outcome === "promoted") { return { @@ -12728,7 +14671,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) enabled: asBoolean(heartbeat.enabled, false), intervalSec: Math.max(0, asNumber(heartbeat.intervalSec, 0)), wakeOnDemand: isHeartbeatWakeOnDemandEnabled(agent), - maxConcurrentRuns: normalizeMaxConcurrentRuns(heartbeat.maxConcurrentRuns), + maxConcurrentRuns: normalizeMaxConcurrentRuns( + heartbeat.maxConcurrentRuns, + ), skipTimerWhenNoActionableWork: asBoolean( heartbeat.skipTimerWhenNoActionableWork ?? heartbeat.requireActionableTimerWork ?? @@ -12736,7 +14681,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) false, ), maxDailyRuns: normalizeOptionalNonNegativeInteger( - heartbeat.maxDailyRuns ?? heartbeat.dailyRunLimit ?? heartbeat.dailyRunCap ?? heartbeat.maxRunsPerDay, + heartbeat.maxDailyRuns ?? + heartbeat.dailyRunLimit ?? + heartbeat.dailyRunCap ?? + heartbeat.maxRunsPerDay, ), maxDailyCostCents: normalizeOptionalNonNegativeInteger( heartbeat.maxDailyCostCents ?? @@ -12754,15 +14702,39 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } function currentUtcDayWindow(now = new Date()) { - const start = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate(), 0, 0, 0, 0)); - const end = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate() + 1, 0, 0, 0, 0)); + const start = new Date( + Date.UTC( + now.getUTCFullYear(), + now.getUTCMonth(), + now.getUTCDate(), + 0, + 0, + 0, + 0, + ), + ); + const end = new Date( + Date.UTC( + now.getUTCFullYear(), + now.getUTCMonth(), + now.getUTCDate() + 1, + 0, + 0, + 0, + 0, + ), + ); return { start, end }; } async function getHeartbeatDailyCapBlock( agent: typeof agents.$inferSelect, policy: ReturnType, - options: { checkRunCap?: boolean; checkCostCap?: boolean; excludeRunId?: string | null } = {}, + options: { + checkRunCap?: boolean; + checkCostCap?: boolean; + excludeRunId?: string | null; + } = {}, client: Pick = db, ) { const checkRunCap = options.checkRunCap ?? true; @@ -12795,7 +14767,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) if (checkCostCap && policy.maxDailyCostCents !== null) { const [row] = await client - .select({ total: sql`coalesce(sum(${costEvents.costCents})::bigint, 0)` }) + .select({ + total: sql`coalesce(sum(${costEvents.costCents})::bigint, 0)`, + }) .from(costEvents) .where( and( @@ -12820,10 +14794,13 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) async function cancelQueuedRunForHeartbeatDailyCap( run: typeof heartbeatRuns.$inferSelect, - dailyCapBlock: NonNullable>>, + dailyCapBlock: NonNullable< + Awaited> + >, ) { const now = new Date(); - const reason = "Cancelled because the agent reached a per-day heartbeat budget cap before adapter invocation"; + const reason = + "Cancelled because the agent reached a per-day heartbeat budget cap before adapter invocation"; const cancelled = await setRunStatus(run.id, "cancelled", { finishedAt: now, error: reason, @@ -12858,7 +14835,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) }, }); - await releaseIssueExecutionAndPromote(cancelled, { suppressImmediateRecovery: true }); + await releaseIssueExecutionAndPromote(cancelled, { + suppressImmediateRecovery: true, + }); return cancelled; } @@ -12881,7 +14860,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) return Boolean(row); } - async function markTimerHeartbeatChecked(agentId: string, source: WakeupOptions["source"]) { + async function markTimerHeartbeatChecked( + agentId: string, + source: WakeupOptions["source"], + ) { if (source !== "timer") return; await db .update(agents) @@ -12904,14 +14886,19 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) lastHeartbeatAt: now, updatedAt: now, }) - .where(and( - eq(agents.id, agent.id), - eq(agents.companyId, agent.companyId), - or( - lte(agents.lastHeartbeatAt, dueBefore), - and(isNull(agents.lastHeartbeatAt), lte(agents.createdAt, dueBefore)), + .where( + and( + eq(agents.id, agent.id), + eq(agents.companyId, agent.companyId), + or( + lte(agents.lastHeartbeatAt, dueBefore), + and( + isNull(agents.lastHeartbeatAt), + lte(agents.createdAt, dueBefore), + ), + ), ), - )) + ) .returning({ id: agents.id }) .then((rows) => rows[0] ?? null); if (!claimed) return null; @@ -12926,17 +14913,32 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) : undefined; } - function parseMaxTurnContinuationPolicy(agent: typeof agents.$inferSelect): MaxTurnContinuationPolicy { + function parseMaxTurnContinuationPolicy( + agent: typeof agents.$inferSelect, + ): MaxTurnContinuationPolicy { const runtimeConfig = parseObject(agent.runtimeConfig); const heartbeat = parseObject(runtimeConfig.heartbeat); const configured = parseObject(heartbeat.maxTurnContinuation); - const rawMaxAttempts = Math.floor(asNumber(configured.maxAttempts, MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS)); - const rawDelayMs = Math.floor(asNumber(configured.delayMs, MAX_TURN_CONTINUATION_DEFAULT_DELAY_MS)); + const rawMaxAttempts = Math.floor( + asNumber( + configured.maxAttempts, + MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS, + ), + ); + const rawDelayMs = Math.floor( + asNumber(configured.delayMs, MAX_TURN_CONTINUATION_DEFAULT_DELAY_MS), + ); return { enabled: asBoolean(configured.enabled, true), - maxAttempts: Math.max(0, Math.min(MAX_TURN_CONTINUATION_MAX_ATTEMPTS_CAP, rawMaxAttempts)), - delayMs: Math.max(0, Math.min(MAX_TURN_CONTINUATION_MAX_DELAY_MS, rawDelayMs)), + maxAttempts: Math.max( + 0, + Math.min(MAX_TURN_CONTINUATION_MAX_ATTEMPTS_CAP, rawMaxAttempts), + ), + delayMs: Math.max( + 0, + Math.min(MAX_TURN_CONTINUATION_MAX_DELAY_MS, rawDelayMs), + ), }; } @@ -12959,13 +14961,20 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) companyId: string, queuedRuns: Array, ) { - const issueIds = [...new Set( - queuedRuns - .map((run) => readNonEmptyString(parseObject(run.contextSnapshot).issueId)) - .filter((issueId): issueId is string => Boolean(issueId)), - )]; + const issueIds = [ + ...new Set( + queuedRuns + .map((run) => + readNonEmptyString(parseObject(run.contextSnapshot).issueId), + ) + .filter((issueId): issueId is string => Boolean(issueId)), + ), + ]; if (issueIds.length === 0) { - return new Map>>(); + return new Map< + string, + Awaited> + >(); } return issuesSvc.listDependencyReadiness(companyId, issueIds); } @@ -12974,40 +14983,62 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const [{ count }] = await db .select({ count: sql`count(*)` }) .from(heartbeatRuns) - .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "running"))); + .where( + and( + eq(heartbeatRuns.agentId, agentId), + eq(heartbeatRuns.status, "running"), + ), + ); return Number(count ?? 0); } - async function claimQueuedRun(run: typeof heartbeatRuns.$inferSelect, companyAgents?: AgentOrgRow[]) { + async function claimQueuedRun( + run: typeof heartbeatRuns.$inferSelect, + companyAgents?: AgentOrgRow[], + ) { if (run.status !== "queued") return run; const agent = await getAgent(run.agentId); if (!agent) { - await cancelRunInternal(run.id, "Cancelled because the agent no longer exists"); + await cancelRunInternal( + run.id, + "Cancelled because the agent no longer exists", + ); return null; } const invokability = companyAgents ? evaluateAgentInvokability(toAgentOrgRow(agent), companyAgents) : await getAgentInvokability(agent); if (!invokability.invokable) { - await cancelRunInternal(run.id, `Cancelled because the agent is not invokable: ${invokability.reason}`); + await cancelRunInternal( + run.id, + `Cancelled because the agent is not invokable: ${invokability.reason}`, + ); return null; } const context = parseObject(run.contextSnapshot); - const budgetBlock = await budgets.getInvocationBlock(run.companyId, run.agentId, { - issueId: readNonEmptyString(context.issueId), - projectId: readNonEmptyString(context.projectId), - }); + const budgetBlock = await budgets.getInvocationBlock( + run.companyId, + run.agentId, + { + issueId: readNonEmptyString(context.issueId), + projectId: readNonEmptyString(context.projectId), + }, + ); if (budgetBlock) { await cancelRunInternal(run.id, budgetBlock.reason); return null; } - const dailyCapBlock = await getHeartbeatDailyCapBlock(agent, parseHeartbeatPolicy(agent), { - excludeRunId: run.id, - checkRunCap: true, - checkCostCap: true, - }); + const dailyCapBlock = await getHeartbeatDailyCapBlock( + agent, + parseHeartbeatPolicy(agent), + { + excludeRunId: run.id, + checkRunCap: true, + checkCostCap: true, + }, + ); if (dailyCapBlock) { await cancelQueuedRunForHeartbeatDailyCap(run, dailyCapBlock); return null; @@ -13015,17 +15046,25 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const issueId = readNonEmptyString(context.issueId); if (issueId) { - const activePauseHold = await treeControlSvc.getActivePauseHoldGate(run.companyId, issueId); - const treeHoldInteractionWake = activePauseHold && await isVerifiedIssueTreeControlInteractionWake(db, { - companyId: run.companyId, + const activePauseHold = await treeControlSvc.getActivePauseHoldGate( + run.companyId, issueId, - agentId: run.agentId, - runId: run.id, - wakeupRequestId: run.wakeupRequestId, - contextSnapshot: context, - }); + ); + const treeHoldInteractionWake = + activePauseHold && + (await isVerifiedIssueTreeControlInteractionWake(db, { + companyId: run.companyId, + issueId, + agentId: run.agentId, + runId: run.id, + wakeupRequestId: run.wakeupRequestId, + contextSnapshot: context, + })); if (activePauseHold && !treeHoldInteractionWake) { - await cancelRunInternal(run.id, "Cancelled because issue is held by an active subtree pause hold"); + await cancelRunInternal( + run.id, + "Cancelled because issue is held by an active subtree pause hold", + ); await logActivity(db, { companyId: run.companyId, actorType: "system", @@ -13041,18 +15080,32 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) holdId: activePauseHold.holdId, rootIssueId: activePauseHold.rootIssueId, source: "heartbeat.claim_queued_run", - securityPrinciples: ["Complete Mediation", "Fail Securely", "Secure Defaults"], + securityPrinciples: [ + "Complete Mediation", + "Fail Securely", + "Secure Defaults", + ], }, }); return null; } - const dependencyReadiness = await issuesSvc.listDependencyReadiness(run.companyId, [issueId]); + const dependencyReadiness = await issuesSvc.listDependencyReadiness( + run.companyId, + [issueId], + ); const readiness = dependencyReadiness.get(issueId); const unresolvedBlockerCount = readiness?.unresolvedBlockerCount ?? 0; if (unresolvedBlockerCount > 0 && !allowsIssueInteractionWake(context)) { - await cancelQueuedRunForBlockedDependencies(run, issueId, readiness?.unresolvedBlockerIssueIds ?? []); - logger.info({ runId: run.id, issueId, unresolvedBlockerCount }, "claimQueuedRun: cancelled blocked queued run"); + await cancelQueuedRunForBlockedDependencies( + run, + issueId, + readiness?.unresolvedBlockerIssueIds ?? [], + ); + logger.info( + { runId: run.id, issueId, unresolvedBlockerCount }, + "claimQueuedRun: cancelled blocked queued run", + ); return null; } @@ -13071,8 +15124,14 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const responsibleUserId = await resolveResponsibleUserIdForRun({ run, contextSnapshot: context, - issueContext: issueId ? await getIssueExecutionContext(run.companyId, issueId) : null, - routineEnvContext: { routineId: null, env: null, responsibleUserId: null }, + issueContext: issueId + ? await getIssueExecutionContext(run.companyId, issueId) + : null, + routineEnvContext: { + routineId: null, + env: null, + responsibleUserId: null, + }, }); const queuedCommentIds = queuedCommentIdsFromRunContext(context); const queuedCommentClaim = @@ -13286,8 +15345,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) triggerDetail: claimed.triggerDetail, error: claimed.error ?? null, errorCode: claimed.errorCode ?? null, - startedAt: claimed.startedAt ? new Date(claimed.startedAt).toISOString() : null, - finishedAt: claimed.finishedAt ? new Date(claimed.finishedAt).toISOString() : null, + startedAt: claimed.startedAt + ? new Date(claimed.startedAt).toISOString() + : null, + finishedAt: claimed.finishedAt + ? new Date(claimed.finishedAt).toISOString() + : null, }, }); publishRunLifecyclePluginEvent(claimed); @@ -13299,7 +15362,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const claimedContext = parseObject(claimed.contextSnapshot); const claimedIssueId = readNonEmptyString(claimedContext.issueId); const claimedWakeReason = readNonEmptyString(claimedContext.wakeReason); - if (claimedIssueId && claimedWakeReason !== "source_scoped_recovery_action") { + if ( + claimedIssueId && + claimedWakeReason !== "source_scoped_recovery_action" + ) { const claimedAgent = await getAgent(claimed.agentId); await db .update(issues) @@ -13316,7 +15382,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) // Mention/context runs can touch an issue, but only the current assignee // owns the issue execution lock shown as the active run. eq(issues.assigneeAgentId, claimed.agentId), - or(isNull(issues.executionRunId), eq(issues.executionRunId, claimed.id)), + or( + isNull(issues.executionRunId), + eq(issues.executionRunId, claimed.id), + ), ), ); } @@ -13478,7 +15547,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const wakeCommentId = deriveCommentId(context, null); const isInteractionWake = allowsIssueInteractionWake(context); - const resumeIntent = context.resumeIntent === true || context.followUpRequested === true; + const resumeIntent = + context.resumeIntent === true || context.followUpRequested === true; const wakeReason = readNonEmptyString(context.wakeReason); const retryReason = readNonEmptyString(context.retryReason) ?? run.scheduledRetryReason ?? null; const interactionResolvedAt = readNonEmptyString(context.interactionResolvedAt); @@ -13512,11 +15582,14 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) issue.status === "in_progress" && !wakeCommentId && !hasResolvedInteractionEvidence && - (wakeReason === "issue_continuation_needed" || retryReason === "issue_continuation_needed") + (wakeReason === "issue_continuation_needed" || + retryReason === "issue_continuation_needed") ) { const queuedWake = parseObject(context.paperclipWake); const queuedContinuationSummary = - readNonEmptyString(parseObject(context.paperclipContinuationSummary).body) ?? + readNonEmptyString( + parseObject(context.paperclipContinuationSummary).body, + ) ?? readNonEmptyString(parseObject(queuedWake.continuationSummary).body); const currentContinuationSummary = queuedContinuationSummary ? null @@ -13538,11 +15611,13 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } } - const reviewExecutionState = issue.status === "in_review" - ? parseIssueExecutionState(issue.executionState) - : null; + const reviewExecutionState = + issue.status === "in_review" + ? parseIssueExecutionState(issue.executionState) + : null; const reviewParticipant = reviewExecutionState?.currentParticipant ?? null; - const isCurrentReviewParticipant = reviewParticipant?.type === "agent" && + const isCurrentReviewParticipant = + reviewParticipant?.type === "agent" && reviewParticipant.agentId === run.agentId; const recoveryActionId = readNonEmptyString(context.recoveryActionId); @@ -13592,16 +15667,26 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } } - if (retryReason === MAX_TURN_CONTINUATION_RETRY_REASON && issue.status !== "in_progress") { + if ( + retryReason === MAX_TURN_CONTINUATION_RETRY_REASON && + issue.status !== "in_progress" + ) { return { stale: true, errorCode: "issue_not_in_progress", reason: `Cancelled because max-turn continuation issue is no longer in_progress (current status: ${issue.status}) before the queued run could start`, - details: { issueId, currentStatus: issue.status, requiredStatus: "in_progress" }, + details: { + issueId, + currentStatus: issue.status, + requiredStatus: "in_progress", + }, }; } - if (retryReason === MAX_TURN_CONTINUATION_RETRY_REASON && issue.executionRunId !== run.id) { + if ( + retryReason === MAX_TURN_CONTINUATION_RETRY_REASON && + issue.executionRunId !== run.id + ) { return { stale: true, errorCode: "issue_execution_lock_changed", @@ -13616,10 +15701,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } if (issue.status === "in_review") { - const currentParticipant = reviewExecutionState?.currentParticipant ?? null; + const currentParticipant = + reviewExecutionState?.currentParticipant ?? null; if (currentParticipant) { const participantMatches = - currentParticipant.type === "agent" && currentParticipant.agentId === run.agentId; + currentParticipant.type === "agent" && + currentParticipant.agentId === run.agentId; if (!participantMatches && !wakeCommentId) { return { stale: true, @@ -13692,7 +15779,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) return cancelled; } - function truncateAgentErrorReason(reason: string | null | undefined): string | null { + function truncateAgentErrorReason( + reason: string | null | undefined, + ): string | null { if (!reason) return null; const trimmed = reason.trim(); if (!trimmed) return null; @@ -13712,13 +15801,17 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) return; } - const isFirstHeartbeat = options?.wasFirstHeartbeat ?? !existing.lastHeartbeatAt; + const isFirstHeartbeat = + options?.wasFirstHeartbeat ?? !existing.lastHeartbeatAt; const runningCount = await countRunningRunsForAgent(agentId); const nextStatus = runningCount > 0 ? "running" - : outcome === "succeeded" || outcome === "interrupted" || outcome === "cancelled" || (outcome === "failed" && options?.keepIdleOnFailure) + : outcome === "succeeded" || + outcome === "interrupted" || + outcome === "cancelled" || + (outcome === "failed" && options?.keepIdleOnFailure) ? "idle" : "error"; @@ -13729,7 +15822,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) // Persist a human-readable reason on the agent record when it enters // error so operators see it on the agent page without digging into run // events; clear it whenever the agent leaves error. - errorReason: nextStatus === "error" ? truncateAgentErrorReason(failureReason) : null, + errorReason: + nextStatus === "error" + ? truncateAgentErrorReason(failureReason) + : null, lastHeartbeatAt: new Date(), updatedAt: new Date(), }) @@ -13739,7 +15835,11 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) if (isFirstHeartbeat && updated) { const tc = getTelemetryClient(); - if (tc) trackAgentFirstHeartbeat(tc, { agentRole: updated.role, agentId: updated.id }); + if (tc) + trackAgentFirstHeartbeat(tc, { + agentRole: updated.role, + agentId: updated.id, + }); } if (updated) { @@ -13774,7 +15874,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) errorCode: options?.errorCode ?? null, errorMessage: options?.errorMessage ?? null, }); - return mergeHeartbeatRunStopMetadata(options?.resultJson ?? null, stopMetadata); + return mergeHeartbeatRunStopMetadata( + options?.resultJson ?? null, + stopMetadata, + ); } function countValue(value: unknown) { @@ -13783,7 +15886,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } function dateValue(value: unknown) { - if (value instanceof Date) return Number.isNaN(value.getTime()) ? null : value; + if (value instanceof Date) + return Number.isNaN(value.getTime()) ? null : value; if (typeof value === "string" || typeof value === "number") { const parsed = new Date(value); return Number.isNaN(parsed.getTime()) ? null : parsed; @@ -13807,51 +15911,59 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) ): Promise { const context = parseObject(run.contextSnapshot); const contextIssueId = readNonEmptyString(context.issueId); - const continuationAttempt = asNumber(context.continuationAttempt, run.continuationAttempt ?? 0); + const continuationAttempt = asNumber( + context.continuationAttempt, + run.continuationAttempt ?? 0, + ); const issue = contextIssueId ? await db - .select({ - status: issues.status, - title: issues.title, - description: issues.description, - }) - .from(issues) - .where(and(eq(issues.companyId, run.companyId), eq(issues.id, contextIssueId))) - .then((rows) => rows[0] ?? null) + .select({ + status: issues.status, + title: issues.title, + description: issues.description, + }) + .from(issues) + .where( + and( + eq(issues.companyId, run.companyId), + eq(issues.id, contextIssueId), + ), + ) + .then((rows) => rows[0] ?? null) : null; const [commentStats] = contextIssueId ? await db - .select({ - count: sql`count(*)::int`, - latestAt: sql`max(${issueComments.createdAt})`, - }) - .from(issueComments) - .where( - and( - eq(issueComments.companyId, run.companyId), - eq(issueComments.issueId, contextIssueId), - eq(issueComments.createdByRunId, run.id), - isNull(issueComments.deletedAt), - ), - ) + .select({ + count: sql`count(*)::int`, + latestAt: sql`max(${issueComments.createdAt})`, + }) + .from(issueComments) + .where( + and( + eq(issueComments.companyId, run.companyId), + eq(issueComments.issueId, contextIssueId), + eq(issueComments.createdByRunId, run.id), + isNull(issueComments.deletedAt), + ), + ) : [{ count: 0, latestAt: null }]; const issueCommentBodies = contextIssueId ? await db - .select({ body: issueComments.body }) - .from(issueComments) - .where( - and( - eq(issueComments.companyId, run.companyId), - eq(issueComments.issueId, contextIssueId), - eq(issueComments.createdByRunId, run.id), - ), - ) - .orderBy(desc(issueComments.createdAt), desc(issueComments.id)) - .limit(5) - .then((rows) => rows.reverse().map((row) => row.body)) + .select({ body: issueComments.body }) + .from(issueComments) + .where( + and( + eq(issueComments.companyId, run.companyId), + eq(issueComments.issueId, contextIssueId), + eq(issueComments.createdByRunId, run.id), + ), + ) + .orderBy(desc(issueComments.createdAt), desc(issueComments.id)) + .limit(5) + .then((rows) => rows.reverse().map((row) => row.body)) : []; const continuationSummary = contextIssueId @@ -13860,38 +15972,41 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const [documentStats] = contextIssueId ? await db - .select({ - count: sql`count(*)::int`, - planCount: sql`count(*) filter (where ${issueDocuments.key} = 'plan')::int`, - latestAt: sql`max(${documentRevisions.createdAt})`, - }) - .from(documentRevisions) - .innerJoin(issueDocuments, eq(documentRevisions.documentId, issueDocuments.documentId)) - .where( - and( - eq(documentRevisions.companyId, run.companyId), - eq(documentRevisions.createdByRunId, run.id), - eq(issueDocuments.companyId, run.companyId), - eq(issueDocuments.issueId, contextIssueId), - sql`${issueDocuments.key} != ${ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY}`, - ), - ) + .select({ + count: sql`count(*)::int`, + planCount: sql`count(*) filter (where ${issueDocuments.key} = 'plan')::int`, + latestAt: sql`max(${documentRevisions.createdAt})`, + }) + .from(documentRevisions) + .innerJoin( + issueDocuments, + eq(documentRevisions.documentId, issueDocuments.documentId), + ) + .where( + and( + eq(documentRevisions.companyId, run.companyId), + eq(documentRevisions.createdByRunId, run.id), + eq(issueDocuments.companyId, run.companyId), + eq(issueDocuments.issueId, contextIssueId), + sql`${issueDocuments.key} != ${ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY}`, + ), + ) : [{ count: 0, planCount: 0, latestAt: null }]; const [workProductStats] = contextIssueId ? await db - .select({ - count: sql`count(*)::int`, - latestAt: sql`max(${issueWorkProducts.createdAt})`, - }) - .from(issueWorkProducts) - .where( - and( - eq(issueWorkProducts.companyId, run.companyId), - eq(issueWorkProducts.issueId, contextIssueId), - eq(issueWorkProducts.createdByRunId, run.id), - ), - ) + .select({ + count: sql`count(*)::int`, + latestAt: sql`max(${issueWorkProducts.createdAt})`, + }) + .from(issueWorkProducts) + .where( + and( + eq(issueWorkProducts.companyId, run.companyId), + eq(issueWorkProducts.issueId, contextIssueId), + eq(issueWorkProducts.createdByRunId, run.id), + ), + ) : [{ count: 0, latestAt: null }]; const [workspaceOperationStats] = await db @@ -13900,7 +16015,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) latestAt: sql`max(${workspaceOperations.startedAt})`, }) .from(workspaceOperations) - .where(and(eq(workspaceOperations.companyId, run.companyId), eq(workspaceOperations.heartbeatRunId, run.id))); + .where( + and( + eq(workspaceOperations.companyId, run.companyId), + eq(workspaceOperations.heartbeatRunId, run.id), + ), + ); const [activityStats] = await db .select({ @@ -13922,7 +16042,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) latestAt: sql`max(${heartbeatRunEvents.createdAt}) filter (where ${heartbeatRunEvents.eventType} not in ('lifecycle', 'adapter.invoke', 'error'))`, }) .from(heartbeatRunEvents) - .where(and(eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id))); + .where( + and( + eq(heartbeatRunEvents.companyId, run.companyId), + eq(heartbeatRunEvents.runId, run.id), + ), + ); return { runStatus: run.status, @@ -13959,7 +16084,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) run: typeof heartbeatRuns.$inferSelect, resultJson?: Record | null, ) { - const classification = classifyRunLiveness(await buildRunLivenessInput(run, resultJson)); + const classification = classifyRunLiveness( + await buildRunLivenessInput(run, resultJson), + ); return db .update(heartbeatRuns) .set({ @@ -13979,9 +16106,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) // `pendingCleanupAttemptsSql` clamps to the same range, so both readers yield // the same value for every input. The claim predicate compares the two values, // so this alignment lets the claim match for a malformed lease. - function readPendingCleanupRetryAttempts(metadata: Record): number { + function readPendingCleanupRetryAttempts( + metadata: Record, + ): number { const value = metadata[PENDING_CLEANUP_ATTEMPTS_METADATA_KEY]; - if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) return 0; + if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) + return 0; return Math.min(Math.floor(value), PENDING_CLEANUP_SWEEP_ATTEMPT_CAP); } @@ -14031,7 +16161,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) // flag write to a lease that left pending_cleanup or dropped below the cap // between the read and this claim. The update writes only the warned key with // `jsonb_set`, so a concurrent write to an unrelated metadata key survives. - async function claimPendingCleanupCapWarning(leaseId: string): Promise { + async function claimPendingCleanupCapWarning( + leaseId: string, + ): Promise { const now = new Date(); const claimed = await db .update(environmentLeases) @@ -14079,7 +16211,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) // for that period between attempts. The sweep reads and writes the attempt // count in the lease metadata. It warns once when a lease reaches the attempt // cap and then stops the retries for that lease. - async function sweepPendingCleanupLeases(opts?: { backoffMs?: number }): Promise<{ + async function sweepPendingCleanupLeases(opts?: { + backoffMs?: number; + }): Promise<{ swept: number; destroyed: number; capped: number; @@ -14106,7 +16240,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) // later tick, and the database rows below still need this sweep. The caught // exception never enters the log, because a write error can carry a // credential in its message, code, cause, or stack. - logger.warn("orphan sandbox cleanup buffer flush failed; the sweep continues"); + logger.warn( + "orphan sandbox cleanup buffer flush failed; the sweep continues", + ); } const rows = await db @@ -14179,7 +16315,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) // method treats the lease as ready, so the sweep keeps its earlier // behavior. const workerReady = environmentRuntime.isPendingCleanupWorkerReady - ? await environmentRuntime.isPendingCleanupWorkerReady({ environment, lease }) + ? await environmentRuntime.isPendingCleanupWorkerReady({ + environment, + lease, + }) : true; if (!workerReady) { // Move the unavailable lease to the back of the sweep queue. Otherwise @@ -14203,7 +16342,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) // Tear the sandbox down from the recorded provider config and the // cleanup-authorized secret versions. The teardown returns no value // and throws on failure, so the sweep releases the lease itself. - await environmentRuntime.retryPendingSandboxTeardown({ environment, lease }); + await environmentRuntime.retryPendingSandboxTeardown({ + environment, + lease, + }); await environmentsSvc.releaseLease(lease.id, "expired", { cleanupStatus: "success", failureReason: "pending_cleanup_retry", @@ -14248,10 +16390,63 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) return { swept: rows.length, destroyed, capped }; } + async function markNativeOwnershipUnverified( + run: typeof heartbeatRuns.$inferSelect, + evidence: { + reason: "live_process_identifier" | "observed_owner_unverified"; + processPidAlive?: boolean; + processGroupAlive?: boolean; + }, + ) { + if ( + run.errorCode === NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE && + run.error === NATIVE_OWNERSHIP_UNVERIFIED_MESSAGE + ) return run; + const blockedStatus = run.status === "failed" ? "failed" : "running"; + const blockedWrite = await setRunStatusFromLive( + run.id, + blockedStatus, + [blockedStatus], + { + error: NATIVE_OWNERSHIP_UNVERIFIED_MESSAGE, + errorCode: NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE, + }, + ); + if (!blockedWrite.updated || !blockedWrite.run) { + return blockedWrite.run ?? run; + } + const blocked = blockedWrite.run; + await appendRunEvent(blocked, { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: NATIVE_OWNERSHIP_UNVERIFIED_MESSAGE, + payload: { + reason: evidence.reason, + ...(evidence.processPidAlive === true + ? { processPidAlive: true } + : {}), + ...(evidence.processGroupAlive === true + ? { processGroupAlive: true } + : {}), + }, + }); + return blocked; + } + async function reapOrphanedRuns(opts?: { staleThresholdMs?: number }) { const staleThresholdMs = opts?.staleThresholdMs ?? 0; const now = new Date(); + // Complete persisted native results before generic orphan recovery. The + // reconciler reads the durable workspace barrier and persisted runtime + // mode, never the current feature flag. + await reconcileNativeFinalizations(db).catch((error) => { + logger.warn( + { err: error }, + "failed to reconcile persisted native finalizations before orphan reaping", + ); + }); await dispatchPendingNativeStatusWakeups().catch((error) => { logger.warn( { err: error }, @@ -14259,6 +16454,104 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) ); }); + // A retryable native run can retain process identifiers from the failed + // attempt. Inspect them before the recovery claim: a live identifier is + // unowned and blocks recovery, while identifiers that are all dead can be + // cleared with a compare-and-set so the explicit retryable failure becomes + // claimable in this same sweep. + const retryableNativeProcesses = await db + .select({ run: heartbeatRuns }) + .from(heartbeatRuns) + .innerJoin( + nativeRunFinalizations, + eq(nativeRunFinalizations.runId, heartbeatRuns.id), + ) + .where(and( + inArray(heartbeatRuns.status, ["running", "failed"]), + eq(heartbeatRuns.runtimeMode, "native"), + eq(nativeRunFinalizations.phase, "retryable_failure"), + isNull(nativeRunFinalizations.resultId), + )); + const claimableNativeRunIds = new Set(); + for (const { run } of retryableNativeProcesses) { + if (!run.processPid && !run.processGroupId) { + claimableNativeRunIds.add(run.id); + continue; + } + const processPidAlive = !!run.processPid && isProcessAlive(run.processPid); + const processGroupAlive = !!run.processGroupId + && isProcessGroupAlive(run.processGroupId); + if (processPidAlive || processGroupAlive) { + await markNativeOwnershipUnverified(run, { + reason: "live_process_identifier", + processPidAlive, + processGroupAlive, + }); + continue; + } + const cleared = await db + .update(heartbeatRuns) + .set({ + processPid: null, + processGroupId: null, + processStartedAt: null, + updatedAt: now, + }) + .where(and( + eq(heartbeatRuns.id, run.id), + eq(heartbeatRuns.runtimeMode, "native"), + run.processPid === null + ? isNull(heartbeatRuns.processPid) + : eq(heartbeatRuns.processPid, run.processPid), + run.processGroupId === null + ? isNull(heartbeatRuns.processGroupId) + : eq(heartbeatRuns.processGroupId, run.processGroupId), + run.processStartedAt === null + ? isNull(heartbeatRuns.processStartedAt) + : eq(heartbeatRuns.processStartedAt, run.processStartedAt), + )) + .returning({ id: heartbeatRuns.id }) + .then((rows) => rows[0] ?? null); + if (cleared) claimableNativeRunIds.add(cleared.id); + } + + // An explicit result-less retryable failure resumes on the original run. + // The database lease is claimed before dispatch so concurrent service + // instances cannot open competing recoveries; executeRun receives the exact + // claimed owner. Expired `observed` ownership never enters this set. + const nativeResumeClaims = claimableNativeRunIds.size === 0 + ? [] + : await dispatchNativeSessionResumptions({ + db, + runnerInstanceId: + runtimeEnv.PAPERCLIP_INSTANCE_ID?.trim() || "paperclip-heartbeat", + now, + runIds: [...claimableNativeRunIds], + dispatch: (claim) => { + const execution = executeRun(claim.runId, { + nativeLeaseOwner: claim.leaseOwner, + }).catch((error) => { + logger.error( + { err: error, runId: claim.runId }, + "persisted native session resume failed", + ); + }); + activeRunExecutionPromises.add(execution); + void execution.finally(() => + activeRunExecutionPromises.delete(execution), + ); + }, + }).catch((error) => { + logger.warn( + { err: error }, + "failed to claim persisted native session resumptions", + ); + return []; + }); + const resumedRunIds = new Set( + nativeResumeClaims.map((claim) => claim.runId), + ); + // A terminal issue transition writes this intent in the same transaction // that expires the native question. Consume it before generic orphan // recovery so a restart preserves the requested cancellation outcome. @@ -14315,27 +16608,38 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) run: heartbeatRuns, adapterType: agents.adapterType, adapterConfig: agents.adapterConfig, + nativeCoordinatorPhase: nativeRunFinalizations.phase, }) .from(heartbeatRuns) .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id)) + .leftJoin( + nativeRunFinalizations, + eq(nativeRunFinalizations.runId, heartbeatRuns.id), + ) .where(eq(heartbeatRuns.status, "running")); - const monitorIssueIds = [...new Set(activeRuns.flatMap(({ run }) => { - const runContext = parseObject(run.contextSnapshot); - if (readNonEmptyString(runContext.wakeReason) !== "issue_monitor_due") return []; - const issueId = readNonEmptyString(runContext.issueId); - return issueId ? [issueId] : []; - }))]; - const monitorIssues = monitorIssueIds.length > 0 - ? await db - .select({ - id: issues.id, - companyId: issues.companyId, - monitorNextCheckAt: issues.monitorNextCheckAt, - }) - .from(issues) - .where(inArray(issues.id, monitorIssueIds)) - : []; + const monitorIssueIds = [ + ...new Set( + activeRuns.flatMap(({ run }) => { + const runContext = parseObject(run.contextSnapshot); + if (readNonEmptyString(runContext.wakeReason) !== "issue_monitor_due") + return []; + const issueId = readNonEmptyString(runContext.issueId); + return issueId ? [issueId] : []; + }), + ), + ]; + const monitorIssues = + monitorIssueIds.length > 0 + ? await db + .select({ + id: issues.id, + companyId: issues.companyId, + monitorNextCheckAt: issues.monitorNextCheckAt, + }) + .from(issues) + .where(inArray(issues.id, monitorIssueIds)) + : []; const monitorNextCheckAtByIssue = new Map( monitorIssues.map((issue) => [ `${issue.companyId}:${issue.id}`, @@ -14345,8 +16649,53 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const reaped: string[] = []; - for (const { run, adapterType, adapterConfig } of activeRuns) { - if (runningProcesses.has(run.id) || activeRunExecutions.has(run.id)) continue; + for (const { + run, + adapterType, + adapterConfig, + nativeCoordinatorPhase, + } of activeRuns) { + const nativeRun = run.runtimeMode === "native"; + const nativeProcessPidAlive = + nativeRun && !!run.processPid && isProcessAlive(run.processPid); + const nativeProcessGroupAlive = + nativeRun && + !!run.processGroupId && + isProcessGroupAlive(run.processGroupId); + const locallyTracked = + runningProcesses.has(run.id) || activeRunExecutions.has(run.id); + const observedOwnerUnverified = + nativeRun && + ( + nativeCoordinatorPhase === "observed" || + (nativeCoordinatorPhase === null && run.nativePhase === "observed") + ) && + !resumedRunIds.has(run.id) && + !locallyTracked; + // Persisted numeric process identifiers prove only that some process is + // alive, not that Paperclip still owns it. Likewise an observed native + // coordinator without a live in-process execution has no durable proof + // that its prior provider owner stopped. Keep both cases running but + // blocked: never signal, finalize, or retry them automatically. This gate + // intentionally precedes resumedRunIds so a claim cannot bypass the + // ownership check. + if ( + nativeProcessPidAlive || + nativeProcessGroupAlive || + observedOwnerUnverified + ) { + await markNativeOwnershipUnverified(run, { + reason: + nativeProcessPidAlive || nativeProcessGroupAlive + ? "live_process_identifier" + : "observed_owner_unverified", + processPidAlive: nativeProcessPidAlive, + processGroupAlive: nativeProcessGroupAlive, + }); + continue; + } + if (resumedRunIds.has(run.id)) continue; + if (locallyTracked) continue; // Apply staleness threshold to avoid false positives if (staleThresholdMs > 0) { @@ -14425,7 +16774,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) { resultJson: parseObject(run.resultJson), errorCode: "process_lost", - errorMessage: shouldRetry ? `${baseMessage}; retrying once` : baseMessage, + errorMessage: shouldRetry + ? `${baseMessage}; retrying once` + : baseMessage, }, ); return result; @@ -14437,7 +16788,11 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) }); if (!finalizedRun) finalizedRun = await getRun(run.id); if (!finalizedRun) continue; - finalizedRun = await classifyAndPersistRunLiveness(finalizedRun, parseObject(finalizedRun.resultJson)) ?? finalizedRun; + finalizedRun = + (await classifyAndPersistRunLiveness( + finalizedRun, + parseObject(finalizedRun.resultJson), + )) ?? finalizedRun; await releaseEnvironmentLeasesForRun({ runId: finalizedRun.id, companyId: finalizedRun.companyId, @@ -14450,10 +16805,18 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const retryAgent = await getAgent(run.agentId); if (shouldRetry) { if (retryAgent) { - retriedRun = await enqueueProcessLossRetry(finalizedRun, retryAgent, now); + retriedRun = await enqueueProcessLossRetry( + finalizedRun, + retryAgent, + now, + ); } } else if (retryAgent) { - const scheduled = await scheduleInteractionContinuationInfrastructureRetryIfEligible(finalizedRun, retryAgent); + const scheduled = + await scheduleInteractionContinuationInfrastructureRetryIfEligible( + finalizedRun, + retryAgent, + ); retriedRun = scheduled?.outcome === "scheduled" ? scheduled.run : null; } @@ -14484,17 +16847,26 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } if (reaped.length > 0) { - logger.warn({ reapedCount: reaped.length, runIds: reaped }, "reaped orphaned heartbeat runs"); + logger.warn( + { reapedCount: reaped.length, runIds: reaped }, + "reaped orphaned heartbeat runs", + ); } // Retry stranded pending_cleanup leases on the same tick. Isolate the sweep // so its failure never hides the reaper result. The backoff equals the // reaper staleness threshold. try { - const sweep = await sweepPendingCleanupLeases({ backoffMs: staleThresholdMs }); + const sweep = await sweepPendingCleanupLeases({ + backoffMs: staleThresholdMs, + }); if (sweep.destroyed > 0 || sweep.capped > 0) { logger.warn( - { destroyed: sweep.destroyed, capped: sweep.capped, swept: sweep.swept }, + { + destroyed: sweep.destroyed, + capped: sweep.capped, + swept: sweep.swept, + }, "swept pending_cleanup environment leases", ); } @@ -14518,11 +16890,13 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .select({ agentId: heartbeatRuns.agentId }) .from(heartbeatRuns) .innerJoin(companies, eq(companies.id, heartbeatRuns.companyId)) - .where(and( - eq(heartbeatRuns.status, "queued"), - eq(companies.status, "active"), - cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined, - )); + .where( + and( + eq(heartbeatRuns.status, "queued"), + eq(companies.status, "active"), + cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined, + ), + ); const agentIds = [...new Set(queuedRuns.map((r) => r.agentId))]; for (const agentId of agentIds) { @@ -14531,7 +16905,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } async function reconcileStrandedAssignedIssues() { - return recovery.reconcileStrandedAssignedIssues({ issueCreatedAtGte: await getWorktreeExecutionCutoff() }); + return recovery.reconcileStrandedAssignedIssues({ + issueCreatedAtGte: await getWorktreeExecutionCutoff(), + }); } async function sweepStaleIssueLocks() { @@ -14540,40 +16916,73 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) function issueIdFromRunContext(contextSnapshot: unknown) { const context = parseObject(contextSnapshot); - return readNonEmptyString(context.issueId) ?? readNonEmptyString(context.taskId); + return ( + readNonEmptyString(context.issueId) ?? readNonEmptyString(context.taskId) + ); } function issueIdFromWakePayload(payload: unknown) { const parsed = parseObject(payload); const nestedContext = parseObject(parsed[DEFERRED_WAKE_CONTEXT_KEY]); - return readNonEmptyString(parsed.issueId) ?? + return ( + readNonEmptyString(parsed.issueId) ?? readNonEmptyString(nestedContext.issueId) ?? - readNonEmptyString(nestedContext.taskId); + readNonEmptyString(nestedContext.taskId) + ); } - async function scanSilentActiveRuns(opts?: { now?: Date; companyId?: string }) { - return recovery.scanSilentActiveRuns({ ...opts, issueCreatedAtGte: await getWorktreeExecutionCutoff() }); + async function scanSilentActiveRuns(opts?: { + now?: Date; + companyId?: string; + }) { + return recovery.scanSilentActiveRuns({ + ...opts, + issueCreatedAtGte: await getWorktreeExecutionCutoff(), + }); } - async function reconcileProductivityReviews(opts?: { now?: Date; companyId?: string }) { - return productivityReviews.reconcileProductivityReviews({ ...opts, issueCreatedAtGte: await getWorktreeExecutionCutoff() }); + async function reconcileProductivityReviews(opts?: { + now?: Date; + companyId?: string; + }) { + return productivityReviews.reconcileProductivityReviews({ + ...opts, + issueCreatedAtGte: await getWorktreeExecutionCutoff(), + }); } - async function reconcileTaskWatchdogs(opts?: { companyId?: string | null; runId?: string | null }) { - return taskWatchdogs.reconcileTaskWatchdogs({ ...opts, issueCreatedAtGte: await getWorktreeExecutionCutoff() }); + async function reconcileTaskWatchdogs(opts?: { + companyId?: string | null; + runId?: string | null; + }) { + return taskWatchdogs.reconcileTaskWatchdogs({ + ...opts, + issueCreatedAtGte: await getWorktreeExecutionCutoff(), + }); } async function buildRunOutputSilence( run: Pick< typeof heartbeatRuns.$inferSelect, - "id" | "companyId" | "status" | "lastOutputAt" | "lastOutputSeq" | "lastOutputStream" | "processStartedAt" | "startedAt" | "createdAt" + | "id" + | "companyId" + | "status" + | "lastOutputAt" + | "lastOutputSeq" + | "lastOutputStream" + | "processStartedAt" + | "startedAt" + | "createdAt" >, now = new Date(), ) { return recovery.buildRunOutputSilence(run, now); } - async function buildIssueGraphLivenessAutoRecoveryPreview(opts?: { lookbackHours?: number; now?: Date }) { + async function buildIssueGraphLivenessAutoRecoveryPreview(opts?: { + lookbackHours?: number; + now?: Date; + }) { return recovery.buildIssueGraphLivenessAutoRecoveryPreview(opts); } @@ -14584,7 +16993,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) now?: Date; reescalationCooldownMs?: number; }) { - return recovery.reconcileIssueGraphLiveness({ ...opts, issueCreatedAtGte: await getWorktreeExecutionCutoff() }); + return recovery.reconcileIssueGraphLiveness({ + ...opts, + issueCreatedAtGte: await getWorktreeExecutionCutoff(), + }); } async function updateRuntimeState( @@ -14601,8 +17013,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const cachedInputTokens = usage?.cachedInputTokens ?? 0; const billingType = normalizeLedgerBillingType(result.billingType); const billedCostUsd = resolveCacheAdjustedCostUsd(result); - const additionalCostCents = normalizeBilledCostCents(billedCostUsd, billingType); - const hasTokenUsage = inputTokens > 0 || outputTokens > 0 || cachedInputTokens > 0; + const additionalCostCents = normalizeBilledCostCents( + billedCostUsd, + billingType, + ); + const hasTokenUsage = + inputTokens > 0 || outputTokens > 0 || cachedInputTokens > 0; const costStatus = resolveLedgerCostStatus({ costUsd: billedCostUsd, inputTokens, @@ -14611,7 +17027,11 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) }); const provider = result.provider ?? "unknown"; const biller = resolveLedgerBiller(result); - const ledgerScope = await resolveLedgerScopeForRun(db, agent.companyId, run); + const ledgerScope = await resolveLedgerScopeForRun( + db, + agent.companyId, + run, + ); await db .update(agentRuntimeState) @@ -14661,32 +17081,47 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const invokability = await getAgentInvokability(agent); if (!invokability.invokable) { if (shouldCancelRunsForNonInvokableAgent(invokability)) { - await cancelActiveForAgentInternal(agentId, `Cancelled because the agent is not invokable: ${invokability.reason}`); + await cancelActiveForAgentInternal( + agentId, + `Cancelled because the agent is not invokable: ${invokability.reason}`, + ); } return []; } const policy = parseHeartbeatPolicy(agent); const runningCount = await countRunningRunsForAgent(agentId); - const availableSlots = Math.max(0, policy.maxConcurrentRuns - runningCount); + const availableSlots = Math.max( + 0, + policy.maxConcurrentRuns - runningCount, + ); if (availableSlots <= 0) return []; const queuedRuns = await db .select() .from(heartbeatRuns) - .where(and( - eq(heartbeatRuns.agentId, agentId), - eq(heartbeatRuns.status, "queued"), - cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined, - )) + .where( + and( + eq(heartbeatRuns.agentId, agentId), + eq(heartbeatRuns.status, "queued"), + cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined, + ), + ) .orderBy(asc(heartbeatRuns.createdAt)); if (queuedRuns.length === 0) return []; - const dependencyReadiness = await listQueuedRunDependencyReadiness(agent.companyId, queuedRuns); - const queuedIssueIds = [...new Set( - queuedRuns - .map((run) => readNonEmptyString(parseObject(run.contextSnapshot).issueId)) - .filter((issueId): issueId is string => Boolean(issueId)), - )]; + const dependencyReadiness = await listQueuedRunDependencyReadiness( + agent.companyId, + queuedRuns, + ); + const queuedIssueIds = [ + ...new Set( + queuedRuns + .map((run) => + readNonEmptyString(parseObject(run.contextSnapshot).issueId), + ) + .filter((issueId): issueId is string => Boolean(issueId)), + ), + ]; const issueRows = await db .select({ id: issues.id, @@ -14696,26 +17131,54 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .from(issues) .where( queuedIssueIds.length > 0 - ? and(eq(issues.companyId, agent.companyId), inArray(issues.id, queuedIssueIds)) + ? and( + eq(issues.companyId, agent.companyId), + inArray(issues.id, queuedIssueIds), + ) : sql`false`, ); const issueById = new Map(issueRows.map((row) => [row.id, row])); const companyAgents = await listCompanyAgentOrgRows(agent.companyId); const prioritizedRuns = [...queuedRuns].sort((left, right) => { - const leftIssueId = readNonEmptyString(parseObject(left.contextSnapshot).issueId); - const rightIssueId = readNonEmptyString(parseObject(right.contextSnapshot).issueId); - const leftReadiness = leftIssueId ? dependencyReadiness.get(leftIssueId) : null; - const rightReadiness = rightIssueId ? dependencyReadiness.get(rightIssueId) : null; - const leftReady = leftIssueId ? (leftReadiness?.isDependencyReady ?? true) : true; - const rightReady = rightIssueId ? (rightReadiness?.isDependencyReady ?? true) : true; + const leftIssueId = readNonEmptyString( + parseObject(left.contextSnapshot).issueId, + ); + const rightIssueId = readNonEmptyString( + parseObject(right.contextSnapshot).issueId, + ); + const leftReadiness = leftIssueId + ? dependencyReadiness.get(leftIssueId) + : null; + const rightReadiness = rightIssueId + ? dependencyReadiness.get(rightIssueId) + : null; + const leftReady = leftIssueId + ? (leftReadiness?.isDependencyReady ?? true) + : true; + const rightReady = rightIssueId + ? (rightReadiness?.isDependencyReady ?? true) + : true; const leftIssue = leftIssueId ? issueById.get(leftIssueId) : null; const rightIssue = rightIssueId ? issueById.get(rightIssueId) : null; - const leftRank = leftIssueId ? (leftReady ? (leftIssue?.status === "in_progress" ? 0 : 1) : 3) : 2; - const rightRank = rightIssueId ? (rightReady ? (rightIssue?.status === "in_progress" ? 0 : 1) : 3) : 2; + const leftRank = leftIssueId + ? leftReady + ? leftIssue?.status === "in_progress" + ? 0 + : 1 + : 3 + : 2; + const rightRank = rightIssueId + ? rightReady + ? rightIssue?.status === "in_progress" + ? 0 + : 1 + : 3 + : 2; if (leftRank !== rightRank) return leftRank - rightRank; const leftPriorityRank = issueRunPriorityRank(leftIssue?.priority); const rightPriorityRank = issueRunPriorityRank(rightIssue?.priority); - if (leftPriorityRank !== rightPriorityRank) return leftPriorityRank - rightPriorityRank; + if (leftPriorityRank !== rightPriorityRank) + return leftPriorityRank - rightPriorityRank; return left.createdAt.getTime() - right.createdAt.getTime(); }); @@ -14729,7 +17192,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) for (const claimedRun of claimedRuns) { const execution = executeRun(claimedRun.id).catch((err) => { - logger.error({ err, runId: claimedRun.id }, "queued heartbeat execution failed"); + logger.error( + { err, runId: claimedRun.id }, + "queued heartbeat execution failed", + ); }); // Register the in-flight execution so drainActiveRunExecutions() can await // it. executeRun resolves only after its finally block finishes flushing @@ -14760,12 +17226,62 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) // activeRunExecutionPromises, and the second await drains that run. A wakeup or // a run can add more entries as it settles, so loop until both sets are empty. async function drainActiveRunExecutions() { - while (activeWakeupPromises.size > 0 || activeRunExecutionPromises.size > 0) { + for (const timer of nativeSessionResumeDispatchTimers.values()) { + clearTimeout(timer); + } + nativeSessionResumeDispatchTimers.clear(); + while ( + activeWakeupPromises.size > 0 || + activeRunExecutionPromises.size > 0 + ) { await Promise.allSettled([...activeWakeupPromises]); await Promise.all([...activeRunExecutionPromises]); } } + function scheduleNativeSessionResumeDispatch( + runId: string, + nextAttemptAt: Date, + ) { + const prior = nativeSessionResumeDispatchTimers.get(runId); + if (prior) clearTimeout(prior); + const delayMs = Math.max(0, nextAttemptAt.getTime() - Date.now()); + const timer = setTimeout(() => { + if (nativeSessionResumeDispatchTimers.get(runId) !== timer) return; + nativeSessionResumeDispatchTimers.delete(runId); + void (async () => { + if ((await getSchedulingSuppression()).suppressed) return; + await dispatchNativeSessionResumptions({ + db, + runnerInstanceId: + runtimeEnv.PAPERCLIP_INSTANCE_ID?.trim() || "paperclip-heartbeat", + runIds: [runId], + dispatch: (claim) => { + const execution = executeRun(claim.runId, { + nativeLeaseOwner: claim.leaseOwner, + }).catch((error) => { + logger.error( + { err: error, runId: claim.runId }, + "scheduled native session resume failed", + ); + }); + activeRunExecutionPromises.add(execution); + void execution.finally(() => + activeRunExecutionPromises.delete(execution), + ); + }, + }); + })().catch((error) => { + logger.error( + { err: error, runId }, + "failed to dispatch scheduled native session resume", + ); + }); + }, delayMs); + timer.unref?.(); + nativeSessionResumeDispatchTimers.set(runId, timer); + } + // Public wakeup entry point. Callers dispatch it fire-and-forget, so register // the promise in activeWakeupPromises before it starts its asynchronous // prologue. drainActiveRunExecutions can then await a wake that is still before @@ -14777,13 +17293,18 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) ): ReturnType { const promise = enqueueWakeup(agentId, opts); activeWakeupPromises.add(promise); - void promise.catch(() => {}).finally(() => { - activeWakeupPromises.delete(promise); - }); + void promise + .catch(() => {}) + .finally(() => { + activeWakeupPromises.delete(promise); + }); return promise; } - async function executeRun(runId: string) { + async function executeRun( + runId: string, + runOptions: { nativeLeaseOwner?: string } = {}, + ) { if ((await getSchedulingSuppression()).suppressed) { try { await releaseRunClaimedJustBeforeSuppression(runId); @@ -14809,8 +17330,88 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) run = claimed; } + if (runOptions.nativeLeaseOwner && run.runtimeMode === "native") { + // A numeric PID or process-group ID is a liveness signal, never an + // ownership capability: the OS may have recycled it after the service + // restart. A still-active in-memory child handle is also insufficient to + // authorize recovery to kill it. Any live or active-looking process + // therefore blocks replacement recovery without receiving a signal. + const tracked = runningProcesses.get(run.id); + const trackedChildIsActive = !!tracked + && tracked.child.exitCode === null + && tracked.child.signalCode === null; + const trackedPid = tracked?.child.pid ?? null; + const trackedProcessGroupId = tracked?.processGroupId ?? null; + const trackedPidAlive = trackedPid ? isProcessAlive(trackedPid) : false; + const trackedProcessGroupAlive = trackedProcessGroupId + ? isProcessGroupAlive(trackedProcessGroupId) + : false; + const persistedPidAlive = !!run.processPid + && isProcessAlive(run.processPid); + const persistedProcessGroupAlive = !!run.processGroupId + && isProcessGroupAlive(run.processGroupId); + if ( + trackedChildIsActive || + trackedPidAlive || + trackedProcessGroupAlive || + persistedPidAlive || + persistedProcessGroupAlive + ) { + await markNativeOwnershipUnverified(run, { + reason: "live_process_identifier", + processPidAlive: trackedPidAlive || persistedPidAlive, + processGroupAlive: + trackedProcessGroupAlive || persistedProcessGroupAlive, + }); + throw new Error(NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE); + } + runningProcesses.delete(run.id); + if (run.processPid || run.processGroupId || run.processStartedAt) { + const cleared = await db + .update(heartbeatRuns) + .set({ + processPid: null, + processGroupId: null, + processStartedAt: null, + updatedAt: new Date(), + }) + .where( + and( + eq(heartbeatRuns.id, run.id), + eq(heartbeatRuns.runtimeMode, "native"), + run.processPid === null + ? isNull(heartbeatRuns.processPid) + : eq(heartbeatRuns.processPid, run.processPid), + run.processGroupId === null + ? isNull(heartbeatRuns.processGroupId) + : eq(heartbeatRuns.processGroupId, run.processGroupId), + run.processStartedAt === null + ? isNull(heartbeatRuns.processStartedAt) + : eq(heartbeatRuns.processStartedAt, run.processStartedAt), + ), + ) + .returning() + .then((rows) => rows[0] ?? null); + if (!cleared) { + const current = await getRun(run.id); + if (current) { + await markNativeOwnershipUnverified(current, { + reason: "live_process_identifier", + }); + } + throw new Error(NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE); + } + run = cleared; + } + } + activeRunExecutions.add(run.id); let runScratch: HeartbeatRunScratch | null = null; + let nativeSessionResumeScheduled = false; + let providerTraceCapture: Awaited< + ReturnType + > | null = null; + let providerTraceFinalized = false; try { const agent = await getAgent(run.agentId); @@ -14831,14 +17432,68 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const runtime = await ensureRuntimeState(agent); const context = parseObject(run.contextSnapshot); + const providerTraceRequested = + parseObject(context.debug).providerTrace === "raw"; + if (providerTraceRequested) { + if (context.providerTraceRequestSource === "agent_debug_setting") { + try { + await logActivity(db, { + companyId: run.companyId, + actorType: "system", + actorId: "system", + agentId: run.agentId, + runId: run.id, + action: "provider_trace.capture_requested", + entityType: "heartbeat_run", + entityId: run.id, + details: { + mode: "raw", + source: "agent_debug_setting", + retentionHours: 24, + maxBytes: 64 * 1024 * 1024, + }, + }); + } catch (error) { + logger.warn( + { error, runId: run.id }, + "provider trace capture audit could not be recorded", + ); + } + } + try { + providerTraceCapture = await traceStore.prepare({ + runId: run.id, + companyId: run.companyId, + provider: + readNonEmptyString(parseObject(agent.adapterConfig).provider) ?? + agent.adapterType, + requestedBy: + readNonEmptyString(context.providerTraceRequestedBy) ?? + "local-admin", + }); + } catch (error) { + logger.warn( + { error, runId: run.id }, + "provider trace sidecar could not be prepared", + ); + } + } const taskKey = deriveTaskKeyWithHeartbeatFallback(context, null); const sessionCodec = getAdapterSessionCodec(agent.adapterType); const issueId = readNonEmptyString(context.issueId); - let issueContext = issueId ? await getIssueExecutionContext(agent.companyId, issueId) : null; - const issueDependencyReadiness = issueId - ? await issuesSvc.listDependencyReadiness(agent.companyId, [issueId]).then((rows) => rows.get(issueId) ?? null) + let issueContext = issueId + ? await getIssueExecutionContext(agent.companyId, issueId) : null; - if (issueId && issueContext && isResolvedInteractionContinuationWakeContext(context)) { + const issueDependencyReadiness = issueId + ? await issuesSvc + .listDependencyReadiness(agent.companyId, [issueId]) + .then((rows) => rows.get(issueId) ?? null) + : null; + if ( + issueId && + issueContext && + isResolvedInteractionContinuationWakeContext(context) + ) { try { // Claim the issue under the same in_progress predicate used by the // queued-run staleness gate. This is the final atomic guard before @@ -14848,7 +17503,11 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) context[PAPERCLIP_HARNESS_CHECKOUT_KEY] = true; } catch (error) { if (!isCheckoutConflictError(error)) throw error; - const staleness = await evaluateQueuedRunStaleness(run, issueId, context); + const staleness = await evaluateQueuedRunStaleness( + run, + issueId, + context, + ); if (staleness.stale) { await cancelRunForStaleIssue(run, issueId, staleness); return; @@ -14871,7 +17530,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) }) ) { try { - await issuesSvc.checkout(issueId, agent.id, ["todo", "backlog", "blocked"], run.id); + await issuesSvc.checkout( + issueId, + agent.id, + ["todo", "backlog", "blocked"], + run.id, + ); context[PAPERCLIP_HARNESS_CHECKOUT_KEY] = true; } catch (error) { if (!isCheckoutConflictError(error)) throw error; @@ -14879,1727 +17543,1401 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } issueContext = await getIssueExecutionContext(agent.companyId, issueId); } - const wakeCommentId = deriveCommentId(context, null); - const wakeCommentContext = - issueContext && wakeCommentId + const wakeCommentId = deriveCommentId(context, null); + const wakeCommentContext = + issueContext && wakeCommentId + ? await db + .select({ + id: issueComments.id, + body: issueComments.body, + authorType: issueComments.authorType, + authorAgentId: issueComments.authorAgentId, + authorUserId: issueComments.authorUserId, + presentation: issueComments.presentation, + metadata: issueComments.metadata, + deletedAt: issueComments.deletedAt, + deletedByType: issueComments.deletedByType, + deletedByAgentId: issueComments.deletedByAgentId, + deletedByUserId: issueComments.deletedByUserId, + deletedByRunId: issueComments.deletedByRunId, + sourceTrust: issueComments.sourceTrust, + }) + .from(issueComments) + .where( + and( + eq(issueComments.id, wakeCommentId), + eq(issueComments.issueId, issueContext.id), + eq(issueComments.companyId, agent.companyId), + ), + ) + .then((rows) => { + const row = rows[0] ?? null; + return row?.deletedAt + ? { + ...row, + body: "", + presentation: null, + metadata: null, + } + : row; + }) + : null; + const issueAssigneeOverrides = + issueContext && issueContext.assigneeAgentId === agent.id + ? parseIssueAssigneeAdapterOverrides( + issueContext.assigneeAdapterOverrides, + ) + : null; + const experimentalInstanceSettings = + await instanceSettings.getExperimental(); + const isolatedWorkspacesEnabled = + experimentalInstanceSettings.enableIsolatedWorkspaces; + const parsedIssueExecutionWorkspaceSettings = + parseIssueExecutionWorkspaceSettings( + issueContext?.executionWorkspaceSettings, + ); + const issueExecutionWorkspaceSettings = isolatedWorkspacesEnabled + ? parsedIssueExecutionWorkspaceSettings + : null; + const environmentExecutionWorkspaceSettings = + selectEnvironmentExecutionWorkspaceSettings( + parsedIssueExecutionWorkspaceSettings, + isolatedWorkspacesEnabled, + ); + const contextProjectId = readNonEmptyString(context.projectId); + const executionProjectId = issueContext?.projectId ?? contextProjectId; + const projectContext = executionProjectId ? await db .select({ - id: issueComments.id, - body: issueComments.body, - authorType: issueComments.authorType, - authorAgentId: issueComments.authorAgentId, - authorUserId: issueComments.authorUserId, - presentation: issueComments.presentation, - metadata: issueComments.metadata, - deletedAt: issueComments.deletedAt, - deletedByType: issueComments.deletedByType, - deletedByAgentId: issueComments.deletedByAgentId, - deletedByUserId: issueComments.deletedByUserId, - deletedByRunId: issueComments.deletedByRunId, - sourceTrust: issueComments.sourceTrust, - }) - .from(issueComments) - .where(and( - eq(issueComments.id, wakeCommentId), - eq(issueComments.issueId, issueContext.id), - eq(issueComments.companyId, agent.companyId), - )) - .then((rows) => { - const row = rows[0] ?? null; - return row?.deletedAt - ? { - ...row, - body: "", - presentation: null, - metadata: null, - } - : row; + id: projects.id, + executionWorkspacePolicy: projects.executionWorkspacePolicy, + env: projects.env, + updatedAt: projects.updatedAt, }) + .from(projects) + .where( + and( + eq(projects.id, executionProjectId), + eq(projects.companyId, agent.companyId), + ), + ) + .then((rows) => rows[0] ?? null) : null; - const issueAssigneeOverrides = - issueContext && issueContext.assigneeAgentId === agent.id - ? parseIssueAssigneeAdapterOverrides( - issueContext.assigneeAdapterOverrides, + const acceptedPlanContinuationWake = issueContext + ? readNonEmptyString(context.workspaceRefreshReason) === + "accepted_plan_confirmation" || + (issueContext.workMode === "planning" && + readNonEmptyString(context.interactionKind) === + "request_confirmation" && + readNonEmptyString(context.interactionStatus) === "accepted") + : false; + const acceptedPlanWakeRoutingDecision = issueContext + ? await resolveAcceptedPlanWakeRoutingDecision({ + db, + companyId: agent.companyId, + agentId: agent.id, + issueId, + acceptedPlanContinuationWake, + contextSnapshot: context, + }) + : null; + if (acceptedPlanWakeRoutingDecision) { + context.forceFreshSession = true; + context.acceptedPlanWakeRouting = { + reason: "other_issue_claim_in_flight", + otherActiveClaimIssueId: + acceptedPlanWakeRoutingDecision.otherActiveClaimIssueId, + otherActiveClaimIdentifier: + acceptedPlanWakeRoutingDecision.otherActiveClaimIdentifier, + otherActiveClaimTitle: + acceptedPlanWakeRoutingDecision.otherActiveClaimTitle, + }; + if (acceptedPlanWakeRoutingDecision.suppressAcceptedContinuation) { + clearInteractionContinuationWakeContext(context); + delete context.workspaceRefreshReason; + } + } else { + delete context.acceptedPlanWakeRouting; + } + const routineEnvContext = await getRoutineEnvForExecutionIssue( + agent.companyId, + issueContext, + ); + const responsibleUserId = await resolveResponsibleUserIdForRun({ + run, + contextSnapshot: context, + issueContext, + routineEnvContext, + }); + if (responsibleUserId && run.responsibleUserId !== responsibleUserId) { + await db + .update(heartbeatRuns) + .set({ responsibleUserId, updatedAt: new Date() }) + .where(eq(heartbeatRuns.id, run.id)); + run = { ...run, responsibleUserId }; + } + if ( + responsibleUserId && + issueContext && + !issueContext.responsibleUserId + ) { + await db + .update(issues) + .set({ responsibleUserId, updatedAt: new Date() }) + .where( + and( + eq(issues.companyId, agent.companyId), + eq(issues.id, issueContext.id), + isNull(issues.responsibleUserId), + ), + ); + issueContext = { ...issueContext, responsibleUserId }; + } + const parsedProjectExecutionWorkspacePolicy = + parseProjectExecutionWorkspacePolicy( + projectContext?.executionWorkspacePolicy, + ); + const projectExecutionWorkspacePolicy = + gateProjectExecutionWorkspacePolicy( + parsedProjectExecutionWorkspacePolicy, + isolatedWorkspacesEnabled, + ); + const trustPreset = resolveCoreTrustPreset({ + companyId: agent.companyId, + agent: { + companyId: agent.companyId, + permissions: agent.permissions, + }, + project: projectContext + ? { + companyId: agent.companyId, + executionWorkspacePolicy: projectExecutionWorkspacePolicy, + } + : null, + issue: issueContext + ? { + companyId: agent.companyId, + executionPolicy: issueContext.executionPolicy, + } + : null, + }); + const config = parseObject(agent.adapterConfig); + const taskSession = taskKey + ? await getTaskSession( + agent.companyId, + agent.id, + agent.adapterType, + taskKey, ) : null; - const experimentalInstanceSettings = await instanceSettings.getExperimental(); - const isolatedWorkspacesEnabled = experimentalInstanceSettings.enableIsolatedWorkspaces; - const parsedIssueExecutionWorkspaceSettings = parseIssueExecutionWorkspaceSettings( - issueContext?.executionWorkspaceSettings, - ); - const issueExecutionWorkspaceSettings = isolatedWorkspacesEnabled - ? parsedIssueExecutionWorkspaceSettings - : null; - const environmentExecutionWorkspaceSettings = selectEnvironmentExecutionWorkspaceSettings( - parsedIssueExecutionWorkspaceSettings, - isolatedWorkspacesEnabled, - ); - const contextProjectId = readNonEmptyString(context.projectId); - const executionProjectId = issueContext?.projectId ?? contextProjectId; - const projectContext = executionProjectId - ? await db - .select({ - id: projects.id, - executionWorkspacePolicy: projects.executionWorkspacePolicy, - env: projects.env, - updatedAt: projects.updatedAt, - }) - .from(projects) - .where(and(eq(projects.id, executionProjectId), eq(projects.companyId, agent.companyId))) - .then((rows) => rows[0] ?? null) - : null; - const acceptedPlanContinuationWake = issueContext - ? readNonEmptyString(context.workspaceRefreshReason) === "accepted_plan_confirmation" - || ( - issueContext.workMode === "planning" - && readNonEmptyString(context.interactionKind) === "request_confirmation" - && readNonEmptyString(context.interactionStatus) === "accepted" - ) - : false; - const acceptedPlanWakeRoutingDecision = issueContext - ? await resolveAcceptedPlanWakeRoutingDecision({ - db, - companyId: agent.companyId, - agentId: agent.id, - issueId, - acceptedPlanContinuationWake, - contextSnapshot: context, - }) - : null; - if (acceptedPlanWakeRoutingDecision) { - context.forceFreshSession = true; - context.acceptedPlanWakeRouting = { - reason: "other_issue_claim_in_flight", - otherActiveClaimIssueId: acceptedPlanWakeRoutingDecision.otherActiveClaimIssueId, - otherActiveClaimIdentifier: acceptedPlanWakeRoutingDecision.otherActiveClaimIdentifier, - otherActiveClaimTitle: acceptedPlanWakeRoutingDecision.otherActiveClaimTitle, - }; - if (acceptedPlanWakeRoutingDecision.suppressAcceptedContinuation) { - clearInteractionContinuationWakeContext(context); - delete context.workspaceRefreshReason; - } - } else { - delete context.acceptedPlanWakeRouting; - } - const routineEnvContext = await getRoutineEnvForExecutionIssue(agent.companyId, issueContext); - const responsibleUserId = await resolveResponsibleUserIdForRun({ - run, - contextSnapshot: context, - issueContext, - routineEnvContext, - }); - if (responsibleUserId && run.responsibleUserId !== responsibleUserId) { - await db - .update(heartbeatRuns) - .set({ responsibleUserId, updatedAt: new Date() }) - .where(eq(heartbeatRuns.id, run.id)); - run = { ...run, responsibleUserId }; - } - if (responsibleUserId && issueContext && !issueContext.responsibleUserId) { - await db - .update(issues) - .set({ responsibleUserId, updatedAt: new Date() }) - .where(and(eq(issues.companyId, agent.companyId), eq(issues.id, issueContext.id), isNull(issues.responsibleUserId))); - issueContext = { ...issueContext, responsibleUserId }; - } - const parsedProjectExecutionWorkspacePolicy = parseProjectExecutionWorkspacePolicy( - projectContext?.executionWorkspacePolicy, - ); - const projectExecutionWorkspacePolicy = gateProjectExecutionWorkspacePolicy( - parsedProjectExecutionWorkspacePolicy, - isolatedWorkspacesEnabled, - ); - const trustPreset = resolveCoreTrustPreset({ - companyId: agent.companyId, - agent: { - companyId: agent.companyId, - permissions: agent.permissions, - }, - project: projectContext + const taskSessionDecodedParams = normalizeSessionParams( + sessionCodec.deserialize(taskSession?.sessionParamsJson ?? null), + ); + const explicitResumeSessionParams = normalizeResumeParamsForAdapter( + agent.adapterType, + sessionCodec.deserialize(parseObject(context.resumeSessionParams)), + ); + const explicitResumeSessionDisplayId = truncateDisplayId( + readNonEmptyString(context.resumeSessionDisplayId) ?? + (sessionCodec.getDisplayId + ? sessionCodec.getDisplayId(explicitResumeSessionParams) + : null) ?? + readNonEmptyString(explicitResumeSessionParams?.sessionId), + ); + const resolvedExecutionWorkspaceMode = resolveExecutionWorkspaceMode({ + projectPolicy: projectExecutionWorkspacePolicy, + issueSettings: issueExecutionWorkspaceSettings, + legacyUseProjectWorkspace: + issueAssigneeOverrides?.useProjectWorkspace ?? null, + }); + const requestedExecutionWorkspaceMode = + trustPreset.kind === "low_trust_review" && + resolvedExecutionWorkspaceMode === "shared_workspace" + ? "isolated_workspace" + : resolvedExecutionWorkspaceMode; + const issueRef = issueContext ? { - companyId: agent.companyId, - executionWorkspacePolicy: projectExecutionWorkspacePolicy, + id: issueContext.id, + identifier: issueContext.identifier, + title: issueContext.title, + status: issueContext.status, + priority: issueContext.priority, + workMode: issueContext.workMode, + reviewPolicy: issueContext.reviewPolicy, + description: issueContext.description, + projectId: issueContext.projectId, + projectWorkspaceId: issueContext.projectWorkspaceId, + executionWorkspaceId: issueContext.executionWorkspaceId, + executionWorkspacePreference: + issueContext.executionWorkspacePreference, } - : null, - issue: issueContext - ? { - companyId: agent.companyId, - executionPolicy: issueContext.executionPolicy, - } - : null, - }); - const config = parseObject(agent.adapterConfig); - const taskSession = taskKey - ? await getTaskSession(agent.companyId, agent.id, agent.adapterType, taskKey) - : null; - const taskSessionDecodedParams = normalizeSessionParams( - sessionCodec.deserialize(taskSession?.sessionParamsJson ?? null), - ); - const explicitResumeSessionParams = normalizeResumeParamsForAdapter( - agent.adapterType, - sessionCodec.deserialize(parseObject(context.resumeSessionParams)), - ); - const explicitResumeSessionDisplayId = truncateDisplayId( - readNonEmptyString(context.resumeSessionDisplayId) ?? - (sessionCodec.getDisplayId ? sessionCodec.getDisplayId(explicitResumeSessionParams) : null) ?? - readNonEmptyString(explicitResumeSessionParams?.sessionId), - ); - const resolvedExecutionWorkspaceMode = resolveExecutionWorkspaceMode({ - projectPolicy: projectExecutionWorkspacePolicy, - issueSettings: issueExecutionWorkspaceSettings, - legacyUseProjectWorkspace: issueAssigneeOverrides?.useProjectWorkspace ?? null, - }); - const requestedExecutionWorkspaceMode = - trustPreset.kind === "low_trust_review" && resolvedExecutionWorkspaceMode === "shared_workspace" - ? "isolated_workspace" - : resolvedExecutionWorkspaceMode; - const issueRef = issueContext - ? { - id: issueContext.id, - identifier: issueContext.identifier, - title: issueContext.title, - status: issueContext.status, - priority: issueContext.priority, - workMode: issueContext.workMode, - description: issueContext.description, - reviewPolicy: issueContext.reviewPolicy, - projectId: issueContext.projectId, - projectWorkspaceId: issueContext.projectWorkspaceId, - executionWorkspaceId: issueContext.executionWorkspaceId, - executionWorkspacePreference: issueContext.executionWorkspacePreference, - } - : null; - const continuationSummary = issueRef - ? await getIssueContinuationSummaryDocument(db, issueRef.id) - : null; - const exposeLowTrustRaw = trustPreset.kind === "low_trust_review"; - const safeContinuationSummary = - continuationSummary && !exposeLowTrustRaw - ? redactQuarantinedBodyForHigherTrust(continuationSummary) - : continuationSummary; - const safeWakeCommentContext = - wakeCommentContext && !exposeLowTrustRaw - ? sanitizeQuarantinedCommentForHigherTrust(wakeCommentContext) - : wakeCommentContext; - const issueAncestors = issueRef - ? await issuesSvc.getAncestors(issueRef.id) - : []; - if (continuationSummary) { - context.paperclipContinuationSummary = { - key: safeContinuationSummary!.key, - title: safeContinuationSummary!.title, - body: safeContinuationSummary!.body, - sourceTrust: safeContinuationSummary!.sourceTrust ?? null, - updatedAt: safeContinuationSummary!.updatedAt.toISOString(), - }; - } else { - delete context.paperclipContinuationSummary; - } - const pinnedSkillTestContext = - issueRef?.workMode === "skill_test" - ? await getPinnedSkillTestContext(agent.companyId, issueRef.id) : null; - if (pinnedSkillTestContext) { - context.paperclipSkillTest = { - ...pinnedSkillTestContext, - directive: "Use this pinned file inventory as the exact skill revision under test, regardless of synced runtime skills.", + const continuationSummary = issueRef + ? await getIssueContinuationSummaryDocument(db, issueRef.id) + : null; + const exposeLowTrustRaw = trustPreset.kind === "low_trust_review"; + const safeContinuationSummary = + continuationSummary && !exposeLowTrustRaw + ? redactQuarantinedBodyForHigherTrust(continuationSummary) + : continuationSummary; + const safeWakeCommentContext = + wakeCommentContext && !exposeLowTrustRaw + ? sanitizeQuarantinedCommentForHigherTrust(wakeCommentContext) + : wakeCommentContext; + const issueAncestors = issueRef + ? await issuesSvc.getAncestors(issueRef.id) + : []; + if (continuationSummary) { + context.paperclipContinuationSummary = { + key: safeContinuationSummary!.key, + title: safeContinuationSummary!.title, + body: safeContinuationSummary!.body, + sourceTrust: safeContinuationSummary!.sourceTrust ?? null, + updatedAt: safeContinuationSummary!.updatedAt.toISOString(), + }; + } else { + delete context.paperclipContinuationSummary; + } + const pinnedSkillTestContext = + issueRef?.workMode === "skill_test" + ? await getPinnedSkillTestContext(agent.companyId, issueRef.id) + : null; + if (pinnedSkillTestContext) { + context.paperclipSkillTest = { + ...pinnedSkillTestContext, + directive: + "Use this pinned file inventory as the exact skill revision under test, regardless of synced runtime skills.", + }; + } else { + delete context.paperclipSkillTest; + } + const paperclipWakePayload = await buildPaperclipWakePayload({ + db, + companyId: agent.companyId, + contextSnapshot: context, + continuationSummary, + issueSummary: issueRef + ? { + id: issueRef.id, + identifier: issueRef.identifier, + title: issueRef.title, + description: issueContext?.description ?? null, + status: issueRef.status, + priority: issueRef.priority, + workMode: issueRef.workMode, + projectId: issueRef.projectId, + executionPolicy: issueContext?.executionPolicy ?? null, + } + : null, + exposeLowTrustRaw, + simplifiedEnglishInteractions: + experimentalInstanceSettings.enableSimplifiedEnglishInteractions === + true, + }); + if (paperclipWakePayload) { + context[PAPERCLIP_WAKE_PAYLOAD_KEY] = paperclipWakePayload; + } else { + delete context[PAPERCLIP_WAKE_PAYLOAD_KEY]; + } + const taskMarkdownInput = { + issue: issueRef + ? { + id: issueRef.id, + identifier: issueRef.identifier, + title: issueRef.title, + workMode: issueRef.workMode, + description: issueRef.description, + } + : null, + ancestors: issueAncestors, + wakeComment: safeWakeCommentContext, + interaction: { + kind: readNonEmptyString(context.interactionKind), + status: readNonEmptyString(context.interactionStatus), + }, + acceptedPlanContinuation: + readNonEmptyString(context.workspaceRefreshReason) === + "accepted_plan_confirmation" && + Object.keys(parseObject(context.acceptedPlanWakeRouting)).length === + 0, + acceptedPlan: (() => { + const accepted = parseObject( + parseObject(context.planReviewInteraction).acceptedTargetRevision, + ); + const revisionId = readNonEmptyString(accepted.revisionId); + if (!revisionId) return null; + return { + documentId: readNonEmptyString(accepted.documentId), + revisionId, + revisionNumber: + typeof accepted.revisionNumber === "number" + ? accepted.revisionNumber + : null, + }; + })(), }; - } else { - delete context.paperclipSkillTest; - } - const paperclipWakePayload = await buildPaperclipWakePayload({ - db, - companyId: agent.companyId, - contextSnapshot: context, - continuationSummary, - issueSummary: issueRef - ? { - id: issueRef.id, - identifier: issueRef.identifier, - title: issueRef.title, - description: issueContext?.description ?? null, - status: issueRef.status, - priority: issueRef.priority, - workMode: issueRef.workMode, - projectId: issueRef.projectId, - executionPolicy: issueContext?.executionPolicy ?? null, - } - : null, - exposeLowTrustRaw, - simplifiedEnglishInteractions: experimentalInstanceSettings.enableSimplifiedEnglishInteractions === true, - }); - if (paperclipWakePayload) { - context[PAPERCLIP_WAKE_PAYLOAD_KEY] = paperclipWakePayload; - } else { - delete context[PAPERCLIP_WAKE_PAYLOAD_KEY]; - } - const taskMarkdownInput = { - issue: issueRef - ? { - id: issueRef.id, - identifier: issueRef.identifier, - title: issueRef.title, - workMode: issueRef.workMode, - description: issueRef.description, - } - : null, - ancestors: issueAncestors, - wakeComment: safeWakeCommentContext, - interaction: { - kind: readNonEmptyString(context.interactionKind), - status: readNonEmptyString(context.interactionStatus), - }, - acceptedPlanContinuation: - readNonEmptyString(context.workspaceRefreshReason) === "accepted_plan_confirmation" - && Object.keys(parseObject(context.acceptedPlanWakeRouting)).length === 0, - }; - const taskMarkdown = buildPaperclipTaskMarkdown(taskMarkdownInput); - const taskMarkdownCompact = buildPaperclipTaskMarkdown({ ...taskMarkdownInput, includeDescription: false }); - if (issueRef) { - context.paperclipIssue = { - id: issueRef.id, - identifier: issueRef.identifier, - title: issueRef.title, - description: issueRef.description, - workMode: issueRef.workMode, - }; - } else { - delete context.paperclipIssue; - } - if (wakeCommentContext) { - context.paperclipWakeComment = safeWakeCommentContext; - } else { - delete context.paperclipWakeComment; - } - if (taskMarkdown) { - context.paperclipTaskMarkdown = taskMarkdown; - } else { - delete context.paperclipTaskMarkdown; - } - if (taskMarkdownCompact && taskMarkdownCompact !== taskMarkdown) { - context.paperclipTaskMarkdownCompact = taskMarkdownCompact; - } else { - delete context.paperclipTaskMarkdownCompact; - } - if (issueRef) { - const redactedWakeContext = await createRunSecretRedactionRegistry(db).redactForIssue( - agent.companyId, - issueRef.id, - { + const taskMarkdown = buildPaperclipTaskMarkdown(taskMarkdownInput); + const taskMarkdownCompact = buildPaperclipTaskMarkdown({ + ...taskMarkdownInput, + includeDescription: false, + }); + if (issueRef) { + context.paperclipIssue = { + id: issueRef.id, + identifier: issueRef.identifier, + title: issueRef.title, + description: issueRef.description, + workMode: issueRef.workMode, + }; + } else { + delete context.paperclipIssue; + } + if (wakeCommentContext) { + context.paperclipWakeComment = safeWakeCommentContext; + } else { + delete context.paperclipWakeComment; + } + if (taskMarkdown) { + context.paperclipTaskMarkdown = taskMarkdown; + } else { + delete context.paperclipTaskMarkdown; + } + if (taskMarkdownCompact && taskMarkdownCompact !== taskMarkdown) { + context.paperclipTaskMarkdownCompact = taskMarkdownCompact; + } else { + delete context.paperclipTaskMarkdownCompact; + } + if (issueRef) { + const redactedWakeContext = await createRunSecretRedactionRegistry( + db, + ).redactForIssue(agent.companyId, issueRef.id, { paperclipIssue: context.paperclipIssue, paperclipWakeComment: context.paperclipWakeComment, paperclipTaskMarkdown: context.paperclipTaskMarkdown, paperclipTaskMarkdownCompact: context.paperclipTaskMarkdownCompact, - }, - ); - context.paperclipIssue = redactedWakeContext.paperclipIssue; - if (redactedWakeContext.paperclipWakeComment) { - context.paperclipWakeComment = redactedWakeContext.paperclipWakeComment; - } - if (redactedWakeContext.paperclipTaskMarkdown) { - context.paperclipTaskMarkdown = redactedWakeContext.paperclipTaskMarkdown; - } - if (redactedWakeContext.paperclipTaskMarkdownCompact) { - context.paperclipTaskMarkdownCompact = redactedWakeContext.paperclipTaskMarkdownCompact; - } - } - const requestedExecutionWorkspaceId = readNonEmptyString(issueRef?.executionWorkspaceId); - const existingExecutionWorkspace = - requestedExecutionWorkspaceId ? await executionWorkspacesSvc.getById(requestedExecutionWorkspaceId) : null; - const workspaceReuseRequest = resolveExecutionWorkspaceReuseRequestForIssue({ - issueExecutionWorkspaceId: requestedExecutionWorkspaceId, - issueExecutionWorkspacePreference: issueRef?.executionWorkspacePreference ?? null, - existingExecutionWorkspaceStatus: existingExecutionWorkspace?.status ?? null, - requestedExistingBranch: issueExecutionWorkspaceSettings?.workspaceStrategy?.existingBranch ?? null, - existingExecutionWorkspaceBranchName: existingExecutionWorkspace?.branchName ?? null, - }); - const requestedShouldReuseExisting = workspaceReuseRequest.requestedShouldReuseExisting; - const reusableExistingExecutionWorkspace = workspaceReuseRequest.existingExecutionWorkspaceAvailable - ? existingExecutionWorkspace - : null; - const requestedReusableExecutionWorkspaceConfig = reusableExistingExecutionWorkspace?.config ?? null; - const localEnvironment = await environmentsSvc.ensureLocalEnvironment(agent.companyId); - const resolvedInstanceSettings = await instanceSettings.get(); - // Managed-sandbox-only policy: a run that would land on the local - // environment is redirected onto the platform-managed sandbox row, and - // with no active managed row the resolution fails closed - // (ManagedSandboxUnavailableError) — never local. Mirrors the forced - // kubernetes execution mode below, which takes precedence when both - // regimes are active. - const managedSandboxOnly = - (await instanceSettings.getExperimental()).enableManagedSandboxOnly === true; - const managedSandboxEnvironment = managedSandboxOnly - ? await environmentsSvc.findManagedSandboxEnvironment(agent.companyId) - : null; - const environmentResolution = resolveExecutionWorkspaceEnvironmentId({ - agentDefaultEnvironmentId: agent.defaultEnvironmentId, - instanceDefaultEnvironmentId: resolvedInstanceSettings.defaultEnvironmentId ?? null, - localDefaultEnvironmentId: localEnvironment.id, - managedSandboxOnly, - managedSandboxEnvironmentId: managedSandboxEnvironment?.id ?? null, - }); - const effectiveExecutionWorkspaceMode: ReturnType = - requestedExecutionWorkspaceMode; - const executionPolicy = { - executionMode: resolvedInstanceSettings.general.executionMode, - // Backstop behind the resolver's local→managed redirect: the run-time - // allowlist below fails any run that still resolved to a `local` - // environment under managed-sandbox-only, so no selection path or - // tenant-set env var can land untrusted execution on the tenant - // container. - managedSandboxOnly, - }; - const executionForcedToKubernetes = isExecutionForcedToKubernetes(executionPolicy); - let selectedEnvironmentId = environmentResolution.environmentId; - if (executionForcedToKubernetes) { - let kubernetesEnvironment = await environmentsSvc.findKubernetesEnvironment(agent.companyId); - if (!kubernetesEnvironment) { - // Lazy recovery for companies created after the startup bootstrap ran - // (the boot hook only provisions environments for companies that exist - // at boot). Re-derive the managed-env config from the bootstrap env. - // If the process env no longer forces Kubernetes (rollback / config - // drift relative to the persisted executionMode setting), skip the - // provisioning gracefully: the guard below still refuses local - // fallback with the explicit error, instead of crashing here on - // undefined config. - let bootstrap: ReturnType = null; - let bootstrapSkipReason: string | null = null; - try { - bootstrap = parseExecutionPolicyBootstrapEnv(process.env); - if (!bootstrap) { - bootstrapSkipReason = - 'PAPERCLIP_EXECUTION_MODE bootstrap env is not kubernetes-forced (absent or "any")'; - } - } catch (err) { - bootstrapSkipReason = `PAPERCLIP_EXECUTION_MODE bootstrap env failed to parse: ${ - err instanceof Error ? err.message : String(err) - }`; + }); + context.paperclipIssue = redactedWakeContext.paperclipIssue; + if (redactedWakeContext.paperclipWakeComment) { + context.paperclipWakeComment = + redactedWakeContext.paperclipWakeComment; } - if (bootstrap) { - await environmentsSvc.ensureKubernetesEnvironment( - agent.companyId, - bootstrap.kubernetesConfig, + if (redactedWakeContext.paperclipTaskMarkdown) { + context.paperclipTaskMarkdown = + redactedWakeContext.paperclipTaskMarkdown; + } + if (redactedWakeContext.paperclipTaskMarkdownCompact) { + context.paperclipTaskMarkdownCompact = + redactedWakeContext.paperclipTaskMarkdownCompact; + } + } + // A native run's execution input is immutable once persisted. Recovery must therefore + // restore the workspace bound to that input rather than consulting the issue's current + // workspace pointer: a newer run may already have moved or cleared the issue binding while + // this older provider session is still recoverable. + const persistedRunnerProfile = parseObject(run.runnerProfileJson); + const persistedNativeExecutionInput = + run.runtimeMode === "native" && + persistedRunnerProfile.nativeExecutionInput !== undefined + ? parseNativeExecutionInput( + persistedRunnerProfile.nativeExecutionInput, + ) + : null; + const persistedNativeExecutionWorkspaceId = + persistedNativeExecutionInput?.binding.executionWorkspaceId ?? null; + const requestedExecutionWorkspaceId = + persistedNativeExecutionWorkspaceId ?? + readNonEmptyString(issueRef?.executionWorkspaceId); + const existingExecutionWorkspace = requestedExecutionWorkspaceId + ? await executionWorkspacesSvc.getById(requestedExecutionWorkspaceId) + : null; + const nativeRecoveryExecutionWorkspaceId = + resolveNativeRecoveryExecutionWorkspaceBinding({ + bindingId: persistedNativeExecutionWorkspaceId, + persistedWorkspaceFound: existingExecutionWorkspace !== null, + }); + const workspaceReuseRequest = + resolveExecutionWorkspaceReuseRequestForIssue({ + issueExecutionWorkspaceId: requestedExecutionWorkspaceId, + issueExecutionWorkspacePreference: nativeRecoveryExecutionWorkspaceId + ? "reuse_existing" + : (issueRef?.executionWorkspacePreference ?? null), + existingExecutionWorkspaceStatus: + existingExecutionWorkspace?.status ?? null, + }); + const requestedShouldReuseExisting = + workspaceReuseRequest.requestedShouldReuseExisting; + const reusableExistingExecutionWorkspace = + workspaceReuseRequest.existingExecutionWorkspaceAvailable + ? existingExecutionWorkspace + : null; + const requestedReusableExecutionWorkspaceConfig = + reusableExistingExecutionWorkspace?.config ?? null; + const localEnvironment = await environmentsSvc.ensureLocalEnvironment( + agent.companyId, + ); + const resolvedInstanceSettings = await instanceSettings.get(); + // Managed-sandbox-only policy: a run that would land on the local + // environment is redirected onto the platform-managed sandbox row, and + // with no active managed row the resolution fails closed + // (ManagedSandboxUnavailableError) — never local. Mirrors the forced + // kubernetes execution mode below, which takes precedence when both + // regimes are active. + const managedSandboxOnly = + (await instanceSettings.getExperimental()).enableManagedSandboxOnly === + true; + const managedSandboxEnvironment = managedSandboxOnly + ? await environmentsSvc.findManagedSandboxEnvironment(agent.companyId) + : null; + const environmentResolution = resolveExecutionWorkspaceEnvironmentId({ + agentDefaultEnvironmentId: agent.defaultEnvironmentId, + instanceDefaultEnvironmentId: + resolvedInstanceSettings.defaultEnvironmentId ?? null, + localDefaultEnvironmentId: localEnvironment.id, + managedSandboxOnly, + managedSandboxEnvironmentId: managedSandboxEnvironment?.id ?? null, + }); + const effectiveExecutionWorkspaceMode: ReturnType< + typeof resolveExecutionWorkspaceMode + > = requestedExecutionWorkspaceMode; + const executionPolicy = { + executionMode: resolvedInstanceSettings.general.executionMode, + // Backstop behind the resolver's local→managed redirect: the run-time + // allowlist below fails any run that still resolved to a `local` + // environment under managed-sandbox-only, so no selection path or + // tenant-set env var can land untrusted execution on the tenant + // container. + managedSandboxOnly, + }; + const executionForcedToKubernetes = + isExecutionForcedToKubernetes(executionPolicy); + let selectedEnvironmentId = environmentResolution.environmentId; + if (executionForcedToKubernetes) { + let kubernetesEnvironment = + await environmentsSvc.findKubernetesEnvironment(agent.companyId); + if (!kubernetesEnvironment) { + // Lazy recovery for companies created after the startup bootstrap ran + // (the boot hook only provisions environments for companies that exist + // at boot). Re-derive the managed-env config from the bootstrap env. + // If the process env no longer forces Kubernetes (rollback / config + // drift relative to the persisted executionMode setting), skip the + // provisioning gracefully: the guard below still refuses local + // fallback with the explicit error, instead of crashing here on + // undefined config. + let bootstrap: ReturnType = + null; + let bootstrapSkipReason: string | null = null; + try { + bootstrap = parseExecutionPolicyBootstrapEnv(process.env); + if (!bootstrap) { + bootstrapSkipReason = + 'PAPERCLIP_EXECUTION_MODE bootstrap env is not kubernetes-forced (absent or "any")'; + } + } catch (err) { + bootstrapSkipReason = `PAPERCLIP_EXECUTION_MODE bootstrap env failed to parse: ${ + err instanceof Error ? err.message : String(err) + }`; + } + if (bootstrap) { + await environmentsSvc.ensureKubernetesEnvironment( + agent.companyId, + bootstrap.kubernetesConfig, + ); + kubernetesEnvironment = + await environmentsSvc.findKubernetesEnvironment(agent.companyId); + } else { + logger.warn( + { + runId: run.id, + agentId: agent.id, + companyId: agent.companyId, + reason: bootstrapSkipReason, + }, + "executionMode=kubernetes is persisted but the bootstrap env cannot provision a managed Kubernetes environment; skipping lazy provisioning for this company (the run will fail with the explicit no-managed-environment error)", + ); + } + } + if (!kubernetesEnvironment) { + throw new Error( + "Instance execution policy requires the Kubernetes sandbox provider " + + "(executionMode=kubernetes) but no managed Kubernetes environment is " + + "configured for this company. Configure one (PAPERCLIP_K8S_* env on the " + + "cloud instance) before running agents; refusing to fall back to local execution.", ); - kubernetesEnvironment = await environmentsSvc.findKubernetesEnvironment(agent.companyId); - } else { - logger.warn( + } + if (kubernetesEnvironment.id !== selectedEnvironmentId) { + logger.info( { runId: run.id, + issueId, agentId: agent.id, - companyId: agent.companyId, - reason: bootstrapSkipReason, + resolvedEnvironmentId: selectedEnvironmentId, + forcedKubernetesEnvironmentId: kubernetesEnvironment.id, }, - "executionMode=kubernetes is persisted but the bootstrap env cannot provision a managed Kubernetes environment; skipping lazy provisioning for this company (the run will fail with the explicit no-managed-environment error)", + "Forcing run onto the managed Kubernetes environment (executionMode=kubernetes)", ); } + selectedEnvironmentId = kubernetesEnvironment.id; } - if (!kubernetesEnvironment) { - throw new Error( - "Instance execution policy requires the Kubernetes sandbox provider " + - "(executionMode=kubernetes) but no managed Kubernetes environment is " + - "configured for this company. Configure one (PAPERCLIP_K8S_* env on the " + - "cloud instance) before running agents; refusing to fall back to local execution.", - ); - } - if (kubernetesEnvironment.id !== selectedEnvironmentId) { - logger.info( - { - runId: run.id, - issueId, - agentId: agent.id, - resolvedEnvironmentId: selectedEnvironmentId, - forcedKubernetesEnvironmentId: kubernetesEnvironment.id, - }, - "Forcing run onto the managed Kubernetes environment (executionMode=kubernetes)", - ); - } - selectedEnvironmentId = kubernetesEnvironment.id; - } - const selectedEnvironmentForConfig = selectedEnvironmentId === localEnvironment.id - ? localEnvironment - : selectedEnvironmentId - ? await environmentsSvc.getById(selectedEnvironmentId) - : null; - const sharedWorkspaceConcurrency = resolveSharedWorkspaceConcurrency({ - projectPolicy: projectExecutionWorkspacePolicy, - issueSettings: issueExecutionWorkspaceSettings, - }); - // A live holder is always consulted for shared workspaces. Depending on policy and the final - // execution target it either remains the existing deferral gate or becomes dispatch context. - // Holder staleness and the workspace_busy retry ladder are intentionally unchanged for every - // path that serializes. - if (issueRef?.projectWorkspaceId && effectiveExecutionWorkspaceMode === "shared_workspace") { - const workspaceHolder = await findSharedWorkspaceHolder({ - companyId: agent.companyId, - projectWorkspaceId: issueRef.projectWorkspaceId, - excludeIssueId: issueRef.id, - excludeRunId: run.id, - honorIsolatedWorkspaceModes: isolatedWorkspacesEnabled, + const selectedEnvironmentForConfig = + selectedEnvironmentId === localEnvironment.id + ? localEnvironment + : selectedEnvironmentId + ? await environmentsSvc.getById(selectedEnvironmentId) + : null; + const sharedWorkspaceConcurrency = resolveSharedWorkspaceConcurrency({ + projectPolicy: projectExecutionWorkspacePolicy, + issueSettings: issueExecutionWorkspaceSettings, }); - if (workspaceHolder) { - const environmentDriver = selectedEnvironmentForConfig?.driver ?? null; - const shouldSerialize = sharedWorkspaceConcurrency === "serialize" - || ( - sharedWorkspaceConcurrency === "auto" - && ( - executionForcedToKubernetes - || (environmentDriver !== "local" && environmentDriver !== "ssh") - ) - ); - if (shouldSerialize) { - throw new WorkspaceBusyDeferral({ - holder: workspaceHolder, - projectWorkspaceId: issueRef.projectWorkspaceId, - deferralAttempt: - run.scheduledRetryReason === WORKSPACE_BUSY_RETRY_REASON - ? (run.scheduledRetryAttempt ?? 0) - : 0, - wasIssueAssignee: issueContext?.assigneeAgentId === agent.id, - }); - } + // A live holder is always consulted for shared workspaces. Depending on policy and the final + // execution target it either remains the existing deferral gate or becomes dispatch context. + // Holder staleness and the workspace_busy retry ladder are intentionally unchanged for every + // path that serializes. + if ( + issueRef?.projectWorkspaceId && + effectiveExecutionWorkspaceMode === "shared_workspace" + ) { + const workspaceHolder = await findSharedWorkspaceHolder({ + companyId: agent.companyId, + projectWorkspaceId: issueRef.projectWorkspaceId, + excludeIssueId: issueRef.id, + excludeRunId: run.id, + honorIsolatedWorkspaceModes: isolatedWorkspacesEnabled, + }); + if (workspaceHolder) { + const environmentDriver = + selectedEnvironmentForConfig?.driver ?? null; + const shouldSerialize = + sharedWorkspaceConcurrency === "serialize" || + (sharedWorkspaceConcurrency === "auto" && + (executionForcedToKubernetes || + (environmentDriver !== "local" && + environmentDriver !== "ssh"))); + if (shouldSerialize) { + throw new WorkspaceBusyDeferral({ + holder: workspaceHolder, + projectWorkspaceId: issueRef.projectWorkspaceId, + deferralAttempt: + run.scheduledRetryReason === WORKSPACE_BUSY_RETRY_REASON + ? (run.scheduledRetryAttempt ?? 0) + : 0, + wasIssueAssignee: issueContext?.assigneeAgentId === agent.id, + }); + } - const holderIssueLabel = workspaceHolder.issueIdentifier ?? workspaceHolder.issueId; - const concurrentWorkspaceNote = - `shared workspace is concurrently held by run ${workspaceHolder.runId} (issue ${holderIssueLabel}); ` - + "expect concurrent mutations, coordinate via commits"; - const appendConcurrentWorkspaceNote = (value: unknown) => { - const existing = typeof value === "string" ? value.trimEnd() : ""; - return existing ? `${existing}\n${concurrentWorkspaceNote}` : concurrentWorkspaceNote; - }; - context.paperclipTaskMarkdown = appendConcurrentWorkspaceNote(context.paperclipTaskMarkdown); - if (typeof context.paperclipTaskMarkdownCompact === "string") { - context.paperclipTaskMarkdownCompact = appendConcurrentWorkspaceNote( - context.paperclipTaskMarkdownCompact, + const holderIssueLabel = + workspaceHolder.issueIdentifier ?? workspaceHolder.issueId; + const concurrentWorkspaceNote = + `shared workspace is concurrently held by run ${workspaceHolder.runId} (issue ${holderIssueLabel}); ` + + "expect concurrent mutations, coordinate via commits"; + const appendConcurrentWorkspaceNote = (value: unknown) => { + const existing = typeof value === "string" ? value.trimEnd() : ""; + return existing + ? `${existing}\n${concurrentWorkspaceNote}` + : concurrentWorkspaceNote; + }; + context.paperclipTaskMarkdown = appendConcurrentWorkspaceNote( + context.paperclipTaskMarkdown, + ); + if (typeof context.paperclipTaskMarkdownCompact === "string") { + context.paperclipTaskMarkdownCompact = + appendConcurrentWorkspaceNote( + context.paperclipTaskMarkdownCompact, + ); + } + logger.info( + { + event: "shared_workspace_concurrent_dispatch", + runId: run.id, + issueId: issueRef.id, + projectWorkspaceId: issueRef.projectWorkspaceId, + holderRunId: workspaceHolder.runId, + holderIssueId: workspaceHolder.issueId, + sharedWorkspaceConcurrency, + environmentDriver, + executionForcedToKubernetes, + }, + "Dispatching alongside a live shared-workspace holder", ); } - logger.info( + } + const workspaceManagedConfig = buildExecutionWorkspaceAdapterConfig({ + agentConfig: config, + projectPolicy: projectExecutionWorkspacePolicy, + issueSettings: issueExecutionWorkspaceSettings, + mode: requestedExecutionWorkspaceMode, + legacyUseProjectWorkspace: + issueAssigneeOverrides?.useProjectWorkspace ?? null, + }); + let adapterModelProfiles: AdapterModelProfileDefinition[] = []; + let profileResolutionFallbackReason: string | null = null; + try { + adapterModelProfiles = await listAdapterModelProfiles( + agent.adapterType, + ); + } catch (error) { + profileResolutionFallbackReason = "adapter_profile_resolution_failed"; + logger.warn( { - event: "shared_workspace_concurrent_dispatch", + err: error, + companyId: agent.companyId, + agentId: agent.id, + adapterType: agent.adapterType, runId: run.id, - issueId: issueRef.id, - projectWorkspaceId: issueRef.projectWorkspaceId, - holderRunId: workspaceHolder.runId, - holderIssueId: workspaceHolder.issueId, - sharedWorkspaceConcurrency, - environmentDriver, - executionForcedToKubernetes, }, - "Dispatching alongside a live shared-workspace holder", + "Failed to resolve adapter model profiles; falling back to primary adapter config", ); } - } - const workspaceManagedConfig = buildExecutionWorkspaceAdapterConfig({ - agentConfig: config, - projectPolicy: projectExecutionWorkspacePolicy, - issueSettings: issueExecutionWorkspaceSettings, - mode: requestedExecutionWorkspaceMode, - legacyUseProjectWorkspace: issueAssigneeOverrides?.useProjectWorkspace ?? null, - }); - let adapterModelProfiles: AdapterModelProfileDefinition[] = []; - let profileResolutionFallbackReason: string | null = null; - try { - adapterModelProfiles = await listAdapterModelProfiles(agent.adapterType); - } catch (error) { - profileResolutionFallbackReason = "adapter_profile_resolution_failed"; - logger.warn( - { - err: error, + const modelProfileApplication = resolveModelProfileApplication({ + adapterModelProfiles, + agentRuntimeConfig: agent.runtimeConfig, + issueModelProfile: issueAssigneeOverrides?.modelProfile ?? null, + contextSnapshot: context, + profileResolutionFallbackReason, + }); + const modelProfileMetadata = modelProfileRunMetadata( + modelProfileApplication, + ); + if (modelProfileMetadata) { + context.paperclipModelProfile = modelProfileMetadata; + if (modelProfileApplication.requested) + context.modelProfile = modelProfileApplication.requested; + } else { + delete context.paperclipModelProfile; + } + const mergedConfig = mergeModelProfileAdapterConfig({ + baseConfig: workspaceManagedConfig, + modelProfile: modelProfileApplication, + issueAdapterConfig: issueAssigneeOverrides?.adapterConfig ?? null, + }); + const configSnapshot = buildExecutionWorkspaceConfigSnapshot( + mergedConfig, + selectedEnvironmentId, + ); + const executionRunConfig = + stripWorkspaceRuntimeFromExecutionRunConfig(mergedConfig); + const runScopedMentionedSkillKeys = + await resolveRunScopedMentionedSkillKeys({ + db, + companyId: agent.companyId, + issueId, + }); + const runScopedSkillKeys = acceptedPlanContinuationWake + && !acceptedPlanWakeRoutingDecision?.suppressAcceptedContinuation + ? [ + ...runScopedMentionedSkillKeys, + ACCEPTED_PLAN_CONVERSION_SKILL_KEY, + ] + : runScopedMentionedSkillKeys; + const pushCapabilityPreflightRequired = requiresPushCapabilityPreflight({ + adapterType: agent.adapterType, + issueId, + explicitRunScopedSkillKeys: runScopedMentionedSkillKeys, + }); + const { resolvedConfig, secretKeys, secretManifest } = + await resolveExecutionRunAdapterConfig({ companyId: agent.companyId, agentId: agent.id, adapterType: agent.adapterType, - runId: run.id, - }, - "Failed to resolve adapter model profiles; falling back to primary adapter config", - ); - } - const modelProfileApplication = resolveModelProfileApplication({ - adapterModelProfiles, - agentRuntimeConfig: agent.runtimeConfig, - issueModelProfile: issueAssigneeOverrides?.modelProfile ?? null, - contextSnapshot: context, - profileResolutionFallbackReason, - }); - const modelProfileMetadata = modelProfileRunMetadata(modelProfileApplication); - if (modelProfileMetadata) { - context.paperclipModelProfile = modelProfileMetadata; - if (modelProfileApplication.requested) context.modelProfile = modelProfileApplication.requested; - } else { - delete context.paperclipModelProfile; - } - const mergedConfig = mergeModelProfileAdapterConfig({ - baseConfig: workspaceManagedConfig, - modelProfile: modelProfileApplication, - issueAdapterConfig: issueAssigneeOverrides?.adapterConfig ?? null, - }); - const configSnapshot = buildExecutionWorkspaceConfigSnapshot(mergedConfig, selectedEnvironmentId); - const executionRunConfig = stripWorkspaceRuntimeFromExecutionRunConfig(mergedConfig); - const runScopedMentionedSkillKeys = await resolveRunScopedMentionedSkillKeys({ - db, - companyId: agent.companyId, - issueId, - }); - const pushCapabilityPreflightRequired = requiresPushCapabilityPreflight({ - adapterType: agent.adapterType, - issueId, - explicitRunScopedSkillKeys: runScopedMentionedSkillKeys, - }); - const { resolvedConfig, secretKeys, secretManifest } = await resolveExecutionRunAdapterConfig({ - companyId: agent.companyId, - agentId: agent.id, - adapterType: agent.adapterType, - issueId, - heartbeatRunId: run.id, - environmentId: selectedEnvironmentForConfig?.id ?? null, - environmentEnv: selectedEnvironmentForConfig?.envVars ?? null, - environmentDriver: selectedEnvironmentForConfig?.driver ?? null, - projectId: projectContext?.id ?? null, - routineId: routineEnvContext.routineId, - responsibleUserId, - executionRunConfig, - projectEnv: projectContext?.env ?? null, - routineEnv: routineEnvContext.env, - secretsSvc, - trustPreset, - requiredScopedEnvBinding: pushCapabilityPreflightRequired - ? { - keys: [...PUSH_CAPABILITY_ENV_KEYS], - consumerScopes: ["agent", "project"], - reason: "push_write_credential_missing", - remediation: - "GitHub PR workflow requires GH_TOKEN or GITHUB_TOKEN bound at project or agent scope.", - } - : undefined, - }); - if (secretManifest.length > 0) { - context.paperclipSecrets = { - manifest: secretManifest, - }; - } else { - delete context.paperclipSecrets; - } - const effectiveResolvedConfig = applyRunScopedMentionedSkillKeys( - resolvedConfig, - runScopedMentionedSkillKeys, - ); - const runtimeSkillPreference = readPaperclipSkillSyncPreference(effectiveResolvedConfig); - const runtimeSkillEntries = await companySkills.listRuntimeSkillEntries(agent.companyId, { - versionSelections: skillVersionSelectionMap(runtimeSkillPreference.desiredSkillEntries, { - versionPinsEnabled: resolvedInstanceSettings.experimental.enableBetaSkills === true, - }), - }); - let runtimeConfig: Record = { - ...effectiveResolvedConfig, - paperclipRuntimeSkills: runtimeSkillEntries, - }; - const latestAgentConfigRevision = await getLatestAgentConfigRevision(agent.companyId, agent.id); - const sessionConfigMetadata = await buildEffectiveRunSessionConfigMetadata({ - adapterType: agent.adapterType, - effectiveAdapterConfig: runtimeConfig, - agentRuntimeConfig: agent.runtimeConfig, - modelProfile: modelProfileMetadata, - issueOverrides: issueAssigneeOverrides, - workspaceConfig: { - requestedMode: requestedExecutionWorkspaceMode, - effectiveMode: effectiveExecutionWorkspaceMode, - issueConfigRevisionAt: issueContext?.updatedAt instanceof Date - ? issueContext.updatedAt.toISOString() - : issueContext?.updatedAt ?? null, - projectConfigRevisionAt: projectContext?.updatedAt instanceof Date - ? projectContext.updatedAt.toISOString() - : projectContext?.updatedAt ?? null, - projectPolicy: projectExecutionWorkspacePolicy, - issueSettings: issueExecutionWorkspaceSettings, - reusableExecutionWorkspaceConfig: requestedReusableExecutionWorkspaceConfig, - existingExecutionWorkspace: reusableExistingExecutionWorkspace - ? { - id: reusableExistingExecutionWorkspace.id, - mode: reusableExistingExecutionWorkspace.mode, - strategyType: reusableExistingExecutionWorkspace.strategyType, - projectWorkspaceId: reusableExistingExecutionWorkspace.projectWorkspaceId, - repoUrl: reusableExistingExecutionWorkspace.repoUrl, - baseRef: reusableExistingExecutionWorkspace.baseRef, - branchName: reusableExistingExecutionWorkspace.branchName, - config: reusableExistingExecutionWorkspace.config, - } - : null, - }, - environment: { - selectionSource: environmentResolution.source, - selectedEnvironmentId, - selectedEnvironment: selectedEnvironmentForConfig - ? { - id: selectedEnvironmentForConfig.id, - driver: selectedEnvironmentForConfig.driver, - config: selectedEnvironmentForConfig.config, - configRevisionAt: selectedEnvironmentForConfig.updatedAt instanceof Date - ? selectedEnvironmentForConfig.updatedAt.toISOString() - : selectedEnvironmentForConfig.updatedAt ?? null, - } - : null, - executionPolicy, - }, - environmentEnv: selectedEnvironmentForConfig?.envVars ?? null, - projectEnv: projectContext?.env ?? null, - routineEnv: routineEnvContext.env, - secretManifest, - runtimeSkills: runtimeSkillEntries, - agentConfigRevision: latestAgentConfigRevision - ? { - id: latestAgentConfigRevision.id, - changedKeys: latestAgentConfigRevision.changedKeys, - configRevisionAt: latestAgentConfigRevision.createdAt.toISOString(), - } - : null, - }); - const configuredModel = readConfiguredModelFromAdapterConfig(runtimeConfig); - const wakeSessionResetReason = describeSessionResetReason(context); - const sessionConfigFreshness = resolveTaskSessionConfigFreshness({ - hasTaskSession: taskSession != null, - configuredModel, - taskSessionParams: taskSession?.sessionParamsJson ?? taskSessionDecodedParams, - configMetadata: sessionConfigMetadata, - wakeResetReason: wakeSessionResetReason, - preserveLegacySessionWithoutConfigMetadata: acceptedPlanContinuationWake && !acceptedPlanWakeRoutingDecision, - }); - const resetTaskSession = shouldResetTaskSessionForWake(context) || sessionConfigFreshness.reset; - const sessionResetReason = sessionConfigFreshness.reasons.join("; ") || null; - const taskSessionForRun = resetTaskSession ? null : taskSession; - const previousSessionParams = - explicitResumeSessionParams ?? - (isCanonicalSessionIdForAdapter(agent.adapterType, explicitResumeSessionDisplayId) - ? { sessionId: explicitResumeSessionDisplayId } - : null) ?? - normalizeResumeParamsForAdapter( - agent.adapterType, - stripPaperclipSessionMetadataFromSessionParams( - sessionCodec.deserialize(taskSessionForRun?.sessionParamsJson ?? null), - ), - ); - const { - selectedEnvironmentDriver: lowTrustPreflightEnvironmentDriver, - workspace: resolvedWorkspace, - } = await resolveWorkspaceAfterLowTrustPreflight({ - db, - trustPreset, - isolatedWorkspacesEnabled, - effectiveExecutionWorkspaceMode, - issue: issueRef - ? { - companyId: agent.companyId, - id: issueRef.id, - projectId: issueRef.projectId, - } - : null, - resolveSelectedEnvironmentDriver: async () => { - const preflightEnvironment = await envOrchestrator.resolveEnvironment({ - companyId: agent.companyId, - selectedEnvironmentId, - localEnvironmentId: localEnvironment.id, + issueId, + heartbeatRunId: run.id, + environmentId: selectedEnvironmentForConfig?.id ?? null, + environmentEnv: selectedEnvironmentForConfig?.envVars ?? null, + environmentDriver: selectedEnvironmentForConfig?.driver ?? null, + projectId: projectContext?.id ?? null, + routineId: routineEnvContext.routineId, + responsibleUserId, + executionRunConfig, + projectEnv: projectContext?.env ?? null, + routineEnv: routineEnvContext.env, + secretsSvc, + trustPreset, + requiredScopedEnvBinding: pushCapabilityPreflightRequired + ? { + keys: [...PUSH_CAPABILITY_ENV_KEYS], + consumerScopes: ["agent", "project"], + reason: "push_write_credential_missing", + remediation: + "GitHub PR workflow requires GH_TOKEN or GITHUB_TOKEN bound at project or agent scope.", + } + : undefined, }); - return preflightEnvironment.driver; - }, - resolveWorkspace: () => - resolveWorkspaceForRun( - agent, - context, - previousSessionParams, - { - useProjectWorkspace: requestedExecutionWorkspaceMode !== "agent_default", + if (secretManifest.length > 0) { + context.paperclipSecrets = { + manifest: secretManifest, + }; + } else { + delete context.paperclipSecrets; + } + const effectiveResolvedConfig = applyRunScopedMentionedSkillKeys( + resolvedConfig, + runScopedSkillKeys, + ); + const runtimeSkillPreference = readPaperclipSkillSyncPreference( + effectiveResolvedConfig, + ); + const runtimeSkillEntries = await companySkills.listRuntimeSkillEntries( + agent.companyId, + { + versionSelections: skillVersionSelectionMap( + runtimeSkillPreference.desiredSkillEntries, + { + versionPinsEnabled: + resolvedInstanceSettings.experimental.enableBetaSkills === true, + }, + ), + }, + ); + let runtimeConfig: Record = { + ...effectiveResolvedConfig, + paperclipRuntimeSkills: runtimeSkillEntries, + }; + const latestAgentConfigRevision = await getLatestAgentConfigRevision( + agent.companyId, + agent.id, + ); + const sessionConfigMetadata = + await buildEffectiveRunSessionConfigMetadata({ + adapterType: agent.adapterType, + effectiveAdapterConfig: runtimeConfig, + agentRuntimeConfig: agent.runtimeConfig, + modelProfile: modelProfileMetadata, + issueOverrides: issueAssigneeOverrides, + workspaceConfig: { + requestedMode: requestedExecutionWorkspaceMode, + effectiveMode: effectiveExecutionWorkspaceMode, + issueConfigRevisionAt: + issueContext?.updatedAt instanceof Date + ? issueContext.updatedAt.toISOString() + : (issueContext?.updatedAt ?? null), + projectConfigRevisionAt: + projectContext?.updatedAt instanceof Date + ? projectContext.updatedAt.toISOString() + : (projectContext?.updatedAt ?? null), + projectPolicy: projectExecutionWorkspacePolicy, + issueSettings: issueExecutionWorkspaceSettings, + reusableExecutionWorkspaceConfig: + requestedReusableExecutionWorkspaceConfig, + existingExecutionWorkspace: reusableExistingExecutionWorkspace + ? { + id: reusableExistingExecutionWorkspace.id, + mode: reusableExistingExecutionWorkspace.mode, + strategyType: reusableExistingExecutionWorkspace.strategyType, + projectWorkspaceId: + reusableExistingExecutionWorkspace.projectWorkspaceId, + repoUrl: reusableExistingExecutionWorkspace.repoUrl, + baseRef: reusableExistingExecutionWorkspace.baseRef, + branchName: reusableExistingExecutionWorkspace.branchName, + config: reusableExistingExecutionWorkspace.config, + } + : null, + }, + environment: { + selectionSource: environmentResolution.source, + selectedEnvironmentId, + selectedEnvironment: selectedEnvironmentForConfig + ? { + id: selectedEnvironmentForConfig.id, + driver: selectedEnvironmentForConfig.driver, + config: selectedEnvironmentForConfig.config, + configRevisionAt: + selectedEnvironmentForConfig.updatedAt instanceof Date + ? selectedEnvironmentForConfig.updatedAt.toISOString() + : (selectedEnvironmentForConfig.updatedAt ?? null), + } + : null, + executionPolicy, + }, + environmentEnv: selectedEnvironmentForConfig?.envVars ?? null, + projectEnv: projectContext?.env ?? null, + routineEnv: routineEnvContext.env, + secretManifest, + runtimeSkills: runtimeSkillEntries, + agentConfigRevision: latestAgentConfigRevision + ? { + id: latestAgentConfigRevision.id, + changedKeys: latestAgentConfigRevision.changedKeys, + configRevisionAt: + latestAgentConfigRevision.createdAt.toISOString(), + } + : null, + }); + const configuredModel = + readConfiguredModelFromAdapterConfig(runtimeConfig); + const wakeSessionResetReason = describeSessionResetReason(context); + const sessionConfigFreshness = resolveTaskSessionConfigFreshness({ + hasTaskSession: taskSession != null, + configuredModel, + taskSessionParams: + taskSession?.sessionParamsJson ?? taskSessionDecodedParams, + configMetadata: sessionConfigMetadata, + wakeResetReason: wakeSessionResetReason, + preserveLegacySessionWithoutConfigMetadata: + acceptedPlanContinuationWake && !acceptedPlanWakeRoutingDecision, + }); + const resetTaskSession = + shouldResetTaskSessionForWake(context) || sessionConfigFreshness.reset; + const sessionResetReason = + sessionConfigFreshness.reasons.join("; ") || null; + const taskSessionForRun = resetTaskSession ? null : taskSession; + const previousSessionParams = + explicitResumeSessionParams ?? + (isCanonicalSessionIdForAdapter( + agent.adapterType, + explicitResumeSessionDisplayId, + ) + ? { sessionId: explicitResumeSessionDisplayId } + : null) ?? + normalizeResumeParamsForAdapter( + agent.adapterType, + stripPaperclipSessionMetadataFromSessionParams( + sessionCodec.deserialize( + taskSessionForRun?.sessionParamsJson ?? null, + ), + ), + ); + const { + selectedEnvironmentDriver: lowTrustPreflightEnvironmentDriver, + workspace: resolvedWorkspace, + } = await resolveWorkspaceAfterLowTrustPreflight({ + db, + trustPreset, + isolatedWorkspacesEnabled, + effectiveExecutionWorkspaceMode, + issue: issueRef + ? { + companyId: agent.companyId, + id: issueRef.id, + projectId: issueRef.projectId, + } + : null, + resolveSelectedEnvironmentDriver: async () => { + const preflightEnvironment = await envOrchestrator.resolveEnvironment( + { + companyId: agent.companyId, + selectedEnvironmentId, + localEnvironmentId: localEnvironment.id, + }, + ); + return preflightEnvironment.driver; + }, + resolveWorkspace: () => + resolveWorkspaceForRun(agent, context, previousSessionParams, { + useProjectWorkspace: + requestedExecutionWorkspaceMode !== "agent_default", // Thread the selected environment driver so run-workspace resolution can tell a local // target from a remote one, and a confined sandbox target from an unconfined remote // target. A remote run resolves referenced projects only for the confined sandbox // transport with the remote flag on. This never changes the anchor workspace. - executionEnvironmentDriver: selectedEnvironmentForConfig?.driver ?? null, - }, + executionEnvironmentDriver: + selectedEnvironmentForConfig?.driver ?? null, + }), + }); + const hostExecutionWorkspaceConfig = + stripHostWorkspaceProvisionForLowTrustSandbox({ + config: mergedConfig, + trustPreset, + selectedEnvironmentDriver: lowTrustPreflightEnvironmentDriver, + }); + const executionWorkspaceBase = { + baseCwd: resolvedWorkspace.cwd, + source: resolvedWorkspace.source, + projectId: resolvedWorkspace.projectId, + workspaceId: resolvedWorkspace.workspaceId, + repoUrl: resolvedWorkspace.repoUrl, + repoRef: resolvedWorkspace.repoRef, + additionalWorkspaces: resolvedWorkspace.additionalWorkspaces, + } satisfies ExecutionWorkspaceInput; + await assertGitWorktreeBaseWorkspaceReady({ + requestedExecutionWorkspaceMode, + config: hostExecutionWorkspaceConfig, + issue: issueRef, + base: executionWorkspaceBase, + anchor: { + baseCwdFallback: resolvedWorkspace.baseCwdFallback, + materializationFailures: resolvedWorkspace.materializationFailures, + }, + }); + const workspaceStrategyForFingerprint = parseObject( + hostExecutionWorkspaceConfig.workspaceStrategy, + ); + const workspaceStrategyFingerprintValue = + Object.keys(workspaceStrategyForFingerprint).length > 0 + ? workspaceStrategyForFingerprint + : null; + const latestWorkspaceStrategyType = resolveEffectiveWorkspaceStrategyType( + requestedExecutionWorkspaceMode, + hostExecutionWorkspaceConfig, + ); + const selectedEnvironmentConfigForFingerprint = parseObject( + selectedEnvironmentForConfig?.config, + ); + const workspaceEnvironmentFingerprint = selectedEnvironmentForConfig + ? { + selectionSource: environmentResolution.source, + selectedEnvironmentId, + driver: selectedEnvironmentForConfig.driver, + provider: readNonEmptyString( + selectedEnvironmentConfigForFingerprint.provider, + ), + config: selectedEnvironmentForConfig.config, + configRevisionAt: + selectedEnvironmentForConfig.updatedAt instanceof Date + ? selectedEnvironmentForConfig.updatedAt.toISOString() + : (selectedEnvironmentForConfig.updatedAt ?? null), + executionPolicy, + } + : null; + const workspaceRealizationFingerprint = { + environmentDriver: selectedEnvironmentForConfig?.driver ?? null, + environmentProvider: readNonEmptyString( + selectedEnvironmentConfigForFingerprint.provider, ), - }); - const hostExecutionWorkspaceConfig = stripHostWorkspaceProvisionForLowTrustSandbox({ - config: mergedConfig, - trustPreset, - selectedEnvironmentDriver: lowTrustPreflightEnvironmentDriver, - }); - const executionWorkspaceBase = { - baseCwd: resolvedWorkspace.cwd, - source: resolvedWorkspace.source, - projectId: resolvedWorkspace.projectId, - workspaceId: resolvedWorkspace.workspaceId, - repoUrl: resolvedWorkspace.repoUrl, - repoRef: resolvedWorkspace.repoRef, - additionalWorkspaces: resolvedWorkspace.additionalWorkspaces, - } satisfies ExecutionWorkspaceInput; - await assertGitWorktreeBaseWorkspaceReady({ - requestedExecutionWorkspaceMode, - config: hostExecutionWorkspaceConfig, - issue: issueRef, - base: executionWorkspaceBase, - anchor: { - baseCwdFallback: resolvedWorkspace.baseCwdFallback, - materializationFailures: resolvedWorkspace.materializationFailures, - }, - }); - const workspaceStrategyForFingerprint = parseObject(hostExecutionWorkspaceConfig.workspaceStrategy); - const workspaceStrategyFingerprintValue = - Object.keys(workspaceStrategyForFingerprint).length > 0 ? workspaceStrategyForFingerprint : null; - const latestWorkspaceStrategyType = resolveEffectiveWorkspaceStrategyType( - requestedExecutionWorkspaceMode, - hostExecutionWorkspaceConfig, - ); - const selectedEnvironmentConfigForFingerprint = parseObject(selectedEnvironmentForConfig?.config); - const workspaceEnvironmentFingerprint = selectedEnvironmentForConfig - ? { - selectionSource: environmentResolution.source, - selectedEnvironmentId, - driver: selectedEnvironmentForConfig.driver, - provider: readNonEmptyString(selectedEnvironmentConfigForFingerprint.provider), - config: selectedEnvironmentForConfig.config, - configRevisionAt: selectedEnvironmentForConfig.updatedAt instanceof Date - ? selectedEnvironmentForConfig.updatedAt.toISOString() - : selectedEnvironmentForConfig.updatedAt ?? null, - executionPolicy, - } - : null; - const workspaceRealizationFingerprint = { - environmentDriver: selectedEnvironmentForConfig?.driver ?? null, - environmentProvider: readNonEmptyString(selectedEnvironmentConfigForFingerprint.provider), - trustPreset: trustPreset.kind, - lowTrustSandboxDriver: lowTrustPreflightEnvironmentDriver, - }; - const latestWorkspaceConfigMetadata = buildEffectiveRunWorkspaceConfigMetadata({ - mode: requestedExecutionWorkspaceMode, - projectId: executionWorkspaceBase.projectId, - projectWorkspaceId: executionWorkspaceBase.workspaceId, - strategyType: latestWorkspaceStrategyType, - workspaceStrategy: workspaceStrategyFingerprintValue, - repoUrl: executionWorkspaceBase.repoUrl, - repoRef: readNonEmptyString(workspaceStrategyForFingerprint.baseRef) ?? executionWorkspaceBase.repoRef, - configSnapshot, - environment: workspaceEnvironmentFingerprint, - realization: workspaceRealizationFingerprint, - secretManifest, - }); - const inferredExistingWorkspaceConfigMetadata = reusableExistingExecutionWorkspace - ? buildEffectiveRunWorkspaceConfigMetadata({ - mode: issueExecutionWorkspaceModeForPersistedWorkspace(reusableExistingExecutionWorkspace.mode), - projectId: reusableExistingExecutionWorkspace.projectId, - projectWorkspaceId: reusableExistingExecutionWorkspace.projectWorkspaceId, - strategyType: reusableExistingExecutionWorkspace.strategyType, - workspaceStrategy: workspaceStrategyFingerprintValue - ? { - ...workspaceStrategyFingerprintValue, - type: reusableExistingExecutionWorkspace.strategyType, - ...(reusableExistingExecutionWorkspace.baseRef - ? { baseRef: reusableExistingExecutionWorkspace.baseRef } - : {}), - } - : { type: reusableExistingExecutionWorkspace.strategyType }, - repoUrl: reusableExistingExecutionWorkspace.repoUrl, - repoRef: reusableExistingExecutionWorkspace.baseRef, - configSnapshot: reusableExistingExecutionWorkspace.config, + trustPreset: trustPreset.kind, + lowTrustSandboxDriver: lowTrustPreflightEnvironmentDriver, + }; + const latestWorkspaceConfigMetadata = + buildEffectiveRunWorkspaceConfigMetadata({ + mode: requestedExecutionWorkspaceMode, + projectId: executionWorkspaceBase.projectId, + projectWorkspaceId: executionWorkspaceBase.workspaceId, + strategyType: latestWorkspaceStrategyType, + workspaceStrategy: workspaceStrategyFingerprintValue, + repoUrl: executionWorkspaceBase.repoUrl, + repoRef: + readNonEmptyString(workspaceStrategyForFingerprint.baseRef) ?? + executionWorkspaceBase.repoRef, + configSnapshot, environment: workspaceEnvironmentFingerprint, realization: workspaceRealizationFingerprint, secretManifest, - evaluatedAt: latestWorkspaceConfigMetadata.evaluatedAt, - }) - : null; - const workspaceConfigFreshness = resolveExecutionWorkspaceConfigFreshness({ - hasExistingWorkspace: requestedShouldReuseExisting && Boolean(reusableExistingExecutionWorkspace), - existingWorkspaceMetadata: reusableExistingExecutionWorkspace?.metadata ?? null, - inferredMetadata: inferredExistingWorkspaceConfigMetadata, - nextMetadata: latestWorkspaceConfigMetadata, - }); - const workspaceReuseProvisioningPolicy = resolveExecutionWorkspaceReuseProvisioningPolicy({ - requestedShouldReuseExisting, - workspaceConfigFreshness, - }); - const workspaceOperationRecorder = workspaceOperationsSvc.createRecorder({ - companyId: agent.companyId, - heartbeatRunId: run.id, - executionWorkspaceId: workspaceReuseProvisioningPolicy.shouldRestoreExistingWorkspace - ? workspaceReuseRequest.requestedExecutionWorkspaceId - : null, - issueId, - }); - // One credential provider per run: base-ref refreshes during workspace realization and - // restore authenticate against private GitHub remotes with the same company-secret token - // the managed clone uses. - const workspaceGitAuthProvider = createGitRemoteAuthProvider(db, agent.companyId, { - issueId, - heartbeatRunId: run.id, - }); - const { executionWorkspace, reusedExecutionWorkspace, policy: resolvedWorkspaceReusePolicy } = - await provisionExecutionWorkspaceForFreshnessDecision({ - requestedShouldReuseExisting, - existingExecutionWorkspaceId: workspaceReuseRequest.requestedExecutionWorkspaceId, - issueRef, - runId: run.id, - workspaceConfigFreshness, - restoreExistingWorkspace: reusableExistingExecutionWorkspace - ? () => ensurePersistedExecutionWorkspaceAvailable({ + }); + const inferredExistingWorkspaceConfigMetadata = + reusableExistingExecutionWorkspace + ? buildEffectiveRunWorkspaceConfigMetadata({ + mode: issueExecutionWorkspaceModeForPersistedWorkspace( + reusableExistingExecutionWorkspace.mode, + ), + projectId: reusableExistingExecutionWorkspace.projectId, + projectWorkspaceId: + reusableExistingExecutionWorkspace.projectWorkspaceId, + strategyType: reusableExistingExecutionWorkspace.strategyType, + workspaceStrategy: workspaceStrategyFingerprintValue + ? { + ...workspaceStrategyFingerprintValue, + type: reusableExistingExecutionWorkspace.strategyType, + ...(reusableExistingExecutionWorkspace.baseRef + ? { baseRef: reusableExistingExecutionWorkspace.baseRef } + : {}), + } + : { type: reusableExistingExecutionWorkspace.strategyType }, + repoUrl: reusableExistingExecutionWorkspace.repoUrl, + repoRef: reusableExistingExecutionWorkspace.baseRef, + configSnapshot: reusableExistingExecutionWorkspace.config, + environment: workspaceEnvironmentFingerprint, + realization: workspaceRealizationFingerprint, + secretManifest, + evaluatedAt: latestWorkspaceConfigMetadata.evaluatedAt, + }) + : null; + const workspaceConfigFreshness = resolveExecutionWorkspaceConfigFreshness( + { + hasExistingWorkspace: + requestedShouldReuseExisting && + Boolean(reusableExistingExecutionWorkspace), + existingWorkspaceMetadata: + reusableExistingExecutionWorkspace?.metadata ?? null, + inferredMetadata: inferredExistingWorkspaceConfigMetadata, + nextMetadata: latestWorkspaceConfigMetadata, + }, + ); + const workspaceReuseProvisioningPolicy = + resolveExecutionWorkspaceReuseProvisioningPolicy({ + requestedShouldReuseExisting, + workspaceConfigFreshness, + }); + const workspaceOperationRecorder = workspaceOperationsSvc.createRecorder({ + companyId: agent.companyId, + heartbeatRunId: run.id, + executionWorkspaceId: + workspaceReuseProvisioningPolicy.shouldRestoreExistingWorkspace + ? workspaceReuseRequest.requestedExecutionWorkspaceId + : null, + issueId, + }); + // One credential provider per run: base-ref refreshes during workspace realization and + // restore authenticate against private GitHub remotes with the same company-secret token + // the managed clone uses. + const workspaceGitAuthProvider = createGitRemoteAuthProvider( + db, + agent.companyId, + { + issueId, + heartbeatRunId: run.id, + }, + ); + const { + executionWorkspace, + reusedExecutionWorkspace, + policy: resolvedWorkspaceReusePolicy, + } = await provisionExecutionWorkspaceForFreshnessDecision( + { + requestedShouldReuseExisting, + existingExecutionWorkspaceId: + workspaceReuseRequest.requestedExecutionWorkspaceId, + issueRef, + runId: run.id, + workspaceConfigFreshness, + restoreExistingWorkspace: reusableExistingExecutionWorkspace + ? () => + ensurePersistedExecutionWorkspaceAvailable({ + db, + base: executionWorkspaceBase, + workspace: { + id: reusableExistingExecutionWorkspace.id, + mode: reusableExistingExecutionWorkspace.mode, + strategyType: + reusableExistingExecutionWorkspace.strategyType, + cwd: reusableExistingExecutionWorkspace.cwd, + providerRef: reusableExistingExecutionWorkspace.providerRef, + projectId: reusableExistingExecutionWorkspace.projectId, + projectWorkspaceId: + reusableExistingExecutionWorkspace.projectWorkspaceId, + repoUrl: reusableExistingExecutionWorkspace.repoUrl, + baseRef: reusableExistingExecutionWorkspace.baseRef, + branchName: reusableExistingExecutionWorkspace.branchName, + metadata: + reusableExistingExecutionWorkspace.metadata as Record< + string, + unknown + > | null, + config: { + provisionCommand: + configSnapshot?.provisionCommand ?? + reusableExistingExecutionWorkspace.config + ?.provisionCommand ?? + projectExecutionWorkspacePolicy?.workspaceStrategy + ?.provisionCommand ?? + null, + runtimeProvisionCommand: + configSnapshot?.runtimeProvisionCommand ?? + reusableExistingExecutionWorkspace.config + ?.runtimeProvisionCommand ?? + projectExecutionWorkspacePolicy?.workspaceStrategy + ?.runtimeProvisionCommand ?? + null, + }, + }, + issue: issueRef, + agent: { + id: agent.id, + name: agent.name, + companyId: agent.companyId, + }, + heartbeatRunId: run.id, + enableWorkspaceBranchReconcileForward: + resolvedInstanceSettings.experimental + .enableWorkspaceBranchReconcileForward, + enableWorkspaceDirtyQuarantineRepair: + resolvedInstanceSettings.experimental + .enableWorkspaceDirtyQuarantineRepair, + recorder: workspaceOperationRecorder, + resolveGitAuth: workspaceGitAuthProvider, + }) + : null, + realizeWorkspace: () => + realizeExecutionWorkspace({ db, base: executionWorkspaceBase, - workspace: { - id: reusableExistingExecutionWorkspace.id, - mode: reusableExistingExecutionWorkspace.mode, - strategyType: reusableExistingExecutionWorkspace.strategyType, - cwd: reusableExistingExecutionWorkspace.cwd, - providerRef: reusableExistingExecutionWorkspace.providerRef, - projectId: reusableExistingExecutionWorkspace.projectId, - projectWorkspaceId: reusableExistingExecutionWorkspace.projectWorkspaceId, - repoUrl: reusableExistingExecutionWorkspace.repoUrl, - baseRef: reusableExistingExecutionWorkspace.baseRef, - branchName: reusableExistingExecutionWorkspace.branchName, - metadata: reusableExistingExecutionWorkspace.metadata as Record | null, - config: { - provisionCommand: - configSnapshot?.provisionCommand - ?? reusableExistingExecutionWorkspace.config?.provisionCommand - ?? projectExecutionWorkspacePolicy?.workspaceStrategy?.provisionCommand - ?? null, - runtimeProvisionCommand: - configSnapshot?.runtimeProvisionCommand - ?? reusableExistingExecutionWorkspace.config?.runtimeProvisionCommand - ?? projectExecutionWorkspacePolicy?.workspaceStrategy?.runtimeProvisionCommand - ?? null, - }, - }, + config: hostExecutionWorkspaceConfig, issue: issueRef, agent: { id: agent.id, name: agent.name, companyId: agent.companyId, }, + recordedBranchOwnership: + existingExecutionWorkspace?.status !== "archived" && + existingExecutionWorkspace?.branchName + ? { + branchName: existingExecutionWorkspace.branchName, + createdByRuntime: isRuntimeOwnedGitBranch( + existingExecutionWorkspace.metadata, + ), + } + : null, heartbeatRunId: run.id, enableWorkspaceBranchReconcileForward: - resolvedInstanceSettings.experimental.enableWorkspaceBranchReconcileForward, + resolvedInstanceSettings.experimental + .enableWorkspaceBranchReconcileForward, enableWorkspaceDirtyQuarantineRepair: - resolvedInstanceSettings.experimental.enableWorkspaceDirtyQuarantineRepair, + resolvedInstanceSettings.experimental + .enableWorkspaceDirtyQuarantineRepair, recorder: workspaceOperationRecorder, resolveGitAuth: workspaceGitAuthProvider, - }) - : null, - realizeWorkspace: () => realizeExecutionWorkspace({ - db, - base: executionWorkspaceBase, - config: hostExecutionWorkspaceConfig, - issue: issueRef, - agent: { - id: agent.id, - name: agent.name, - companyId: agent.companyId, - }, - recordedBranchOwnership: - existingExecutionWorkspace?.status !== "archived" - && existingExecutionWorkspace?.branchName - ? { - branchName: existingExecutionWorkspace.branchName, - createdByRuntime: isRuntimeOwnedGitBranch( - existingExecutionWorkspace.metadata, - ), - } + }), + }, + ); + const resolvedProjectId = + executionWorkspace.projectId ?? + issueRef?.projectId ?? + executionProjectId ?? + null; + const resolvedProjectWorkspaceId = + issueRef?.projectWorkspaceId ?? resolvedWorkspace.workspaceId ?? null; + let persistedExecutionWorkspace: ExecutionWorkspace | null = null; + const baseExecutionWorkspaceMetadata = + mergeExecutionWorkspaceMetadataForPersistence({ + existingMetadata: + resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace + ? (reusableExistingExecutionWorkspace?.metadata ?? null) : null, - heartbeatRunId: run.id, - enableWorkspaceBranchReconcileForward: - resolvedInstanceSettings.experimental.enableWorkspaceBranchReconcileForward, - enableWorkspaceDirtyQuarantineRepair: - resolvedInstanceSettings.experimental.enableWorkspaceDirtyQuarantineRepair, - recorder: workspaceOperationRecorder, - resolveGitAuth: workspaceGitAuthProvider, - }), - }); - const resolvedProjectId = executionWorkspace.projectId ?? issueRef?.projectId ?? executionProjectId ?? null; - const resolvedProjectWorkspaceId = issueRef?.projectWorkspaceId ?? resolvedWorkspace.workspaceId ?? null; - let persistedExecutionWorkspace: ExecutionWorkspace | null = null; - const baseExecutionWorkspaceMetadata = mergeExecutionWorkspaceMetadataForPersistence({ - existingMetadata: resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace - ? reusableExistingExecutionWorkspace?.metadata ?? null - : null, - source: executionWorkspace.source, - // Branch ownership, not worktree freshness: attaching a worktree to a - // pre-existing branch reports created=true but must never make terminal - // cleanup delete that operator-owned branch. - createdByRuntime: resolveExecutionWorkspaceBranchOwnership(executionWorkspace), - strategyType: executionWorkspace.strategy, - configSnapshot, - shouldReuseExisting: resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace, - shouldRefreshConfigSnapshot: resolvedWorkspaceReusePolicy.shouldRefreshWorkspaceConfigSnapshot, - workspaceConfigMetadata: resolvedWorkspaceReusePolicy.shouldPersistLatestWorkspaceConfigMetadata - ? latestWorkspaceConfigMetadata - : null, - baseRef: executionWorkspace.repoRef, - baseRefSha: executionWorkspace.baseRefSha ?? null, - }); - let persistedWorktreeInstanceRoot = - resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace - && typeof reusableExistingExecutionWorkspace?.metadata?.[WORKTREE_INSTANCE_ROOT_METADATA_KEY] === "string" - ? reusableExistingExecutionWorkspace.metadata[WORKTREE_INSTANCE_ROOT_METADATA_KEY] - : null; - if ( - !persistedWorktreeInstanceRoot - && executionWorkspace.strategy === "git_worktree" - && executionWorkspace.worktreePath - ) { - try { - persistedWorktreeInstanceRoot = ( - await readManagedWorktreeInstanceOwnership(executionWorkspace.worktreePath) - )?.instanceRoot ?? null; - } catch (error) { - logger.warn( - { - runId: run.id, - issueId, - executionWorkspaceCwd: executionWorkspace.cwd, - error: error instanceof Error ? error.message : String(error), - }, - "Could not record managed worktree instance ownership", - ); - } - } - const nextExecutionWorkspaceMetadata = { - ...baseExecutionWorkspaceMetadata, - ...(persistedWorktreeInstanceRoot - ? { [WORKTREE_INSTANCE_ROOT_METADATA_KEY]: persistedWorktreeInstanceRoot } - : {}), - }; - const pendingForwardBranchReconcile = executionWorkspace.pendingForwardBranchReconcile ?? null; - const branchNameForInitialPersistence = - pendingForwardBranchReconcile?.recordedBranchName ?? executionWorkspace.branchName; - try { - persistedExecutionWorkspace = resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace && reusableExistingExecutionWorkspace - ? await executionWorkspacesSvc.update(reusableExistingExecutionWorkspace.id, { - cwd: executionWorkspace.cwd, - repoUrl: executionWorkspace.repoUrl, - baseRef: executionWorkspace.repoRef, - branchName: branchNameForInitialPersistence, - providerType: executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "local_fs", - providerRef: executionWorkspace.worktreePath, - status: "active", - lastUsedAt: new Date(), - metadata: nextExecutionWorkspaceMetadata, - projectWorkspaceId: reconcileReusedExecutionWorkspaceProjectWorkspaceId( - reusableExistingExecutionWorkspace.projectWorkspaceId, - resolvedProjectWorkspaceId, - ), - }) - : resolvedProjectId - ? await executionWorkspacesSvc.create({ - companyId: agent.companyId, - projectId: resolvedProjectId, - projectWorkspaceId: resolvedProjectWorkspaceId, - sourceIssueId: issueRef?.id ?? null, - mode: - requestedExecutionWorkspaceMode === "isolated_workspace" - ? "isolated_workspace" - : requestedExecutionWorkspaceMode === "operator_branch" - ? "operator_branch" - : requestedExecutionWorkspaceMode === "agent_default" - ? "adapter_managed" - : "shared_workspace", - strategyType: executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "project_primary", - name: branchNameForInitialPersistence ?? issueRef?.identifier ?? `workspace-${agent.id.slice(0, 8)}`, - status: "active", - cwd: executionWorkspace.cwd, - repoUrl: executionWorkspace.repoUrl, - baseRef: executionWorkspace.repoRef, - branchName: branchNameForInitialPersistence, - providerType: executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "local_fs", - providerRef: executionWorkspace.worktreePath, - lastUsedAt: new Date(), - openedAt: new Date(), - metadata: nextExecutionWorkspaceMetadata, - }) + source: executionWorkspace.source, + // Attaching a new worktree to a pre-existing branch reports a fresh + // workspace, but must not make cleanup own the operator's branch. + createdByRuntime: + resolveExecutionWorkspaceBranchOwnership(executionWorkspace), + strategyType: executionWorkspace.strategy, + configSnapshot, + shouldReuseExisting: + resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace, + shouldRefreshConfigSnapshot: + resolvedWorkspaceReusePolicy.shouldRefreshWorkspaceConfigSnapshot, + workspaceConfigMetadata: + resolvedWorkspaceReusePolicy.shouldPersistLatestWorkspaceConfigMetadata + ? latestWorkspaceConfigMetadata + : null, + baseRef: executionWorkspace.repoRef, + baseRefSha: executionWorkspace.baseRefSha ?? null, + }); + let persistedWorktreeInstanceRoot = + resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace && + typeof reusableExistingExecutionWorkspace?.metadata?.[ + WORKTREE_INSTANCE_ROOT_METADATA_KEY + ] === "string" + ? reusableExistingExecutionWorkspace.metadata[ + WORKTREE_INSTANCE_ROOT_METADATA_KEY + ] : null; - } catch (error) { - if (executionWorkspace.created) { + if ( + !persistedWorktreeInstanceRoot && + executionWorkspace.strategy === "git_worktree" && + executionWorkspace.worktreePath + ) { try { - await cleanupExecutionWorkspaceArtifacts({ - workspace: { - id: - reusableExistingExecutionWorkspace?.id - ?? workspaceReuseRequest.requestedExecutionWorkspaceId - ?? `transient-${run.id}`, - cwd: executionWorkspace.cwd, - providerType: executionWorkspace.strategy === "git_worktree" ? "git_worktree" : "local_fs", - providerRef: executionWorkspace.worktreePath, - branchName: executionWorkspace.branchName, - repoUrl: executionWorkspace.repoUrl, - baseRef: executionWorkspace.repoRef, - projectId: resolvedProjectId, - projectWorkspaceId: resolvedProjectWorkspaceId, - sourceIssueId: issueRef?.id ?? null, - metadata: nextExecutionWorkspaceMetadata, - }, - projectWorkspace: { - cwd: resolvedWorkspace.cwd, - cleanupCommand: null, - }, - cleanupCommand: configSnapshot?.cleanupCommand ?? null, - teardownCommand: configSnapshot?.teardownCommand ?? projectExecutionWorkspacePolicy?.workspaceStrategy?.teardownCommand ?? null, - recorder: workspaceOperationRecorder, - }); - } catch (cleanupError) { + persistedWorktreeInstanceRoot = + ( + await readManagedWorktreeInstanceOwnership( + executionWorkspace.worktreePath, + ) + )?.instanceRoot ?? null; + } catch (error) { logger.warn( { runId: run.id, issueId, executionWorkspaceCwd: executionWorkspace.cwd, - cleanupError: cleanupError instanceof Error ? cleanupError.message : String(cleanupError), + error: error instanceof Error ? error.message : String(error), }, - "Failed to cleanup realized execution workspace after persistence failure", + "Could not record managed worktree instance ownership", ); } } - throw error; - } - await workspaceOperationRecorder.attachExecutionWorkspaceId(persistedExecutionWorkspace?.id ?? null); - await recordWorkspaceConfigFreshnessOperation({ - recorder: workspaceOperationRecorder, - runId: run.id, - decision: workspaceConfigFreshness, - hasExistingWorkspace: Boolean(reusableExistingExecutionWorkspace), - reuseRequested: requestedShouldReuseExisting, - workspaceReused: Boolean(reusedExecutionWorkspace), - configSnapshotRefreshed: resolvedWorkspaceReusePolicy.shouldRefreshWorkspaceConfigSnapshot, - previousWorkspaceId: workspaceReuseRequest.requestedExecutionWorkspaceId, - activeWorkspaceId: persistedExecutionWorkspace?.id ?? null, - }); - if ( - reusableExistingExecutionWorkspace && - persistedExecutionWorkspace && - reusableExistingExecutionWorkspace.id !== persistedExecutionWorkspace.id && - reusableExistingExecutionWorkspace.status === "active" - ) { - await executionWorkspacesSvc.update(reusableExistingExecutionWorkspace.id, { - status: "idle", - cleanupReason: null, - }); - } - if (issueId && persistedExecutionWorkspace) { - const nextIssueWorkspaceMode = issueExecutionWorkspaceModeForPersistedWorkspace(persistedExecutionWorkspace.mode); - const shouldSwitchIssueToExistingWorkspace = - issueRef?.executionWorkspacePreference === "reuse_existing" || - requestedExecutionWorkspaceMode === "isolated_workspace" || - requestedExecutionWorkspaceMode === "operator_branch"; - const nextIssuePatch: Record = {}; - if (issueRef?.executionWorkspaceId !== persistedExecutionWorkspace.id) { - nextIssuePatch.executionWorkspaceId = persistedExecutionWorkspace.id; - } - if (resolvedProjectWorkspaceId && issueRef?.projectWorkspaceId !== resolvedProjectWorkspaceId) { - nextIssuePatch.projectWorkspaceId = resolvedProjectWorkspaceId; - } - if (shouldSwitchIssueToExistingWorkspace) { - nextIssuePatch.executionWorkspacePreference = "reuse_existing"; - nextIssuePatch.executionWorkspaceSettings = { - ...(issueExecutionWorkspaceSettings ?? {}), - mode: nextIssueWorkspaceMode, - }; - } - if (Object.keys(nextIssuePatch).length > 0) { - await issuesSvc.update(issueId, nextIssuePatch); - } - } - if (persistedExecutionWorkspace) { - context.executionWorkspaceId = persistedExecutionWorkspace.id; - await db - .update(heartbeatRuns) - .set({ - contextSnapshot: context, - updatedAt: new Date(), - }) - .where(eq(heartbeatRuns.id, run.id)); - } - const acquiredEnvironment = await envOrchestrator.acquireForRun({ - companyId: agent.companyId, - selectedEnvironmentId, - localEnvironmentId: localEnvironment.id, - adapterType: agent.adapterType, - issueId: issueId ?? null, - heartbeatRunId: run.id, - agentId: agent.id, - persistedExecutionWorkspace, - executionWorkspaceSettings: environmentExecutionWorkspaceSettings, - }); - const selectedEnvironment = acquiredEnvironment.environment; - // Defense-in-depth: re-check the actually-acquired environment against the - // execution allowlist. Even if selection were bypassed, a denied (local/ssh/ - // non-k8s) environment FAILS the run here rather than executing untrusted. - const allowlistDecision = evaluateExecutionAllowlist(executionPolicy, { - driver: selectedEnvironment.driver, - provider: - typeof selectedEnvironment.config?.provider === "string" - ? selectedEnvironment.config.provider - : null, - }); - if (!allowlistDecision.allowed) { - logger.error( - { - runId: run.id, - issueId, - agentId: agent.id, - environmentId: selectedEnvironment.id, - deniedDriver: allowlistDecision.deniedDriver, - deniedProvider: allowlistDecision.deniedProvider, - }, - "Execution allowlist denied the resolved environment; failing run", - ); - throw new Error(allowlistDecision.reason); - } - let activeEnvironmentLease = { - environment: acquiredEnvironment.environment, - lease: acquiredEnvironment.lease, - leaseContext: acquiredEnvironment.leaseContext, - }; - // The host duplex observability recorder for this run. It binds the fixed duplex - // observability surface to real sinks: the spans to the OTel tracer, the - // guarded counters to the tool-runtime metric store, and the transport event - // to the run-event path. Each sink runs guarded and fire-and-forget, so a - // telemetry failure never breaks the run. The orchestrator stamps it on the - // sandbox target; a non-duplex run keeps the safe no-op default in the bridge. - const duplexObservabilityRecorder = createHostDuplexObservabilityRecorder({ - tracer: getStartupTracer(), - incrementCounter: (metric) => { - void incrementToolRuntimeMetricCounter(db, { - companyId: run.companyId, - metric, - }).catch(() => {}); - }, - emitTransportEvent: (event) => { - void (async () => { - const eventRun = run; - await appendRunEvent(eventRun, { - eventType: event.name, - stream: "system", - level: event.dimensions.outcome === "error" ? "warn" : "info", - payload: { ...event.dimensions }, - }); - })().catch(() => {}); - }, - }); - const realizationResult = await envOrchestrator.realizeForRun({ - environment: selectedEnvironment, - lease: activeEnvironmentLease.lease, - adapterType: agent.adapterType, - companyId: agent.companyId, - issueId: issueId ?? null, - heartbeatRunId: run.id, - executionWorkspace, - effectiveExecutionWorkspaceMode, - persistedExecutionWorkspace, - duplexObservabilityRecorder, - }); - activeEnvironmentLease = { - ...activeEnvironmentLease, - lease: realizationResult.lease, - }; - persistedExecutionWorkspace = realizationResult.persistedExecutionWorkspace; - const workspaceRealization = realizationResult.workspaceRealization; - const executionTarget = realizationResult.executionTarget; - const remoteExecution = realizationResult.remoteExecution; - if (!executionTarget || executionTarget.kind === "local") { - try { - runScratch = await prepareHeartbeatRunScratch({ - companyId: agent.companyId, - agentId: agent.id, - runId: run.id, - issueId: issueRef?.id ?? null, - issueIdentifier: issueRef?.identifier ?? null, - }); - const existingRuntimeEnv = parseObject(runtimeConfig.env); - const scratchEnv = buildHeartbeatRunScratchEnv(existingRuntimeEnv, runScratch); - runtimeConfig = { - ...runtimeConfig, - env: { - ...existingRuntimeEnv, - ...scratchEnv.env, - }, - }; - context.paperclipScratch = { - type: "heartbeat_run", - dir: runScratch.dir, - cleanupPolicy: "terminal_run", - marker: HEARTBEAT_RUN_SCRATCH_MARKER, - tempKeysApplied: scratchEnv.tempKeysApplied, - }; - } catch (scratchPrepareError) { - runScratch = null; - delete context.paperclipScratch; - logger.warn( - { - err: scratchPrepareError, - runId: run.id, - issueId, - agentId: agent.id, - }, - "failed to prepare heartbeat run scratch directory; continuing without scratch env", - ); - } - } else { - delete context.paperclipScratch; - } - context.paperclipEnvironment = { - id: selectedEnvironment.id, - name: selectedEnvironment.name, - driver: selectedEnvironment.driver, - leaseId: activeEnvironmentLease.lease.id, - workspaceRealization, - ...(typeof activeEnvironmentLease.lease.metadata?.remoteCwd === "string" - ? { - remoteCwd: activeEnvironmentLease.lease.metadata.remoteCwd, - host: - typeof activeEnvironmentLease.lease.metadata?.host === "string" - ? activeEnvironmentLease.lease.metadata.host - : undefined, - port: - typeof activeEnvironmentLease.lease.metadata?.port === "number" - ? activeEnvironmentLease.lease.metadata.port - : undefined, - username: - typeof activeEnvironmentLease.lease.metadata?.username === "string" - ? activeEnvironmentLease.lease.metadata.username - : undefined, - } - : {}), - }; - await db - .update(heartbeatRuns) - .set({ - contextSnapshot: context, - updatedAt: new Date(), - }) - .where(eq(heartbeatRuns.id, run.id)); - const runtimeSessionResolution = resolveRuntimeSessionParamsForWorkspace({ - agentId: agent.id, - previousSessionParams, - resolvedWorkspace: { - ...resolvedWorkspace, - cwd: executionWorkspace.cwd, - }, - }); - const runtimeSessionParams = runtimeSessionResolution.sessionParams; - const runtimeWorkspaceWarnings = [ - ...resolvedWorkspace.warnings, - ...executionWorkspace.warnings, - ...(runtimeSessionResolution.warning ? [runtimeSessionResolution.warning] : []), - ...(requestedShouldReuseExisting && workspaceConfigFreshness.reasons.length > 0 - ? [ - `Execution workspace reuse freshness action "${workspaceConfigFreshness.action}" because ${workspaceConfigFreshness.reasons.join("; ")}.`, - ] - : []), - ...(resetTaskSession && sessionResetReason - ? [ - taskKey - ? `Skipping saved session resume for task "${taskKey}" because ${sessionResetReason}.` - : `Skipping saved session resume because ${sessionResetReason}.`, - ] - : []), - ]; - context.paperclipWorkspace = { - cwd: executionWorkspace.cwd, - source: executionWorkspace.source, - mode: effectiveExecutionWorkspaceMode, - strategy: executionWorkspace.strategy, - projectId: executionWorkspace.projectId, - workspaceId: executionWorkspace.workspaceId, - repoUrl: executionWorkspace.repoUrl, - repoRef: executionWorkspace.repoRef, - branchName: executionWorkspace.branchName, - worktreePath: executionWorkspace.worktreePath, - realization: workspaceRealization, - agentHome: await (async () => { - const home = resolveDefaultAgentWorkspaceDir(agent.id); - await fs.mkdir(home, { recursive: true }); - return home; - })(), - }; - context.paperclipWorkspaces = buildRunWorkspaceHints(resolvedWorkspace); - // Emit exactly one requested-vs-synced observability line for the referenced-project set. A run - // with no referenced project stays silent, so this adds no noise to the anchor-only default. The - // per-drop human warning already rides `runtimeWorkspaceWarnings`; this line carries the counts - // and the per-failure reason for a partial sync. - const referencedProjectObservability = buildReferencedProjectRunObservability({ - syncedProjectIds: resolvedWorkspace.additionalWorkspaces.map( - (additional) => additional.projectId, - ), - failures: resolvedWorkspace.referencedProjectFailures, - }); - if (referencedProjectObservability.referenced_projects_requested > 0) { - logger.info( - { - runId: run.id, - companyId: agent.companyId, - issueId: issueRef?.id ?? null, - ...referencedProjectObservability, - }, - "run referenced-project sync", - ); - } - // The wake payload is built before the execution workspace is resolved, so - // attach the branch pin here; the shared wake-prompt renderer surfaces it as - // a one-time "stay on this branch" hint on non-resumed sessions. - if (executionWorkspace.branchName) { - const wakePayloadForWorkspace = parseObject(context[PAPERCLIP_WAKE_PAYLOAD_KEY]); - context[PAPERCLIP_WAKE_PAYLOAD_KEY] = { - ...wakePayloadForWorkspace, - executionWorkspace: { branchName: executionWorkspace.branchName }, + const nextExecutionWorkspaceMetadata = { + ...baseExecutionWorkspaceMetadata, + ...(persistedWorktreeInstanceRoot + ? { + [WORKTREE_INSTANCE_ROOT_METADATA_KEY]: + persistedWorktreeInstanceRoot, + } + : {}), }; - } - const runtimeServiceIntents = (() => { - const runtimeConfig = parseObject(hostExecutionWorkspaceConfig.workspaceRuntime); - return Array.isArray(runtimeConfig.services) - ? runtimeConfig.services.filter( - (value): value is Record => typeof value === "object" && value !== null, - ) - : []; - })(); - assertLowTrustRuntimeServicesAllowed({ - resolution: trustPreset, - runtimeServiceCount: runtimeServiceIntents.length, - }); - if (runtimeServiceIntents.length > 0) { - context.paperclipRuntimeServiceIntents = runtimeServiceIntents; - } else { - delete context.paperclipRuntimeServiceIntents; - } - if (executionWorkspace.projectId && !readNonEmptyString(context.projectId)) { - context.projectId = executionWorkspace.projectId; - } - const runtimeSessionFallback = taskKey || resetTaskSession - ? null - : isCanonicalSessionIdForAdapter(agent.adapterType, runtime.sessionId) - ? runtime.sessionId - : null; - const runtimeSessionDisplayId = truncateDisplayId( - explicitResumeSessionDisplayId ?? - taskSessionForRun?.sessionDisplayId ?? - (sessionCodec.getDisplayId ? sessionCodec.getDisplayId(runtimeSessionParams) : null) ?? - readNonEmptyString(runtimeSessionParams?.sessionId) ?? - runtimeSessionFallback, - ); - let previousSessionDisplayId = requiresCanonicalSessionIds(agent.adapterType) - ? truncateDisplayId( - readNonEmptyString(previousSessionParams?.sessionId) ?? - (isCanonicalSessionIdForAdapter(agent.adapterType, runtimeSessionDisplayId) ? runtimeSessionDisplayId : null) ?? - runtimeSessionFallback, - ) - : runtimeSessionDisplayId; - let runtimeSessionIdForAdapter = - readNonEmptyString(runtimeSessionParams?.sessionId) ?? runtimeSessionFallback; - let runtimeSessionParamsForAdapter = normalizeSessionParams( - stripPaperclipSessionMetadataFromSessionParams(runtimeSessionParams), - ); - - const sessionCompaction = await evaluateSessionCompaction({ - agent, - sessionId: previousSessionDisplayId ?? runtimeSessionIdForAdapter, - issueId, - continuationSummaryBody: continuationSummary?.body ?? null, - }); - if (sessionCompaction.rotate) { - context.paperclipSessionHandoffMarkdown = sessionCompaction.handoffMarkdown; - context.paperclipSessionRotationReason = sessionCompaction.reason; - context.paperclipPreviousSessionId = previousSessionDisplayId ?? runtimeSessionIdForAdapter; - runtimeSessionIdForAdapter = null; - runtimeSessionParamsForAdapter = null; - previousSessionDisplayId = null; - if (sessionCompaction.reason) { - runtimeWorkspaceWarnings.push( - `Starting a fresh session because ${sessionCompaction.reason}.`, - ); + const pendingForwardBranchReconcile = + executionWorkspace.pendingForwardBranchReconcile ?? null; + const branchNameForInitialPersistence = + pendingForwardBranchReconcile?.recordedBranchName ?? + executionWorkspace.branchName; + try { + persistedExecutionWorkspace = + resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace && + reusableExistingExecutionWorkspace + ? await executionWorkspacesSvc.update( + reusableExistingExecutionWorkspace.id, + { + cwd: executionWorkspace.cwd, + repoUrl: executionWorkspace.repoUrl, + baseRef: executionWorkspace.repoRef, + branchName: branchNameForInitialPersistence, + providerType: + executionWorkspace.strategy === "git_worktree" + ? "git_worktree" + : "local_fs", + providerRef: executionWorkspace.worktreePath, + status: "active", + lastUsedAt: new Date(), + metadata: nextExecutionWorkspaceMetadata, + projectWorkspaceId: + reconcileReusedExecutionWorkspaceProjectWorkspaceId( + reusableExistingExecutionWorkspace.projectWorkspaceId, + resolvedProjectWorkspaceId, + ), + }, + ) + : resolvedProjectId + ? await executionWorkspacesSvc.create({ + companyId: agent.companyId, + projectId: resolvedProjectId, + projectWorkspaceId: resolvedProjectWorkspaceId, + sourceIssueId: issueRef?.id ?? null, + mode: + requestedExecutionWorkspaceMode === "isolated_workspace" + ? "isolated_workspace" + : requestedExecutionWorkspaceMode === "operator_branch" + ? "operator_branch" + : requestedExecutionWorkspaceMode === "agent_default" + ? "adapter_managed" + : "shared_workspace", + strategyType: + executionWorkspace.strategy === "git_worktree" + ? "git_worktree" + : "project_primary", + name: + branchNameForInitialPersistence ?? + issueRef?.identifier ?? + `workspace-${agent.id.slice(0, 8)}`, + status: "active", + cwd: executionWorkspace.cwd, + repoUrl: executionWorkspace.repoUrl, + baseRef: executionWorkspace.repoRef, + branchName: branchNameForInitialPersistence, + providerType: + executionWorkspace.strategy === "git_worktree" + ? "git_worktree" + : "local_fs", + providerRef: executionWorkspace.worktreePath, + lastUsedAt: new Date(), + openedAt: new Date(), + metadata: nextExecutionWorkspaceMetadata, + }) + : null; + } catch (error) { + if (executionWorkspace.created) { + try { + await cleanupExecutionWorkspaceArtifacts({ + workspace: { + id: + reusableExistingExecutionWorkspace?.id ?? + workspaceReuseRequest.requestedExecutionWorkspaceId ?? + `transient-${run.id}`, + cwd: executionWorkspace.cwd, + providerType: + executionWorkspace.strategy === "git_worktree" + ? "git_worktree" + : "local_fs", + providerRef: executionWorkspace.worktreePath, + branchName: executionWorkspace.branchName, + repoUrl: executionWorkspace.repoUrl, + baseRef: executionWorkspace.repoRef, + projectId: resolvedProjectId, + projectWorkspaceId: resolvedProjectWorkspaceId, + sourceIssueId: issueRef?.id ?? null, + metadata: nextExecutionWorkspaceMetadata, + }, + projectWorkspace: { + cwd: resolvedWorkspace.cwd, + cleanupCommand: null, + }, + cleanupCommand: configSnapshot?.cleanupCommand ?? null, + teardownCommand: + configSnapshot?.teardownCommand ?? + projectExecutionWorkspacePolicy?.workspaceStrategy + ?.teardownCommand ?? + null, + recorder: workspaceOperationRecorder, + }); + } catch (cleanupError) { + logger.warn( + { + runId: run.id, + issueId, + executionWorkspaceCwd: executionWorkspace.cwd, + cleanupError: + cleanupError instanceof Error + ? cleanupError.message + : String(cleanupError), + }, + "Failed to cleanup realized execution workspace after persistence failure", + ); + } + } + throw error; } - } else { - delete context.paperclipSessionHandoffMarkdown; - delete context.paperclipSessionRotationReason; - delete context.paperclipPreviousSessionId; - } - - const runtimeForAdapter = { - sessionId: runtimeSessionIdForAdapter, - sessionParams: runtimeSessionParamsForAdapter, - sessionDisplayId: previousSessionDisplayId, - taskKey, - }; - const configFreshnessResultMetadata = { - version: sessionConfigMetadata.version, - session: { - fingerprintVersion: sessionConfigMetadata.version, - categories: sessionConfigMetadata.categories, - reset: resetTaskSession, - resetReasons: sessionConfigFreshness.reasons, - changedCategories: sessionConfigFreshness.changedCategories, - taskSessionAvailable: taskSession != null, - taskSessionReused: taskSessionForRun != null, - storedFingerprintPresent: Boolean(sessionConfigFreshness.storedFingerprint), - nextFingerprint: sessionConfigFreshness.nextFingerprint, - }, - workspace: { - fingerprintVersion: latestWorkspaceConfigMetadata.version, - categories: latestWorkspaceConfigMetadata.categories, - action: workspaceConfigFreshness.action, - changedCategories: workspaceConfigFreshness.changedCategories, - reasons: workspaceConfigFreshness.reasons, + await workspaceOperationRecorder.attachExecutionWorkspaceId( + persistedExecutionWorkspace?.id ?? null, + ); + await recordWorkspaceConfigFreshnessOperation({ + recorder: workspaceOperationRecorder, + runId: run.id, + decision: workspaceConfigFreshness, + hasExistingWorkspace: Boolean(reusableExistingExecutionWorkspace), reuseRequested: requestedShouldReuseExisting, workspaceReused: Boolean(reusedExecutionWorkspace), - configSnapshotRefreshed: resolvedWorkspaceReusePolicy.shouldRefreshWorkspaceConfigSnapshot, - storedFingerprintPresent: workspaceConfigFreshness.storedFingerprintPresent, - storedFingerprint: workspaceConfigFreshness.storedFingerprint, - inferredFingerprint: workspaceConfigFreshness.inferredFingerprint, - nextFingerprint: workspaceConfigFreshness.nextFingerprint, - previousWorkspaceId: workspaceReuseRequest.requestedExecutionWorkspaceId, + configSnapshotRefreshed: + resolvedWorkspaceReusePolicy.shouldRefreshWorkspaceConfigSnapshot, + previousWorkspaceId: + workspaceReuseRequest.requestedExecutionWorkspaceId, activeWorkspaceId: persistedExecutionWorkspace?.id ?? null, - }, - }; - let handle: RunLogHandle | null = null; - let stdoutExcerpt = ""; - let stderrExcerpt = ""; - let outputSeq = Number(run.lastOutputSeq ?? 0); - let lastOutputFlushAt: Date | null = run.lastOutputAt ?? null; - let lastLogRuntimeStatusTouchMs = 0; - const outputProgressState: { - pending: { - at: Date; - seq: number; - stream: "stdout" | "stderr"; - bytes: number; - } | null; - } = { pending: null }; - let persistedLogBytes = Number(run.logBytes ?? 0); - const flushOutputProgress = async (opts?: { force?: boolean }) => { - const pendingOutputProgress = outputProgressState.pending; - if (!pendingOutputProgress) return; - const shouldFlush = - opts?.force === true || - !lastOutputFlushAt || - pendingOutputProgress.at.getTime() - lastOutputFlushAt.getTime() >= ACTIVE_RUN_OUTPUT_PROGRESS_FLUSH_INTERVAL_MS; - if (!shouldFlush) return; - await db - .update(heartbeatRuns) - .set({ - lastOutputAt: pendingOutputProgress.at, - lastOutputSeq: pendingOutputProgress.seq, - lastOutputStream: pendingOutputProgress.stream, - lastOutputBytes: pendingOutputProgress.bytes, - updatedAt: new Date(), - }) - .where(eq(heartbeatRuns.id, run.id)); - lastOutputFlushAt = pendingOutputProgress.at; - outputProgressState.pending = null; - }; - try { - const startedAt = run.startedAt ?? new Date(); - const runningWithSession = await db - .update(heartbeatRuns) - .set({ - startedAt, - sessionIdBefore: runtimeForAdapter.sessionDisplayId ?? runtimeForAdapter.sessionId, - contextSnapshot: context, - updatedAt: new Date(), - }) - .where(eq(heartbeatRuns.id, run.id)) - .returning() - .then((rows) => rows[0] ?? null); - if (runningWithSession) run = runningWithSession; - - // Pause Durability: flip to "running" ONLY if the agent is still invokable. - // Atomic conditional UPDATE is the sole gate (no read-then-write); 0 rows => abort. - const runningAgent = await db - .update(agents) - .set({ status: "running", updatedAt: new Date() }) - .where(and(eq(agents.id, agent.id), notInArray(agents.status, [...DIRECT_NON_INVOKABLE_STATUSES]))) - .returning() - .then((rows) => rows[0] ?? null); - - if (!runningAgent) { - logger.warn( - { agentId: agent.id, runId: run.id, previousStatus: agent.status }, - "execution-start aborted: agent not invokable", - ); - const abortReason = "Cancelled: agent not invokable at execution-start"; - await setRunStatus(run.id, "cancelled", { - finishedAt: new Date(), - error: abortReason, - errorCode: "agent_not_invokable", - ...(agent ? { - resultJson: mergeRunStopMetadataForAgent(agent, "cancelled", { - resultJson: parseObject(run.resultJson), - errorCode: "agent_not_invokable", - errorMessage: abortReason, - }), - } : {}), - }); - await setWakeupStatus(run.wakeupRequestId, "cancelled", { - finishedAt: new Date(), - error: abortReason, - }); - await releaseIssueExecutionAndPromote(run); - return; - } - - publishLiveEvent({ - companyId: runningAgent.companyId, - type: "agent.status", - payload: { - agentId: runningAgent.id, - status: runningAgent.status, - outcome: "running", - }, }); - - const currentRun = run; - await appendRunEvent(currentRun, { - eventType: "lifecycle", - stream: "system", - level: "info", - message: "run started", - }); - - handle = await runLogStore.begin({ - companyId: run.companyId, - agentId: run.agentId, - runId, - }); - - await db - .update(heartbeatRuns) - .set({ - logStore: handle.store, - logRef: handle.logRef, - updatedAt: new Date(), - }) - .where(eq(heartbeatRuns.id, runId)); - - const currentUserRedactionOptions = await getCurrentUserRedactionOptions(); - const onLog = async (stream: "stdout" | "stderr", chunk: string) => { - const sanitizedChunk = compactRunLogChunk( - redactCurrentUserText(chunk, currentUserRedactionOptions), - ); - if (stream === "stdout") stdoutExcerpt = appendExcerpt(stdoutExcerpt, sanitizedChunk); - if (stream === "stderr") stderrExcerpt = appendExcerpt(stderrExcerpt, sanitizedChunk); - const ts = new Date().toISOString(); - - outputSeq += 1; - const chunkSeq = outputSeq; - let appendedBytes = 0; - if (handle) { - appendedBytes = await runLogStore.append(handle, { - stream, - chunk: sanitizedChunk, - ts, - seq: chunkSeq, - }); - persistedLogBytes += appendedBytes; - } - outputProgressState.pending = { - at: new Date(ts), - seq: chunkSeq, - stream, - bytes: persistedLogBytes, - }; - await flushOutputProgress(); - - // Streamed CLI output is real run activity: keep the in-memory - // runtime status ("Working... / X ago") fresh between structured - // events so sandbox runs with mid-run log streaming never show a - // minutes-stale timestamp. Throttled to avoid churning the live - // event stream on every 250ms tail chunk. - const logActivityAt = new Date(ts); - if ( - isHeartbeatRunRuntimeStatusActive(run.status) && - logActivityAt.getTime() - lastLogRuntimeStatusTouchMs >= - ACTIVE_RUN_LOG_RUNTIME_STATUS_REFRESH_INTERVAL_MS - ) { - lastLogRuntimeStatusTouchMs = logActivityAt.getTime(); - const touchedStatus = touchHeartbeatRunRuntimeStatus({ - companyId: run.companyId, - issueId, - agentId: run.agentId, - runId: run.id, - at: logActivityAt, - }); - if (touchedStatus) publishHeartbeatRunRuntimeProgress(touchedStatus); - } - - const payloadChunk = - sanitizedChunk.length > MAX_LIVE_LOG_CHUNK_BYTES - ? sanitizedChunk.slice(sanitizedChunk.length - MAX_LIVE_LOG_CHUNK_BYTES) - : sanitizedChunk; - - publishLiveEvent({ - companyId: run.companyId, - type: "heartbeat.run.log", - payload: { - runId: run.id, - agentId: run.agentId, - issueId, - ts, - seq: chunkSeq, - stream, - chunk: payloadChunk, - truncated: payloadChunk.length !== sanitizedChunk.length, + if ( + reusableExistingExecutionWorkspace && + persistedExecutionWorkspace && + reusableExistingExecutionWorkspace.id !== + persistedExecutionWorkspace.id && + reusableExistingExecutionWorkspace.status === "active" + ) { + await executionWorkspacesSvc.update( + reusableExistingExecutionWorkspace.id, + { + status: "idle", + cleanupReason: null, }, - }); - }; - if (runScopedMentionedSkillKeys.length > 0) { - await onLog( - "stdout", - `[paperclip] Enabled run-scoped skills from issue mentions: ${runScopedMentionedSkillKeys.join(", ")}\n`, ); } - for (const warning of runtimeWorkspaceWarnings) { - const logEntry = formatRuntimeWorkspaceWarningLog(warning); - await onLog(logEntry.stream, logEntry.chunk); + if ( + issueId && + persistedExecutionWorkspace && + !nativeRecoveryExecutionWorkspaceId + ) { + const nextIssueWorkspaceMode = + issueExecutionWorkspaceModeForPersistedWorkspace( + persistedExecutionWorkspace.mode, + ); + const shouldSwitchIssueToExistingWorkspace = + issueRef?.executionWorkspacePreference === "reuse_existing" || + requestedExecutionWorkspaceMode === "isolated_workspace" || + requestedExecutionWorkspaceMode === "operator_branch"; + const nextIssuePatch: Record = {}; + if (issueRef?.executionWorkspaceId !== persistedExecutionWorkspace.id) { + nextIssuePatch.executionWorkspaceId = persistedExecutionWorkspace.id; + } + if ( + resolvedProjectWorkspaceId && + issueRef?.projectWorkspaceId !== resolvedProjectWorkspaceId + ) { + nextIssuePatch.projectWorkspaceId = resolvedProjectWorkspaceId; + } + if (shouldSwitchIssueToExistingWorkspace) { + nextIssuePatch.executionWorkspacePreference = "reuse_existing"; + nextIssuePatch.executionWorkspaceSettings = { + ...(issueExecutionWorkspaceSettings ?? {}), + mode: nextIssueWorkspaceMode, + }; + } + if (Object.keys(nextIssuePatch).length > 0) { + await issuesSvc.update(issueId, nextIssuePatch); + } } - await assertGitSensitiveAdapterWorkspaceValid({ - adapterType: agent.adapterType, - agentId: agent.id, - issue: issueRef - ? { - id: issueRef.id, - identifier: issueRef.identifier, - projectId: issueRef.projectId, - projectWorkspaceId: issueRef.projectWorkspaceId, - } - : null, - resolvedWorkspace, - executionWorkspace, - persistedExecutionWorkspace, - executionTarget, - environmentDriver: selectedEnvironment.driver, - leaseMetadata: activeEnvironmentLease.lease.metadata, - }); - await assertPushCapabilityCheckoutValid({ - enabled: pushCapabilityPreflightRequired && executionTarget?.kind === "local", - issue: issueRef - ? { - id: issueRef.id, - identifier: issueRef.identifier, - } - : null, - cwd: executionWorkspace.cwd, - }); - const adapterEnv = Object.fromEntries( - Object.entries(parseObject(resolvedConfig.env)).filter( - (entry): entry is [string, string] => typeof entry[0] === "string" && typeof entry[1] === "string", - ), - ); - const runtimeServices = await ensureRuntimeServicesForRun({ - db, - runId: run.id, - agent: { - id: agent.id, - name: agent.name, - companyId: agent.companyId, - }, - issue: issueRef, - workspace: executionWorkspace, - executionWorkspaceId: persistedExecutionWorkspace?.id ?? issueRef?.executionWorkspaceId ?? null, - config: hostExecutionWorkspaceConfig, - adapterEnv, - onLog, - recorder: workspaceOperationRecorder, - }); - if (runtimeServices.length > 0) { - context.paperclipRuntimeServices = runtimeServices; - context.paperclipRuntimePrimaryUrl = - runtimeServices.find((service) => readNonEmptyString(service.url))?.url ?? null; + if (persistedExecutionWorkspace) { + context.executionWorkspaceId = persistedExecutionWorkspace.id; await db .update(heartbeatRuns) .set({ @@ -16608,85 +18946,132 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) }) .where(eq(heartbeatRuns.id, run.id)); } - if (issueId && (executionWorkspace.created || runtimeServices.some((service) => !service.reused))) { - try { - await postWorkspaceReadyComment({ - issuesSvc, + const nativeRunnerPreparationSpans: NativeRunHistoricalSpan[] = []; + const environmentAcquireStartedAtMs = Date.now(); + let acquiredEnvironment: Awaited>; + try { + acquiredEnvironment = await envOrchestrator.acquireForRun({ + companyId: agent.companyId, + selectedEnvironmentId, + localEnvironmentId: localEnvironment.id, + adapterType: agent.adapterType, + issueId: issueId ?? null, + heartbeatRunId: run.id, + agentId: agent.id, + persistedExecutionWorkspace, + executionWorkspaceSettings: environmentExecutionWorkspaceSettings, + }); + nativeRunnerPreparationSpans.push({ + name: "environment.acquire", + parentName: "task.run", + startedAtMs: environmentAcquireStartedAtMs, + endedAtMs: Date.now(), + attributes: { adapter: agent.adapterType }, + }); + } catch (error) { + nativeRunnerPreparationSpans.push({ + name: "environment.acquire", + parentName: "task.run", + startedAtMs: environmentAcquireStartedAtMs, + endedAtMs: Date.now(), + outcome: "failed", + attributes: { adapter: agent.adapterType }, + }); + throw error; + } + const selectedEnvironment = acquiredEnvironment.environment; + // Defense-in-depth: re-check the actually-acquired environment against the + // execution allowlist. Even if selection were bypassed, a denied (local/ssh/ + // non-k8s) environment FAILS the run here rather than executing untrusted. + const allowlistDecision = evaluateExecutionAllowlist(executionPolicy, { + driver: selectedEnvironment.driver, + provider: + typeof selectedEnvironment.config?.provider === "string" + ? selectedEnvironment.config.provider + : null, + }); + if (!allowlistDecision.allowed) { + logger.error( + { + runId: run.id, issueId, agentId: agent.id, - runId: run.id, - workspace: executionWorkspace, - runtimeServices, - }); - } catch (err) { - await onLog( - "stderr", - `[paperclip] Failed to post workspace-ready comment: ${err instanceof Error ? err.message : String(err)}\n`, - ); - } - } - const onAdapterMeta = async (meta: AdapterInvocationMeta) => { - if (meta.env && secretKeys.size > 0) { - for (const key of secretKeys) { - if (key in meta.env) meta.env[key] = "***REDACTED***"; - } - } - const modelProfileMetadata = modelProfileRunMetadata(modelProfileApplication); - await appendRunEvent(currentRun, { - eventType: "adapter.invoke", - stream: "system", - level: "info", - message: "adapter invocation", - payload: { - ...(meta as unknown as Record), - ...(modelProfileMetadata ? { modelProfile: modelProfileMetadata } : {}), + environmentId: selectedEnvironment.id, + deniedDriver: allowlistDecision.deniedDriver, + deniedProvider: allowlistDecision.deniedProvider, }, - }); - }; - - const onAdapterEvent = async (event: AdapterRuntimeEvent) => { - const eventType = event.eventType.trim(); - if (!eventType) return; - await appendRunEvent(currentRun, { - eventType: eventType.slice(0, 120), - stream: event.stream, - level: event.level, - color: event.color, - message: event.message, - payload: event.payload, - }); - }; - - const runtimeResolution = resolveHeartbeatRuntimeMode({ - persisted: { - runtimeMode: run.runtimeMode, - runtimeModeResolvedAt: run.runtimeModeResolvedAt, - }, - enabled: resolvedInstanceSettings.experimental.enableNativeRunner === true, - adapterType: agent.adapterType, - adapterConfig: agent.adapterConfig, - agentStatus: runningAgent.status, - issue: issueRef ? { workMode: issueRef.workMode } : null, - executionTarget, - }); - if (runtimeResolution.kind === "legacy") { - const runtimeModeReason = - !resolvedInstanceSettings.experimental.enableNativeRunner - && runtimeResolution.reason === "direct_adapter" - ? "instance_flag_disabled" - : (run.runtimeModeReason ?? runtimeResolution.reason); - await db - .update(heartbeatRuns) - .set({ - runtimeMode: "legacy", - runtimeModeResolverVersion: runtimeResolution.resolverVersion, - runtimeModeReason, - runtimeModeResolvedAt: run.runtimeModeResolvedAt ?? new Date(), - updatedAt: new Date(), - }) - .where(eq(heartbeatRuns.id, run.id)); + "Execution allowlist denied the resolved environment; failing run", + ); + throw new Error(allowlistDecision.reason); } - const adapter = getServerAdapter(agent.adapterType); + let activeEnvironmentLease = { + environment: acquiredEnvironment.environment, + lease: acquiredEnvironment.lease, + leaseContext: acquiredEnvironment.leaseContext, + }; + const duplexObservabilityRecorder = createHostDuplexObservabilityRecorder( + { + tracer: getStartupTracer(), + incrementCounter: (metric) => { + void incrementToolRuntimeMetricCounter(db, { + companyId: run.companyId, + metric, + }).catch(() => {}); + }, + emitTransportEvent: (event) => { + void (async () => { + await appendRunEvent(run, { + eventType: event.name, + stream: "system", + level: event.dimensions.outcome === "error" ? "warn" : "info", + payload: { ...event.dimensions }, + }); + })().catch(() => {}); + }, + }, + ); + const environmentRealizeStartedAtMs = Date.now(); + let realizationResult: Awaited>; + try { + realizationResult = await envOrchestrator.realizeForRun({ + environment: selectedEnvironment, + lease: activeEnvironmentLease.lease, + adapterType: agent.adapterType, + companyId: agent.companyId, + issueId: issueId ?? null, + heartbeatRunId: run.id, + executionWorkspace, + effectiveExecutionWorkspaceMode, + persistedExecutionWorkspace, + duplexObservabilityRecorder, + }); + nativeRunnerPreparationSpans.push({ + name: "environment.workspace.realize", + parentName: "task.run", + startedAtMs: environmentRealizeStartedAtMs, + endedAtMs: Date.now(), + attributes: { driver: selectedEnvironment.driver }, + }); + } catch (error) { + nativeRunnerPreparationSpans.push({ + name: "environment.workspace.realize", + parentName: "task.run", + startedAtMs: environmentRealizeStartedAtMs, + endedAtMs: Date.now(), + outcome: "failed", + attributes: { driver: selectedEnvironment.driver }, + }); + throw error; + } + const environmentRealizeEndedAtMs = Date.now(); + activeEnvironmentLease = { + ...activeEnvironmentLease, + lease: realizationResult.lease, + }; + persistedExecutionWorkspace = realizationResult.persistedExecutionWorkspace; + const workspaceRealization = realizationResult.workspaceRealization; + const executionTarget = realizationResult.executionTarget; + const remoteExecution = realizationResult.remoteExecution; const dispatchResolvedInteractionContinuationWithAtomicGate = async ( dispatch: (markDispatchStarted: () => void) => Promise, ): Promise< @@ -16757,110 +19142,1176 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) await cancelRunForStaleIssue(run, issueId, gate.staleness); return { dispatched: false }; }; - const localAgentJwtScope = - issueRef?.workMode === "skill_test" - ? { kind: "skill_test" as const, issueId: issueRef.id } - : { kind: "standard" as const }; - const authToken = adapter.supportsLocalAgentJwt - ? createLocalAgentJwt( - agent.id, - agent.companyId, - agent.adapterType, - run.id, - run.responsibleUserId, - localAgentJwtScope, - ) - : null; - if (adapter.supportsLocalAgentJwt && !authToken) { - logger.warn( - { + if (!executionTarget || executionTarget.kind === "local") { + try { + runScratch = await prepareHeartbeatRunScratch({ companyId: agent.companyId, agentId: agent.id, runId: run.id, - adapterType: agent.adapterType, + issueId: issueRef?.id ?? null, + issueIdentifier: issueRef?.identifier ?? null, + }); + const existingRuntimeEnv = parseObject(runtimeConfig.env); + const scratchEnv = buildHeartbeatRunScratchEnv( + existingRuntimeEnv, + runScratch, + ); + runtimeConfig = { + ...runtimeConfig, + env: { + ...existingRuntimeEnv, + ...scratchEnv.env, + }, + }; + context.paperclipScratch = { + type: "heartbeat_run", + dir: runScratch.dir, + cleanupPolicy: "terminal_run", + marker: HEARTBEAT_RUN_SCRATCH_MARKER, + tempKeysApplied: scratchEnv.tempKeysApplied, + }; + } catch (scratchPrepareError) { + runScratch = null; + delete context.paperclipScratch; + logger.warn( + { + err: scratchPrepareError, + runId: run.id, + issueId, + agentId: agent.id, + }, + "failed to prepare heartbeat run scratch directory; continuing without scratch env", + ); + } + } else { + delete context.paperclipScratch; + } + context.paperclipEnvironment = { + id: selectedEnvironment.id, + name: selectedEnvironment.name, + driver: selectedEnvironment.driver, + leaseId: activeEnvironmentLease.lease.id, + workspaceRealization, + ...(typeof activeEnvironmentLease.lease.metadata?.remoteCwd === "string" + ? { + remoteCwd: activeEnvironmentLease.lease.metadata.remoteCwd, + host: + typeof activeEnvironmentLease.lease.metadata?.host === "string" + ? activeEnvironmentLease.lease.metadata.host + : undefined, + port: + typeof activeEnvironmentLease.lease.metadata?.port === "number" + ? activeEnvironmentLease.lease.metadata.port + : undefined, + username: + typeof activeEnvironmentLease.lease.metadata?.username === + "string" + ? activeEnvironmentLease.lease.metadata.username + : undefined, + } + : {}), + }; + await db + .update(heartbeatRuns) + .set({ + contextSnapshot: context, + updatedAt: new Date(), + }) + .where(eq(heartbeatRuns.id, run.id)); + const runtimeSessionResolution = resolveRuntimeSessionParamsForWorkspace({ + agentId: agent.id, + previousSessionParams, + resolvedWorkspace: { + ...resolvedWorkspace, + cwd: executionWorkspace.cwd, + }, + }); + const runtimeSessionParams = runtimeSessionResolution.sessionParams; + const runtimeWorkspaceWarnings = [ + ...resolvedWorkspace.warnings, + ...executionWorkspace.warnings, + ...(runtimeSessionResolution.warning + ? [runtimeSessionResolution.warning] + : []), + ...(requestedShouldReuseExisting && + workspaceConfigFreshness.reasons.length > 0 + ? [ + `Execution workspace reuse freshness action "${workspaceConfigFreshness.action}" because ${workspaceConfigFreshness.reasons.join("; ")}.`, + ] + : []), + ...(resetTaskSession && sessionResetReason + ? [ + taskKey + ? `Skipping saved session resume for task "${taskKey}" because ${sessionResetReason}.` + : `Skipping saved session resume because ${sessionResetReason}.`, + ] + : []), + ]; + context.paperclipWorkspace = { + cwd: executionWorkspace.cwd, + source: executionWorkspace.source, + mode: effectiveExecutionWorkspaceMode, + strategy: executionWorkspace.strategy, + projectId: executionWorkspace.projectId, + workspaceId: executionWorkspace.workspaceId, + repoUrl: executionWorkspace.repoUrl, + repoRef: executionWorkspace.repoRef, + branchName: executionWorkspace.branchName, + worktreePath: executionWorkspace.worktreePath, + realization: workspaceRealization, + agentHome: await (async () => { + const home = resolveDefaultAgentWorkspaceDir(agent.id); + await fs.mkdir(home, { recursive: true }); + return home; + })(), + }; + context.paperclipWorkspaces = buildRunWorkspaceHints(resolvedWorkspace); + // Emit exactly one requested-vs-synced observability line for the referenced-project set. A run + // with no referenced project stays silent, so this adds no noise to the anchor-only default. The + // per-drop human warning already rides `runtimeWorkspaceWarnings`; this line carries the counts + // and the per-failure reason for a partial sync. + const referencedProjectObservability = + buildReferencedProjectRunObservability({ + syncedProjectIds: resolvedWorkspace.additionalWorkspaces.map( + (additional) => additional.projectId, + ), + failures: resolvedWorkspace.referencedProjectFailures, + }); + if (referencedProjectObservability.referenced_projects_requested > 0) { + logger.info( + { + runId: run.id, + companyId: agent.companyId, + issueId: issueRef?.id ?? null, + ...referencedProjectObservability, }, - "local agent jwt secret missing or invalid; running without injected PAPERCLIP_API_KEY", + "run referenced-project sync", ); } - let adapterFinalizeOutcome: "succeeded" | "failed" | null = null; - const inspectFinalizeWorkspaceBranch = async () => { - const workspaceRecord = persistedExecutionWorkspace?.id - ? await executionWorkspacesSvc.getById(persistedExecutionWorkspace.id) - : persistedExecutionWorkspace; - if (workspaceRecord?.strategyType !== "git_worktree") return null; + // The wake payload is built before the execution workspace is resolved, so + // attach the branch pin here; the shared wake-prompt renderer surfaces it as + // a one-time "stay on this branch" hint on non-resumed sessions. + if (executionWorkspace.branchName) { + const wakePayloadForWorkspace = parseObject( + context[PAPERCLIP_WAKE_PAYLOAD_KEY], + ); + context[PAPERCLIP_WAKE_PAYLOAD_KEY] = { + ...wakePayloadForWorkspace, + executionWorkspace: { branchName: executionWorkspace.branchName }, + }; + } + const runtimeServiceIntents = (() => { + const runtimeConfig = parseObject( + hostExecutionWorkspaceConfig.workspaceRuntime, + ); + return Array.isArray(runtimeConfig.services) + ? runtimeConfig.services.filter( + (value): value is Record => + typeof value === "object" && value !== null, + ) + : []; + })(); + assertLowTrustRuntimeServicesAllowed({ + resolution: trustPreset, + runtimeServiceCount: runtimeServiceIntents.length, + }); + if (runtimeServiceIntents.length > 0) { + context.paperclipRuntimeServiceIntents = runtimeServiceIntents; + } else { + delete context.paperclipRuntimeServiceIntents; + } + if ( + executionWorkspace.projectId && + !readNonEmptyString(context.projectId) + ) { + context.projectId = executionWorkspace.projectId; + } + const runtimeSessionFallback = + taskKey || resetTaskSession + ? null + : isCanonicalSessionIdForAdapter(agent.adapterType, runtime.sessionId) + ? runtime.sessionId + : null; + const runtimeSessionDisplayId = truncateDisplayId( + explicitResumeSessionDisplayId ?? + taskSessionForRun?.sessionDisplayId ?? + (sessionCodec.getDisplayId + ? sessionCodec.getDisplayId(runtimeSessionParams) + : null) ?? + readNonEmptyString(runtimeSessionParams?.sessionId) ?? + runtimeSessionFallback, + ); + let previousSessionDisplayId = requiresCanonicalSessionIds( + agent.adapterType, + ) + ? truncateDisplayId( + readNonEmptyString(previousSessionParams?.sessionId) ?? + (isCanonicalSessionIdForAdapter( + agent.adapterType, + runtimeSessionDisplayId, + ) + ? runtimeSessionDisplayId + : null) ?? + runtimeSessionFallback, + ) + : runtimeSessionDisplayId; + let runtimeSessionIdForAdapter = + readNonEmptyString(runtimeSessionParams?.sessionId) ?? + runtimeSessionFallback; + let runtimeSessionParamsForAdapter = normalizeSessionParams( + stripPaperclipSessionMetadataFromSessionParams(runtimeSessionParams), + ); - const worktreePath = - readNonEmptyString(workspaceRecord.providerRef) ?? - readNonEmptyString(workspaceRecord.cwd) ?? - readNonEmptyString(executionWorkspace.worktreePath) ?? - readNonEmptyString(executionWorkspace.cwd); - const expectedBranchName = - readNonEmptyString(workspaceRecord.branchName) ?? - readNonEmptyString(executionWorkspace.branchName); - if (!worktreePath || !expectedBranchName) return null; + const sessionCompaction = await evaluateSessionCompaction({ + agent, + sessionId: previousSessionDisplayId ?? runtimeSessionIdForAdapter, + issueId, + continuationSummaryBody: continuationSummary?.body ?? null, + }); + if (sessionCompaction.rotate) { + context.paperclipSessionHandoffMarkdown = + sessionCompaction.handoffMarkdown; + context.paperclipSessionRotationReason = sessionCompaction.reason; + context.paperclipPreviousSessionId = + previousSessionDisplayId ?? runtimeSessionIdForAdapter; + runtimeSessionIdForAdapter = null; + runtimeSessionParamsForAdapter = null; + previousSessionDisplayId = null; + if (sessionCompaction.reason) { + runtimeWorkspaceWarnings.push( + `Starting a fresh session because ${sessionCompaction.reason}.`, + ); + } + } else { + delete context.paperclipSessionHandoffMarkdown; + delete context.paperclipSessionRotationReason; + delete context.paperclipPreviousSessionId; + } - const inspection = await inspectManagedGitWorktreeBranch({ - worktreePath, - expectedBranchName, - }); - return { workspaceRecord, inspection }; + const runtimeForAdapter = { + sessionId: runtimeSessionIdForAdapter, + sessionParams: runtimeSessionParamsForAdapter, + sessionDisplayId: previousSessionDisplayId, + taskKey, }; - const recordWorkspaceFinalize = async ( - status: "succeeded" | "failed", - metadata?: Record, - ) => { - if (adapterFinalizeOutcome) return; - let finalizeBranchMetadata: Record | null = null; - let finalizeBranchRepairMetadata: Record | null = null; - if (status === "succeeded") { - const branchInspection = await inspectFinalizeWorkspaceBranch(); - if (branchInspection) { - let inspection = branchInspection.inspection; - const initialManagedGitWorktreeBranch = formatManagedGitWorktreeBranchInspection(inspection); - if (!inspection.valid && inspection.reasonCode === "branch_mismatch" && inspection.repoRoot) { - let repairedExpectedBranchName = inspection.expectedBranchName; - try { - const coherence = await ensureGitWorktreeBranchCoherent({ - db, - repoRoot: inspection.repoRoot, - worktreePath: inspection.worktreePath, - expectedBranchName: inspection.expectedBranchName, - actualBranchName: inspection.actualBranchName, - sourceIssue: issueRef - ? { - id: issueRef.id, - identifier: issueRef.identifier, - title: issueRef.title, - workMode: issueRef.workMode, - } - : null, - executionWorkspaceId: branchInspection.workspaceRecord.id, - heartbeatRunId: run.id, - enableWorkspaceBranchReconcileForward: - resolvedInstanceSettings.experimental.enableWorkspaceBranchReconcileForward, - enableWorkspaceDirtyQuarantineRepair: - resolvedInstanceSettings.experimental.enableWorkspaceDirtyQuarantineRepair, - persistForwardReconcile: false, - reconcileOperationPhase: "workspace_finalize", - recorder: workspaceOperationRecorder, - }); - if (coherence.branchName && coherence.branchName !== branchInspection.workspaceRecord.branchName) { - repairedExpectedBranchName = coherence.branchName; - executionWorkspace.branchName = coherence.branchName; - executionWorkspace.warnings.push(...coherence.warnings); + const configFreshnessResultMetadata = { + version: sessionConfigMetadata.version, + session: { + fingerprintVersion: sessionConfigMetadata.version, + categories: sessionConfigMetadata.categories, + reset: resetTaskSession, + resetReasons: sessionConfigFreshness.reasons, + changedCategories: sessionConfigFreshness.changedCategories, + taskSessionAvailable: taskSession != null, + taskSessionReused: taskSessionForRun != null, + storedFingerprintPresent: Boolean( + sessionConfigFreshness.storedFingerprint, + ), + nextFingerprint: sessionConfigFreshness.nextFingerprint, + }, + workspace: { + fingerprintVersion: latestWorkspaceConfigMetadata.version, + categories: latestWorkspaceConfigMetadata.categories, + action: workspaceConfigFreshness.action, + changedCategories: workspaceConfigFreshness.changedCategories, + reasons: workspaceConfigFreshness.reasons, + reuseRequested: requestedShouldReuseExisting, + workspaceReused: Boolean(reusedExecutionWorkspace), + configSnapshotRefreshed: + resolvedWorkspaceReusePolicy.shouldRefreshWorkspaceConfigSnapshot, + storedFingerprintPresent: + workspaceConfigFreshness.storedFingerprintPresent, + storedFingerprint: workspaceConfigFreshness.storedFingerprint, + inferredFingerprint: workspaceConfigFreshness.inferredFingerprint, + nextFingerprint: workspaceConfigFreshness.nextFingerprint, + previousWorkspaceId: + workspaceReuseRequest.requestedExecutionWorkspaceId, + activeWorkspaceId: persistedExecutionWorkspace?.id ?? null, + }, + }; + + let handle: RunLogHandle | null = null; + let stdoutExcerpt = ""; + let stderrExcerpt = ""; + let outputSeq = Number(run.lastOutputSeq ?? 0); + let lastOutputFlushAt: Date | null = run.lastOutputAt ?? null; + let lastLogRuntimeStatusTouchMs = 0; + const outputProgressState: { + pending: { + at: Date; + seq: number; + stream: "stdout" | "stderr"; + bytes: number; + } | null; + } = { pending: null }; + let persistedLogBytes = Number(run.logBytes ?? 0); + const flushOutputProgress = async (opts?: { force?: boolean }) => { + const pendingOutputProgress = outputProgressState.pending; + if (!pendingOutputProgress) return; + const shouldFlush = + opts?.force === true || + !lastOutputFlushAt || + pendingOutputProgress.at.getTime() - lastOutputFlushAt.getTime() >= + ACTIVE_RUN_OUTPUT_PROGRESS_FLUSH_INTERVAL_MS; + if (!shouldFlush) return; + await db + .update(heartbeatRuns) + .set({ + lastOutputAt: pendingOutputProgress.at, + lastOutputSeq: pendingOutputProgress.seq, + lastOutputStream: pendingOutputProgress.stream, + lastOutputBytes: pendingOutputProgress.bytes, + updatedAt: new Date(), + }) + .where(eq(heartbeatRuns.id, run.id)); + lastOutputFlushAt = pendingOutputProgress.at; + outputProgressState.pending = null; + }; + try { + const startedAt = run.startedAt ?? new Date(); + const runningWithSession = await db + .update(heartbeatRuns) + .set({ + startedAt, + sessionIdBefore: + runtimeForAdapter.sessionDisplayId ?? runtimeForAdapter.sessionId, + contextSnapshot: context, + updatedAt: new Date(), + }) + .where(eq(heartbeatRuns.id, run.id)) + .returning() + .then((rows) => rows[0] ?? null); + if (runningWithSession) run = runningWithSession; + + // Pause Durability: flip to "running" ONLY if the agent is still invokable. + // Atomic conditional UPDATE is the sole gate (no read-then-write); 0 rows => abort. + const runningAgent = await db + .update(agents) + .set({ status: "running", updatedAt: new Date() }) + .where( + and( + eq(agents.id, agent.id), + notInArray(agents.status, [...DIRECT_NON_INVOKABLE_STATUSES]), + ), + ) + .returning() + .then((rows) => rows[0] ?? null); + + if (!runningAgent) { + logger.warn( + { agentId: agent.id, runId: run.id, previousStatus: agent.status }, + "execution-start aborted: agent not invokable", + ); + const abortReason = + "Cancelled: agent not invokable at execution-start"; + await setRunStatus(run.id, "cancelled", { + finishedAt: new Date(), + error: abortReason, + errorCode: "agent_not_invokable", + ...(agent + ? { + resultJson: mergeRunStopMetadataForAgent(agent, "cancelled", { + resultJson: parseObject(run.resultJson), + errorCode: "agent_not_invokable", + errorMessage: abortReason, + }), } - } catch (repairErr) { - const workspaceValidationFailure = isWorkspaceValidationFailure(repairErr) ? repairErr : null; - finalizeBranchMetadata = { - executionWorkspaceId: branchInspection.workspaceRecord.id, - ...initialManagedGitWorktreeBranch, - }; + : {}), + }); + await setWakeupStatus(run.wakeupRequestId, "cancelled", { + finishedAt: new Date(), + error: abortReason, + }); + await releaseIssueExecutionAndPromote(run); + return; + } + + publishLiveEvent({ + companyId: runningAgent.companyId, + type: "agent.status", + payload: { + agentId: runningAgent.id, + status: runningAgent.status, + outcome: "running", + }, + }); + + const currentRun = run; + await appendRunEvent(currentRun, { + eventType: "lifecycle", + stream: "system", + level: "info", + message: "run started", + }); + + handle = await runLogStore.begin({ + companyId: run.companyId, + agentId: run.agentId, + runId, + }); + + await db + .update(heartbeatRuns) + .set({ + logStore: handle.store, + logRef: handle.logRef, + updatedAt: new Date(), + }) + .where(eq(heartbeatRuns.id, runId)); + + const currentUserRedactionOptions = + await getCurrentUserRedactionOptions(); + const onLog = async (stream: "stdout" | "stderr", chunk: string) => { + const sanitizedChunk = compactRunLogChunk( + redactCurrentUserText(chunk, currentUserRedactionOptions), + ); + if (stream === "stdout") + stdoutExcerpt = appendExcerpt(stdoutExcerpt, sanitizedChunk); + if (stream === "stderr") + stderrExcerpt = appendExcerpt(stderrExcerpt, sanitizedChunk); + const ts = new Date().toISOString(); + + outputSeq += 1; + const chunkSeq = outputSeq; + let appendedBytes = 0; + if (handle) { + appendedBytes = await runLogStore.append(handle, { + stream, + chunk: sanitizedChunk, + ts, + seq: chunkSeq, + }); + persistedLogBytes += appendedBytes; + } + outputProgressState.pending = { + at: new Date(ts), + seq: chunkSeq, + stream, + bytes: persistedLogBytes, + }; + await flushOutputProgress(); + + // Streamed CLI output is real run activity: keep the in-memory + // runtime status ("Working... / X ago") fresh between structured + // events so sandbox runs with mid-run log streaming never show a + // minutes-stale timestamp. Throttled to avoid churning the live + // event stream on every 250ms tail chunk. + const logActivityAt = new Date(ts); + if ( + isHeartbeatRunRuntimeStatusActive(run.status) && + logActivityAt.getTime() - lastLogRuntimeStatusTouchMs >= + ACTIVE_RUN_LOG_RUNTIME_STATUS_REFRESH_INTERVAL_MS + ) { + lastLogRuntimeStatusTouchMs = logActivityAt.getTime(); + const touchedStatus = touchHeartbeatRunRuntimeStatus({ + companyId: run.companyId, + issueId, + agentId: run.agentId, + runId: run.id, + at: logActivityAt, + }); + if (touchedStatus) + publishHeartbeatRunRuntimeProgress(touchedStatus); + } + + const payloadChunk = + sanitizedChunk.length > MAX_LIVE_LOG_CHUNK_BYTES + ? sanitizedChunk.slice( + sanitizedChunk.length - MAX_LIVE_LOG_CHUNK_BYTES, + ) + : sanitizedChunk; + + publishLiveEvent({ + companyId: run.companyId, + type: "heartbeat.run.log", + payload: { + runId: run.id, + agentId: run.agentId, + issueId, + ts, + seq: chunkSeq, + stream, + chunk: payloadChunk, + truncated: payloadChunk.length !== sanitizedChunk.length, + }, + }); + }; + if (runScopedMentionedSkillKeys.length > 0) { + await onLog( + "stdout", + `[paperclip] Enabled run-scoped skills from issue mentions: ${runScopedMentionedSkillKeys.join(", ")}\n`, + ); + } + for (const warning of runtimeWorkspaceWarnings) { + const logEntry = formatRuntimeWorkspaceWarningLog(warning); + await onLog(logEntry.stream, logEntry.chunk); + } + await assertGitSensitiveAdapterWorkspaceValid({ + adapterType: agent.adapterType, + agentId: agent.id, + issue: issueRef + ? { + id: issueRef.id, + identifier: issueRef.identifier, + projectId: issueRef.projectId, + projectWorkspaceId: issueRef.projectWorkspaceId, + } + : null, + resolvedWorkspace, + executionWorkspace, + persistedExecutionWorkspace, + executionTarget, + environmentDriver: selectedEnvironment.driver, + leaseMetadata: activeEnvironmentLease.lease.metadata, + }); + await assertPushCapabilityCheckoutValid({ + enabled: + pushCapabilityPreflightRequired && + executionTarget?.kind === "local", + issue: issueRef + ? { + id: issueRef.id, + identifier: issueRef.identifier, + } + : null, + cwd: executionWorkspace.cwd, + }); + const adapterEnv = Object.fromEntries( + Object.entries(parseObject(resolvedConfig.env)).filter( + (entry): entry is [string, string] => + typeof entry[0] === "string" && typeof entry[1] === "string", + ), + ); + const runtimeServices = await ensureRuntimeServicesForRun({ + db, + runId: run.id, + agent: { + id: agent.id, + name: agent.name, + companyId: agent.companyId, + }, + issue: issueRef, + workspace: executionWorkspace, + executionWorkspaceId: + persistedExecutionWorkspace?.id ?? + issueRef?.executionWorkspaceId ?? + null, + config: hostExecutionWorkspaceConfig, + adapterEnv, + onLog, + recorder: workspaceOperationRecorder, + }); + if (runtimeServices.length > 0) { + context.paperclipRuntimeServices = runtimeServices; + context.paperclipRuntimePrimaryUrl = + runtimeServices.find((service) => readNonEmptyString(service.url)) + ?.url ?? null; + await db + .update(heartbeatRuns) + .set({ + contextSnapshot: context, + updatedAt: new Date(), + }) + .where(eq(heartbeatRuns.id, run.id)); + } + if ( + issueId && + (executionWorkspace.created || + runtimeServices.some((service) => !service.reused)) + ) { + try { + await postWorkspaceReadyComment({ + issuesSvc, + issueId, + agentId: agent.id, + runId: run.id, + workspace: executionWorkspace, + runtimeServices, + }); + } catch (err) { + await onLog( + "stderr", + `[paperclip] Failed to post workspace-ready comment: ${err instanceof Error ? err.message : String(err)}\n`, + ); + } + } + const onAdapterMeta = async (meta: AdapterInvocationMeta) => { + if (meta.env && secretKeys.size > 0) { + for (const key of secretKeys) { + if (key in meta.env) meta.env[key] = "***REDACTED***"; + } + } + const modelProfileMetadata = modelProfileRunMetadata( + modelProfileApplication, + ); + await appendRunEvent(currentRun, { + eventType: "adapter.invoke", + stream: "system", + level: "info", + message: "adapter invocation", + payload: { + ...(meta as unknown as Record), + ...(modelProfileMetadata + ? { modelProfile: modelProfileMetadata } + : {}), + }, + }); + }; + + const onAdapterEvent = async (event: AdapterRuntimeEvent) => { + const eventType = event.eventType.trim(); + if (!eventType) return; + await appendRunEvent(currentRun, { + eventType: eventType.slice(0, 120), + stream: event.stream, + level: event.level, + color: event.color, + message: event.message, + payload: event.payload, + }); + }; + + const adapter = getServerAdapter(agent.adapterType); + // Runtime selection is immutable once persisted. In particular, turning the instance flag + // off prevents new native runs without changing the recovery path for an already-native run. + const nativeRuntimeResolution = resolveHeartbeatNativeRuntimeMode({ + persisted: run, + enabled: + resolvedInstanceSettings.experimental.enableNativeRunner === true, + runtimeConfig: agent.runtimeConfig, + adapterConfig: agent.adapterConfig, + agent: { + id: agent.id, + status: runningAgent.status, + adapterType: agent.adapterType, + }, + issue: issueRef, + target: executionTarget, + workspaceId: persistedExecutionWorkspace?.id ?? null, + }); + let nativeExecution: NativeExecutionInput | null = null; + let nativeRunnerInstanceId: string | null = null; + if (nativeRuntimeResolution.kind === "native") { + if (!issueRef) { + throw new Error("native_runtime_ineligible: issue is required"); + } + const nativeExecutionWorkspaceId = + persistedExecutionWorkspace?.id ?? run.id; + const persistedContract = run.completionContractId + ? await db + .select() + .from(completionContracts) + .where( + and( + eq(completionContracts.id, run.completionContractId), + eq(completionContracts.companyId, agent.companyId), + eq(completionContracts.issueId, issueRef.id), + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null) + : null; + const completionContract = persistedContract + ? { + row: persistedContract, + contract: persistedContract.contractJson as never, + } + : await ensureNativeCompletionContract({ + db, + companyId: agent.companyId, + issue: issueRef, + actorId: agent.id, + immediateRequest: safeWakeCommentContext?.body ?? null, + }); + const taskNativeSessionId = readNonEmptyString( + taskSessionDecodedParams?.sessionId, + ); + const resumableTaskSessionId = + taskSessionForRun?.lastRunId && + taskSessionForRun.lastRunId !== run.id && + isNativeSessionId(taskNativeSessionId) + ? taskNativeSessionId + : null; + const previousNativeRun = + resumableTaskSessionId && taskSessionForRun?.lastRunId + ? await db + .select({ + id: heartbeatRuns.id, + companyId: heartbeatRuns.companyId, + agentId: heartbeatRuns.agentId, + runnerInstanceId: heartbeatRuns.runnerInstanceId, + nativeSessionId: heartbeatRuns.nativeSessionId, + runnerProfileJson: heartbeatRuns.runnerProfileJson, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.id, taskSessionForRun.lastRunId), + eq(heartbeatRuns.companyId, agent.companyId), + eq(heartbeatRuns.agentId, agent.id), + eq(heartbeatRuns.nativeSessionId, resumableTaskSessionId), + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null) + : null; + nativeRunnerInstanceId = + previousNativeRun?.runnerInstanceId && + previousNativeRun.nativeSessionId === + (run.nativeSessionId ?? resumableTaskSessionId) + ? previousNativeRun.runnerInstanceId + : (run.runnerInstanceId ?? randomUUID()); + let nativeSessionId = + run.nativeSessionId ?? resumableTaskSessionId ?? randomUUID(); + let nativeResumeCheckpoint: ReturnType< + typeof rebindNativeSessionCheckpoint + > = null; + const agentLifecyclePolicy = + parseObject(agent.adapterConfig).lifecycleMode === "warm" + ? { + mode: "warm" as const, + idleTimeoutMs: + Number.isSafeInteger( + parseObject(agent.adapterConfig).idleTimeoutMs, + ) && + Number(parseObject(agent.adapterConfig).idleTimeoutMs) > 0 + ? Number(parseObject(agent.adapterConfig).idleTimeoutMs) + : 300_000, + } + : { mode: "per_turn" as const, idleTimeoutMs: null }; + const effectiveLifecyclePolicy = agentLifecyclePolicy; + const persistedProfile = persistedRunnerProfile; + if (persistedNativeExecutionInput) { + nativeExecution = persistedNativeExecutionInput; + if ( + nativeExecution.binding.companyId !== agent.companyId || + nativeExecution.binding.runId !== run.id || + nativeExecution.binding.issueId !== issueRef.id || + nativeExecution.binding.agentId !== agent.id || + nativeExecution.binding.executionWorkspaceId !== + nativeExecutionWorkspaceId || + nativeExecution.completionContract.id !== + completionContract.row.id || + nativeExecution.completionContract.sha256 !== + completionContract.row.canonicalSha256 + ) + throw new Error( + "native_execution_input_persisted_binding_mismatch", + ); + } else { + const interactionId = readNonEmptyString(context.interactionId); + const interactionResponses = + await materializeNativeInteractionResponses({ + db, + companyId: agent.companyId, + issueId: issueRef.id, + runId: run.id, + agentId: agent.id, + interactionIds: interactionId ? [interactionId] : [], + }); + const executionMode = + issueRef.workMode === "planning" && !acceptedPlanContinuationWake + ? ("plan" as const) + : ("default" as const); + const pinnedPlan = + executionMode === "plan" + ? await documentService(db).getIssueDocumentByKey( + issueRef.id, + "plan", + ) + : null; + const pinnedReviewContext = + executionMode === "plan" + ? await buildPlanReviewContext({ + db, + companyId: agent.companyId, + issueId: issueRef.id, + issueWorkMode: issueRef.workMode, + interactionId: readNonEmptyString(context.interactionId), + }) + : null; + const pinnedPlanMarkdown = pinnedPlan?.body ?? ""; + const nativeRuntimeContext = await buildNativeRuntimeContext({ + db, + agent, + runId: run.id, + runtimeConfig, + runtimeSkillEntries, + }); + nativeExecution = buildNativeExecutionInput({ + companyId: agent.companyId, + runId: run.id, + issue: issueRef, + taskPrompt: + readNonEmptyString(context.paperclipTaskMarkdown) ?? + `# ${issueRef.identifier ?? issueRef.id}: ${issueRef.title}`, + wakePayload: context.paperclipWake, + resumedSession: previousNativeRun !== null, + agentId: agent.id, + workspace: { + // Projectless paperclip_runner tasks still have a resolved local cwd. Bind that + // transient workspace to the run id so the native input remains durable and replayable + // without fabricating a project-scoped execution_workspaces row. + id: nativeExecutionWorkspaceId, + cwd: executionWorkspace.cwd, + repoUrl: executionWorkspace.repoUrl, + repoRef: executionWorkspace.repoRef, + branchName: executionWorkspace.branchName, + }, + normalizedSessionId: nativeSessionId, + executionMode, + planningContext: + executionMode === "plan" + ? { + documentId: pinnedPlan?.id ?? null, + baseRevisionId: pinnedPlan?.latestRevisionId ?? null, + baseRevisionNumber: pinnedPlan?.latestRevisionNumber ?? 0, + markdown: pinnedPlanMarkdown, + sha256: createHash("sha256") + .update(pinnedPlanMarkdown) + .digest("hex"), + reviewContext: pinnedReviewContext + ? (structuredClone( + pinnedReviewContext, + ) as unknown as Record) + : {}, + } + : null, + codexApprovalPolicy: resolvePaperclipRunnerPermissionMode( + "codex", + parseObject(agent.adapterConfig).codexPermissionMode, + ) as "never" | "on-request" | "untrusted", + model: + typeof parseObject(agent.adapterConfig).model === "string" + ? String(parseObject(agent.adapterConfig).model) + : null, + lifecyclePolicy: effectiveLifecyclePolicy, + interactionResponses, + completionContract: { + id: completionContract.row.id, + sha256: completionContract.row.canonicalSha256, + schemaVersion: completionContract.row.schemaVersion, + contract: completionContract.contract, + }, + runtimeContext: nativeRuntimeContext, + }); + if ( + previousNativeRun && + nativeSessionId === resumableTaskSessionId + ) { + nativeResumeCheckpoint = rebindNativeSessionCheckpoint({ + previousRun: previousNativeRun, + currentExecution: nativeExecution, + }); + if (!nativeResumeCheckpoint) { + nativeSessionId = randomUUID(); + nativeExecution = parseNativeExecutionInput({ + ...nativeExecution, + session: { + ...nativeExecution.session, + normalizedSessionId: nativeSessionId, + }, + }); + } + } + } + await db.transaction(async (tx) => { + const lockedRun = await tx + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, run.id)) + .for("update") + .limit(1) + .then((rows) => rows[0] ?? null); + if (!lockedRun) throw new Error("native_runtime_run_missing"); + if ( + lockedRun.runtimeModeResolvedAt && + lockedRun.runtimeMode !== "native" + ) { + throw new Error("native_runtime_mode_conflict"); + } + const lockedProfile = parseObject(lockedRun.runnerProfileJson); + await tx + .update(heartbeatRuns) + .set({ + runtimeMode: "native", + runtimeModeResolverVersion: + lockedRun.runtimeModeResolverVersion ?? + nativeRuntimeResolution.resolverVersion, + runtimeModeReason: + lockedRun.runtimeModeReason ?? nativeRuntimeResolution.reason, + runtimeModeResolvedAt: + lockedRun.runtimeModeResolvedAt ?? new Date(), + runnerProfileJson: { + ...nativeRuntimeResolution.profile, + ...lockedProfile, + ...(providerTraceRequested + ? { + providerTrace: { + mode: "raw", + traceId: providerTraceCapture?.metadata.id ?? null, + maxBytes: PROVIDER_TRACE_MAX_BYTES, + }, + } + : {}), + nativeExecutionInput: + lockedProfile.nativeExecutionInput ?? nativeExecution, + ...(lockedProfile.sessionCheckpoint !== undefined + ? { sessionCheckpoint: lockedProfile.sessionCheckpoint } + : nativeResumeCheckpoint + ? { + sessionCheckpoint: + nativeResumeCheckpoint as unknown as Record< + string, + unknown + >, + } + : {}), + }, + runnerInstanceId: + previousNativeRun?.runnerInstanceId && + lockedRun.nativeSessionId !== null && + lockedRun.nativeSessionId === + previousNativeRun.nativeSessionId + ? previousNativeRun.runnerInstanceId + : (lockedRun.runnerInstanceId ?? nativeRunnerInstanceId), + nativeSessionId: lockedRun.nativeSessionId ?? nativeSessionId, + nativeIssueId: lockedRun.nativeIssueId ?? issueRef.id, + driverKind: + lockedRun.driverKind ?? + nativeExecution?.session.driverKind ?? + "codex_app_server", + driverVersion: lockedRun.driverVersion ?? "phase6-v1", + completionContractId: + lockedRun.completionContractId ?? completionContract.row.id, + completionContractSha256: + lockedRun.completionContractSha256 ?? + completionContract.row.canonicalSha256, + nativePhase: lockedRun.nativePhase ?? "observed", + nativePhaseUpdatedAt: + lockedRun.nativePhaseUpdatedAt ?? new Date(), + updatedAt: new Date(), + }) + .where(eq(heartbeatRuns.id, run.id)); + await tx + .insert(nativeRunFinalizations) + .values({ + runId: run.id, + companyId: agent.companyId, + issueId: issueRef.id, + phase: "observed", + }) + .onConflictDoNothing(); + }); + } else { + await db + .update(heartbeatRuns) + .set({ + runtimeMode: "legacy", + runtimeModeResolverVersion: + nativeRuntimeResolution.resolverVersion, + runtimeModeReason: nativeRuntimeResolution.reason, + runtimeModeResolvedAt: run.runtimeModeResolvedAt ?? new Date(), + runnerProfileJson: providerTraceRequested + ? { + providerTrace: { + mode: "raw", + traceId: providerTraceCapture?.metadata.id ?? null, + maxBytes: PROVIDER_TRACE_MAX_BYTES, + }, + } + : null, + updatedAt: new Date(), + }) + .where(eq(heartbeatRuns.id, run.id)); + } + const localAgentJwtScope = + issueRef?.workMode === "skill_test" + ? { kind: "skill_test" as const, issueId: issueRef.id } + : { kind: "standard" as const }; + const authToken = + nativeRuntimeResolution.kind === "legacy" && + adapter.supportsLocalAgentJwt + ? createLocalAgentJwt( + agent.id, + agent.companyId, + agent.adapterType, + run.id, + run.responsibleUserId, + localAgentJwtScope, + ) + : null; + if ( + nativeRuntimeResolution.kind === "legacy" && + adapter.supportsLocalAgentJwt && + !authToken + ) { + logger.warn( + { + companyId: agent.companyId, + agentId: agent.id, + runId: run.id, + adapterType: agent.adapterType, + }, + "local agent jwt secret missing or invalid; running without injected PAPERCLIP_API_KEY", + ); + } + let adapterFinalizeOutcome: "succeeded" | "failed" | null = null; + const inspectFinalizeWorkspaceBranch = async () => { + const workspaceRecord = persistedExecutionWorkspace?.id + ? await executionWorkspacesSvc.getById( + persistedExecutionWorkspace.id, + ) + : persistedExecutionWorkspace; + if (workspaceRecord?.strategyType !== "git_worktree") return null; + + const worktreePath = + readNonEmptyString(workspaceRecord.providerRef) ?? + readNonEmptyString(workspaceRecord.cwd) ?? + readNonEmptyString(executionWorkspace.worktreePath) ?? + readNonEmptyString(executionWorkspace.cwd); + const expectedBranchName = + readNonEmptyString(workspaceRecord.branchName) ?? + readNonEmptyString(executionWorkspace.branchName); + if (!worktreePath || !expectedBranchName) return null; + + const inspection = await inspectManagedGitWorktreeBranch({ + worktreePath, + expectedBranchName, + }); + return { workspaceRecord, inspection }; + }; + const recordWorkspaceFinalize = async ( + status: "succeeded" | "failed", + metadata?: Record, + ) => { + if (adapterFinalizeOutcome) return; + let finalizeBranchMetadata: Record | null = null; + let finalizeBranchRepairMetadata: Record | null = + null; + if (status === "succeeded") { + const branchInspection = await inspectFinalizeWorkspaceBranch(); + if (branchInspection) { + let inspection = branchInspection.inspection; + const initialManagedGitWorktreeBranch = + formatManagedGitWorktreeBranchInspection(inspection); + if ( + !inspection.valid && + inspection.reasonCode === "branch_mismatch" && + inspection.repoRoot + ) { + let repairedExpectedBranchName = inspection.expectedBranchName; + try { + const coherence = await ensureGitWorktreeBranchCoherent({ + db, + repoRoot: inspection.repoRoot, + worktreePath: inspection.worktreePath, + expectedBranchName: inspection.expectedBranchName, + actualBranchName: inspection.actualBranchName, + sourceIssue: issueRef + ? { + id: issueRef.id, + identifier: issueRef.identifier, + title: issueRef.title, + workMode: issueRef.workMode, + } + : null, + executionWorkspaceId: branchInspection.workspaceRecord.id, + heartbeatRunId: run.id, + enableWorkspaceBranchReconcileForward: + resolvedInstanceSettings.experimental + .enableWorkspaceBranchReconcileForward, + enableWorkspaceDirtyQuarantineRepair: + resolvedInstanceSettings.experimental + .enableWorkspaceDirtyQuarantineRepair, + persistForwardReconcile: false, + reconcileOperationPhase: "workspace_finalize", + recorder: workspaceOperationRecorder, + }); + if ( + coherence.branchName && + coherence.branchName !== + branchInspection.workspaceRecord.branchName + ) { + repairedExpectedBranchName = coherence.branchName; + executionWorkspace.branchName = coherence.branchName; + executionWorkspace.warnings.push(...coherence.warnings); + } + } catch (repairErr) { + const workspaceValidationFailure = + isWorkspaceValidationFailure(repairErr) ? repairErr : null; + finalizeBranchMetadata = { + executionWorkspaceId: branchInspection.workspaceRecord.id, + ...initialManagedGitWorktreeBranch, + }; + finalizeBranchRepairMetadata = { + attempted: true, + succeeded: false, + initial: initialManagedGitWorktreeBranch, + reason: + repairErr instanceof Error + ? repairErr.message + : String(repairErr), + }; + await workspaceOperationRecorder.recordOperation({ + phase: "workspace_finalize", + cwd: executionWorkspace.cwd, + metadata: { + adapterType: agent.adapterType, + executionTargetKind: executionTarget?.kind ?? "local", + ...metadata, + managedGitWorktreeBranch: finalizeBranchMetadata, + managedGitWorktreeBranchRepair: + finalizeBranchRepairMetadata, + ...(workspaceValidationFailure?.resultJson + ? { + workspaceValidation: + workspaceValidationFailure.resultJson + .workspaceValidation ?? + workspaceValidationFailure.resultJson, + } + : {}), + }, + run: async () => ({ + status: "failed", + stderr: `Managed git worktree branch check failed: ${repairErr instanceof Error ? repairErr.message : String(repairErr)}\n`, + }), + }); + adapterFinalizeOutcome = "failed"; + throw repairErr; + } + + const repairedInspection = + await inspectManagedGitWorktreeBranch({ + worktreePath: inspection.worktreePath, + expectedBranchName: repairedExpectedBranchName, + repoRoot: inspection.repoRoot, + }); finalizeBranchRepairMetadata = { attempted: true, - succeeded: false, + succeeded: repairedInspection.valid, initial: initialManagedGitWorktreeBranch, - reason: repairErr instanceof Error ? repairErr.message : String(repairErr), + repaired: + formatManagedGitWorktreeBranchInspection( + repairedInspection, + ), }; + inspection = repairedInspection; + } + + const managedGitWorktreeBranch = + formatManagedGitWorktreeBranchInspection(inspection); + finalizeBranchMetadata = { + executionWorkspaceId: branchInspection.workspaceRecord.id, + ...managedGitWorktreeBranch, + }; + if (!inspection.valid) { + const workspaceValidationFingerprint = + fingerprintFinalizeWorkspaceBranchValidation({ + issueId: issueRef?.id ?? null, + executionWorkspaceId: branchInspection.workspaceRecord.id, + inspection: managedGitWorktreeBranch, + }); await workspaceOperationRecorder.recordOperation({ phase: "workspace_finalize", cwd: executionWorkspace.cwd, @@ -16869,1083 +20320,1539 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) executionTargetKind: executionTarget?.kind ?? "local", ...metadata, managedGitWorktreeBranch: finalizeBranchMetadata, - managedGitWorktreeBranchRepair: finalizeBranchRepairMetadata, - ...(workspaceValidationFailure?.resultJson - ? { workspaceValidation: workspaceValidationFailure.resultJson.workspaceValidation ?? workspaceValidationFailure.resultJson } + ...(finalizeBranchRepairMetadata + ? { + managedGitWorktreeBranchRepair: + finalizeBranchRepairMetadata, + } : {}), }, run: async () => ({ status: "failed", - stderr: `Managed git worktree branch check failed: ${repairErr instanceof Error ? repairErr.message : String(repairErr)}\n`, + stderr: `Managed git worktree branch check failed: ${inspection.reason ?? "unknown branch mismatch"}\n`, }), }); adapterFinalizeOutcome = "failed"; - throw repairErr; - } - - const repairedInspection = await inspectManagedGitWorktreeBranch({ - worktreePath: inspection.worktreePath, - expectedBranchName: repairedExpectedBranchName, - repoRoot: inspection.repoRoot, - }); - finalizeBranchRepairMetadata = { - attempted: true, - succeeded: repairedInspection.valid, - initial: initialManagedGitWorktreeBranch, - repaired: formatManagedGitWorktreeBranchInspection(repairedInspection), - }; - inspection = repairedInspection; - } - - const managedGitWorktreeBranch = formatManagedGitWorktreeBranchInspection(inspection); - finalizeBranchMetadata = { - executionWorkspaceId: branchInspection.workspaceRecord.id, - ...managedGitWorktreeBranch, - }; - if (!inspection.valid) { - const workspaceValidationFingerprint = fingerprintFinalizeWorkspaceBranchValidation({ - issueId: issueRef?.id ?? null, - executionWorkspaceId: branchInspection.workspaceRecord.id, - inspection: managedGitWorktreeBranch, - }); - await workspaceOperationRecorder.recordOperation({ - phase: "workspace_finalize", - cwd: executionWorkspace.cwd, - metadata: { - adapterType: agent.adapterType, - executionTargetKind: executionTarget?.kind ?? "local", - ...metadata, - managedGitWorktreeBranch: finalizeBranchMetadata, - ...(finalizeBranchRepairMetadata ? { managedGitWorktreeBranchRepair: finalizeBranchRepairMetadata } : {}), - }, - run: async () => ({ - status: "failed", - stderr: `Managed git worktree branch check failed: ${inspection.reason ?? "unknown branch mismatch"}\n`, - }), - }); - adapterFinalizeOutcome = "failed"; - throw new WorkspaceValidationFailure( - `Execution workspace ${branchInspection.workspaceRecord.id} expected git worktree branch "${inspection.expectedBranchName}" at "${inspection.worktreePath}", but ${inspection.reason ?? "the checked-out branch could not be verified"}. Record a sanctioned execution-workspace branch transition or restore the workspace branch before completing the run.`, - { - workspaceValidation: { - reason: "git_worktree_branch_incoherence", - fingerprint: workspaceValidationFingerprint, - adapterType: agent.adapterType, - issueId: issueRef?.id ?? null, - issueIdentifier: issueRef?.identifier ?? null, - persistedExecutionWorkspaceId: branchInspection.workspaceRecord.id, - executionWorkspaceCwd: executionWorkspace.cwd, - managedGitWorktreeBranch: finalizeBranchMetadata, + throw new WorkspaceValidationFailure( + `Execution workspace ${branchInspection.workspaceRecord.id} expected git worktree branch "${inspection.expectedBranchName}" at "${inspection.worktreePath}", but ${inspection.reason ?? "the checked-out branch could not be verified"}. Record a sanctioned execution-workspace branch transition or restore the workspace branch before completing the run.`, + { + workspaceValidation: { + reason: "git_worktree_branch_incoherence", + fingerprint: workspaceValidationFingerprint, + adapterType: agent.adapterType, + issueId: issueRef?.id ?? null, + issueIdentifier: issueRef?.identifier ?? null, + persistedExecutionWorkspaceId: + branchInspection.workspaceRecord.id, + executionWorkspaceCwd: executionWorkspace.cwd, + managedGitWorktreeBranch: finalizeBranchMetadata, + }, }, + ); + } + } + } + await workspaceOperationRecorder.recordOperation({ + phase: "workspace_finalize", + cwd: executionWorkspace.cwd, + metadata: { + adapterType: agent.adapterType, + executionTargetKind: executionTarget?.kind ?? "local", + ...metadata, + ...(finalizeBranchMetadata + ? { managedGitWorktreeBranch: finalizeBranchMetadata } + : {}), + ...(finalizeBranchRepairMetadata + ? { + managedGitWorktreeBranchRepair: + finalizeBranchRepairMetadata, + } + : {}), + }, + run: async () => ({ status }), + }); + // Only mark the outcome after the row landed, so a transient write + // failure on the succeeded path can still be recovered by recording + // finalize=failed from the catch path below. + adapterFinalizeOutcome = status; + }; + + let adapterResult: AdapterExecutionResult; + try { + if (nativeRuntimeResolution.kind === "native") { + if (!nativeExecution || !nativeRunnerInstanceId) + throw new Error("native_runtime_selection_not_persisted"); + const nativeMcpServers = await buildPaperclipRuntimeMcpServers({ + db, + agent, + runId: run.id, + failOnUnavailableAssignedConnection: true, + }); + if (!("runtimeContext" in nativeExecution) && nativeMcpServers.length) { + throw new Error("historical native runs cannot acquire newly assigned MCP access"); + } + if ("runtimeContext" in nativeExecution) { + if (nativeMcpServers.length > 1) throw new Error("native MCP realization must produce one aggregate gateway"); + const server = nativeMcpServers[0] ?? null; + const digest = server?.connectionId.startsWith("assignment:") ? server.connectionId.slice("assignment:".length) : null; + if (digest !== (nativeExecution.runtimeContext.mcp.bindingId ? nativeExecution.runtimeContext.mcp.digest : null)) { + throw new Error("native MCP assignment digest mismatch"); + } + } + const nativeMcpServer = nativeMcpServers[0] ?? null; + const nativeDispatchAtMs = Date.now(); + const runCreatedAtMs = run.createdAt.getTime(); + const runStartedAtMs = (run.startedAt ?? run.createdAt).getTime(); + const wakeComments = Array.isArray(parseObject(context.paperclipWake).comments) + ? (parseObject(context.paperclipWake).comments as unknown[]) + : []; + const wakeCommentCreatedAtMs = wakeComments + .map((value) => Date.parse(readNonEmptyString(parseObject(value).createdAt) ?? "")) + .filter(Number.isFinite) + .sort((a, b) => a - b)[0]; + if (wakeCommentCreatedAtMs !== undefined) { + nativeRunnerPreparationSpans.unshift({ + name: "comment.to_run_created", + parentName: "task.run", + startedAtMs: wakeCommentCreatedAtMs, + endedAtMs: Math.max(wakeCommentCreatedAtMs, runCreatedAtMs), + }); + } + nativeRunnerPreparationSpans.push( + { + name: "heartbeat.queue", + parentName: "task.run", + startedAtMs: runCreatedAtMs, + endedAtMs: Math.max(runCreatedAtMs, runStartedAtMs), + }, + { + name: "heartbeat.prepare_before_environment", + parentName: "task.run", + startedAtMs: runStartedAtMs, + endedAtMs: Math.max(runStartedAtMs, environmentAcquireStartedAtMs), + }, + { + name: "heartbeat.prepare_after_environment", + parentName: "task.run", + startedAtMs: Math.min(nativeDispatchAtMs, environmentRealizeEndedAtMs), + endedAtMs: nativeDispatchAtMs, + }, + ); + const guardedDispatch = + await dispatchResolvedInteractionContinuationWithAtomicGate( + (markDispatchStarted) => + executePaperclipNativeSession({ + db, + execution: nativeExecution, + runnerInstanceId: nativeRunnerInstanceId, + leaseOwner: runOptions.nativeLeaseOwner, + backend: + options.nativeSessionBackendFactory?.(nativeExecution), + useRunnerd: agent.adapterType === "paperclip_runner", + onLog, + onEvent: onAdapterEvent, + preparationSpans: nativeRunnerPreparationSpans, + // Bootstrap the provider with executable/home discovery while + // keeping the agent's configured provider values authoritative. + runnerEnvironment: { + ...buildNativeProviderEnvironment(adapterEnv), + ...(nativeMcpServer + ? { + PAPERCLIP_NATIVE_MCP_NAME: nativeMcpServer.name, + PAPERCLIP_NATIVE_MCP_URL: nativeMcpServer.url, + PAPERCLIP_NATIVE_MCP_TOKEN: nativeMcpServer.token, + } + : {}), + ...(providerTraceCapture + ? { + PAPERCLIP_PROVIDER_TRACE_PATH: + providerTraceCapture.path, + PAPERCLIP_PROVIDER_TRACE_MAX_BYTES: String( + PROVIDER_TRACE_MAX_BYTES, + ), + } + : {}), + }, + enqueueWakeup, + onSpawn: async (meta) => { + markDispatchStarted(); + await persistRunProcessMetadata(run.id, meta); + }, + }), + ); + if (!guardedDispatch.dispatched) return; + adapterResult = await guardedDispatch.resultPromise; + } else { + const interactionId = readNonEmptyString(context.interactionId); + const legacyQuestionResponse = + issueRef + && interactionId + && readNonEmptyString(context.interactionKind) === "ask_user_questions" + && readNonEmptyString(context.interactionStatus) === "answered" + ? await materializeLegacyQuestionResponseWakeProjection({ + db, + companyId: agent.companyId, + issueId: issueRef.id, + runId: run.id, + agentId: agent.id, + interactionId, + }) + : null; + // Do not write the answer projection back to `context`: legacy + // adapters need it in their prompt, but the authoritative answers + // remain on the interaction instead of being duplicated in the + // heartbeat run snapshot. + const adapterContext: Record = { + ...context, + ...(legacyQuestionResponse + ? { + [PAPERCLIP_WAKE_PAYLOAD_KEY]: { + ...parseObject(context[PAPERCLIP_WAKE_PAYLOAD_KEY]), + questionResponse: legacyQuestionResponse, + }, + } + : {}), + }; + const runtimeTools = createAdapterRuntimeToolAccess({ + agentId: agent.id, + companyId: agent.companyId, + runId: run.id, + responsibleUserId: run.responsibleUserId, + }); + if (!runtimeTools) { + logger.warn( + { + companyId: agent.companyId, + agentId: agent.id, + runId: run.id, }, + "runtime connection tools could not be delivered", + ); + } + const runtimeMcpServers = await buildPaperclipRuntimeMcpServers({ + db, + agent, + runId: run.id, + }); + const runtimeToolDelivery = + adapter.runtimeToolDelivery ?? "invocation_context"; + if (runtimeTools && runtimeToolDelivery === "native_mcp") { + runtimeMcpServers.unshift({ + name: "Paperclip connections", + url: runtimeTools.mcpEndpoint, + token: runtimeTools.bearerToken, + connectionId: "paperclip-runtime-tools", + }); + } + const runtimeMcp = createAdapterRuntimeMcpAccess(runtimeMcpServers); + if ( + runtimeTools && + runtimeToolDelivery === "invocation_context" + ) { + adapterContext.paperclipRuntimeTools = runtimeTools; + } + const managedMcpConfig = await createManagedMcpRunConfig({ + db, + agent, + runId: run.id, + config: runtimeConfig, + projectId: issueRef?.projectId ?? null, + issueId: issueRef?.id ?? null, + }); + if (managedMcpConfig) { + adapterContext.paperclipManagedMcp = managedMcpConfig; + } + const guardedDispatch = + await dispatchResolvedInteractionContinuationWithAtomicGate( + (markDispatchStarted) => + adapter.execute({ + runId: run.id, + agent, + runtime: runtimeForAdapter, + config: runtimeConfig, + context: adapterContext, + runtimeCommandSpec: + adapter.getRuntimeCommandSpec?.(runtimeConfig) ?? null, + executionTarget, + executionTransport: remoteExecution + ? { + remoteExecution: + remoteExecution as unknown as Record< + string, + unknown + >, + } + : undefined, + runtimeMcp, + runtimeTools, + onLog, + onMeta: onAdapterMeta, + onEvent: onAdapterEvent, + startupTraceContext: getStartupTraceContext(), + onRuntimeProgress: async (progress) => { + await recordCurrentHeartbeatRunRuntimeProgress( + run, + progress, + issueId, + ); + }, + onDispatch: markDispatchStarted, + onSpawn: async (meta) => { + markDispatchStarted(); + await persistRunProcessMetadata(run.id, { + pid: meta.pid, + processGroupId: + "processGroupId" in meta && + typeof meta.processGroupId === "number" + ? meta.processGroupId + : null, + startedAt: meta.startedAt, + }); + }, + authToken: authToken ?? undefined, + }), + ); + if (!guardedDispatch.dispatched) return; + adapterResult = await guardedDispatch.resultPromise; + } + // Adapter returned cleanly, which means its workspace-restore finally + // block also ran without throwing. Record the workspace_finalize + // barrier so dependents that share this executionWorkspace can wake. + // If recording the barrier itself fails, propagate as a run failure + // rather than silently leaving dependents stranded behind a missing + // finalize row. + await recordWorkspaceFinalize("succeeded"); + if (adapterResult.nativeFinalization) { + adapterResult.nativeFinalization.workspaceFinalizeStatus = + "succeeded"; + try { + await finalizeNativeRun({ + db, + runId: run.id, + workspaceFinalizeStatus: "succeeded", + }); + await dispatchPendingNativeStatusWakeups({ + companyId: run.companyId, + }); + } catch (finalizeErr) { + logger.warn( + { err: finalizeErr, runId: run.id }, + "native result persisted but finalization did not apply; the reconciliation loop will retry", ); } } - } - await workspaceOperationRecorder.recordOperation({ - phase: "workspace_finalize", - cwd: executionWorkspace.cwd, - metadata: { - adapterType: agent.adapterType, - executionTargetKind: executionTarget?.kind ?? "local", - ...metadata, - ...(finalizeBranchMetadata ? { managedGitWorktreeBranch: finalizeBranchMetadata } : {}), - ...(finalizeBranchRepairMetadata ? { managedGitWorktreeBranchRepair: finalizeBranchRepairMetadata } : {}), - }, - run: async () => ({ status }), - }); - // Only mark the outcome after the row landed, so a transient write - // failure on the succeeded path can still be recovered by recording - // finalize=failed from the catch path below. - adapterFinalizeOutcome = status; - }; - - let adapterResult: Awaited>; - try { - const onSpawn = async (meta: { - pid: number; - processGroupId: number | null; - startedAt: string; - }) => { - await persistRunProcessMetadata(run.id, { - pid: meta.pid, - processGroupId: meta.processGroupId, - startedAt: meta.startedAt, - }); - }; - if (runtimeResolution.kind === "native") { - if (!issueRef) throw new Error("paperclip_runner_issue_required"); - const native = await prepareNativeHeartbeatRun({ - db, - run, - issue: issueRef, - environmentLeaseId: activeEnvironmentLease.lease.id, - }); - const prompt = readNonEmptyString(context.paperclipTaskMarkdown) - ?? `# ${issueRef.identifier ?? issueRef.id}: ${issueRef.title}`; - const configuredTimeoutSec = Number(runtimeConfig.timeoutSec); - const timeoutMs = Number.isFinite(configuredTimeoutSec) && configuredTimeoutSec > 0 - ? Math.min(configuredTimeoutSec * 1_000, 24 * 60 * 60 * 1_000) - : 60 * 60 * 1_000; - const environment = Object.fromEntries( - Object.entries(parseObject(runtimeConfig.env)).filter( - (entry): entry is [string, string] => typeof entry[1] === "string", - ), - ); - await onAdapterMeta({ - adapterType: "paperclip_runner", - command: "paperclip-runnerd", - cwd: executionWorkspace.cwd, - promptMetrics: { promptChars: prompt.length }, - context: { provider: "codex", protocolVersion: 1 }, - }); - const guardedDispatch = await dispatchResolvedInteractionContinuationWithAtomicGate((markDispatchStarted) => - executeNativeCodexRunner({ - db, - companyId: agent.companyId, - issueId: issueRef.id, - runId: run.id, - agentId: agent.id, - runnerInstanceId: native.runnerInstanceId, - environmentLeaseId: native.environmentLeaseId, - normalizedSessionId: native.normalizedSessionId, - turnId: native.turnId, - itemId: native.itemId, - cwd: executionWorkspace.cwd, - prompt, - model: readNonEmptyString(runtimeConfig.model), - resumeProviderSessionId: runtimeSessionIdForAdapter, - completionContract: native.completionContract, - timeoutMs, - environment, - onLog, - onSpawn: async (meta) => { - markDispatchStarted(); - await onSpawn(meta); - }, - }), - ); - if (!guardedDispatch.dispatched) return; - adapterResult = await guardedDispatch.resultPromise; - } else { - const interactionId = readNonEmptyString(context.interactionId); - const legacyQuestionResponse = - issueRef - && interactionId - && readNonEmptyString(context.interactionKind) === "ask_user_questions" - && readNonEmptyString(context.interactionStatus) === "answered" - ? await materializeLegacyQuestionResponseWakeProjection({ - db, - companyId: agent.companyId, - issueId: issueRef.id, - runId: run.id, - agentId: agent.id, - interactionId, - }) - : null; - // Keep the legacy answer projection ephemeral: the interaction is - // the authoritative copy, while direct adapters receive it in the - // wake prompt for this invocation only. - const adapterContext: Record = { - ...context, - ...(legacyQuestionResponse - ? { - [PAPERCLIP_WAKE_PAYLOAD_KEY]: { - ...parseObject(context[PAPERCLIP_WAKE_PAYLOAD_KEY]), - questionResponse: legacyQuestionResponse, - }, - } - : {}), - }; - const runtimeTools = createAdapterRuntimeToolAccess({ - agentId: agent.id, - companyId: agent.companyId, - runId: run.id, - responsibleUserId: run.responsibleUserId, - }); - if (!runtimeTools) { + } catch (adapterErr) { + const nativeResumeScheduled = + nativeRuntimeResolution.kind === "native" + ? await db + .select({ + phase: nativeRunFinalizations.phase, + resultId: nativeRunFinalizations.resultId, + }) + .from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, run.id)) + .limit(1) + .then( + (rows) => + rows[0]?.phase === "retryable_failure" && + rows[0]?.resultId === null, + ) + : false; + if (nativeResumeScheduled) { + nativeSessionResumeScheduled = true; + throw new NativeSessionResumeScheduledError(adapterErr); + } + // Adapter (or its restore finally) threw — or the finalize record + // write itself threw. Either way the workspace may be in a partial + // state. Best-effort record finalize=failed so the dependent readiness + // check keeps the gate closed instead of waking on stale local state, + // and surface the original error to the caller. + try { + await recordWorkspaceFinalize("failed", { + errorMessage: + adapterErr instanceof Error + ? adapterErr.message + : String(adapterErr), + }); + } catch (recordErr) { logger.warn( { - companyId: agent.companyId, - agentId: agent.id, + err: recordErr, runId: run.id, + executionWorkspaceId: persistedExecutionWorkspace?.id ?? null, }, - "runtime connection tools could not be delivered", + "failed to record workspace_finalize=failed operation; dependents may remain gated", ); } - const runtimeMcpServers = await buildPaperclipRuntimeMcpServers({ - db, - agent, - runId: run.id, - }); - const runtimeToolDelivery = adapter.runtimeToolDelivery ?? "invocation_context"; - if (runtimeTools && runtimeToolDelivery === "native_mcp") { - runtimeMcpServers.unshift({ - name: "Paperclip connections", - url: runtimeTools.mcpEndpoint, - token: runtimeTools.bearerToken, - connectionId: "paperclip-runtime-tools", - }); - } - const runtimeMcp = createAdapterRuntimeMcpAccess(runtimeMcpServers); - if (runtimeTools && runtimeToolDelivery === "invocation_context") { - adapterContext.paperclipRuntimeTools = runtimeTools; - } - const managedMcpConfig = await createManagedMcpRunConfig({ - db, - agent, - runId: run.id, - config: runtimeConfig, - projectId: issueRef?.projectId ?? null, - issueId: issueRef?.id ?? null, - }); - if (managedMcpConfig) { - adapterContext.paperclipManagedMcp = managedMcpConfig; - } - const guardedDispatch = await dispatchResolvedInteractionContinuationWithAtomicGate((markDispatchStarted) => - adapter.execute({ - runId: run.id, - agent, - runtime: runtimeForAdapter, - config: runtimeConfig, - context: adapterContext, - runtimeCommandSpec: adapter.getRuntimeCommandSpec?.(runtimeConfig) ?? null, - executionTarget, - executionTransport: remoteExecution - ? { remoteExecution: remoteExecution as unknown as Record } - : undefined, - runtimeMcp, - runtimeTools, - onLog, - onMeta: onAdapterMeta, - onEvent: onAdapterEvent, - // The endpoint-gated OpenTelemetry startup trace context. It is a - // no-op unless `OTEL_EXPORTER_OTLP_ENDPOINT` is set and the OTel - // packages are installed, so the sandbox-start span path stays inert - // by default. - startupTraceContext: getStartupTraceContext(), - onRuntimeProgress: async (progress) => { - await recordCurrentHeartbeatRunRuntimeProgress(run, progress, issueId); - }, - onDispatch: markDispatchStarted, - onSpawn: async (meta) => { - markDispatchStarted(); - await onSpawn(meta); - }, - authToken: authToken ?? undefined, - }), - ); - if (!guardedDispatch.dispatched) return; - adapterResult = await guardedDispatch.resultPromise; - } - // Adapter returned cleanly, which means its workspace-restore finally - // block also ran without throwing. Record the workspace_finalize - // barrier so dependents that share this executionWorkspace can wake. - // If recording the barrier itself fails, propagate as a run failure - // rather than silently leaving dependents stranded behind a missing - // finalize row. - await recordWorkspaceFinalize("succeeded"); - } catch (adapterErr) { - // Adapter (or its restore finally) threw — or the finalize record - // write itself threw. Either way the workspace may be in a partial - // state. Best-effort record finalize=failed so the dependent readiness - // check keeps the gate closed instead of waking on stale local state, - // and surface the original error to the caller. - try { - await recordWorkspaceFinalize("failed", { - errorMessage: adapterErr instanceof Error ? adapterErr.message : String(adapterErr), - }); - } catch (recordErr) { - logger.warn( - { err: recordErr, runId: run.id, executionWorkspaceId: persistedExecutionWorkspace?.id ?? null }, - "failed to record workspace_finalize=failed operation; dependents may remain gated", - ); - } - throw adapterErr; - } finally { - try { - await revokeHeartbeatRunGatewayTokens({ - db, - companyId: agent.companyId, - runId: run.id, - }); - } catch (revokeErr) { - logger.warn( - { err: revokeErr, runId: run.id, companyId: agent.companyId }, - "failed to revoke heartbeat-run MCP gateway tokens", - ); - } - } - // Reconcile the referenced-project set against the real remote staging outcome. A referenced - // project can pass authorization and clone locally at run prep, then fail to stage into the - // sandbox during execution. The run-prep observability above counts such a project as synced, - // so emit a second, stage-time line that counts each staging failure as a first-class - // `staging` failure. The synced set is the resolved referenced projects minus the ones that - // failed to stage. A run with no staging failure stays silent, so the anchor-only and - // fully-synced paths add no noise. - const referencedProjectStagingFailures = adapterResult.referencedProjectStagingFailures ?? []; - if (referencedProjectStagingFailures.length > 0) { - const stagingFailedProjectIds = new Set( - referencedProjectStagingFailures.map((failure) => failure.projectId), - ); - const stagedProjectObservability = buildReferencedProjectRunObservability({ - syncedProjectIds: resolvedWorkspace.additionalWorkspaces - .map((additional) => additional.projectId) - .filter((projectId) => !stagingFailedProjectIds.has(projectId)), - failures: referencedProjectStagingFailures.map((failure) => ({ - projectId: failure.projectId, - reason: "staging" as const, - error: failure.error, - })), - }); - logger.info( - { - runId: run.id, - companyId: agent.companyId, - issueId: issueRef?.id ?? null, - ...stagedProjectObservability, - }, - "run referenced-project remote staging", - ); - } - const adapterManagedRuntimeServices = adapterResult.runtimeServices - ? await persistAdapterManagedRuntimeServices({ - db, - adapterType: agent.adapterType, - runId: run.id, - agent: { - id: agent.id, - name: agent.name, - companyId: agent.companyId, - }, - issue: issueRef, - workspace: executionWorkspace, - reports: adapterResult.runtimeServices, - }) - : []; - if (adapterManagedRuntimeServices.length > 0) { - const combinedRuntimeServices = [ - ...runtimeServices, - ...adapterManagedRuntimeServices, - ]; - context.paperclipRuntimeServices = combinedRuntimeServices; - context.paperclipRuntimePrimaryUrl = - combinedRuntimeServices.find((service) => readNonEmptyString(service.url))?.url ?? null; - await db - .update(heartbeatRuns) - .set({ - contextSnapshot: context, - updatedAt: new Date(), - }) - .where(eq(heartbeatRuns.id, run.id)); - if (issueId) { - try { - await postWorkspaceReadyComment({ - issuesSvc, - issueId, - agentId: agent.id, - runId: run.id, - workspace: executionWorkspace, - runtimeServices: adapterManagedRuntimeServices, - }); - } catch (err) { - await onLog( - "stderr", - `[paperclip] Failed to post adapter-managed runtime comment: ${err instanceof Error ? err.message : String(err)}\n`, - ); - } - } - } - let outcome: RunSessionOutcome; - const latestRun = await getRun(run.id); - if (isHeartbeatRunTerminalStatus(latestRun?.status)) { - outcome = latestRun.status; - } else if (adapterResult.timedOut) { - outcome = "timed_out"; - } else if ((adapterResult.exitCode ?? 0) === 0 && !adapterResult.errorMessage) { - outcome = "succeeded"; - } else { - outcome = "failed"; - } - - const nextSessionState = resolveNextSessionState({ - adapterType: agent.adapterType, - codec: sessionCodec, - adapterResult, - outcome, - previousParams: previousSessionParams, - previousDisplayId: runtimeForAdapter.sessionDisplayId, - previousLegacySessionId: runtimeForAdapter.sessionId, - }); - const rawUsage = normalizeUsageTotals(adapterResult.usage); - const sessionUsageResolution = await resolveNormalizedUsageForSession({ - agentId: agent.id, - runId: run.id, - sessionId: nextSessionState.displayId ?? nextSessionState.legacySessionId, - rawUsage, - usageBasis: adapterResult.usageBasis ?? null, - }); - const normalizedUsage = sessionUsageResolution.normalizedUsage; - const runErrorMessage = - outcome === "cancelled" - ? (latestRun?.error ?? adapterResult.errorMessage ?? "Cancelled") - : outcome === "succeeded" - ? null - : redactCurrentUserText( - adapterResult.errorMessage ?? (outcome === "timed_out" ? "Timed out" : "Adapter failed"), - currentUserRedactionOptions, + if (nativeRuntimeResolution.kind === "native") { + try { + await finalizeNativeRun({ + db, + runId: run.id, + workspaceFinalizeStatus: "failed", + }); + await dispatchPendingNativeStatusWakeups({ + companyId: run.companyId, + }); + } catch (finalizeErr) { + logger.warn( + { err: finalizeErr, runId: run.id }, + "native result could not be marked workspace_failed; the reconciliation loop will retry persisted results", ); - const recordedResponsibleUserDenialCode = - normalizeResponsibleUserDenialCode(latestRun?.errorCode); - const runErrorCode = - outcome === "timed_out" - ? "timeout" - : outcome === "cancelled" - ? (latestRun?.errorCode ?? "cancelled") - : outcome === "failed" - ? (adapterResult.errorCode ?? recordedResponsibleUserDenialCode ?? "adapter_failed") - : null; - - let logSummary: { bytes: number; sha256?: string; compressed: boolean } | null = null; - if (handle) { - logSummary = await runLogStore.finalize(handle); - } - const finalLogBytes = logSummary?.bytes; - if (outputProgressState.pending && typeof finalLogBytes === "number") { - outputProgressState.pending.bytes = finalLogBytes; - } - await flushOutputProgress({ force: true }); - - const status = - outcome === "succeeded" - ? "succeeded" - : outcome === "cancelled" - ? "cancelled" - : outcome === "timed_out" - ? "timed_out" - : "failed"; - - const cacheAdjustedCostUsd = resolveCacheAdjustedCostUsd(adapterResult); - const usageJson = - normalizedUsage || adapterResult.costUsd != null || cacheAdjustedCostUsd != null - ? ({ - ...(normalizedUsage ?? {}), - ...(rawUsage ? { - rawInputTokens: rawUsage.inputTokens, - rawCachedInputTokens: rawUsage.cachedInputTokens, - rawOutputTokens: rawUsage.outputTokens, - } : {}), - ...(sessionUsageResolution.derivedFromSessionTotals - ? { usageSource: "session_delta" } - : adapterResult.usageBasis === "per_run" - ? { usageSource: "per_run" } - : {}), - ...((nextSessionState.displayId ?? nextSessionState.legacySessionId) - ? { persistedSessionId: nextSessionState.displayId ?? nextSessionState.legacySessionId } - : {}), - sessionReused: runtimeForAdapter.sessionId != null || runtimeForAdapter.sessionDisplayId != null, - taskSessionReused: taskSessionForRun != null, - freshSession: runtimeForAdapter.sessionId == null && runtimeForAdapter.sessionDisplayId == null, - sessionRotated: sessionCompaction.rotate, - sessionRotationReason: sessionCompaction.reason, - configFreshness: configFreshnessResultMetadata, - provider: readNonEmptyString(adapterResult.provider) ?? "unknown", - biller: resolveLedgerBiller(adapterResult), - model: readNonEmptyString(adapterResult.model) ?? "unknown", - ...(adapterResult.costUsd != null ? { costUsd: adapterResult.costUsd } : {}), - ...(cacheAdjustedCostUsd != null ? { cacheAdjustedCostUsd } : {}), - costStatus: resolveLedgerCostStatus({ - costUsd: cacheAdjustedCostUsd, - inputTokens: normalizedUsage?.inputTokens ?? 0, - cachedInputTokens: normalizedUsage?.cachedInputTokens ?? 0, - outputTokens: normalizedUsage?.outputTokens ?? 0, - }), - billingType: normalizeLedgerBillingType(adapterResult.billingType), - } as Record) - : null; - - const persistedResultJson = mergeHeartbeatRunResultJson( - mergeRunStopMetadataForAgent(agent, outcome, { - resultJson: mergeModelProfileRunMetadata( - mergeAdapterRecoveryMetadata({ - resultJson: { - ...parseObject(adapterResult.resultJson), - configFreshness: configFreshnessResultMetadata, + } + } + throw adapterErr; + } finally { + try { + await revokeHeartbeatRunGatewayTokens({ + db, + companyId: agent.companyId, + runId: run.id, + }); + } catch (revokeErr) { + logger.warn( + { err: revokeErr, runId: run.id, companyId: agent.companyId }, + "failed to revoke heartbeat-run MCP gateway tokens", + ); + } + } + // Reconcile the referenced-project set against the real remote staging outcome. A referenced + // project can pass authorization and clone locally at run prep, then fail to stage into the + // sandbox during execution. The run-prep observability above counts such a project as synced, + // so emit a second, stage-time line that counts each staging failure as a first-class + // `staging` failure. The synced set is the resolved referenced projects minus the ones that + // failed to stage. A run with no staging failure stays silent, so the anchor-only and + // fully-synced paths add no noise. + const referencedProjectStagingFailures = + adapterResult.referencedProjectStagingFailures ?? []; + if (referencedProjectStagingFailures.length > 0) { + const stagingFailedProjectIds = new Set( + referencedProjectStagingFailures.map( + (failure) => failure.projectId, + ), + ); + const stagedProjectObservability = + buildReferencedProjectRunObservability({ + syncedProjectIds: resolvedWorkspace.additionalWorkspaces + .map((additional) => additional.projectId) + .filter((projectId) => !stagingFailedProjectIds.has(projectId)), + failures: referencedProjectStagingFailures.map((failure) => ({ + projectId: failure.projectId, + reason: "staging" as const, + error: failure.error, + })), + }); + logger.info( + { + runId: run.id, + companyId: agent.companyId, + issueId: issueRef?.id ?? null, + ...stagedProjectObservability, + }, + "run referenced-project remote staging", + ); + } + const adapterManagedRuntimeServices = adapterResult.runtimeServices + ? await persistAdapterManagedRuntimeServices({ + db, + adapterType: agent.adapterType, + runId: run.id, + agent: { + id: agent.id, + name: agent.name, + companyId: agent.companyId, }, - errorFamily: adapterResult.errorFamily ?? null, - retryNotBefore: adapterResult.retryNotBefore ?? null, - }), - modelProfileApplication, - ), - errorCode: runErrorCode, - errorMessage: runErrorMessage, - }), - adapterResult.summary ?? null, - ); - - const persistedRunWrite = await setRunStatusIfRunning(run.id, status, { - finishedAt: new Date(), - error: runErrorMessage, - errorCode: runErrorCode, - exitCode: adapterResult.exitCode, - signal: adapterResult.signal, - usageJson, - resultJson: persistedResultJson, - sessionIdAfter: nextSessionState.displayId ?? nextSessionState.legacySessionId, - stdoutExcerpt, - stderrExcerpt, - logBytes: logSummary?.bytes, - logSha256: logSummary?.sha256, - logCompressed: logSummary?.compressed ?? false, - }); - if (!persistedRunWrite.updated) { - logger.info( - { - runId: run.id, - attemptedStatus: status, - currentStatus: persistedRunWrite.run?.status ?? null, - }, - "skipping late run finalization because the run already left running state", - ); - return; - } - - if (runtimeResolution.kind === "native") { - const nativePhase = status === "succeeded" ? "completed" : "failed"; - await db.transaction(async (tx) => { - await tx - .update(nativeRunFinalizations) - .set({ - phase: nativePhase, - failureCode: status === "succeeded" ? null : (runErrorCode ?? "provider_failed"), - updatedAt: new Date(), + issue: issueRef, + workspace: executionWorkspace, + reports: adapterResult.runtimeServices, }) - .where(eq(nativeRunFinalizations.runId, run.id)); - await tx + : []; + if (adapterManagedRuntimeServices.length > 0) { + const combinedRuntimeServices = [ + ...runtimeServices, + ...adapterManagedRuntimeServices, + ]; + context.paperclipRuntimeServices = combinedRuntimeServices; + context.paperclipRuntimePrimaryUrl = + combinedRuntimeServices.find((service) => + readNonEmptyString(service.url), + )?.url ?? null; + await db .update(heartbeatRuns) .set({ - nativePhase, - nativePhaseUpdatedAt: new Date(), + contextSnapshot: context, updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, run.id)); - }); - } - - let persistedRun = persistedRunWrite.run; - if (persistedRun) { - persistedRun = await classifyAndPersistRunLiveness(persistedRun, persistedResultJson) ?? persistedRun; - } - - await setWakeupStatus(run.wakeupRequestId, outcome === "succeeded" ? "completed" : status, { - finishedAt: new Date(), - error: runErrorMessage, - }); - - const finalizedRun = persistedRun ?? (await getRun(run.id)); - if (finalizedRun) { - await appendRunEvent(finalizedRun, { - eventType: "lifecycle", - stream: "system", - level: outcome === "succeeded" ? "info" : "error", - message: `run ${outcome}`, - payload: { - status, - exitCode: adapterResult.exitCode, - }, - }); - try { - await completeSkillTestRunForHeartbeatOutcome({ - run: finalizedRun, - issueId, - issueWorkMode: issueRef?.workMode ?? null, - outcome, - error: runErrorMessage, - }); - } catch (err) { - logger.warn( - { err, runId: finalizedRun.id, issueId }, - "failed to complete skill test run after heartbeat finalization", - ); - await onLog( - "stderr", - `[paperclip] Failed to complete skill test run: ${err instanceof Error ? err.message : String(err)}\n`, - ); - } - const livenessRun = finalizedRun; - await refreshContinuationSummaryForRun(livenessRun, agent); - const skipRunIssueComment = parseObject(livenessRun.contextSnapshot).skipIssueComment === true; - if (issueId && outcome === "succeeded" && !skipRunIssueComment) { - try { - const existingRunComment = await findRunIssueComment(livenessRun.id, livenessRun.companyId, issueId); - if (!existingRunComment) { - const issueComment = buildHeartbeatRunIssueComment(persistedResultJson); - if (issueComment) { - await issuesSvc.addComment(issueId, issueComment, { agentId: agent.id, runId: livenessRun.id }); - } + if (issueId) { + try { + await postWorkspaceReadyComment({ + issuesSvc, + issueId, + agentId: agent.id, + runId: run.id, + workspace: executionWorkspace, + runtimeServices: adapterManagedRuntimeServices, + }); + } catch (err) { + await onLog( + "stderr", + `[paperclip] Failed to post adapter-managed runtime comment: ${err instanceof Error ? err.message : String(err)}\n`, + ); } + } + } + let outcome: RunSessionOutcome; + const latestRun = await getRun(run.id); + if (isHeartbeatRunTerminalStatus(latestRun?.status)) { + outcome = latestRun.status; + } else if (adapterResult.nativeFinalization) { + const nativeTerminal = + adapterResult.nativeFinalization.terminal.runTerminalState; + outcome = + nativeTerminal === "succeeded" + ? "succeeded" + : nativeTerminal === "cancelled" + ? "cancelled" + : "failed"; + } else if (adapterResult.timedOut) { + outcome = "timed_out"; + } else if ( + (adapterResult.exitCode ?? 0) === 0 && + !adapterResult.errorMessage + ) { + outcome = "succeeded"; + } else { + outcome = "failed"; + } + + const nextSessionState = resolveNextSessionState({ + adapterType: agent.adapterType, + codec: sessionCodec, + adapterResult, + outcome, + previousParams: previousSessionParams, + previousDisplayId: runtimeForAdapter.sessionDisplayId, + previousLegacySessionId: runtimeForAdapter.sessionId, + }); + const rawUsage = normalizeUsageTotals(adapterResult.usage); + const sessionUsageResolution = await resolveNormalizedUsageForSession({ + agentId: agent.id, + runId: run.id, + sessionId: + nextSessionState.displayId ?? nextSessionState.legacySessionId, + rawUsage, + usageBasis: adapterResult.usageBasis ?? null, + }); + const normalizedUsage = sessionUsageResolution.normalizedUsage; + const runErrorMessage = + outcome === "cancelled" + ? (latestRun?.error ?? adapterResult.errorMessage ?? "Cancelled") + : outcome === "succeeded" + ? null + : redactCurrentUserText( + adapterResult.errorMessage ?? + (outcome === "timed_out" ? "Timed out" : "Adapter failed"), + currentUserRedactionOptions, + ); + const recordedResponsibleUserDenialCode = + normalizeResponsibleUserDenialCode(latestRun?.errorCode); + const runErrorCode = + outcome === "timed_out" + ? "timeout" + : outcome === "cancelled" + ? (latestRun?.errorCode ?? "cancelled") + : outcome === "failed" + ? (adapterResult.errorCode ?? + recordedResponsibleUserDenialCode ?? + "adapter_failed") + : null; + + let logSummary: { + bytes: number; + sha256?: string; + compressed: boolean; + } | null = null; + if (handle) { + logSummary = await runLogStore.finalize(handle); + } + const finalLogBytes = logSummary?.bytes; + if (outputProgressState.pending && typeof finalLogBytes === "number") { + outputProgressState.pending.bytes = finalLogBytes; + } + await flushOutputProgress({ force: true }); + + if (providerTraceCapture) { + try { + await traceStore.finalize(run.id, run.companyId); + providerTraceFinalized = true; + } catch (error) { + logger.warn( + { error, runId: run.id }, + "provider trace finalization failed without affecting run outcome", + ); + } + } + + const status = + outcome === "succeeded" + ? "succeeded" + : outcome === "cancelled" + ? "cancelled" + : outcome === "timed_out" + ? "timed_out" + : "failed"; + + const cacheAdjustedCostUsd = resolveCacheAdjustedCostUsd(adapterResult); + const usageJson = + normalizedUsage || + adapterResult.costUsd != null || + cacheAdjustedCostUsd != null + ? ({ + ...(normalizedUsage ?? {}), + ...(rawUsage + ? { + rawInputTokens: rawUsage.inputTokens, + rawCachedInputTokens: rawUsage.cachedInputTokens, + rawOutputTokens: rawUsage.outputTokens, + } + : {}), + ...(sessionUsageResolution.derivedFromSessionTotals + ? { usageSource: "session_delta" } + : adapterResult.usageBasis === "per_run" + ? { usageSource: "per_run" } + : {}), + ...((nextSessionState.displayId ?? + nextSessionState.legacySessionId) + ? { + persistedSessionId: + nextSessionState.displayId ?? + nextSessionState.legacySessionId, + } + : {}), + sessionReused: + runtimeForAdapter.sessionId != null || + runtimeForAdapter.sessionDisplayId != null, + taskSessionReused: taskSessionForRun != null, + freshSession: + runtimeForAdapter.sessionId == null && + runtimeForAdapter.sessionDisplayId == null, + sessionRotated: sessionCompaction.rotate, + sessionRotationReason: sessionCompaction.reason, + configFreshness: configFreshnessResultMetadata, + provider: + readNonEmptyString(adapterResult.provider) ?? "unknown", + biller: resolveLedgerBiller(adapterResult), + model: readNonEmptyString(adapterResult.model) ?? "unknown", + ...(adapterResult.costUsd != null + ? { costUsd: adapterResult.costUsd } + : {}), + ...(cacheAdjustedCostUsd != null + ? { cacheAdjustedCostUsd } + : {}), + costStatus: resolveLedgerCostStatus({ + costUsd: cacheAdjustedCostUsd, + inputTokens: normalizedUsage?.inputTokens ?? 0, + cachedInputTokens: normalizedUsage?.cachedInputTokens ?? 0, + outputTokens: normalizedUsage?.outputTokens ?? 0, + }), + billingType: normalizeLedgerBillingType( + adapterResult.billingType, + ), + } as Record) + : null; + + const persistedResultJson = mergeHeartbeatRunResultJson( + mergeRunStopMetadataForAgent(agent, outcome, { + resultJson: mergeModelProfileRunMetadata( + mergeAdapterRecoveryMetadata({ + resultJson: { + ...(adapterResult.nativeFinalization + ? parseObject(latestRun?.resultJson) + : {}), + ...parseObject(adapterResult.resultJson), + configFreshness: configFreshnessResultMetadata, + }, + errorFamily: adapterResult.errorFamily ?? null, + retryNotBefore: adapterResult.retryNotBefore ?? null, + }), + modelProfileApplication, + ), + errorCode: runErrorCode, + errorMessage: runErrorMessage, + }), + adapterResult.summary ?? null, + ); + + const finalRunPatch: Partial = { + finishedAt: new Date(), + error: runErrorMessage, + errorCode: runErrorCode, + exitCode: adapterResult.exitCode, + signal: adapterResult.signal, + usageJson, + resultJson: persistedResultJson, + sessionIdAfter: + nextSessionState.displayId ?? nextSessionState.legacySessionId, + stdoutExcerpt, + stderrExcerpt, + logBytes: logSummary?.bytes, + logSha256: logSummary?.sha256, + logCompressed: logSummary?.compressed ?? false, + }; + const persistedRunWrite = await setRunStatusIfRunning( + run.id, + status, + finalRunPatch, + ); + let persistedRun: typeof heartbeatRuns.$inferSelect | null = + persistedRunWrite.run; + if (!persistedRunWrite.updated) { + persistedRun = null; + // Native reconciliation can commit and project the terminal status in + // the narrow window between adapter completion and this live write. + // The status is authoritative, but it must not make us discard the + // adapter's semantic result, usage, logs, or presentation decision. + // Only complete the late metadata write when the reconciler chose the + // same terminal status; a conflicting terminal outcome remains owned + // by the path that won the compare-and-set. + if ( + adapterResult.nativeFinalization && + persistedRunWrite.run?.status === status + ) { + persistedRun = await db + .update(heartbeatRuns) + .set({ + ...finalRunPatch, + finishedAt: + persistedRunWrite.run.finishedAt ?? finalRunPatch.finishedAt, + updatedAt: new Date(), + }) + .where( + and( + eq(heartbeatRuns.id, run.id), + eq(heartbeatRuns.status, status), + ), + ) + .returning() + .then((rows) => rows[0] ?? null); + } + if (!persistedRun) { + logger.info( + { + runId: run.id, + attemptedStatus: status, + currentStatus: persistedRunWrite.run?.status ?? null, + }, + "skipping late run finalization because the run already left running state", + ); + return; + } + } + if (persistedRun) { + persistedRun = + (await classifyAndPersistRunLiveness( + persistedRun, + persistedResultJson, + )) ?? persistedRun; + } + + await setWakeupStatus( + run.wakeupRequestId, + outcome === "succeeded" ? "completed" : status, + { + finishedAt: new Date(), + error: runErrorMessage, + }, + ); + + const finalizedRun = persistedRun ?? (await getRun(run.id)); + if (finalizedRun) { + await appendRunEvent(finalizedRun, { + eventType: "lifecycle", + stream: "system", + level: outcome === "succeeded" ? "info" : "error", + message: `run ${outcome}`, + payload: { + status, + exitCode: adapterResult.exitCode, + }, + }); + try { + await completeSkillTestRunForHeartbeatOutcome({ + run: finalizedRun, + issueId, + issueWorkMode: issueRef?.workMode ?? null, + outcome, + error: runErrorMessage, + }); + } catch (err) { + logger.warn( + { err, runId: finalizedRun.id, issueId }, + "failed to complete skill test run after heartbeat finalization", + ); + await onLog( + "stderr", + `[paperclip] Failed to complete skill test run: ${err instanceof Error ? err.message : String(err)}\n`, + ); + } + const livenessRun = finalizedRun; + await refreshContinuationSummaryForRun(livenessRun, agent); + const skipRunIssueComment = + parseObject(livenessRun.contextSnapshot).skipIssueComment === true; + let resolvedPresentationDecision: RunPresentationDecision | null = + null; + try { + const existingRunComment = issueId + ? await findRunIssueComment( + livenessRun.id, + livenessRun.companyId, + issueId, + persistedResultJson, + ) + : null; + const finalAgentMessage = + await findLatestCompletedFinalAgentMessage( + livenessRun.id, + livenessRun.companyId, + ); + const resolved = resolveHeartbeatRunResponse({ + resultJson: persistedResultJson, + existingComment: existingRunComment, + finalAgentMessage, + }); + let presentationDecision: RunPresentationDecision = + resolved.decision; + + if ( + issueId && + !skipRunIssueComment && + presentationDecision.commentAction === "create" && + resolved.text + ) { + const comment = await issuesSvc.addComment( + issueId, + resolved.text, + { agentId: agent.id, runId: livenessRun.id }, + ); + presentationDecision = { + ...presentationDecision, + commentId: comment.id, + reasonCodes: [ + ...presentationDecision.reasonCodes, + "resolved_response_materialized", + ], + }; + await logActivity(db, { + companyId: livenessRun.companyId, + actorType: "agent", + actorId: agent.id, + agentId: agent.id, + runId: livenessRun.id, + issueId, + action: "issue.comment_added", + entityType: "issue", + entityId: issueId, + details: { + commentId: comment.id, + bodySnippet: comment.body.slice(0, 120), + identifier: issueRef?.identifier ?? null, + issueTitle: issueRef?.title ?? null, + authorizationReason: "internal_agent_write", + source: "run_presentation_resolver", + presentationSource: presentationDecision.chosenSource, + }, + }); + } else if (presentationDecision.commentAction === "create") { + presentationDecision = { + ...presentationDecision, + commentAction: "none", + reasonCodes: [ + ...presentationDecision.reasonCodes, + skipRunIssueComment + ? "issue_comment_suppressed" + : "run_has_no_issue", + ], + }; + } + + await db + .update(heartbeatRuns) + .set({ + resultJson: { + ...persistedResultJson, + presentationDecision, + }, + updatedAt: new Date(), + }) + .where(eq(heartbeatRuns.id, livenessRun.id)); + await appendRunEvent( + livenessRun, + { + eventType: "run.presentation.resolved", + stream: "system", + level: "info", + message: "run presentation resolved", + payload: { presentationDecision }, + }, + ); + resolvedPresentationDecision = presentationDecision; } catch (err) { await onLog( "stderr", - `[paperclip] Failed to post run summary comment: ${err instanceof Error ? err.message : String(err)}\n`, + `[paperclip] Failed to resolve run presentation: ${err instanceof Error ? err.message : String(err)}\n`, ); } - } - if (outcome === "failed" && isMaxTurnExhaustionRun(livenessRun)) { - const policy = parseMaxTurnContinuationPolicy(agent); - if (policy.enabled && policy.maxAttempts > 0) { - await scheduleBoundedRetryForRun(livenessRun, agent, { - retryReason: MAX_TURN_CONTINUATION_RETRY_REASON, - wakeReason: MAX_TURN_CONTINUATION_WAKE_REASON, - maxAttempts: policy.maxAttempts, - delayMs: policy.delayMs, - }); - } else { - await appendRunEvent(livenessRun, { - eventType: "lifecycle", - stream: "system", - level: "warn", - message: "Max-turn continuation suppressed because the policy is disabled", - payload: { + if (outcome === "failed" && isMaxTurnExhaustionRun(livenessRun)) { + const policy = parseMaxTurnContinuationPolicy(agent); + if (policy.enabled && policy.maxAttempts > 0) { + await scheduleBoundedRetryForRun(livenessRun, agent, { retryReason: MAX_TURN_CONTINUATION_RETRY_REASON, - policy, - }, - }); - } - } else if (outcome === "failed" && readTransientRecoveryContractFromRun(livenessRun)) { - await scheduleBoundedRetryForRun(livenessRun, agent); - } - const issueCommentPolicyResult = await finalizeIssueCommentPolicy(livenessRun, agent); - await releaseIssueExecutionAndPromote(livenessRun); - await handleRunLivenessContinuation(livenessRun); - await handleIssueReviewPathDisposition(livenessRun); - await handleSuccessfulRunHandoff( - issueCommentPolicyResult.outcome === "retry_queued" || issueCommentPolicyResult.outcome === "retry_exhausted" - ? { - ...livenessRun, - issueCommentStatus: issueCommentPolicyResult.outcome, + wakeReason: MAX_TURN_CONTINUATION_WAKE_REASON, + maxAttempts: policy.maxAttempts, + delayMs: policy.delayMs, + }); + } else { + await appendRunEvent( + livenessRun, + { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: + "Max-turn continuation suppressed because the policy is disabled", + payload: { + retryReason: MAX_TURN_CONTINUATION_RETRY_REASON, + policy, + }, + }, + ); } - : livenessRun, - agent, - ); + } else if ( + outcome === "failed" && + readTransientRecoveryContractFromRun(livenessRun) + ) { + await scheduleBoundedRetryForRun(livenessRun, agent); + } + const issueCommentPolicyResult = await finalizeIssueCommentPolicy( + livenessRun, + agent, + resolvedPresentationDecision, + ); + await releaseIssueExecutionAndPromote(livenessRun); + await handleRunLivenessContinuation(livenessRun); + await handleIssueReviewPathDisposition(livenessRun); + await handleSuccessfulRunHandoff( + issueCommentPolicyResult.outcome === "retry_queued" || + issueCommentPolicyResult.outcome === "retry_exhausted" + ? { + ...livenessRun, + issueCommentStatus: issueCommentPolicyResult.outcome, + } + : livenessRun, + agent, + ); - // Dependency wake re-check: if this run's issue was marked done mid-run, - // the route-time `issue_blockers_resolved` wake may have been gated by - // workspace finalization or merged into this run. Reuse the level-triggered - // dependency backstop so finalize and periodic recovery share idempotency, - // readiness, active-path, and observability rules. - if (issueId && finalizedRun) { - try { - const blockerIssueStatus = await db - .select({ status: issues.status }) - .from(issues) - .where(eq(issues.id, issueId)) - .then((rows) => rows[0]?.status ?? null); - if (blockerIssueStatus === "done") { - await recovery.reconcileResolvedDependencyWakeBackstop({ - runId: finalizedRun.id, - companyId: finalizedRun.companyId, - blockerIssueId: issueId, - source: "workspace.finalize", + // Dependency wake re-check: if this run's issue was marked done mid-run, + // the route-time `issue_blockers_resolved` wake may have been gated by + // workspace finalization or merged into this run. Reuse the level-triggered + // dependency backstop so finalize and periodic recovery share idempotency, + // readiness, active-path, and observability rules. + if (issueId && finalizedRun) { + try { + const blockerIssueStatus = await db + .select({ status: issues.status }) + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => rows[0]?.status ?? null); + if (blockerIssueStatus === "done") { + await recovery.reconcileResolvedDependencyWakeBackstop({ + runId: finalizedRun.id, + companyId: finalizedRun.companyId, + blockerIssueId: issueId, + source: "workspace.finalize", + }); + } + } catch (finalizeWakeErr) { + logger.warn( + { err: finalizeWakeErr, runId: run.id, issueId }, + "failed to evaluate dependent wakes after workspace_finalize", + ); + } + } + } + + if (finalizedRun) { + await updateRuntimeState( + agent, + finalizedRun, + adapterResult, + { + legacySessionId: nextSessionState.legacySessionId, + }, + normalizedUsage, + ); + if (taskKey) { + if ( + adapterResult.clearSession || + (!nextSessionState.params && !nextSessionState.displayId) + ) { + await clearTaskSessions(agent.companyId, agent.id, { + taskKey, + adapterType: agent.adapterType, + }); + } else { + await upsertTaskSession({ + companyId: agent.companyId, + agentId: agent.id, + adapterType: agent.adapterType, + taskKey, + sessionParamsJson: + attachPaperclipSessionMetadataToSessionParams( + nextSessionState.params, + configuredModel, + sessionConfigMetadata, + ), + sessionDisplayId: nextSessionState.displayId, + lastRunId: finalizedRun.id, + lastError: runErrorMessage, }); } - } catch (finalizeWakeErr) { + } + } + await finalizeAgentStatus(agent.id, outcome, runErrorMessage, { + keepIdleOnFailure: + outcome === "failed" && + ((finalizedRun + ? readHeartbeatRunErrorFamily(finalizedRun) === "provider_quota" + : runErrorCode === "provider_quota") || + isWorkspaceSyncConflictFailure(adapterResult.errorMessage)), + wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run), + }); + } catch (err) { + if (err instanceof NativeCancellationPendingRecoveryError) { + await cancelRunInternal( + run.id, + "Recovered durable native run cancellation", + ); + return; + } + if (err instanceof NativeSessionResumeScheduledError) { + const retryMessage = + err.original instanceof Error + ? err.original.message + : String(err.original ?? ""); + const retryReasonCode = /native_finalization_missing/i.test( + retryMessage, + ) + ? "semantic_result_missing" + : "native_session_interrupted"; + const coordinator = await db + .select({ + nextAttemptAt: nativeRunFinalizations.nextAttemptAt, + attempt: nativeRunFinalizations.attempt, + }) + .from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, run.id)) + .limit(1) + .then((rows) => rows[0] ?? null); + await appendRunEvent(run, { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: + retryReasonCode === "semantic_result_missing" + ? "provider turn completed without a semantic result; same-run disposition recovery persisted" + : "native session transport interrupted; same-run resume persisted", + payload: { + attempt: coordinator?.attempt ?? null, + nextAttemptAt: coordinator?.nextAttemptAt?.toISOString() ?? null, + fallbackSuppressed: true, + retryReasonCode, + }, + }).catch(() => undefined); + if (coordinator?.nextAttemptAt) { + scheduleNativeSessionResumeDispatch( + run.id, + coordinator.nextAttemptAt, + ); + } + return; + } + const message = redactCurrentUserText( + err instanceof Error ? err.message : "Unknown adapter failure", + await getCurrentUserRedactionOptions(), + ); + const workspaceValidationFailure = isWorkspaceValidationFailure(err) + ? err + : null; + const configurationIncompleteFailure = isConfigurationIncompleteFailure( + err, + ) + ? err + : null; + const recordedResponsibleUserDenialCode = + normalizeResponsibleUserDenialCode( + (await getRun(run.id).catch(() => null))?.errorCode, + ); + // The runtime resolution is scoped to the adapter try block. The + // durable coordinator is also the stronger authority here: legacy + // runs simply have no row, while native result-less exhaustion keeps + // its named failure instead of being flattened to `adapter_failed`. + const nativeTerminalFailureCode = await db + .select({ + phase: nativeRunFinalizations.phase, + resultId: nativeRunFinalizations.resultId, + failureCode: nativeRunFinalizations.failureCode, + }) + .from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, run.id)) + .limit(1) + .then((rows) => { + const coordinator = rows[0]; + return coordinator?.phase === "terminal_failure" && + coordinator.resultId === null + ? coordinator.failureCode + : null; + }) + .catch(() => null); + const failureErrorCode = + workspaceValidationFailure?.code ?? + configurationIncompleteFailure?.code ?? + recordedResponsibleUserDenialCode ?? + nativeTerminalFailureCode ?? + "adapter_failed"; + logger.error({ err, runId }, "heartbeat execution failed"); + + let logSummary: { + bytes: number; + sha256?: string; + compressed: boolean; + } | null = null; + if (handle) { + try { + logSummary = await runLogStore.finalize(handle); + } catch (finalizeErr) { logger.warn( - { err: finalizeWakeErr, runId: run.id, issueId }, - "failed to evaluate dependent wakes after workspace_finalize", + { err: finalizeErr, runId }, + "failed to finalize run log after error", ); } } - } + const finalLogBytes = logSummary?.bytes; + if (outputProgressState.pending && typeof finalLogBytes === "number") { + outputProgressState.pending.bytes = finalLogBytes; + } + await flushOutputProgress({ force: true }).catch((flushErr) => { + logger.warn( + { err: flushErr, runId }, + "failed to flush run output progress after error", + ); + }); - if (finalizedRun) { - await updateRuntimeState(agent, finalizedRun, adapterResult, { - legacySessionId: nextSessionState.legacySessionId, - }, normalizedUsage); - if (taskKey) { - if (adapterResult.clearSession || (!nextSessionState.params && !nextSessionState.displayId)) { - await clearTaskSessions(agent.companyId, agent.id, { - taskKey, - adapterType: agent.adapterType, + const failedRunWrite = await setRunStatusIfRunning(run.id, "failed", { + error: message, + errorCode: failureErrorCode, + finishedAt: new Date(), + resultJson: mergeRunStopMetadataForAgent(agent, "failed", { + errorCode: failureErrorCode, + errorMessage: message, + resultJson: + workspaceValidationFailure?.resultJson ?? + configurationIncompleteFailure?.resultJson ?? + null, + }), + stdoutExcerpt, + stderrExcerpt, + logBytes: logSummary?.bytes, + logSha256: logSummary?.sha256, + logCompressed: logSummary?.compressed ?? false, + }); + if (!failedRunWrite.updated) { + logger.info( + { + runId: run.id, + attemptedStatus: "failed", + currentStatus: failedRunWrite.run?.status ?? null, + }, + "skipping late adapter failure finalization because the run already left running state", + ); + return; + } + + const failedRun = failedRunWrite.run; + await setWakeupStatus(run.wakeupRequestId, "failed", { + finishedAt: new Date(), + error: message, + }); + + if (failedRun) { + await appendRunEvent(failedRun, { + eventType: "error", + stream: "system", + level: "error", + message, + }); + const livenessRun = + (await classifyAndPersistRunLiveness(failedRun)) ?? failedRun; + try { + await completeSkillTestRunForHeartbeatOutcome({ + run: livenessRun, + issueId, + issueWorkMode: issueRef?.workMode ?? null, + outcome: "failed", + error: message, }); - } else { + } catch (err) { + logger.warn( + { err, runId: livenessRun.id, issueId }, + "failed to complete skill test run after heartbeat adapter failure", + ); + } + await refreshContinuationSummaryForRun(livenessRun, agent); + if ( + !isWorkspaceValidationFailedRun(livenessRun) && + !isConfigurationIncompleteFailedRun(livenessRun) + ) { + await finalizeIssueCommentPolicy(livenessRun, agent); + } + await scheduleInteractionContinuationInfrastructureRetryIfEligible( + livenessRun, + agent, + ); + await releaseIssueExecutionAndPromote(livenessRun); + await handleIssueReviewPathDisposition(livenessRun); + + await updateRuntimeState( + agent, + livenessRun, + { + exitCode: null, + signal: null, + timedOut: false, + errorMessage: message, + }, + { + legacySessionId: runtimeForAdapter.sessionId, + }, + ); + + if ( + taskKey && + (previousSessionParams || previousSessionDisplayId || taskSession) + ) { await upsertTaskSession({ companyId: agent.companyId, agentId: agent.id, adapterType: agent.adapterType, taskKey, sessionParamsJson: attachPaperclipSessionMetadataToSessionParams( - nextSessionState.params, + previousSessionParams, configuredModel, sessionConfigMetadata, ), - sessionDisplayId: nextSessionState.displayId, - lastRunId: finalizedRun.id, - lastError: runErrorMessage, + sessionDisplayId: previousSessionDisplayId, + lastRunId: failedRun.id, + lastError: message, }); } } - } - await finalizeAgentStatus( - agent.id, - outcome, - runErrorMessage, - { - keepIdleOnFailure: - outcome === "failed" && - ((finalizedRun ? readHeartbeatRunErrorFamily(finalizedRun) === "provider_quota" : runErrorCode === "provider_quota") || - isWorkspaceSyncConflictFailure(adapterResult.errorMessage)), + + await finalizeAgentStatus(agent.id, "failed", message, { wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run), - }, - ); - } catch (err) { - const message = redactCurrentUserText( - err instanceof Error ? err.message : "Unknown adapter failure", - await getCurrentUserRedactionOptions(), - ); - const workspaceValidationFailure = isWorkspaceValidationFailure(err) ? err : null; - const configurationIncompleteFailure = isConfigurationIncompleteFailure(err) ? err : null; - const recordedResponsibleUserDenialCode = - normalizeResponsibleUserDenialCode((await getRun(run.id).catch(() => null))?.errorCode); - const failureErrorCode = - workspaceValidationFailure?.code - ?? configurationIncompleteFailure?.code - ?? nativeRunnerErrorCode(err) - ?? recordedResponsibleUserDenialCode - ?? "adapter_failed"; - logger.error({ err, runId }, "heartbeat execution failed"); - - let logSummary: { bytes: number; sha256?: string; compressed: boolean } | null = null; - if (handle) { - try { - logSummary = await runLogStore.finalize(handle); - } catch (finalizeErr) { - logger.warn({ err: finalizeErr, runId }, "failed to finalize run log after error"); - } - } - const finalLogBytes = logSummary?.bytes; - if (outputProgressState.pending && typeof finalLogBytes === "number") { - outputProgressState.pending.bytes = finalLogBytes; - } - await flushOutputProgress({ force: true }).catch((flushErr) => { - logger.warn({ err: flushErr, runId }, "failed to flush run output progress after error"); - }); - - const failedRunWrite = await setRunStatusIfRunning(run.id, "failed", { - error: message, - errorCode: failureErrorCode, - finishedAt: new Date(), - resultJson: mergeRunStopMetadataForAgent(agent, "failed", { - errorCode: failureErrorCode, - errorMessage: message, - resultJson: workspaceValidationFailure?.resultJson ?? configurationIncompleteFailure?.resultJson ?? null, - }), - stdoutExcerpt, - stderrExcerpt, - logBytes: logSummary?.bytes, - logSha256: logSummary?.sha256, - logCompressed: logSummary?.compressed ?? false, - }); - if (!failedRunWrite.updated) { - logger.info( - { - runId: run.id, - attemptedStatus: "failed", - currentStatus: failedRunWrite.run?.status ?? null, - }, - "skipping late adapter failure finalization because the run already left running state", - ); - return; - } - - const failedRun = failedRunWrite.run; - if (failedRun?.runtimeMode === "native") { - await db - .update(heartbeatRuns) - .set({ nativePhase: "failed", nativePhaseUpdatedAt: new Date(), updatedAt: new Date() }) - .where(eq(heartbeatRuns.id, failedRun.id)); - await db - .update(nativeRunFinalizations) - .set({ phase: "failed", failureCode: failureErrorCode, updatedAt: new Date() }) - .where(eq(nativeRunFinalizations.runId, failedRun.id)); - } - await setWakeupStatus(run.wakeupRequestId, "failed", { - finishedAt: new Date(), - error: message, - }); - - if (failedRun) { - await appendRunEvent(failedRun, { - eventType: "error", - stream: "system", - level: "error", - message, + keepIdleOnFailure: isWorkspaceSyncConflictFailure(message), }); - const livenessRun = await classifyAndPersistRunLiveness(failedRun) ?? failedRun; - try { - await completeSkillTestRunForHeartbeatOutcome({ - run: livenessRun, - issueId, - issueWorkMode: issueRef?.workMode ?? null, - outcome: "failed", - error: message, - }); - } catch (err) { - logger.warn( - { err, runId: livenessRun.id, issueId }, - "failed to complete skill test run after heartbeat adapter failure", - ); - } - await refreshContinuationSummaryForRun(livenessRun, agent); - if (!isWorkspaceValidationFailedRun(livenessRun) && !isConfigurationIncompleteFailedRun(livenessRun)) { - await finalizeIssueCommentPolicy(livenessRun, agent); - } - await scheduleInteractionContinuationInfrastructureRetryIfEligible(livenessRun, agent); - await releaseIssueExecutionAndPromote(livenessRun); - await handleIssueReviewPathDisposition(livenessRun); - - await updateRuntimeState(agent, livenessRun, { - exitCode: null, - signal: null, - timedOut: false, - errorMessage: message, - }, { - legacySessionId: runtimeForAdapter.sessionId, - }); - - if (taskKey && (previousSessionParams || previousSessionDisplayId || taskSession)) { - await upsertTaskSession({ - companyId: agent.companyId, - agentId: agent.id, - adapterType: agent.adapterType, - taskKey, - sessionParamsJson: attachPaperclipSessionMetadataToSessionParams( - previousSessionParams, - configuredModel, - sessionConfigMetadata, - ), - sessionDisplayId: previousSessionDisplayId, - lastRunId: failedRun.id, - lastError: message, - }); - } } - - await finalizeAgentStatus(agent.id, "failed", message, { - wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run), - keepIdleOnFailure: isWorkspaceSyncConflictFailure(message), - }); - } } catch (outerErr) { - if (isWorkspaceBusyDeferral(outerErr)) { - // Expected contention on a shared project workspace, not a - // failure: park the run as a bounded scheduled retry and leave the - // holder undisturbed. The finally block below still releases - // leases, runtime services, and scratch for this run. - await finalizeWorkspaceBusyDeferral(run, outerErr).catch((deferralErr) => { - logger.error( - { err: deferralErr, runId }, - "failed to finalize workspace-busy deferral", + if (isWorkspaceBusyDeferral(outerErr)) { + // Expected contention on a shared project workspace, not a + // failure: park the run as a bounded scheduled retry and leave the + // holder undisturbed. The finally block below still releases + // leases, runtime services, and scratch for this run. + await finalizeWorkspaceBusyDeferral(run, outerErr).catch( + (deferralErr) => { + logger.error( + { err: deferralErr, runId }, + "failed to finalize workspace-busy deferral", + ); + }, + ); + } else { + // Setup code before adapter.execute threw (e.g. ensureRuntimeState, resolveWorkspaceForRun). + // The inner catch did not fire, so we must record the failure here. + const message = redactCurrentUserText( + outerErr instanceof Error + ? outerErr.message + : "Unknown setup failure", + await getCurrentUserRedactionOptions(), + ); + // A missing secret/env binding is a known pre-dispatch configuration gap, + // not an opaque setup crash. Surface it with its own errorCode so the + // recovery path routes it to a human owner instead of looping retries. + const workspaceValidationSetupFailure = isWorkspaceValidationFailure( + outerErr, + ) + ? outerErr + : null; + const configurationIncompleteSetupFailure = + isConfigurationIncompleteFailure(outerErr) ? outerErr : null; + const unresolvedBaseRefSetupFailure = + isUnresolvedWorkspaceBaseRefError(outerErr) ? outerErr : null; + const recordedResponsibleUserDenialCode = + normalizeResponsibleUserDenialCode( + (await getRun(runId).catch(() => null))?.errorCode, + ); + const setupFailureErrorCode = + workspaceValidationSetupFailure?.code ?? + configurationIncompleteSetupFailure?.code ?? + (unresolvedBaseRefSetupFailure + ? CONFIGURATION_INCOMPLETE_FAILURE_CODE + : null) ?? + recordedResponsibleUserDenialCode ?? + "setup_failed"; + logger.error( + { err: outerErr, runId }, + "heartbeat execution setup failed", + ); + const setupFailureAgent = await getAgent(run.agentId).catch(() => null); + const setupFailureWrite = await setRunStatusIfRunning(runId, "failed", { + error: message, + errorCode: setupFailureErrorCode, + finishedAt: new Date(), + ...(setupFailureAgent + ? { + resultJson: mergeRunStopMetadataForAgent( + setupFailureAgent, + "failed", + { + errorCode: setupFailureErrorCode, + errorMessage: message, + resultJson: + workspaceValidationSetupFailure?.resultJson ?? + configurationIncompleteSetupFailure?.resultJson ?? + (unresolvedBaseRefSetupFailure + ? buildUnresolvedWorkspaceBaseRefResultJson( + run, + unresolvedBaseRefSetupFailure, + ) + : null), + }, + ), + } + : {}), + }).catch(() => ({ run: null, updated: false as const })); + if (!setupFailureWrite.updated) { + logger.info( + { + runId, + attemptedStatus: "failed", + currentStatus: setupFailureWrite.run?.status ?? null, + }, + "skipping late setup failure finalization because the run already left running state", + ); + } else { + await setWakeupStatus(run.wakeupRequestId, "failed", { + finishedAt: new Date(), + error: message, + }).catch(() => undefined); + } + const failedRun = await getRun(runId).catch(() => null); + if (setupFailureWrite.updated && failedRun) { + // Emit a run-log event so the failure is visible in the run timeline, + // consistent with what the inner catch block does for adapter failures. + await appendRunEvent(failedRun, { + eventType: "error", + stream: "system", + level: "error", + message, + }).catch(() => undefined); + const livenessRun = await classifyAndPersistRunLiveness( + failedRun, + ).catch(() => failedRun); + const setupFailureIssueId = readNonEmptyString( + parseObject(livenessRun.contextSnapshot).issueId, + ); + if (setupFailureIssueId) { + await completeSkillTestRunForHeartbeatOutcome({ + run: livenessRun, + issueId: setupFailureIssueId, + outcome: "failed", + error: message, + }).catch((completionErr) => { + logger.warn( + { + err: completionErr, + runId: livenessRun.id, + issueId: setupFailureIssueId, + }, + "failed to complete skill test run after heartbeat setup failure", ); }); - } else { - // Setup code before adapter.execute threw (e.g. ensureRuntimeState, resolveWorkspaceForRun). - // The inner catch did not fire, so we must record the failure here. - const message = redactCurrentUserText( - outerErr instanceof Error ? outerErr.message : "Unknown setup failure", - await getCurrentUserRedactionOptions(), - ); - // A missing secret/env binding is a known pre-dispatch configuration gap, - // not an opaque setup crash. Surface it with its own errorCode so the - // recovery path routes it to a human owner instead of looping retries. - const workspaceValidationSetupFailure = isWorkspaceValidationFailure(outerErr) ? outerErr : null; - const configurationIncompleteSetupFailure = isConfigurationIncompleteFailure(outerErr) ? outerErr : null; - // A remote-only base ref that never resolved is a known pre-dispatch - // configuration gap, not an opaque setup crash. Map it to the same - // configuration-incomplete code so the recovery path routes it to a - // human owner and bounds the repeat by its per-ref fingerprint. - const unresolvedBaseRefSetupFailure = isUnresolvedWorkspaceBaseRefError(outerErr) ? outerErr : null; - const recordedResponsibleUserDenialCode = - normalizeResponsibleUserDenialCode((await getRun(runId).catch(() => null))?.errorCode); - const setupFailureErrorCode = - workspaceValidationSetupFailure?.code ?? - configurationIncompleteSetupFailure?.code ?? - (unresolvedBaseRefSetupFailure ? CONFIGURATION_INCOMPLETE_FAILURE_CODE : null) ?? - nativeRunnerErrorCode(outerErr) ?? - recordedResponsibleUserDenialCode ?? - "setup_failed"; - logger.error({ err: outerErr, runId }, "heartbeat execution setup failed"); - const setupFailureAgent = await getAgent(run.agentId).catch(() => null); - const setupFailureWrite = await setRunStatusIfRunning(runId, "failed", { - error: message, - errorCode: setupFailureErrorCode, - finishedAt: new Date(), - ...(setupFailureAgent ? { - resultJson: mergeRunStopMetadataForAgent(setupFailureAgent, "failed", { - errorCode: setupFailureErrorCode, - errorMessage: message, - resultJson: - workspaceValidationSetupFailure?.resultJson ?? - configurationIncompleteSetupFailure?.resultJson ?? - (unresolvedBaseRefSetupFailure - ? buildUnresolvedWorkspaceBaseRefResultJson(run, unresolvedBaseRefSetupFailure) - : null), - }), - } : {}), - }).catch(() => ({ run: null, updated: false as const })); - if (!setupFailureWrite.updated) { - logger.info( - { - runId, - attemptedStatus: "failed", - currentStatus: setupFailureWrite.run?.status ?? null, - }, - "skipping late setup failure finalization because the run already left running state", - ); - } else { - await setWakeupStatus(run.wakeupRequestId, "failed", { - finishedAt: new Date(), - error: message, - }).catch(() => undefined); } - const failedRun = await getRun(runId).catch(() => null); - if (setupFailureWrite.updated && failedRun) { - // Emit a run-log event so the failure is visible in the run timeline, - // consistent with what the inner catch block does for adapter failures. - await appendRunEvent(failedRun, { - eventType: "error", - stream: "system", - level: "error", - message, - }).catch(() => undefined); - const livenessRun = await classifyAndPersistRunLiveness(failedRun).catch(() => failedRun); - const setupFailureIssueId = readNonEmptyString(parseObject(livenessRun.contextSnapshot).issueId); - if (setupFailureIssueId) { - await completeSkillTestRunForHeartbeatOutcome({ - run: livenessRun, - issueId: setupFailureIssueId, - outcome: "failed", - error: message, - }).catch((completionErr) => { - logger.warn( - { err: completionErr, runId: livenessRun.id, issueId: setupFailureIssueId }, - "failed to complete skill test run after heartbeat setup failure", - ); - }); + const failedAgent = + setupFailureAgent ?? + (await getAgent(run.agentId).catch(() => null)); + if (failedAgent) { + await refreshContinuationSummaryForRun( + livenessRun, + failedAgent, + ).catch(() => undefined); + if ( + !isWorkspaceValidationFailedRun(livenessRun) && + !isConfigurationIncompleteFailedRun(livenessRun) + ) { + await finalizeIssueCommentPolicy(livenessRun, failedAgent).catch( + () => undefined, + ); } - const failedAgent = setupFailureAgent ?? await getAgent(run.agentId).catch(() => null); - if (failedAgent) { - await refreshContinuationSummaryForRun(livenessRun, failedAgent).catch(() => undefined); - if (!isWorkspaceValidationFailedRun(livenessRun) && !isConfigurationIncompleteFailedRun(livenessRun)) { - await finalizeIssueCommentPolicy(livenessRun, failedAgent).catch(() => undefined); - } - await scheduleInteractionContinuationInfrastructureRetryIfEligible(livenessRun, failedAgent).catch((retryError) => { - logger.warn( - { err: retryError, runId: livenessRun.id }, - "failed to schedule interaction continuation retry after setup failure", - ); - }); - } - await releaseIssueExecutionAndPromote(livenessRun).catch((releaseError) => { + await scheduleInteractionContinuationInfrastructureRetryIfEligible( + livenessRun, + failedAgent, + ).catch((retryError) => { + logger.warn( + { err: retryError, runId: livenessRun.id }, + "failed to schedule interaction continuation retry after setup failure", + ); + }); + } + await releaseIssueExecutionAndPromote(livenessRun).catch( + (releaseError) => { logger.error( { err: releaseError, runId }, "failed to release issue execution after heartbeat setup failure", ); - }); - await handleIssueReviewPathDisposition(livenessRun).catch((reviewPathError) => { + }, + ); + await handleIssueReviewPathDisposition(livenessRun).catch( + (reviewPathError) => { logger.error( { err: reviewPathError, runId }, "failed to evaluate review-path disposition after heartbeat setup failure", ); - }); - } - // Ensure the agent is not left stuck in "running" if the setup-failure - // path owned the terminal transition. If another path already finalized - // the run, keep that terminal outcome authoritative. - if (setupFailureWrite.updated) { - await finalizeAgentStatus(run.agentId, "failed", message, { - wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run), - }).catch(() => undefined); - } - } - } finally { - let latestRun = await getRun(run.id).catch(() => null); - // Close the invariant "environment lease released implies the run is - // terminal". When the teardown reaches this point with the run still - // running or queued, force a terminal status before the lease is - // released, so the UI never shows a finished task as "Live". - if (latestRun) { - latestRun = await terminalizeRunOnLeaseRelease(latestRun).catch((terminalizeErr) => { - logger.error( - { err: terminalizeErr, runId: run.id }, - "failed to terminalize run before environment lease release", - ); - return latestRun; - }); - } - await releaseEnvironmentLeasesForRun({ - runId: run.id, - companyId: run.companyId, - agentId: run.agentId, - status: latestRun?.status, - failureReason: latestRun?.error ?? undefined, - }); - await releaseRuntimeServicesForRun(run.id).catch(() => undefined); - if (runScratch && latestRun && isHeartbeatRunTerminalStatus(latestRun.status)) { - const scratchForCleanup = runScratch; - let scratchCleanup: Awaited> | null = null; - try { - scratchCleanup = await cleanupHeartbeatRunScratch({ - scratch: scratchForCleanup, - processGroupId: latestRun.processGroupId, - isProcessGroupAlive, - }); - } catch (scratchCleanupError) { - logger.warn( - { - err: scratchCleanupError, - runId: run.id, - scratchDir: scratchForCleanup.dir, - }, - "failed to clean heartbeat run scratch directory", - ); - await appendRunEvent(latestRun, { - eventType: "error", - stream: "system", - level: "warn", - message: "run scratch cleanup failed", - payload: { - dir: scratchForCleanup.dir, - error: scratchCleanupError instanceof Error - ? scratchCleanupError.message - : String(scratchCleanupError), - }, - }).catch(() => undefined); - } - if (scratchCleanup) { - await appendRunEvent(latestRun, { - eventType: "lifecycle", - stream: "system", - level: scratchCleanup.removed ? "info" : "warn", - message: scratchCleanup.removed - ? "run scratch cleaned" - : `run scratch cleanup skipped: ${scratchCleanup.reason}`, - payload: scratchCleanup, - }).catch((scratchCleanupEventError) => { - logger.warn( - { - err: scratchCleanupEventError, - runId: run.id, - scratchDir: scratchForCleanup.dir, - }, - "failed to record heartbeat run scratch cleanup event", - ); - }); - } - } - activeRunExecutions.delete(run.id); - await startNextQueuedRunForAgent(run.agentId); + }, + ); } + // Ensure the agent is not left stuck in "running" if the setup-failure + // path owned the terminal transition. If another path already finalized + // the run, keep that terminal outcome authoritative. + if (setupFailureWrite.updated) { + await finalizeAgentStatus(run.agentId, "failed", message, { + wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run), + }).catch(() => undefined); + } + } + } finally { + let latestRun = await getRun(run.id).catch(() => null); + // Trace capture is debug-only and must settle independently of every + // provider outcome. Adapter/setup failures used to skip the success-path + // finalizer, leaving metadata permanently stuck at `capturing` even when + // runnerd had already closed (or never managed to write) its sidecar. + // Same-run native resumes retain the open capture until the resumed + // execution reaches a true terminal boundary. + if ( + providerTraceCapture && + !providerTraceFinalized && + !nativeSessionResumeScheduled + ) { + try { + await traceStore.finalize(run.id, run.companyId); + providerTraceFinalized = true; + } catch (traceFinalizeError) { + logger.warn( + { err: traceFinalizeError, runId: run.id }, + "provider trace finalization failed during heartbeat teardown", + ); + } + } + // Close the invariant "environment lease released implies the run is + // terminal". When the teardown reaches this point with the run still + // running or queued, force a terminal status before the lease is + // released, so the UI never shows a finished task as "Live". + if (latestRun && !nativeSessionResumeScheduled) { + latestRun = await terminalizeRunOnLeaseRelease(latestRun).catch( + (terminalizeErr) => { + logger.error( + { err: terminalizeErr, runId: run.id }, + "failed to terminalize run before environment lease release", + ); + return latestRun; + }, + ); + } + if (!nativeSessionResumeScheduled) { + await releaseEnvironmentLeasesForRun({ + runId: run.id, + companyId: run.companyId, + agentId: run.agentId, + status: latestRun?.status, + failureReason: latestRun?.error ?? undefined, + }); + await releaseRuntimeServicesForRun(run.id).catch(() => undefined); + } + if ( + runScratch && + latestRun && + isHeartbeatRunTerminalStatus(latestRun.status) + ) { + const scratchForCleanup = runScratch; + let scratchCleanup: Awaited< + ReturnType + > | null = null; + try { + scratchCleanup = await cleanupHeartbeatRunScratch({ + scratch: scratchForCleanup, + processGroupId: latestRun.processGroupId, + isProcessGroupAlive, + }); + } catch (scratchCleanupError) { + logger.warn( + { + err: scratchCleanupError, + runId: run.id, + scratchDir: scratchForCleanup.dir, + }, + "failed to clean heartbeat run scratch directory", + ); + await appendRunEvent(latestRun, { + eventType: "error", + stream: "system", + level: "warn", + message: "run scratch cleanup failed", + payload: { + dir: scratchForCleanup.dir, + error: + scratchCleanupError instanceof Error + ? scratchCleanupError.message + : String(scratchCleanupError), + }, + }).catch(() => undefined); + } + if (scratchCleanup) { + await appendRunEvent(latestRun, { + eventType: "lifecycle", + stream: "system", + level: scratchCleanup.removed ? "info" : "warn", + message: scratchCleanup.removed + ? "run scratch cleaned" + : `run scratch cleanup skipped: ${scratchCleanup.reason}`, + payload: scratchCleanup, + }).catch((scratchCleanupEventError) => { + logger.warn( + { + err: scratchCleanupEventError, + runId: run.id, + scratchDir: scratchForCleanup.dir, + }, + "failed to record heartbeat run scratch cleanup event", + ); + }); + } + } + activeRunExecutions.delete(run.id); + if (!nativeSessionResumeScheduled) { + await startNextQueuedRunForAgent(run.agentId); + } + } } async function releaseIssueExecutionAndPromote( @@ -18013,7 +21920,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) eq(issues.executionRunId, run.id), eq(issues.checkoutRunId, run.id), ) - : or(eq(issues.executionRunId, run.id), eq(issues.checkoutRunId, run.id)), + : or( + eq(issues.executionRunId, run.id), + eq(issues.checkoutRunId, run.id), + ), ), ) .orderBy(asc(issues.id)); @@ -18039,7 +21949,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) updatedAt: promotionUpdateTimestamp, }) .where( - and(eq(issues.companyId, run.companyId), eq(issues.executionRunId, run.id)), + and( + eq(issues.companyId, run.companyId), + eq(issues.executionRunId, run.id), + ), ); // `checkoutRunId` clear is symmetric to #6008's per-issue self-heal, // extended to all siblings: covers paths where the issue's assignee or @@ -18052,7 +21965,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) updatedAt: promotionUpdateTimestamp, }) .where( - and(eq(issues.companyId, run.companyId), eq(issues.checkoutRunId, run.id)), + and( + eq(issues.companyId, run.companyId), + eq(issues.checkoutRunId, run.id), + ), ); // Deferred-wake promotion is bound to a single primary issue: the run's context @@ -18071,7 +21987,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) // Sibling lock cleanup is already done above; only the primary issue carries // the recovery surface because the comment is attached to a single issue. if ( - (isWorkspaceValidationFailedRun(run) || isConfigurationIncompleteFailedRun(run)) && + (isWorkspaceValidationFailedRun(run) || + isConfigurationIncompleteFailedRun(run)) && (issue.status === "todo" || issue.status === "in_progress") && !issue.assigneeUserId && issue.assigneeAgentId === run.agentId @@ -18090,7 +22007,6 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) }; } - while (true) { let deferred = await tx .select() @@ -18209,28 +22125,33 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const companyAgents = deferredAgent ? await tx - .select({ - id: agents.id, - companyId: agents.companyId, - name: agents.name, - reportsTo: agents.reportsTo, - status: agents.status, - }) - .from(agents) - .where(eq(agents.companyId, issue.companyId)) + .select({ + id: agents.id, + companyId: agents.companyId, + name: agents.name, + reportsTo: agents.reportsTo, + status: agents.status, + }) + .from(agents) + .where(eq(agents.companyId, issue.companyId)) : []; const deferredInvokability = deferredAgent?.companyId === issue.companyId ? evaluateAgentInvokability(deferredAgent, companyAgents) : evaluateAgentInvokability(null, companyAgents); - if (!deferredAgent || deferredAgent.companyId !== issue.companyId || !deferredInvokability.invokable) { + if ( + !deferredAgent || + deferredAgent.companyId !== issue.companyId || + !deferredInvokability.invokable + ) { await tx .update(agentWakeupRequests) .set({ status: "failed", finishedAt: new Date(), - error: "Deferred wake could not be promoted: agent is not invokable", + error: + "Deferred wake could not be promoted: agent is not invokable", updatedAt: new Date(), }) .where(eq(agentWakeupRequests.id, deferred.id)); @@ -18238,16 +22159,23 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } const deferredPayload = parseObject(deferred.payload); - const deferredContextSeed = parseObject(deferredPayload[DEFERRED_WAKE_CONTEXT_KEY]); - const activePauseHold = await treeControlSvc.getActivePauseHoldGate(issue.companyId, issue.id); - const treeHoldInteractionWake = activePauseHold && await isVerifiedIssueTreeControlInteractionWake(tx, { - companyId: issue.companyId, - issueId: issue.id, - agentId: deferred.agentId, - contextSnapshot: deferredContextSeed, - requestedByActorType: deferred.requestedByActorType, - requestedByActorId: deferred.requestedByActorId, - }); + const deferredContextSeed = parseObject( + deferredPayload[DEFERRED_WAKE_CONTEXT_KEY], + ); + const activePauseHold = await treeControlSvc.getActivePauseHoldGate( + issue.companyId, + issue.id, + ); + const treeHoldInteractionWake = + activePauseHold && + (await isVerifiedIssueTreeControlInteractionWake(tx, { + companyId: issue.companyId, + issueId: issue.id, + agentId: deferred.agentId, + contextSnapshot: deferredContextSeed, + requestedByActorType: deferred.requestedByActorType, + requestedByActorId: deferred.requestedByActorId, + })); if (activePauseHold && !treeHoldInteractionWake) { await tx .update(agentWakeupRequests) @@ -18261,7 +22189,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) continue; } - const promotedContextSeed: Record = { ...deferredContextSeed }; + const promotedContextSeed: Record = { + ...deferredContextSeed, + }; if (activePauseHold) { promotedContextSeed.treeHoldInteraction = true; promotedContextSeed.activeTreeHold = { @@ -18274,7 +22204,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) }; } const deferredCommentIds = extractWakeCommentIds(deferredContextSeed); - const deferredWakeReason = readNonEmptyString(deferredContextSeed.wakeReason); + const deferredWakeReason = readNonEmptyString( + deferredContextSeed.wakeReason, + ); // Local-CLI agents post comments under user auth, so a self-comment from // the run that is now ending would otherwise look like a real human // comment and trigger a reopen on the very issue this run just closed. @@ -18295,7 +22227,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .then((rows) => rows); deferredCommentWakeIsSelfAuthored = deferredComments.length > 0 && - deferredComments.every((comment) => comment.createdByRunId === run.id); + deferredComments.every( + (comment) => comment.createdByRunId === run.id, + ); } // Only human/comment-reopen interactions should revive completed issues; // system follow-ups such as retry or cleanup wakes must not reopen closed work. @@ -18303,10 +22237,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) deferredCommentIds.length > 0 && !deferredCommentWakeIsSelfAuthored && (issue.status === "done" || issue.status === "cancelled") && - ( - deferred.requestedByActorType === "user" || - deferredWakeReason === "issue_reopened_via_comment" - ); + (deferred.requestedByActorType === "user" || + deferredWakeReason === "issue_reopened_via_comment"); let reopenedActivity: LogActivityInput | null = null; if (shouldReopenDeferredCommentWake) { @@ -18349,11 +22281,15 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } } - const promotedReason = readNonEmptyString(deferred.reason) ?? "issue_execution_promoted"; + const promotedReason = + readNonEmptyString(deferred.reason) ?? "issue_execution_promoted"; const promotedSource = - (readNonEmptyString(deferred.source) as WakeupOptions["source"]) ?? "automation"; + (readNonEmptyString(deferred.source) as WakeupOptions["source"]) ?? + "automation"; const promotedTriggerDetail = - (readNonEmptyString(deferred.triggerDetail) as WakeupOptions["triggerDetail"]) ?? null; + (readNonEmptyString( + deferred.triggerDetail, + ) as WakeupOptions["triggerDetail"]) ?? null; const promotedPayload = deferredPayload; delete promotedPayload[DEFERRED_WAKE_CONTEXT_KEY]; @@ -18370,30 +22306,41 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const sessionBefore = readNonEmptyString(promotedContextSnapshot.resumeSessionDisplayId) ?? - await resolveSessionBeforeForWakeup(deferredAgent, promotedTaskKey); + (await resolveSessionBeforeForWakeup(deferredAgent, promotedTaskKey)); const promotedContinuationAttempt = readContinuationAttempt( promotedContextSnapshot.livenessContinuationAttempt, ); - const promotedResponsibleUserId = await resolveResponsibleUserIdForRunSeed({ - companyId: deferredAgent.companyId, - contextSnapshot: promotedContextSnapshot, - issueContext: issue, - routineEnvContext: await getRoutineEnvForExecutionIssue(deferredAgent.companyId, issue), - requestedByActorType: deferred.requestedByActorType as "user" | "agent" | "system" | null, - requestedByActorId: deferred.requestedByActorId, - source: promotedSource, - triggerDetail: promotedTriggerDetail, - existingRunResponsibleUserId: run.responsibleUserId, - }); - if (!promotedResponsibleUserId) { - throw new HttpError(422, "Unable to resolve responsible user for promoted heartbeat run", { - code: "responsible_user_unresolved", - runId: run.id, - agentId: deferredAgent.id, + const promotedResponsibleUserId = + await resolveResponsibleUserIdForRunSeed({ companyId: deferredAgent.companyId, - issueId: issue.id, - wakeReason: readNonEmptyString(promotedContextSnapshot.wakeReason), + contextSnapshot: promotedContextSnapshot, + issueContext: issue, + routineEnvContext: await getRoutineEnvForExecutionIssue( + deferredAgent.companyId, + issue, + ), + requestedByActorType: deferred.requestedByActorType as + "user" | "agent" | "system" | null, + requestedByActorId: deferred.requestedByActorId, + source: promotedSource, + triggerDetail: promotedTriggerDetail, + existingRunResponsibleUserId: run.responsibleUserId, }); + if (!promotedResponsibleUserId) { + throw new HttpError( + 422, + "Unable to resolve responsible user for promoted heartbeat run", + { + code: "responsible_user_unresolved", + runId: run.id, + agentId: deferredAgent.id, + companyId: deferredAgent.companyId, + issueId: issue.id, + wakeReason: readNonEmptyString( + promotedContextSnapshot.wakeReason, + ), + }, + ); } const now = new Date(); const newRun = await tx @@ -18435,7 +22382,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) updatedAt: now, }) // Promoted mention wakes are issue-scoped, not issue ownership transfers. - .where(and(eq(issues.id, issue.id), eq(issues.assigneeAgentId, deferredAgent.id))); + .where( + and( + eq(issues.id, issue.id), + eq(issues.assigneeAgentId, deferredAgent.id), + ), + ); return { kind: "promoted" as const, @@ -18451,7 +22403,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .where( and( eq(heartbeatRuns.companyId, issue.companyId), - inArray(heartbeatRuns.status, [...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES]), + inArray(heartbeatRuns.status, [ + ...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES, + ]), sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issue.id}`, sql`${heartbeatRuns.id} <> ${run.id}`, agentId ? eq(heartbeatRuns.agentId, agentId) : sql`true`, @@ -18479,9 +22433,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .limit(1) .then((rows) => rows[0] ?? null); const executionState = parseIssueExecutionState(issue.executionState); - const currentParticipant = executionState?.status === "pending" - ? executionState.currentParticipant - : null; + const currentParticipant = + executionState?.status === "pending" + ? executionState.currentParticipant + : null; const issueNeedsReviewParticipantRecovery = issue.status === "in_review" && !issue.assigneeUserId && @@ -18493,12 +22448,18 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) ); if (issueNeedsReviewParticipantRecovery) { - const existingReviewParticipantExecutionPath = await findExistingExecutionPath(currentParticipant.agentId); + const existingReviewParticipantExecutionPath = + await findExistingExecutionPath(currentParticipant.agentId); if ( options.suppressImmediateRecovery || existingReviewParticipantExecutionPath || issueHasPersistedMonitor || - await isAutomaticRecoverySuppressedByPauseHold(db, issue.companyId, issue.id, treeControlSvc) + (await isAutomaticRecoverySuppressedByPauseHold( + db, + issue.companyId, + issue.id, + treeControlSvc, + )) ) { return { kind: "released" as const }; } @@ -18534,13 +22495,16 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) source: "automation", triggerDetail: "system", reason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON, - payload: withRecoveryModelProfileHint({ - issueId: issue.id, - retryOfRunId: run.id, - retryReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON, - currentStageId: executionState?.currentStageId ?? null, - currentStageType: executionState?.currentStageType ?? null, - }, "normal_model"), + payload: withRecoveryModelProfileHint( + { + issueId: issue.id, + retryOfRunId: run.id, + retryReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON, + currentStageId: executionState?.currentStageId ?? null, + currentStageType: executionState?.currentStageType ?? null, + }, + "normal_model", + ), status: "queued", requestedByActorType: "system", requestedByActorId: null, @@ -18558,18 +22522,21 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) triggerDetail: "system", status: "queued", wakeupRequestId: wakeupRequest.id, - contextSnapshot: withRecoveryModelProfileHint({ - issueId: issue.id, - taskId: issue.id, - wakeReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON, - retryReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON, - source: "issue.execution_review_recovery", - retryOfRunId: run.id, - currentStageId: executionState?.currentStageId ?? null, - currentStageType: executionState?.currentStageType ?? null, - reviewRecoveryInstruction: - "The previous reviewer run ended while this execution-review stage was still pending. Submit the review decision now, or mark the issue blocked with the exact unblock action.", - }, "normal_model"), + contextSnapshot: withRecoveryModelProfileHint( + { + issueId: issue.id, + taskId: issue.id, + wakeReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON, + retryReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON, + source: "issue.execution_review_recovery", + retryOfRunId: run.id, + currentStageId: executionState?.currentStageId ?? null, + currentStageType: executionState?.currentStageType ?? null, + reviewRecoveryInstruction: + "The previous reviewer run ended while this execution-review stage was still pending. Submit the review decision now, or mark the issue blocked with the exact unblock action.", + }, + "normal_model", + ), sessionIdBefore: recoverySessionBefore, retryOfRunId: run.id, updatedAt: now, @@ -18605,7 +22572,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) (issue.status === "todo" || issue.status === "in_progress") && !issue.assigneeUserId && issue.assigneeAgentId === run.agentId && - (run.status === "failed" || run.status === "timed_out" || run.status === "cancelled"); + (run.status === "failed" || + run.status === "timed_out" || + run.status === "cancelled"); if ( readNonEmptyString(parseObject(run.contextSnapshot).retryReason) === @@ -18622,11 +22591,22 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } const existingExecutionPath = await findExistingExecutionPath(); - if (existingExecutionPath || issueHasPersistedMonitor || await findExplicitBlockerPath()) { + if ( + existingExecutionPath || + issueHasPersistedMonitor || + (await findExplicitBlockerPath()) + ) { return { kind: "released" as const }; } - if (await isAutomaticRecoverySuppressedByPauseHold(db, issue.companyId, issue.id, treeControlSvc)) { + if ( + await isAutomaticRecoverySuppressedByPauseHold( + db, + issue.companyId, + issue.id, + treeControlSvc, + ) + ) { return { kind: "released" as const }; } @@ -18643,10 +22623,16 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) !recoveryAgent || isWorkspaceValidationFailedRun(run) || isConfigurationIncompleteFailedRun(run) || - didAutomaticRecoveryFail(run, issue.status === "todo" ? "assignment_recovery" : "issue_continuation_needed"); + didAutomaticRecoveryFail( + run, + issue.status === "todo" + ? "assignment_recovery" + : "issue_continuation_needed", + ); if (shouldBlockImmediately) { const workspaceValidationFailure = isWorkspaceValidationFailedRun(run); - const configurationIncompleteFailure = isConfigurationIncompleteFailedRun(run); + const configurationIncompleteFailure = + isConfigurationIncompleteFailedRun(run); const notice = workspaceValidationFailure ? buildWorkspaceValidationRecoveryNoticeSeed() : configurationIncompleteFailure @@ -18667,24 +22653,38 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) }; } - const retryReason = issue.status === "todo" ? "assignment_recovery" : "issue_continuation_needed"; - const recoveryReason = issue.status === "todo" ? "issue_assignment_recovery" : "issue_continuation_needed"; + const retryReason = + issue.status === "todo" + ? "assignment_recovery" + : "issue_continuation_needed"; + const recoveryReason = + issue.status === "todo" + ? "issue_assignment_recovery" + : "issue_continuation_needed"; const recoverySource = - issue.status === "todo" ? "issue.assignment_recovery" : "issue.continuation_recovery"; + issue.status === "todo" + ? "issue.assignment_recovery" + : "issue.continuation_recovery"; const now = new Date(); - const recoveryContextSnapshot = withRecoveryModelProfileHint({ - issueId: issue.id, - taskId: issue.id, - wakeReason: recoveryReason, - retryReason, - source: recoverySource, - retryOfRunId: run.id, - }, "normal_model"); + const recoveryContextSnapshot = withRecoveryModelProfileHint( + { + issueId: issue.id, + taskId: issue.id, + wakeReason: recoveryReason, + retryReason, + source: recoverySource, + retryOfRunId: run.id, + }, + "normal_model", + ); const responsibleUserId = await resolveResponsibleUserIdForRunSeed({ companyId: issue.companyId, contextSnapshot: recoveryContextSnapshot, issueContext: issue, - routineEnvContext: await getRoutineEnvForExecutionIssue(issue.companyId, issue), + routineEnvContext: await getRoutineEnvForExecutionIssue( + issue.companyId, + issue, + ), requestedByActorType: "system", requestedByActorId: null, source: "automation", @@ -18692,14 +22692,18 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) existingRunResponsibleUserId: run.responsibleUserId, }); if (!responsibleUserId) { - throw new HttpError(422, "Unable to resolve responsible user for recovery heartbeat run", { - code: "responsible_user_unresolved", - runId: run.id, - agentId: recoveryAgent.id, - companyId: issue.companyId, - issueId: issue.id, - wakeReason: recoveryReason, - }); + throw new HttpError( + 422, + "Unable to resolve responsible user for recovery heartbeat run", + { + code: "responsible_user_unresolved", + runId: run.id, + agentId: recoveryAgent.id, + companyId: issue.companyId, + issueId: issue.id, + wakeReason: recoveryReason, + }, + ); } const wakeupRequest = await tx .insert(agentWakeupRequests) @@ -18709,10 +22713,13 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) source: "automation", triggerDetail: "system", reason: recoveryReason, - payload: withRecoveryModelProfileHint({ - issueId: issue.id, - retryOfRunId: run.id, - }, "normal_model"), + payload: withRecoveryModelProfileHint( + { + issueId: issue.id, + retryOfRunId: run.id, + }, + "normal_model", + ), status: "queued", requestedByActorType: "system", requestedByActorId: null, @@ -18766,17 +22773,20 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) if (promotionResult?.kind === "blocked") { await recovery.escalateStrandedAssignedIssue({ issue: promotionResult.issue, - previousStatus: promotionResult.previousStatus as "todo" | "in_progress" | "in_review", + previousStatus: promotionResult.previousStatus as + "todo" | "in_progress" | "in_review", latestRun: run, notice: promotionResult.notice, recoveryCause: promotionResult.recoveryCause === WORKSPACE_VALIDATION_RECOVERY_CAUSE ? WORKSPACE_VALIDATION_RECOVERY_CAUSE - : promotionResult.recoveryCause === CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE + : promotionResult.recoveryCause === + CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE ? CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE - : promotionResult.recoveryCause === EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE + : promotionResult.recoveryCause === + EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE ? EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE - : undefined, + : undefined, }); return; } @@ -18784,7 +22794,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) if (promotionResult?.kind === "blocked_recovery_in_place") { await recovery.escalateStrandedRecoveryIssueInPlace({ issue: promotionResult.issue, - previousStatus: promotionResult.previousStatus as "todo" | "in_progress" | "in_review", + previousStatus: promotionResult.previousStatus as + "todo" | "in_progress" | "in_review", latestRun: run, }); return; @@ -18793,7 +22804,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const promotedRun = promotionResult?.run ?? null; if (!promotedRun) return; - if (promotionResult?.kind === "promoted" && promotionResult.reopenedActivity) { + if ( + promotionResult?.kind === "promoted" && + promotionResult.reopenedActivity + ) { await logActivity(db, promotionResult.reopenedActivity); } @@ -18815,7 +22829,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) async function enqueueWakeup(agentId: string, opts: WakeupOptions = {}) { const source = opts.source ?? "on_demand"; const triggerDetail = opts.triggerDetail ?? null; - const contextSnapshot: Record = { ...(opts.contextSnapshot ?? {}) }; + const contextSnapshot: Record = { + ...(opts.contextSnapshot ?? {}), + }; const reason = opts.reason ?? null; const payload = opts.payload ?? null; const { @@ -18830,11 +22846,27 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) triggerDetail, payload, }); - let issueId = readNonEmptyString(enrichedContextSnapshot.issueId) ?? issueIdFromPayload; + let issueId = + readNonEmptyString(enrichedContextSnapshot.issueId) ?? issueIdFromPayload; const agent = await getAgent(agentId); if (!agent) throw notFound("Agent not found"); + const agentDebug = parseObject(parseObject(agent.runtimeConfig).debug); + const runDebug = parseObject(enrichedContextSnapshot.debug); + if ( + agentDebug.providerTrace === "raw" + && runDebug.providerTrace !== "raw" + ) { + enrichedContextSnapshot.debug = { + ...runDebug, + providerTrace: "raw", + }; + enrichedContextSnapshot.providerTraceRequestedBy = + `agent:${agent.id}:debug-setting`; + enrichedContextSnapshot.providerTraceRequestSource = "agent_debug_setting"; + } + const writeSkippedRequest = async ( skipReason: string, patch: Partial = {}, @@ -18854,7 +22886,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) ...patch, }); }; - const writeSkippedHeartbeatRequest = async (skipReason: string, details: Record) => { + const writeSkippedHeartbeatRequest = async ( + skipReason: string, + details: Record, + ) => { await writeSkippedRequest(skipReason, { payload: { ...(payload ?? {}), @@ -18871,9 +22906,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) return null; } - const worktreeExecutionCutoff = opts.requestedByActorType === "user" - ? null - : await getWorktreeExecutionCutoff(); + const worktreeExecutionCutoff = + opts.requestedByActorType === "user" + ? null + : await getWorktreeExecutionCutoff(); const company = await db .select({ status: companies.status }) @@ -18892,29 +22928,47 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) return null; } - const explicitResumeSession = await resolveExplicitResumeSessionOverride(agent, payload, taskKey); + const explicitResumeSession = await resolveExplicitResumeSessionOverride( + agent, + payload, + taskKey, + ); if (explicitResumeSession) { - enrichedContextSnapshot.resumeFromRunId = explicitResumeSession.resumeFromRunId; - enrichedContextSnapshot.resumeSessionDisplayId = explicitResumeSession.sessionDisplayId; - enrichedContextSnapshot.resumeSessionParams = explicitResumeSession.sessionParams; - if (!readNonEmptyString(enrichedContextSnapshot.issueId) && explicitResumeSession.issueId) { + enrichedContextSnapshot.resumeFromRunId = + explicitResumeSession.resumeFromRunId; + enrichedContextSnapshot.resumeSessionDisplayId = + explicitResumeSession.sessionDisplayId; + enrichedContextSnapshot.resumeSessionParams = + explicitResumeSession.sessionParams; + if ( + !readNonEmptyString(enrichedContextSnapshot.issueId) && + explicitResumeSession.issueId + ) { enrichedContextSnapshot.issueId = explicitResumeSession.issueId; } - if (!readNonEmptyString(enrichedContextSnapshot.taskId) && explicitResumeSession.taskId) { + if ( + !readNonEmptyString(enrichedContextSnapshot.taskId) && + explicitResumeSession.taskId + ) { enrichedContextSnapshot.taskId = explicitResumeSession.taskId; } - if (!readNonEmptyString(enrichedContextSnapshot.taskKey) && explicitResumeSession.taskKey) { + if ( + !readNonEmptyString(enrichedContextSnapshot.taskKey) && + explicitResumeSession.taskKey + ) { enrichedContextSnapshot.taskKey = explicitResumeSession.taskKey; } issueId = readNonEmptyString(enrichedContextSnapshot.issueId) ?? issueId; } - const effectiveTaskKey = readNonEmptyString(enrichedContextSnapshot.taskKey) ?? taskKey; + const effectiveTaskKey = + readNonEmptyString(enrichedContextSnapshot.taskKey) ?? taskKey; const sessionBefore = explicitResumeSession?.sessionDisplayId ?? - await resolveSessionBeforeForWakeup(agent, effectiveTaskKey); + (await resolveSessionBeforeForWakeup(agent, effectiveTaskKey)); let hasResolvablePriorSessionWorkspace: boolean | null = null; const resolveHasResolvablePriorSessionWorkspace = async () => { - if (hasResolvablePriorSessionWorkspace !== null) return hasResolvablePriorSessionWorkspace; + if (hasResolvablePriorSessionWorkspace !== null) + return hasResolvablePriorSessionWorkspace; hasResolvablePriorSessionWorkspace = issueId ? await hasResolvablePriorSessionWorkspaceForWake({ agent, @@ -18925,7 +22979,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) : false; return hasResolvablePriorSessionWorkspace; }; - const continuationAttempt = readContinuationAttempt(enrichedContextSnapshot.livenessContinuationAttempt); + const continuationAttempt = readContinuationAttempt( + enrichedContextSnapshot.livenessContinuationAttempt, + ); let projectId = readNonEmptyString(enrichedContextSnapshot.projectId); if (!projectId && issueId) { @@ -18937,20 +22993,33 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) // evaluated. const lookupIsUuid = isUuidLike(issueId); const idMatch = lookupIsUuid - ? or(eq(issues.id, issueId), eq(issues.identifier, issueId.toUpperCase())) + ? or( + eq(issues.id, issueId), + eq(issues.identifier, issueId.toUpperCase()), + ) : eq(issues.identifier, issueId.toUpperCase()); const resolvedIssue = await db - .select({ id: issues.id, projectId: issues.projectId, createdAt: issues.createdAt }) + .select({ + id: issues.id, + projectId: issues.projectId, + createdAt: issues.createdAt, + }) .from(issues) .where(and(eq(issues.companyId, agent.companyId), idMatch)) .then((rows) => rows[0] ?? null); if (resolvedIssue) { - if (worktreeExecutionCutoff && resolvedIssue.createdAt < worktreeExecutionCutoff) { - await writeSkippedHeartbeatRequest("heartbeat.worktree_execution_cutoff", { - reason: "worktree_execution_cutoff", - cutoff: worktreeExecutionCutoff.toISOString(), - issueId: resolvedIssue.id, - }); + if ( + worktreeExecutionCutoff && + resolvedIssue.createdAt < worktreeExecutionCutoff + ) { + await writeSkippedHeartbeatRequest( + "heartbeat.worktree_execution_cutoff", + { + reason: "worktree_execution_cutoff", + cutoff: worktreeExecutionCutoff.toISOString(), + issueId: resolvedIssue.id, + }, + ); return null; } projectId = resolvedIssue.projectId ?? null; @@ -18975,38 +23044,54 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) let queuedResponsibleUserIdPromise: Promise | null = null; const resolveQueuedResponsibleUserId = () => { queuedResponsibleUserIdPromise ??= (async () => { - const queuedIssueContext = issueId ? await getIssueExecutionContext(agent.companyId, issueId) : null; - const queuedRoutineEnvContext = await getRoutineEnvForExecutionIssue(agent.companyId, queuedIssueContext); - const queuedResponsibleUserId = await resolveResponsibleUserIdForRunSeed({ - companyId: agent.companyId, - contextSnapshot: enrichedContextSnapshot, - issueContext: queuedIssueContext, - routineEnvContext: queuedRoutineEnvContext, - requestedByActorType: opts.requestedByActorType ?? null, - requestedByActorId: opts.requestedByActorId ?? null, - source, - triggerDetail, - }); - if (!queuedResponsibleUserId) { - throw new HttpError(422, "Unable to resolve responsible user for heartbeat run dispatch", { - code: "responsible_user_unresolved", - agentId, + const queuedIssueContext = issueId + ? await getIssueExecutionContext(agent.companyId, issueId) + : null; + const queuedRoutineEnvContext = await getRoutineEnvForExecutionIssue( + agent.companyId, + queuedIssueContext, + ); + const queuedResponsibleUserId = + await resolveResponsibleUserIdForRunSeed({ companyId: agent.companyId, - issueId: issueId ?? null, + contextSnapshot: enrichedContextSnapshot, + issueContext: queuedIssueContext, + routineEnvContext: queuedRoutineEnvContext, + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, source, triggerDetail, - wakeReason: readNonEmptyString(enrichedContextSnapshot.wakeReason), }); + if (!queuedResponsibleUserId) { + throw new HttpError( + 422, + "Unable to resolve responsible user for heartbeat run dispatch", + { + code: "responsible_user_unresolved", + agentId, + companyId: agent.companyId, + issueId: issueId ?? null, + source, + triggerDetail, + wakeReason: readNonEmptyString( + enrichedContextSnapshot.wakeReason, + ), + }, + ); } return queuedResponsibleUserId; })(); return queuedResponsibleUserIdPromise; }; - const budgetBlock = await budgets.getInvocationBlock(agent.companyId, agentId, { - issueId, - projectId, - }); + const budgetBlock = await budgets.getInvocationBlock( + agent.companyId, + agentId, + { + issueId, + projectId, + }, + ); if (budgetBlock) { await writeSkippedRequest("budget.blocked"); throw conflict(budgetBlock.reason, { @@ -19047,25 +23132,34 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) !wakeCommentId && !readNonEmptyString(enrichedContextSnapshot.taskId) && !readNonEmptyString(enrichedContextSnapshot.taskKey); - if (policy.skipTimerWhenNoActionableWork && genericTimerWake && !(await hasActionableTimerWork(agent))) { + if ( + policy.skipTimerWhenNoActionableWork && + genericTimerWake && + !(await hasActionableTimerWork(agent)) + ) { await writeSkippedHeartbeatRequest("heartbeat.timer.no_actionable_work", { - reason: "No assigned todo or in_progress issue requires this agent before timer adapter invocation.", + reason: + "No assigned todo or in_progress issue requires this agent before timer adapter invocation.", }); await markTimerHeartbeatChecked(agentId, source); return null; } if (issueId) { - const activePauseHold = await treeControlSvc.getActivePauseHoldGate(agent.companyId, issueId); + const activePauseHold = await treeControlSvc.getActivePauseHoldGate( + agent.companyId, + issueId, + ); if (activePauseHold) { - const treeHoldInteractionWake = await isVerifiedIssueTreeControlInteractionWake(db, { - companyId: agent.companyId, - issueId, - agentId, - contextSnapshot: enrichedContextSnapshot, - requestedByActorType: opts.requestedByActorType, - requestedByActorId: opts.requestedByActorId, - }); + const treeHoldInteractionWake = + await isVerifiedIssueTreeControlInteractionWake(db, { + companyId: agent.companyId, + issueId, + agentId, + contextSnapshot: enrichedContextSnapshot, + requestedByActorType: opts.requestedByActorType, + requestedByActorId: opts.requestedByActorId, + }); if (!treeHoldInteractionWake) { await writeSkippedRequest("issue_tree_hold_active"); @@ -19084,7 +23178,11 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) requestedReason: reason, source, triggerDetail, - securityPrinciples: ["Complete Mediation", "Fail Securely", "Secure Defaults"], + securityPrinciples: [ + "Complete Mediation", + "Fail Securely", + "Secure Defaults", + ], }, }); return null; @@ -19130,7 +23228,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) createdAt: issues.createdAt, }) .from(issues) - .where(and(eq(issues.id, issueId), eq(issues.companyId, agent.companyId))) + .where( + and(eq(issues.id, issueId), eq(issues.companyId, agent.companyId)), + ) .then((rows) => rows[0] ?? null); if (!issue) { @@ -19208,7 +23308,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) return { kind: "skipped" as const }; } - const cancelStaleScheduledRetry = async (scheduledRun: typeof heartbeatRuns.$inferSelect) => { + const cancelStaleScheduledRetry = async ( + scheduledRun: typeof heartbeatRuns.$inferSelect, + ) => { const issueCancelled = issue.status === "cancelled"; if ( scheduledRun.status !== "scheduled_retry" || @@ -19227,10 +23329,17 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) status: "cancelled", finishedAt: now, error: reason, - errorCode: issueCancelled ? "issue_cancelled" : "issue_reassigned", + errorCode: issueCancelled + ? "issue_cancelled" + : "issue_reassigned", updatedAt: now, }) - .where(and(eq(heartbeatRuns.id, scheduledRun.id), eq(heartbeatRuns.status, "scheduled_retry"))) + .where( + and( + eq(heartbeatRuns.id, scheduledRun.id), + eq(heartbeatRuns.status, "scheduled_retry"), + ), + ) .returning() .then((rows) => rows[0] ?? null); @@ -19257,7 +23366,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) executionLockedAt: null, updatedAt: now, }) - .where(and(eq(issues.id, issue.id), eq(issues.executionRunId, scheduledRun.id))); + .where( + and( + eq(issues.id, issue.id), + eq(issues.executionRunId, scheduledRun.id), + ), + ); } const eventSeq = await allocateHeartbeatRunEventSeq( @@ -19280,22 +23394,28 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) issueId: issue.id, issueStatus: issue.status, scheduledRetryAttempt: cancelled.scheduledRetryAttempt, - scheduledRetryAt: cancelled.scheduledRetryAt ? new Date(cancelled.scheduledRetryAt).toISOString() : null, + scheduledRetryAt: cancelled.scheduledRetryAt + ? new Date(cancelled.scheduledRetryAt).toISOString() + : null, scheduledRetryReason: cancelled.scheduledRetryReason, previousRetryAgentId: cancelled.agentId, currentAssigneeAgentId: issue.assigneeAgentId, }, }); + await tx + .update(heartbeatRuns) + .set({ nextEventSeq: eventSeq + 1, updatedAt: now }) + .where(eq(heartbeatRuns.id, cancelled.id)); return true; }; let activeExecutionRun = issue.executionRunId ? await tx - .select() - .from(heartbeatRuns) - .where(eq(heartbeatRuns.id, issue.executionRunId)) - .then((rows) => rows[0] ?? null) + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, issue.executionRunId)) + .then((rows) => rows[0] ?? null) : null; if ( @@ -19307,7 +23427,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) activeExecutionRun = null; } - if (activeExecutionRun && await cancelStaleScheduledRetry(activeExecutionRun)) { + if ( + activeExecutionRun && + (await cancelStaleScheduledRetry(activeExecutionRun)) + ) { activeExecutionRun = null; } @@ -19337,7 +23460,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .set({ status: "cancelled", finishedAt: new Date(), - error: "Execution lock released after issue reassigned to a different agent", + error: + "Execution lock released after issue reassigned to a different agent", errorCode: "lock_released_on_reassignment", updatedAt: new Date(), }) @@ -19355,10 +23479,16 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .set({ status: "cancelled", finishedAt: new Date(), - error: "Execution lock released after issue reassigned to a different agent", + error: + "Execution lock released after issue reassigned to a different agent", updatedAt: new Date(), }) - .where(eq(agentWakeupRequests.id, activeExecutionRun.wakeupRequestId)); + .where( + eq( + agentWakeupRequests.id, + activeExecutionRun.wakeupRequestId, + ), + ); } activeExecutionRun = null; } @@ -19383,7 +23513,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .where( and( eq(heartbeatRuns.companyId, issue.companyId), - inArray(heartbeatRuns.status, [...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES]), + inArray(heartbeatRuns.status, [ + ...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES, + ]), sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issue.id}`, ), ) @@ -19408,7 +23540,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .update(issues) .set({ executionRunId: legacyRun.id, - executionAgentNameKey: normalizeAgentNameKey(legacyAgent?.name), + executionAgentNameKey: normalizeAgentNameKey( + legacyAgent?.name, + ), executionLockedAt: new Date(), updatedAt: new Date(), }) @@ -19417,11 +23551,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } } - const dependencyReadiness = await issuesSvc.listDependencyReadiness( - issue.companyId, - [issue.id], - tx, - ).then((rows) => rows.get(issue.id) ?? null); + const dependencyReadiness = await issuesSvc + .listDependencyReadiness(issue.companyId, [issue.id], tx) + .then((rows) => rows.get(issue.id) ?? null); // Blocked descendants should stay idle until the final blocker resolves. // Human comment/mention wakes are the exception: they may run in a @@ -19433,17 +23565,25 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) if (blockedInteractionWake) { enrichedContextSnapshot.dependencyBlockedInteraction = true; - enrichedContextSnapshot.unresolvedBlockerIssueIds = dependencyReadiness.unresolvedBlockerIssueIds; - enrichedContextSnapshot.unresolvedBlockerCount = dependencyReadiness.unresolvedBlockerCount; - enrichedContextSnapshot.unresolvedBlockerSummaries = await listUnresolvedBlockerSummaries( - tx, - issue.companyId, - issue.id, - dependencyReadiness.unresolvedBlockerIssueIds, - ); + enrichedContextSnapshot.unresolvedBlockerIssueIds = + dependencyReadiness.unresolvedBlockerIssueIds; + enrichedContextSnapshot.unresolvedBlockerCount = + dependencyReadiness.unresolvedBlockerCount; + enrichedContextSnapshot.unresolvedBlockerSummaries = + await listUnresolvedBlockerSummaries( + tx, + issue.companyId, + issue.id, + dependencyReadiness.unresolvedBlockerIssueIds, + ); } - if (!activeExecutionRun && dependencyReadiness && !dependencyReadiness.isDependencyReady && !blockedInteractionWake) { + if ( + !activeExecutionRun && + dependencyReadiness && + !dependencyReadiness.isDependencyReady && + !blockedInteractionWake + ) { await tx.insert(agentWakeupRequests).values({ companyId: agent.companyId, agentId, @@ -19453,7 +23593,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) payload: { ...(payload ?? {}), issueId, - unresolvedBlockerIssueIds: dependencyReadiness.unresolvedBlockerIssueIds, + unresolvedBlockerIssueIds: + dependencyReadiness.unresolvedBlockerIssueIds, }, status: "skipped", requestedByActorType: opts.requestedByActorType ?? null, @@ -19464,8 +23605,15 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) return { kind: "skipped" as const }; } - if (isolatedWorkspacesEnabled && !activeExecutionRun && issue.status !== "done" && issue.status !== "cancelled") { - const issueSettings = parseIssueExecutionWorkspaceSettings(issue.executionWorkspaceSettings); + if ( + isolatedWorkspacesEnabled && + !activeExecutionRun && + issue.status !== "done" && + issue.status !== "cancelled" + ) { + const issueSettings = parseIssueExecutionWorkspaceSettings( + issue.executionWorkspaceSettings, + ); const resolvedMode = resolveExecutionWorkspaceMode({ projectPolicy: null, issueSettings, @@ -19478,23 +23626,30 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) mode: resolvedMode, legacyUseProjectWorkspace: null, }); - const resolvedStrategy = resolveEffectiveWorkspaceStrategyType(resolvedMode, workspaceManagedConfig); + const resolvedStrategy = resolveEffectiveWorkspaceStrategyType( + resolvedMode, + workspaceManagedConfig, + ); const existingExecutionWorkspaceStatus = issue.executionWorkspaceId ? await tx - .select({ status: executionWorkspaces.status }) - .from(executionWorkspaces) - .where(and( - eq(executionWorkspaces.id, issue.executionWorkspaceId), - eq(executionWorkspaces.companyId, issue.companyId), - )) - .then((rows) => rows[0]?.status ?? null) + .select({ status: executionWorkspaces.status }) + .from(executionWorkspaces) + .where( + and( + eq(executionWorkspaces.id, issue.executionWorkspaceId), + eq(executionWorkspaces.companyId, issue.companyId), + ), + ) + .then((rows) => rows[0]?.status ?? null) : null; const reuseRequest = resolveExecutionWorkspaceReuseRequestForIssue({ issueExecutionWorkspaceId: issue.executionWorkspaceId, - issueExecutionWorkspacePreference: issue.executionWorkspacePreference, + issueExecutionWorkspacePreference: + issue.executionWorkspacePreference, existingExecutionWorkspaceStatus, }); - const hasResolvablePriorSessionWorkspace = await resolveHasResolvablePriorSessionWorkspace(); + const hasResolvablePriorSessionWorkspace = + await resolveHasResolvablePriorSessionWorkspace(); if ( isUnrunnableWorktreeCombo({ @@ -19502,16 +23657,21 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) projectId: issue.projectId ?? projectId ?? null, projectWorkspaceId: issue.projectWorkspaceId, executionWorkspaceId: issue.executionWorkspaceId, - executionWorkspacePreference: issue.executionWorkspacePreference, + executionWorkspacePreference: + issue.executionWorkspacePreference, }, resolvedMode, resolvedStrategy, - reusableExecutionWorkspaceAvailable: reuseRequest.existingExecutionWorkspaceAvailable, + reusableExecutionWorkspaceAvailable: + reuseRequest.existingExecutionWorkspaceAvailable, hasResolvablePriorSessionWorkspace, }) ) { const now = new Date(); - const issueLabel = formatIssueIdentifierLink(issue.identifier, issue.id); + const issueLabel = formatIssueIdentifierLink( + issue.identifier, + issue.id, + ); const blockedComment = [ `Paperclip blocked ${issueLabel} before dispatch because its workspace settings are not runnable.`, "", @@ -19593,15 +23753,20 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) normalizeAgentNameKey(issue.executionAgentNameKey) ?? normalizeAgentNameKey(executionAgent?.name); const isSameExecutionAgent = - Boolean(executionAgentNameKey) && executionAgentNameKey === agentNameKey; + Boolean(executionAgentNameKey) && + executionAgentNameKey === agentNameKey; const shouldDeferFollowupWake = shouldDeferFollowupWakeForSameIssue({ activeRunStatus: activeExecutionRun.status, isSameExecutionAgent, wakeCommentId, - forceFreshSession: enrichedContextSnapshot.forceFreshSession === true, + forceFreshSession: + enrichedContextSnapshot.forceFreshSession === true, }); const shouldQueueFollowupForRunningWake = - shouldQueueFollowupForRunningIssueWake({ contextSnapshot: enrichedContextSnapshot, wakeCommentId }) && + shouldQueueFollowupForRunningIssueWake({ + contextSnapshot: enrichedContextSnapshot, + wakeCommentId, + }) && activeExecutionRun.status === "running" && isSameExecutionAgent; const availableActiveExecutionRun = isSameExecutionAgent @@ -19609,10 +23774,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) : activeExecutionRun; if ( - isSameExecutionAgent - && !shouldDeferFollowupWake - && !shouldQueueFollowupForRunningWake - && availableActiveExecutionRun + isSameExecutionAgent && + !shouldDeferFollowupWake && + !shouldQueueFollowupForRunningWake && + availableActiveExecutionRun ) { const mergedContextSnapshot = mergeCoalescedContextSnapshot( availableActiveExecutionRun.contextSnapshot, @@ -19675,8 +23840,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .then((rows) => rows[0] ?? null); if (existingDeferred) { - const existingDeferredPayload = parseObject(existingDeferred.payload); - const existingDeferredContext = parseObject(existingDeferredPayload[DEFERRED_WAKE_CONTEXT_KEY]); + const existingDeferredPayload = parseObject( + existingDeferred.payload, + ); + const existingDeferredContext = parseObject( + existingDeferredPayload[DEFERRED_WAKE_CONTEXT_KEY], + ); const mergedDeferredContext = mergeCoalescedContextSnapshot( existingDeferredContext, enrichedContextSnapshot, @@ -19730,7 +23899,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) reason, wakeCommentId: wakeCommentId ?? null, requestedByActorType: opts.requestedByActorType ?? null, - forceFreshSession: enrichedContextSnapshot.forceFreshSession === true, + forceFreshSession: + enrichedContextSnapshot.forceFreshSession === true, hasExplicitResume: Boolean(explicitResumeSession), }) ) { @@ -19747,7 +23917,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) eq(heartbeatRuns.companyId, agent.companyId), eq(heartbeatRuns.agentId, agentId), sql`${heartbeatRuns.finishedAt} is not null`, - gte(heartbeatRuns.finishedAt, new Date(throttleNow.getTime() - ISSUE_REWAKE_LOOKBACK_MS)), + gte( + heartbeatRuns.finishedAt, + new Date(throttleNow.getTime() - ISSUE_REWAKE_LOOKBACK_MS), + ), sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issue.id}`, ), ) @@ -19755,7 +23928,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .limit(ISSUE_REWAKE_RUN_SAMPLE_LIMIT); if (recentTerminalRuns.length > 0) { - const sampleRunIds = recentTerminalRuns.map((sampleRun) => sampleRun.id); + const sampleRunIds = recentTerminalRuns.map( + (sampleRun) => sampleRun.id, + ); const progressRows = await tx .select({ runId: activityLog.runId }) .from(activityLog) @@ -19771,21 +23946,24 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const lastRunFinishedAt = recentTerminalRuns[0]?.finishedAt ?? null; const newInputRows = lastRunFinishedAt ? await tx - .select({ id: activityLog.id }) - .from(activityLog) - .where( - and( - eq(activityLog.companyId, agent.companyId), - eq(activityLog.entityType, "issue"), - eq(activityLog.entityId, issue.id), - gt(activityLog.createdAt, lastRunFinishedAt), - inArray(activityLog.action, ISSUE_NEW_INPUT_ACTIVITY_ACTIONS), - wakeCommentId && opts.requestedByActorType === "agent" - ? ne(activityLog.actorType, "agent") - : undefined, - ), - ) - .limit(1) + .select({ id: activityLog.id }) + .from(activityLog) + .where( + and( + eq(activityLog.companyId, agent.companyId), + eq(activityLog.entityType, "issue"), + eq(activityLog.entityId, issue.id), + gt(activityLog.createdAt, lastRunFinishedAt), + inArray( + activityLog.action, + ISSUE_NEW_INPUT_ACTIVITY_ACTIONS, + ), + wakeCommentId && opts.requestedByActorType === "agent" + ? ne(activityLog.actorType, "agent") + : undefined, + ), + ) + .limit(1) : []; const throttleDecision = evaluateIssueRewakeThrottle({ @@ -19818,7 +23996,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) requestedReason: reason, noProgressStreak: throttleDecision.noProgressStreak, cooldownMs: throttleDecision.cooldownMs, - lastRunFinishedAt: throttleDecision.lastRunFinishedAt.toISOString(), + lastRunFinishedAt: + throttleDecision.lastRunFinishedAt.toISOString(), nextAllowedAt: throttleDecision.nextAllowedAt.toISOString(), }, }, @@ -19833,7 +24012,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } } - const dailyCapBlock = await getHeartbeatDailyCapBlock(agent, policy, {}, tx); + const dailyCapBlock = await getHeartbeatDailyCapBlock( + agent, + policy, + {}, + tx, + ); if (dailyCapBlock) { const now = new Date(); await tx.insert(agentWakeupRequests).values({ @@ -19845,7 +24029,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) payload: { ...(payload ?? {}), heartbeatSkip: { - reason: "Per-agent heartbeat daily cap reached before adapter invocation.", + reason: + "Per-agent heartbeat daily cap reached before adapter invocation.", observed: dailyCapBlock.observed, limit: dailyCapBlock.limit, }, @@ -19917,7 +24102,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) return { kind: "queued" as const, run: newRun }; }); - if (outcome.kind === "deferred" || outcome.kind === "skipped") return null; + if (outcome.kind === "deferred" || outcome.kind === "skipped") + return null; if (outcome.kind === "coalesced") { await startNextQueuedRunForAgent(agent.id); return outcome.run; @@ -19943,26 +24129,44 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const activeRuns = await db .select() .from(heartbeatRuns) - .where(and(eq(heartbeatRuns.agentId, agentId), inArray(heartbeatRuns.status, [...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES]))) + .where( + and( + eq(heartbeatRuns.agentId, agentId), + inArray(heartbeatRuns.status, [ + ...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES, + ]), + ), + ) .orderBy(desc(heartbeatRuns.createdAt)); const sameScopeQueuedRun = activeRuns.find( - (candidate) => candidate.status === "queued" && isSameTaskScope(runTaskKey(candidate), taskKey), + (candidate) => + candidate.status === "queued" && + isSameTaskScope(runTaskKey(candidate), taskKey), ); const sameScopeScheduledRetryRun = activeRuns.find( - (candidate) => candidate.status === "scheduled_retry" && isSameTaskScope(runTaskKey(candidate), taskKey), + (candidate) => + candidate.status === "scheduled_retry" && + isSameTaskScope(runTaskKey(candidate), taskKey), ); const sameScopeRunningRun = activeRuns.find( - (candidate) => candidate.status === "running" && isSameTaskScope(runTaskKey(candidate), taskKey), + (candidate) => + candidate.status === "running" && + isSameTaskScope(runTaskKey(candidate), taskKey), ); const shouldQueueFollowupForRunningWake = Boolean(sameScopeRunningRun) && !sameScopeQueuedRun && - shouldQueueFollowupForRunningIssueWake({ contextSnapshot: enrichedContextSnapshot, wakeCommentId }); + shouldQueueFollowupForRunningIssueWake({ + contextSnapshot: enrichedContextSnapshot, + wakeCommentId, + }); const rawCoalescedTarget = sameScopeQueuedRun ?? sameScopeScheduledRetryRun ?? - (shouldQueueFollowupForRunningWake ? null : sameScopeRunningRun ?? null); + (shouldQueueFollowupForRunningWake + ? null + : (sameScopeRunningRun ?? null)); const coalescedTargetRun = filterZombieCoalesceTarget( rawCoalescedTarget, @@ -20012,7 +24216,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) sql`select id from agents where id = ${agentId} and company_id = ${agent.companyId} for update`, ); - const dailyCapBlock = await getHeartbeatDailyCapBlock(agent, policy, {}, tx); + const dailyCapBlock = await getHeartbeatDailyCapBlock( + agent, + policy, + {}, + tx, + ); if (dailyCapBlock) { const now = new Date(); await tx.insert(agentWakeupRequests).values({ @@ -20024,7 +24233,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) payload: { ...(payload ?? {}), heartbeatSkip: { - reason: "Per-agent heartbeat daily cap reached before adapter invocation.", + reason: + "Per-agent heartbeat daily cap reached before adapter invocation.", observed: dailyCapBlock.observed, limit: dailyCapBlock.limit, }, @@ -20113,10 +24323,17 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } /** - * Native status commitment persists dependency/parent wake intents in the - * same transaction as the authoritative status projection. Bridge those - * durable intents through the ordinary heartbeat scheduler so its policy, - * workspace, concurrency, and responsible-user checks remain authoritative. + * Native status commitment deliberately persists dependency/parent wake + * intents in the same transaction as the authoritative status projection. + * Those rows are not runnable until the heartbeat scheduler has applied its + * normal policy, workspace, concurrency, and responsible-user checks. Bridge + * the durable intent into that scheduler here instead of treating a bare + * `agent_wakeup_requests` row as if it were already a queued heartbeat run. + * + * The intent is claimed before dispatch. A deterministic dispatcher actor id + * lets a later sweep recover the narrow process-crash window after the real + * wake was inserted but before the intent was linked to it. Dispatch failure + * only requeues the intent; it never changes the provider run outcome. */ async function dispatchPendingNativeStatusWakeups(input: { companyId?: string; @@ -20325,8 +24542,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } async function listProjectScopedRunIds(companyId: string, projectId: string) { - const runIssueId = sql`${heartbeatRuns.contextSnapshot} ->> 'issueId'`; - const effectiveProjectId = sql`coalesce(${heartbeatRuns.contextSnapshot} ->> 'projectId', ${issues.projectId}::text)`; + const runIssueId = sql< + string | null + >`${heartbeatRuns.contextSnapshot} ->> 'issueId'`; + const effectiveProjectId = sql< + string | null + >`coalesce(${heartbeatRuns.contextSnapshot} ->> 'projectId', ${issues.projectId}::text)`; const rows = await db .selectDistinctOn([heartbeatRuns.id], { id: heartbeatRuns.id }) @@ -20341,7 +24562,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .where( and( eq(heartbeatRuns.companyId, companyId), - inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]), + inArray(heartbeatRuns.status, [ + ...CANCELLABLE_HEARTBEAT_RUN_STATUSES, + ]), sql`${effectiveProjectId} = ${projectId}`, ), ); @@ -20349,12 +24572,21 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) return rows.map((row) => row.id); } - async function listProjectScopedWakeupIds(companyId: string, projectId: string) { - const wakeIssueId = sql`${agentWakeupRequests.payload} ->> 'issueId'`; - const effectiveProjectId = sql`coalesce(${agentWakeupRequests.payload} ->> 'projectId', ${issues.projectId}::text)`; + async function listProjectScopedWakeupIds( + companyId: string, + projectId: string, + ) { + const wakeIssueId = sql< + string | null + >`${agentWakeupRequests.payload} ->> 'issueId'`; + const effectiveProjectId = sql< + string | null + >`coalesce(${agentWakeupRequests.payload} ->> 'projectId', ${issues.projectId}::text)`; const rows = await db - .selectDistinctOn([agentWakeupRequests.id], { id: agentWakeupRequests.id }) + .selectDistinctOn([agentWakeupRequests.id], { + id: agentWakeupRequests.id, + }) .from(agentWakeupRequests) .leftJoin( issues, @@ -20366,7 +24598,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .where( and( eq(agentWakeupRequests.companyId, companyId), - inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]), + inArray(agentWakeupRequests.status, [ + "queued", + "deferred_issue_execution", + ]), sql`${agentWakeupRequests.runId} is null`, sql`${effectiveProjectId} = ${projectId}`, ), @@ -20375,7 +24610,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) return rows.map((row) => row.id); } - async function cancelPendingWakeupsForBudgetScope(scope: BudgetEnforcementScope) { + async function cancelPendingWakeupsForBudgetScope( + scope: BudgetEnforcementScope, + ) { const now = new Date(); let wakeupIds: string[] = []; @@ -20386,7 +24623,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .where( and( eq(agentWakeupRequests.companyId, scope.companyId), - inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]), + inArray(agentWakeupRequests.status, [ + "queued", + "deferred_issue_execution", + ]), sql`${agentWakeupRequests.runId} is null`, ), ) @@ -20399,13 +24639,19 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) and( eq(agentWakeupRequests.companyId, scope.companyId), eq(agentWakeupRequests.agentId, scope.scopeId), - inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]), + inArray(agentWakeupRequests.status, [ + "queued", + "deferred_issue_execution", + ]), sql`${agentWakeupRequests.runId} is null`, ), ) .then((rows) => rows.map((row) => row.id)); } else { - wakeupIds = await listProjectScopedWakeupIds(scope.companyId, scope.scopeId); + wakeupIds = await listProjectScopedWakeupIds( + scope.companyId, + scope.scopeId, + ); } if (wakeupIds.length === 0) return 0; @@ -20430,10 +24676,19 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) eventPayload?: Record; }; - async function cancelRunInternal(runId: string, reason = "Cancelled by control plane", options: CancelRunOptions = {}) { + async function cancelRunInternal( + runId: string, + reason = "Cancelled by control plane", + options: CancelRunOptions = {}, + ) { const run = await getRun(runId); if (!run) throw notFound("Heartbeat run not found"); - if (!CANCELLABLE_HEARTBEAT_RUN_STATUSES.includes(run.status as (typeof CANCELLABLE_HEARTBEAT_RUN_STATUSES)[number])) return run; + if ( + !CANCELLABLE_HEARTBEAT_RUN_STATUSES.includes( + run.status as (typeof CANCELLABLE_HEARTBEAT_RUN_STATUSES)[number], + ) + ) + return run; const agent = await getAgent(run.agentId); const errorCode = options.errorCode ?? "cancelled"; const resultJson = agent @@ -20449,6 +24704,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const running = runningProcesses.get(run.id); try { + await cancelHeartbeatNativeRun({ + db, + runId: run.id, + reason, + runtimeMode: run.runtimeMode, + }); if (running) { await terminateHeartbeatRunProcess({ pid: running.child.pid, @@ -20461,11 +24722,37 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } const finishedAt = new Date(); + const persistedCancellationResult = + run.runtimeMode === "native" + ? await getRun(run.id).then((current) => + parseObject(current?.resultJson), + ) + : {}; const cancelled = await setRunStatus(run.id, "cancelled", { finishedAt, error: reason, errorCode, - ...(resultJson ? { resultJson } : {}), + ...(resultJson || Object.keys(persistedCancellationResult).length > 0 + ? { + resultJson: { + ...persistedCancellationResult, + ...(resultJson ?? {}), + // The native cancellation helper may have advanced a durable + // pending intent to its acknowledged state after `run` was + // first read. Never let that stale snapshot overwrite the + // authoritative post-dispatch acknowledgement. + ...(Object.hasOwn( + persistedCancellationResult, + "nativeCancellation", + ) + ? { + nativeCancellation: + persistedCancellationResult.nativeCancellation, + } + : {}), + }, + } + : {}), }); await setWakeupStatus(run.wakeupRequestId, "cancelled", { @@ -20491,25 +24778,52 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) return cancelled; } - async function cancelActiveForAgentInternal(agentId: string, reason = "Cancelled due to agent pause", errorCode = "cancelled") { + async function cancelActiveForAgentInternal( + agentId: string, + reason = "Cancelled due to agent pause", + errorCode = "cancelled", + ) { const agent = await getAgent(agentId); const runs = await db .select() .from(heartbeatRuns) - .where(and(eq(heartbeatRuns.agentId, agentId), inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]))); + .where( + and( + eq(heartbeatRuns.agentId, agentId), + inArray(heartbeatRuns.status, [ + ...CANCELLABLE_HEARTBEAT_RUN_STATUSES, + ]), + ), + ); for (const run of runs) { + if (run.runtimeMode === "native") { + await cancelHeartbeatNativeRun({ + db, + runId: run.id, + reason, + runtimeMode: run.runtimeMode, + }); + } + const persistedCancellationResult = + run.runtimeMode === "native" + ? await getRun(run.id).then((current) => + parseObject(current?.resultJson), + ) + : parseObject(run.resultJson); await setRunStatus(run.id, "cancelled", { finishedAt: new Date(), error: reason, errorCode, - ...(agent ? { - resultJson: mergeRunStopMetadataForAgent(agent, "cancelled", { - resultJson: parseObject(run.resultJson), - errorCode, - errorMessage: reason, - }), - } : {}), + ...(agent + ? { + resultJson: mergeRunStopMetadataForAgent(agent, "cancelled", { + resultJson: persistedCancellationResult, + errorCode, + errorMessage: reason, + }), + } + : {}), }); await setWakeupStatus(run.wakeupRequestId, "cancelled", { @@ -20524,16 +24838,21 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) processGroupId: running.processGroupId, graceMs: Math.max(1, running.graceSec) * 1000, }); - runningProcesses.delete(run.id); } + runningProcesses.delete(run.id); await releaseIssueExecutionAndPromote(run); } return runs.length; } - async function cancelPendingWakeupsForAgentsInternal(agentIds: string[], reason: string) { - const uniqueAgentIds = [...new Set(agentIds)].filter((agentId) => agentId.length > 0); + async function cancelPendingWakeupsForAgentsInternal( + agentIds: string[], + reason: string, + ) { + const uniqueAgentIds = [...new Set(agentIds)].filter( + (agentId) => agentId.length > 0, + ); if (uniqueAgentIds.length === 0) return 0; const now = new Date(); @@ -20543,7 +24862,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .where( and( inArray(agentWakeupRequests.agentId, uniqueAgentIds), - inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]), + inArray(agentWakeupRequests.status, [ + "queued", + "deferred_issue_execution", + ]), sql`${agentWakeupRequests.runId} is null`, ), ) @@ -20564,13 +24886,21 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) return wakeupIds.length; } - async function cancelInvocationsForAgentsInternal(agentIds: string[], reason: string) { - const uniqueAgentIds = [...new Set(agentIds)].filter((agentId) => agentId.length > 0); + async function cancelInvocationsForAgentsInternal( + agentIds: string[], + reason: string, + ) { + const uniqueAgentIds = [...new Set(agentIds)].filter( + (agentId) => agentId.length > 0, + ); let runsCancelled = 0; for (const agentId of uniqueAgentIds) { runsCancelled += await cancelActiveForAgentInternal(agentId, reason); } - const wakeupsCancelled = await cancelPendingWakeupsForAgentsInternal(uniqueAgentIds, reason); + const wakeupsCancelled = await cancelPendingWakeupsForAgentsInternal( + uniqueAgentIds, + reason, + ); return { agentIds: uniqueAgentIds, runsCancelled, @@ -20580,7 +24910,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) async function cancelBudgetScopeWork(scope: BudgetEnforcementScope) { if (scope.scopeType === "agent") { - await cancelActiveForAgentInternal(scope.scopeId, "Cancelled due to budget pause"); + await cancelActiveForAgentInternal( + scope.scopeId, + "Cancelled due to budget pause", + ); await cancelPendingWakeupsForBudgetScope(scope); return; } @@ -20588,15 +24921,17 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const runIds = scope.scopeType === "company" ? await db - .select({ id: heartbeatRuns.id }) - .from(heartbeatRuns) - .where( - and( - eq(heartbeatRuns.companyId, scope.companyId), - inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]), - ), - ) - .then((rows) => rows.map((row) => row.id)) + .select({ id: heartbeatRuns.id }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, scope.companyId), + inArray(heartbeatRuns.status, [ + ...CANCELLABLE_HEARTBEAT_RUN_STATUSES, + ]), + ), + ) + .then((rows) => rows.map((row) => row.id)) : await listProjectScopedRunIds(scope.companyId, scope.scopeId); for (const runId of runIds) { @@ -20617,7 +24952,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) while (liveRunExecutions.has(runId)) { if (Date.now() >= deadline) { - throw new Error(`Timed out waiting for heartbeat run ${runId} execution to drain`); + throw new Error( + `Timed out waiting for heartbeat run ${runId} execution to drain`, + ); } await new Promise((resolve) => setTimeout(resolve, intervalMs)); } @@ -20638,21 +24975,24 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) ...heartbeatRunListContextColumns, } : safeForLegacyEncoding - ? { - ...heartbeatRunListColumns, - error: sql`NULL`.as("error"), - ...heartbeatRunListContextColumns, - } - : { - ...heartbeatRunListColumns, - ...heartbeatRunListContextColumns, - ...heartbeatRunListResultColumns, - }, + ? { + ...heartbeatRunListColumns, + error: sql`NULL`.as("error"), + ...heartbeatRunListContextColumns, + } + : { + ...heartbeatRunListColumns, + ...heartbeatRunListContextColumns, + ...heartbeatRunListResultColumns, + }, ) .from(heartbeatRuns) .where( agentId - ? and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.agentId, agentId)) + ? and( + eq(heartbeatRuns.companyId, companyId), + eq(heartbeatRuns.agentId, agentId), + ) : eq(heartbeatRuns.companyId, companyId), ) .orderBy(desc(heartbeatRuns.createdAt)); @@ -20698,17 +25038,18 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) wakeSource: contextWakeSource, wakeTriggerDetail: contextWakeTriggerDetail, }), - resultJson: safeForLegacyEncoding || summary - ? null - : summarizeHeartbeatRunListResultJson({ - summary: resultSummary, - result: resultResult, - message: resultMessage, - error: resultError, - totalCostUsd: resultTotalCostUsd, - costUsd: resultCostUsd, - costUsdCamel: resultCostUsdCamel, - }), + resultJson: + safeForLegacyEncoding || summary + ? null + : summarizeHeartbeatRunListResultJson({ + summary: resultSummary, + result: resultResult, + message: resultMessage, + error: resultError, + totalCostUsd: resultTotalCostUsd, + costUsd: resultCostUsd, + costUsdCamel: resultCostUsdCamel, + }), }; }); }, @@ -20729,13 +25070,19 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const latestTaskSession = await db .select() .from(agentTaskSessions) - .where(and(eq(agentTaskSessions.companyId, agent.companyId), eq(agentTaskSessions.agentId, agent.id))) + .where( + and( + eq(agentTaskSessions.companyId, agent.companyId), + eq(agentTaskSessions.agentId, agent.id), + ), + ) .orderBy(desc(agentTaskSessions.updatedAt)) .limit(1) .then((rows) => rows[0] ?? null); return { ...ensured, - sessionDisplayId: latestTaskSession?.sessionDisplayId ?? ensured.sessionId, + sessionDisplayId: + latestTaskSession?.sessionDisplayId ?? ensured.sessionId, sessionParamsJson: latestTaskSession?.sessionParamsJson ?? null, }; }, @@ -20747,11 +25094,22 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) return db .select() .from(agentTaskSessions) - .where(and(eq(agentTaskSessions.companyId, agent.companyId), eq(agentTaskSessions.agentId, agentId))) - .orderBy(desc(agentTaskSessions.updatedAt), desc(agentTaskSessions.createdAt)); + .where( + and( + eq(agentTaskSessions.companyId, agent.companyId), + eq(agentTaskSessions.agentId, agentId), + ), + ) + .orderBy( + desc(agentTaskSessions.updatedAt), + desc(agentTaskSessions.createdAt), + ); }, - resetRuntimeSession: async (agentId: string, opts?: { taskKey?: string | null }) => { + resetRuntimeSession: async ( + agentId: string, + opts?: { taskKey?: string | null }, + ) => { const agent = await getAgent(agentId); if (!agent) throw notFound("Agent not found"); await ensureRuntimeState(agent); @@ -20790,7 +25148,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) db .select() .from(heartbeatRunEvents) - .where(and(eq(heartbeatRunEvents.runId, runId), gt(heartbeatRunEvents.seq, afterSeq))) + .where( + and( + eq(heartbeatRunEvents.runId, runId), + gt(heartbeatRunEvents.seq, afterSeq), + ), + ) .orderBy(asc(heartbeatRunEvents.seq)) .limit(Math.max(1, Math.min(limit, 1000))), @@ -20814,16 +25177,22 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) }, readLog: async ( - runOrLookup: string | { - id: string; - companyId: string; - logStore: string | null; - logRef: string | null; - }, + runOrLookup: + | string + | { + id: string; + companyId: string; + logStore: string | null; + logRef: string | null; + }, opts?: { offset?: number; limitBytes?: number }, ) => { - const run = typeof runOrLookup === "string" ? await getRunLogAccess(runOrLookup) : runOrLookup; - const runId = typeof runOrLookup === "string" ? runOrLookup : runOrLookup.id; + const run = + typeof runOrLookup === "string" + ? await getRunLogAccess(runOrLookup) + : runOrLookup; + const runId = + typeof runOrLookup === "string" ? runOrLookup : runOrLookup.id; if (!run) throw notFound("Heartbeat run not found"); if (!run.logStore || !run.logRef) throw notFound("Run log not found"); @@ -20851,7 +25220,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) source: "timer" | "assignment" | "on_demand" | "automation" = "on_demand", contextSnapshot: Record = {}, triggerDetail: "manual" | "ping" | "callback" | "system" = "manual", - actor?: { actorType?: "user" | "agent" | "system"; actorId?: string | null }, + actor?: { + actorType?: "user" | "agent" | "system"; + actorId?: string | null; + }, ) => trackWakeup(agentId, { source, @@ -20941,13 +25313,18 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .from(agents) .innerJoin(companies, eq(companies.id, agents.companyId)) .where(eq(companies.status, "active")); - const agentsByCompany = groupAgentOrgRowsByCompany(allAgents.map(toAgentOrgRow)); + const agentsByCompany = groupAgentOrgRowsByCompany( + allAgents.map(toAgentOrgRow), + ); let checked = 0; let enqueued = 0; let skipped = 0; for (const agent of allAgents) { - const invokability = evaluateAgentInvokability(toAgentOrgRow(agent), agentsByCompany.get(agent.companyId) ?? []); + const invokability = evaluateAgentInvokability( + toAgentOrgRow(agent), + agentsByCompany.get(agent.companyId) ?? [], + ); if (!invokability.invokable) continue; const policy = parseHeartbeatPolicy(agent); if (!policy.enabled || policy.intervalSec <= 0) continue; @@ -20956,22 +25333,30 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const eligibleIssue = await db .select({ id: issues.id }) .from(issues) - .where(and( - eq(issues.companyId, agent.companyId), - eq(issues.assigneeAgentId, agent.id), - inArray(issues.status, ["todo", "in_progress"]), - gte(issues.createdAt, cutoff), - )) + .where( + and( + eq(issues.companyId, agent.companyId), + eq(issues.assigneeAgentId, agent.id), + inArray(issues.status, ["todo", "in_progress"]), + gte(issues.createdAt, cutoff), + ), + ) .limit(1) .then((rows) => rows[0] ?? null); if (!eligibleIssue) continue; } checked += 1; - const baseline = new Date(agent.lastHeartbeatAt ?? agent.createdAt).getTime(); + const baseline = new Date( + agent.lastHeartbeatAt ?? agent.createdAt, + ).getTime(); const elapsedMs = now.getTime() - baseline; if (elapsedMs < policy.intervalSec * 1000) continue; - const timerClaim = await claimDueTimerHeartbeat(agent, now, policy.intervalSec); + const timerClaim = await claimDueTimerHeartbeat( + agent, + now, + policy.intervalSec, + ); if (!timerClaim) continue; const run = await enqueueWakeup(agent.id, { @@ -21000,14 +25385,16 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) }; }, - cancelRun: (runId: string, reason?: string, options?: CancelRunOptions) => cancelRunInternal(runId, reason, options), + cancelRun: (runId: string, reason?: string, options?: CancelRunOptions) => + cancelRunInternal(runId, reason, options), /** * Pause-only. Emits errorCode "agent_paused" unconditionally; its sole caller is the * agent pause route. For non-pause cancellations use cancelRun, or call the internal * cancelActiveForAgentInternal(agentId, reason, errorCode) with an explicit errorCode. */ - cancelActiveForAgent: (agentId: string, reason?: string) => cancelActiveForAgentInternal(agentId, reason, "agent_paused"), + cancelActiveForAgent: (agentId: string, reason?: string) => + cancelActiveForAgentInternal(agentId, reason, "agent_paused"), cancelInvocationsForAgents: (agentIds: string[], reason: string) => cancelInvocationsForAgentsInternal(agentIds, reason), diff --git a/server/src/services/native-runtime/index.ts b/server/src/services/native-runtime/index.ts index 821574bbdc..3666cb32fb 100644 --- a/server/src/services/native-runtime/index.ts +++ b/server/src/services/native-runtime/index.ts @@ -1,6 +1,9 @@ export * from "./runtime-mode.js"; export * from "./completion-contracts.js"; +export * from "./native-execution-input.js"; export * from "./runtime-context.js"; +export * from "./native-session-executor.js"; +export * from "./native-session-resume.js"; export * from "./native-interaction-bridge.js"; export * from "./paperclip-control-plane-port.js"; export * from "./native-run-finalizer.js"; diff --git a/server/src/services/native-runtime/native-execution-input.ts b/server/src/services/native-runtime/native-execution-input.ts new file mode 100644 index 0000000000..ff54df3f6f --- /dev/null +++ b/server/src/services/native-runtime/native-execution-input.ts @@ -0,0 +1,107 @@ +import type { + NativeCodexApprovalPolicy, + NativeExecutionInputV4, + NativeInteractionResponseEnvelope, + NativePlanningContext, + NativeRuntimeContextSnapshot, + StrictCompletionContractInput, +} from "../../vendor/paperclip-runner/index.js"; +import { parseNativeExecutionInput } from "../../vendor/paperclip-runner/index.js"; +import { renderPaperclipWakePrompt } from "@paperclipai/adapter-utils/server-utils"; + +/** Closed constructor: callers cannot spread legacy context or environment data. */ +export function buildNativeExecutionInput(input: { + companyId: string; + runId: string; + issue: { + id: string; + identifier: string | null; + title: string; + description: string | null; + workMode: string; + }; + taskPrompt: string; + /** + * The already-sanitized Paperclip wake envelope for this run. Native drivers + * receive a closed execution input rather than the legacy adapter context, + * so the constructor must deliberately project the same bounded wake delta + * that legacy adapters place in their provider prompt. + */ + wakePayload?: unknown; + resumedSession?: boolean; + agentId: string; + workspace: { + id: string; + cwd: string; + repoUrl: string | null; + repoRef: string | null; + branchName: string | null; + }; + normalizedSessionId: string | null; + codexApprovalPolicy?: NativeCodexApprovalPolicy; + model?: string | null; + lifecyclePolicy?: NativeExecutionInputV4["session"]["lifecyclePolicy"]; + executionMode?: "default" | "plan"; + planningContext?: NativePlanningContext | null; + interactionResponses?: NativeInteractionResponseEnvelope[]; + completionContract: { + id: string; + sha256: string; + schemaVersion: string; + contract: StrictCompletionContractInput; + }; + runtimeContext: NativeRuntimeContextSnapshot; +}): NativeExecutionInputV4 { + if (input.issue.workMode !== "standard" && input.issue.workMode !== "planning" && input.issue.workMode !== "ask") { + throw new Error("native_execution_input_invalid: issue work mode must be standard, planning, or ask"); + } + const executionMode = input.executionMode + ?? (input.issue.workMode === "planning" ? "plan" : "default"); + const wakePrompt = renderPaperclipWakePrompt(input.wakePayload, { + resumedSession: input.resumedSession === true, + suppressIssueDescription: input.taskPrompt.trim().length > 0, + }); + const taskPrompt = [wakePrompt, input.taskPrompt.trim()] + .filter((section) => section.length > 0) + .join("\n\n"); + return parseNativeExecutionInput({ + schema: "paperclip.native-execution-input.v4", + executionMode, + planningContext: input.planningContext ?? null, + binding: { + companyId: input.companyId, + runId: input.runId, + issueId: input.issue.id, + agentId: input.agentId, + executionWorkspaceId: input.workspace.id, + }, + task: { + identifier: input.issue.identifier ?? input.issue.id, + title: input.issue.title, + description: input.issue.description, + prompt: taskPrompt, + workMode: input.issue.workMode, + }, + workspace: { + cwd: input.workspace.cwd, + repoUrl: input.workspace.repoUrl, + repoRef: input.workspace.repoRef, + branchName: input.workspace.branchName, + }, + session: { + normalizedSessionId: input.normalizedSessionId, + driverKind: "codex_app_server", + protocolVersion: 1, + lifecyclePolicy: input.lifecyclePolicy ?? { mode: "per_turn", idleTimeoutMs: null }, + }, + provider: { + kind: "codex", + model: input.model ?? null, + approvalPolicy: input.codexApprovalPolicy ?? "never", + }, + completionContract: input.completionContract, + interactionResponses: input.interactionResponses ?? [], + credentialBindings: [], + runtimeContext: input.runtimeContext, + }) as NativeExecutionInputV4; +} diff --git a/server/src/services/native-runtime/native-question-bridge.test.ts b/server/src/services/native-runtime/native-question-bridge.test.ts index 3c04925ebf..e444ff9eda 100644 --- a/server/src/services/native-runtime/native-question-bridge.test.ts +++ b/server/src/services/native-runtime/native-question-bridge.test.ts @@ -11,6 +11,7 @@ import { issueQuestionResponseDeliveries, issueThreadInteractions, issues, + nativeRunFinalizations, } from "@paperclipai/db"; import type { PrpEvent } from "@paperclipai/paperclip-runner"; @@ -105,6 +106,7 @@ describeEmbeddedPostgres("native question bridge", () => { title: "Answer a native question", status: "in_progress", assigneeAgentId: agentId, + responsibleUserId: "operator-1", }); await db.insert(heartbeatRuns).values({ id: runId, @@ -119,6 +121,12 @@ describeEmbeddedPostgres("native question bridge", () => { driverKind: "codex", contextSnapshot: { issueId }, }); + await db.insert(nativeRunFinalizations).values({ + runId, + companyId, + issueId, + phase: "observed", + }); } function runtimeRequestEvent(): PrpEvent { diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts new file mode 100644 index 0000000000..c480b9dc89 --- /dev/null +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -0,0 +1,1952 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + heartbeatRuns, + issues, + nativeRunFinalizations, + type Db, +} from "@paperclipai/db"; +import type { + NativeExecutionInputV1, + PrpEvent, +} from "@paperclipai/paperclip-runner"; +import { createHash } from "node:crypto"; + +type BackendFactoryOptions = { + runnerInstanceId?: string; + codexTransportFactory?: () => unknown; + dynamicToolHandler?: (call: unknown) => Promise; + onSpawn?: (meta: { + pid: number; + processGroupId: number | null; + startedAt: string; + }) => Promise; +}; + +const state = vi.hoisted(() => ({ + execute: vi.fn(), + createTransport: vi.fn( + (_options: { stateDirectory?: string; runnerBinary?: string }) => ({ + transport: {}, + }), + ), + createBackend: vi.fn( + (_input: NativeExecutionInputV1, _options: BackendFactoryOptions) => ({ + kind: "test", + }), + ), + cancel: vi.fn(), + toolAuthorityExecute: vi.fn(), + persistActivity: vi.fn( + async (_db: unknown, input: { action: string }) => ({ + activity: { + id: + input.action === "native.cancellation_intent_recorded" + ? "native-cancellation-audit" + : "native-cancellation-ack-audit", + }, + publication: { + companyId: "company", + payload: { action: input.action }, + pluginEvent: null, + }, + }), + ), + publishActivity: vi.fn(), + resolveRunnerBinary: vi.fn(() => "/tmp/paperclip-runnerd"), + release: null as null | (() => void), +})); + +vi.mock( + "../../vendor/paperclip-runner/index.js", + async (importOriginal) => ({ + ...(await importOriginal< + typeof import("../../vendor/paperclip-runner/index.js") + >()), + createNativeSessionBackend: state.createBackend, + createRunnerdCodexTransport: state.createTransport, + executeNativeSession: state.execute, + parsePaperclipQuestionSet: (value: unknown) => value, + }), +); + +vi.mock("./paperclip-runner-tool-authority.js", () => ({ + PaperclipRunnerToolAuthority: class { + readonly binding: Record; + + constructor(_db: unknown, binding: Record) { + this.binding = binding; + } + + async definitions() { + return []; + } + + async execute(call: unknown) { + return state.toolAuthorityExecute(this.binding, call); + } + }, +})); + +vi.mock("../activity-log.js", () => ({ + persistActivity: state.persistActivity, + publishActivity: state.publishActivity, +})); + +vi.mock("./native-codex-runner.js", () => ({ + resolvePaperclipRunnerBinary: state.resolveRunnerBinary, +})); + +import { + continuingPendingInteractionIds, + buildNativeProviderEnvironment, + cancelNativeSession, + createGovernedWaitEventObservation, + createRunnerdBackend, + executePaperclipNativeSession, + getNativeSessionSteeringState, + NativeSessionSteeringError, + nativeSessionFailureDisposition, + nativeSessionFailureSourceCode, + nativeSessionRecoveryProjection, + nativeGovernedWaitResult, + providerPlanMarkdown, + renewNativeSessionExecutionLease, + runtimeInputLifecycleMetric, + runtimeQuestionFallbackFromEvent, + resolveNativeRuntimeRequest, + semanticProviderPlanMarkdown, + steerNativeSession, +} from "./native-session-executor.js"; + +describe("runtime question fallback", () => { + const questionSet = { + schema: "paperclip.question_set.v1" as const, + title: "Configure deployment", + description: "These answers are required before work can continue.", + submitLabel: "Continue", + questions: [ + { + id: "region", + prompt: "Which region?", + required: true, + answerMode: "single_select" as const, + options: [ + { id: "us", label: "US" }, + { id: "eu", label: "Europe" }, + ], + }, + { + id: "replicas", + prompt: "How many replicas?", + required: true, + answerMode: "text" as const, + textValidation: { inputType: "integer" as const, minimum: 1 }, + }, + ], + }; + + it.each(["provider_process_lost", "durable_handoff"])( + "materializes one idempotent durable interaction after %s", + (reason) => { + const fallback = runtimeQuestionFallbackFromEvent({ + eventType: "runtime_request.expired", + runId: "00000000-0000-4000-8000-000000000001", + payload: { + requestId: "elicitation-1", + requestKind: "runtime", + requestType: "input", + reason, + replayAllowed: false, + request: { + schema: "paperclip.runtime_request.v2", + requestKind: "runtime", + requestId: "elicitation-1", + type: "input", + status: "pending", + prompt: "Configure deployment", + turnId: "turn-1", + itemId: "item-1", + input: questionSet, + }, + }, + }); + expect(fallback).toMatchObject({ + kind: "ask_user_questions", + idempotencyKey: + "runtime-input-durable:v1:00000000-0000-4000-8000-000000000001:elicitation-1", + sourceRunId: "00000000-0000-4000-8000-000000000001", + continuationPolicy: "wake_assignee", + payload: { + runtimeRequestId: "elicitation-1", + questionSet, + supersedeOnUserComment: false, + questions: [ + { + id: "region", + selectionMode: "single", + options: [ + { id: "us", label: "US" }, + { id: "eu", label: "Europe" }, + ], + }, + { + id: "replicas", + selectionMode: "single", + options: [{ id: "__paperclip_text__", freeText: true }], + }, + ], + }, + }); + }, + ); + + it.each([ + ["runtime_request.resolved", "provider_process_lost", false], + ["runtime_request.cancelled", "provider_process_lost", false], + ["runtime_request.expired", "explicit_cancellation", false], + ["runtime_request.expired", "provider_process_lost", true], + ])( + "does not fall back for %s / %s / replay=%s", + (eventType, reason, replayAllowed) => { + expect( + runtimeQuestionFallbackFromEvent({ + eventType: eventType as never, + runId: "00000000-0000-4000-8000-000000000001", + payload: { + reason, + replayAllowed, + request: { + schema: "paperclip.runtime_request.v2", + requestKind: "runtime", + requestId: "elicitation-1", + type: "input", + status: "pending", + turnId: "turn-1", + itemId: "item-1", + input: questionSet, + }, + }, + }), + ).toBeNull(); + }, + ); + + it("emits content-free lifecycle metric dimensions", () => { + expect( + runtimeInputLifecycleMetric({ + eventType: "runtime_request.created", + payload: { + request: { + type: "input", + requestId: "input-1", + origin: { adapter: "codex-app-server" }, + input: questionSet, + }, + }, + }), + ).toEqual({ + outcome: "normalized", + adapter: "codex-app-server", + requestId: "input-1", + }); + expect( + runtimeInputLifecycleMetric({ + eventType: "runtime_request.expired", + payload: { + requestId: "input-1", + requestType: "input", + reason: "durable_handoff", + adapter: "codex-app-server", + }, + }), + ).toEqual({ + outcome: "durable_handoff", + adapter: "codex-app-server", + requestId: "input-1", + }); + expect( + runtimeInputLifecycleMetric({ + eventType: "runtime_request.expired", + payload: { + requestId: "input-1", + requestType: "input", + reason: "provider_process_lost", + adapter: "codex-app-server", + }, + }), + ).toEqual({ + outcome: "provider_loss_handoff", + adapter: "codex-app-server", + requestId: "input-1", + }); + }); +}); + +describe("native provider bootstrap environment", () => { + it("inherits the host executable and credential-home context", () => { + expect( + buildNativeProviderEnvironment( + {}, + { + PATH: "/opt/homebrew/bin:/usr/bin", + HOME: "/Users/runner", + CODEX_HOME: "/Users/runner/.codex", + PAPERCLIP_INTERNAL_SECRET: "must-not-leak", + }, + ), + ).toEqual({ + PATH: "/opt/homebrew/bin:/usr/bin", + HOME: "/Users/runner", + CODEX_HOME: "/Users/runner/.codex", + }); + }); + + it("lets explicitly configured agent env override host defaults", () => { + expect( + buildNativeProviderEnvironment( + { + PATH: "/agent/bin", + OPENAI_API_KEY: "configured-provider-key", + }, + { + PATH: "/host/bin", + HOME: "/Users/runner", + }, + ), + ).toEqual({ + PATH: "/agent/bin", + HOME: "/Users/runner", + OPENAI_API_KEY: "configured-provider-key", + }); + }); +}); + +const execution = { + schema: "paperclip.native-execution-input.v1", + provider: { kind: "codex", model: null }, + binding: { + companyId: "company", + runId: "run-native-cancel", + issueId: "issue", + agentId: "agent", + executionWorkspaceId: "workspace", + }, + task: { + identifier: "PAP-NATIVE", + title: "Exercise the native session", + description: null, + prompt: "Complete the native session test task.", + workMode: "standard", + }, + workspace: { + cwd: "/tmp/paperclip-native-session-test", + repoUrl: null, + repoRef: null, + branchName: null, + }, + session: { + normalizedSessionId: "session-native-cancel", + driverKind: "codex_app_server", + protocolVersion: 1, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, + }, + completionContract: { + id: "contract", + sha256: "sha", + schemaVersion: "paperclip.completion-contract.v1", + contract: { + revision: "1", + objective: "Exercise the native session.", + criteria: [{ id: "objective", requirement: "The session completes." }], + }, + }, + interactionResponses: [], + credentialBindings: [], +} as NativeExecutionInputV1; + +describe("provider plan synchronization", () => { + it("prefers the provider's completed Markdown when it is available", () => { + expect( + providerPlanMarkdown({ + markdown: "# Release plan\n\n1. Prepare\n2. Deploy", + explanation: "This fallback must not replace the completed plan.", + steps: [{ body: "Fallback", status: "pending" }], + }), + ).toBe("# Release plan\n\n1. Prepare\n2. Deploy"); + }); + + it("extracts a completed plan from the semantic result artifact", () => { + expect( + semanticProviderPlanMarkdown({ + artifacts: [ + { + kind: "native_provider_plan", + ref: "\n# Health check\n\n1. Add endpoint\n2. Verify it\n", + }, + ], + }), + ).toBe("# Health check\n\n1. Add endpoint\n2. Verify it"); + }); + + it("decodes the native provider's compact plan reference into readable Markdown", () => { + expect( + semanticProviderPlanMarkdown({ + artifacts: [ + { + kind: "native_provider_plan", + ref: "native-provider-plan:health-check-endpoint-v1#1-register-GET-health-return-200-json-status-ok;2-add-API-tests", + }, + ], + }), + ).toBe( + [ + "# Health check endpoint", + "", + "1. Register GET /health return 200 JSON status ok", + "2. Add API tests", + ].join("\n"), + ); + }); + + it("decodes a task-scoped native plan URI", () => { + expect( + semanticProviderPlanMarkdown({ + artifacts: [ + { + kind: "native_provider_plan", + ref: "native-plan://DOT-13/health-check#1-add-GET-health;2-add-tests", + }, + ], + }), + ).toBe("# Health check\n\n1. Add GET /health\n2. Add tests"); + }); + + it("retains readable Markdown embedded after a native provider plan reference", () => { + expect( + semanticProviderPlanMarkdown({ + artifacts: [ + { + kind: "native_provider_plan", + ref: "native-provider-plan:DOT-14-health-check-v1\n1. Add `GET /health`.\n2. Add tests.", + }, + ], + }), + ).toBe("# Health check\n\n1. Add `GET /health`.\n2. Add tests."); + }); + + it("normalizes a plain numbered native provider plan", () => { + expect( + semanticProviderPlanMarkdown({ + artifacts: [ + { + kind: "native_provider_plan", + ref: "1. Add GET /health. | 2. Add tests. | 3. Document it.", + }, + ], + }), + ).toBe("# Plan\n\n1. Add GET /health.\n2. Add tests.\n3. Document it."); + }); + + it("normalizes a task-labelled inline numbered plan", () => { + expect( + semanticProviderPlanMarkdown({ + artifacts: [ + { + kind: "native_provider_plan", + ref: "DOT-16 plan: (1) add GET /health; (2) add tests; (3) document it.", + }, + ], + }), + ).toBe("# Plan\n\n1. add GET /health\n2. add tests\n3. document it."); + }); + + it("uses an explicitly numbered semantic summary when the artifact is opaque", () => { + expect( + semanticProviderPlanMarkdown({ + summary: + "Native provider plan completed: 1) add GET /health; 2) add tests; 3) document it.", + artifacts: [ + { + kind: "native_provider_plan", + ref: "native-provider-plan:DOT-18:health-check", + }, + ], + }), + ).toBe("# Plan\n\n1. add GET /health\n2. add tests\n3. document it."); + }); + + it("renders a bounded Markdown checklist without embedding provenance", () => { + const markdown = providerPlanMarkdown({ + explanation: "Release safely", + steps: [ + { body: "Prepare", status: "completed" }, + { body: "Deploy", status: "in_progress" }, + { body: "Verify", status: "blocked" }, + ], + runId: "must-not-appear", + providerThreadId: "native-secret", + }); + expect(markdown).toBe( + [ + "Release safely", + "", + "- [x] Prepare", + "- [ ] Deploy _(in progress)_", + "- [ ] Verify _(blocked)_", + ].join("\n"), + ); + expect(markdown).not.toContain("must-not-appear"); + expect(markdown).not.toContain("native-secret"); + }); +}); + +describe("native governed waits", () => { + it("turns a durable pending interaction into a response-wake result", () => { + expect( + nativeGovernedWaitResult({ + interaction: { + id: "interaction-1", + title: "Choose an output format", + summary: null, + }, + completionContract: { + revision: "contract-v3", + objective: "Create the requested output", + criteria: [{ id: "objective", requirement: "The output is created" }], + }, + }), + ).toEqual( + expect.objectContaining({ + schema: "paperclip.run_result.v1", + reportedWorkDisposition: "yielded", + summary: "Waiting for Choose an output format.", + completionClaim: expect.objectContaining({ + contractRevision: "contract-v3", + objectiveSatisfied: false, + criteria: [ + { + criterionId: "objective", + status: "unknown", + evidenceRefs: ["interaction:interaction-1"], + }, + ], + }), + evidence: [{ ref: "interaction:interaction-1" }], + attentionRequests: [], + continuation: { + kind: "response_wake", + summary: + "Resume from the resolved interaction response without repeating prior work.", + idempotencyKey: "interaction-response:interaction-1", + }, + }), + ); + }); + + it("keeps an authority-checked partial item-verdict interaction as the wait target", () => { + const partial = structuredClone(execution); + partial.interactionResponses = [ + { + interactionId: "interaction-partial", + kind: "request_item_verdicts", + response: { + status: "pending", + result: { + version: 1, + complete: false, + items: [{ id: "alpha", verdict: "approve" }], + }, + }, + }, + ]; + expect(continuingPendingInteractionIds(partial)).toEqual([ + "interaction-partial", + ]); + + partial.interactionResponses[0]!.response.status = "answered"; + expect(continuingPendingInteractionIds(partial)).toEqual([]); + }); + + it("consumes an exact replay observation once without leaking stale state", async () => { + const waitResult = nativeGovernedWaitResult({ + interaction: { + id: "interaction-replayed", + title: "Approve the replayed operation", + summary: null, + }, + completionContract: { + revision: "contract-v3", + objective: "Complete the approved operation", + criteria: [{ id: "objective", requirement: "Complete it" }], + }, + }); + const observation = createGovernedWaitEventObservation( + async () => waitResult, + ); + const replayedEvent: PrpEvent = { + schema: "paperclip.prp.event.v1" as const, + sourceInstanceId: "runner-recovered", + sourceEventId: "runner-recovered:item:7", + sourceSeq: 7, + sourceKind: "runner" as const, + runId: "run-recovered", + normalizedSessionId: "session-recovered", + turnId: "turn-recovered", + eventType: "item.completed" as const, + schemaVersion: 1, + priority: 0 as const, + emittedAt: "2026-08-31T00:00:00.000Z", + payload: {}, + }; + + await observation.observe(replayedEvent, true); + expect(observation.consume(replayedEvent)).toEqual(waitResult); + expect(observation.consume(replayedEvent)).toBeNull(); + + await observation.observe(replayedEvent, true); + expect( + observation.consume({ + ...replayedEvent, + sourceEventId: "runner-recovered:item:8", + sourceSeq: 8, + }), + ).toBeNull(); + expect(observation.consume(replayedEvent)).toBeNull(); + + let resolveLookup!: (value: typeof waitResult) => void; + const delayedObservation = createGovernedWaitEventObservation( + () => + new Promise((resolve) => { + resolveLookup = resolve; + }), + ); + const observing = delayedObservation.observe(replayedEvent, true); + expect(delayedObservation.consume(replayedEvent)).toBeNull(); + resolveLookup(waitResult); + await observing; + expect(delayedObservation.consume(replayedEvent)).toBeNull(); + }); +}); + +type LeaseCoordinator = { + runId: string; + companyId: string; + issueId: string; + phase: string; + attempt: number; + leaseOwner: string | null; + leaseExpiresAt: Date | null; + resultId: string | null; +}; + +function leaseDb( + boundExecution: NativeExecutionInputV1 = execution, + coordinatorOverrides: Partial = {}, + runResultJson: Record = {}, +): Db { + const coordinator: LeaseCoordinator = { + runId: boundExecution.binding.runId, + companyId: boundExecution.binding.companyId, + issueId: boundExecution.binding.issueId, + phase: "observed", + attempt: 0, + leaseOwner: null, + leaseExpiresAt: null, + resultId: null, + ...coordinatorOverrides, + }; + const update = () => ({ + set: () => ({ + where: () => { + const result = Promise.resolve([]) as unknown as Promise & { + returning: () => Promise>; + }; + result.returning = () => Promise.resolve([{ runId: coordinator.runId }]); + return result; + }, + }), + }); + const tx = { + select: () => ({ + from: (table: unknown) => ({ + where: () => ({ + for: () => ({ + limit: () => + Promise.resolve([ + table === nativeRunFinalizations + ? coordinator + : { + agentId: boundExecution.binding.agentId, + companyId: boundExecution.binding.companyId, + nativeIssueId: boundExecution.binding.issueId, + resultJson: runResultJson, + runtimeMode: "native", + }, + ]), + }), + }), + }), + }), + update, + }; + return { + transaction: async (operation: (transaction: Db) => Promise) => + operation(tx as unknown as Db), + update, + } as unknown as Db; +} + +function cancellationDb(options?: { + coordinator?: { + runId: string; + assessmentId: string | null; + decisionId?: string | null; + } | null; + failResultJsonUpdateAt?: number; +}) { + const initialRun = { + id: execution.binding.runId, + agentId: execution.binding.agentId, + companyId: execution.binding.companyId, + nativeIssueId: execution.binding.issueId, + runtimeMode: "native", + contextSnapshot: { issueId: "untrusted-context-issue" }, + resultJson: { staleSnapshot: true }, + }; + let currentResultJson: Record = { + durableReceipt: { operationId: "operation-1" }, + }; + const issue = { + status: "in_progress", + statusVersion: 3, + lastStatusDecisionId: null, + }; + const coordinator = + options && "coordinator" in options + ? options.coordinator + : { runId: execution.binding.runId, assessmentId: null }; + let forUpdateCount = 0; + let resultJsonUpdateCount = 0; + const updates: Array<{ table: unknown; values: Record }> = []; + const select = vi.fn(() => ({ + from: (table: unknown) => { + const rows = + table === heartbeatRuns + ? [{ ...initialRun, resultJson: currentResultJson }] + : table === issues + ? [issue] + : table === nativeRunFinalizations && coordinator + ? [coordinator] + : []; + const result = Promise.resolve(rows); + type Query = { + where: () => Query; + for: () => Query; + limit: () => Promise; + }; + const query = {} as Query; + Object.assign(query, { + where: () => query, + for: () => { + forUpdateCount += 1; + return query; + }, + limit: () => result, + }); + return query; + }, + })); + const update = vi.fn((table: unknown) => ({ + set: (values: Record) => ({ + where: () => { + updates.push({ table, values }); + const updatesResultJson = "resultJson" in values; + if (updatesResultJson) resultJsonUpdateCount += 1; + const shouldFail = + updatesResultJson && + resultJsonUpdateCount === options?.failResultJsonUpdateAt; + if (updatesResultJson && !shouldFail) { + currentResultJson = values.resultJson as Record; + } + const result = Promise.resolve([]) as unknown as Promise & { + returning: () => Promise>; + }; + result.returning = () => + shouldFail + ? Promise.reject(new Error("post_dispatch_db_failure")) + : Promise.resolve([{ id: execution.binding.runId }]); + return result; + }, + }), + })); + const tx = { select, update }; + const db = { + select, + update, + transaction: async (operation: (transaction: Db) => Promise) => + operation(tx as unknown as Db), + } as unknown as Db; + return { + db, + updates, + getForUpdateCount: () => forUpdateCount, + getResultJson: () => currentResultJson, + getResultJsonUpdateCount: () => resultJsonUpdateCount, + tx, + }; +} + +describe("native session cancellation", () => { + beforeEach(() => { + state.cancel.mockReset().mockReturnValue({ cleanup: Promise.resolve() }); + state.persistActivity.mockClear(); + state.publishActivity.mockClear(); + state.release = null; + state.execute.mockReset().mockImplementation(async (options) => { + options.onSession?.({ cancel: state.cancel }); + await new Promise((resolve) => { + state.release = resolve; + }); + options.onSession?.(null); + return { + result: { summary: "cancelled" }, + terminal: { runTerminalState: "cancelled" }, + turnId: "turn", + normalizedSessionId: "session", + providerSessionId: null, + driverKind: "test", + driverVersion: "1", + nativeEventCount: 1, + highestContiguousSourceSeq: 1, + }; + }); + }); + + it("routes control-plane cancellation to the active normalized session and removes the handle", async () => { + const running = executePaperclipNativeSession({ + db: leaseDb(), + execution, + runnerInstanceId: "runner", + }); + await vi.waitFor(() => expect(state.release).toBeTypeOf("function")); + + await expect( + cancelNativeSession(execution.binding.runId, "budget hard stop"), + ).resolves.toBe(true); + await expect( + cancelNativeSession(execution.binding.runId, "duplicate budget stop"), + ).resolves.toBe(true); + expect(state.cancel).toHaveBeenCalledWith({ + reason: "budget hard stop", + signal: expect.any(AbortSignal), + }); + expect(state.cancel).toHaveBeenCalledTimes(1); + + state.release?.(); + await running; + await expect( + cancelNativeSession(execution.binding.runId, "late cancel"), + ).resolves.toBe(false); + }); + + it("allows cancellation to be retried when the session dispatch fails", async () => { + state.cancel.mockImplementationOnce(() => { + throw new Error("transport unavailable"); + }); + const running = executePaperclipNativeSession({ + db: leaseDb(), + execution, + runnerInstanceId: "runner", + }); + await vi.waitFor(() => expect(state.release).toBeTypeOf("function")); + + await expect( + cancelNativeSession(execution.binding.runId, "budget hard stop"), + ).rejects.toThrow("transport unavailable"); + await expect( + cancelNativeSession(execution.binding.runId, "retry budget stop"), + ).resolves.toBe(true); + expect(state.cancel).toHaveBeenNthCalledWith(2, { + reason: "retry budget stop", + signal: expect.any(AbortSignal), + }); + + state.release?.(); + await running; + }); + + it("observes cleanup failure after cancellation authority is committed", async () => { + state.cancel.mockImplementationOnce(() => ({ + cleanup: Promise.reject(new Error("provider cleanup failed")), + })); + const running = executePaperclipNativeSession({ + db: leaseDb(), + execution, + runnerInstanceId: "runner", + }); + await vi.waitFor(() => expect(state.release).toBeTypeOf("function")); + + await expect( + cancelNativeSession(execution.binding.runId, "budget hard stop"), + ).resolves.toBe(true); + + state.release?.(); + await running; + }); + + it("binds cancellation to nativeIssueId and merges metadata under a row lock", async () => { + const persistence = cancellationDb(); + + await expect( + cancelNativeSession(execution.binding.runId, "budget hard stop", { + db: persistence.db, + scope: "run", + }), + ).resolves.toMatchObject({ + dispatched: false, + decision: expect.any(Object), + auditId: "native-cancellation-audit", + }); + + expect(persistence.getForUpdateCount()).toBe(2); + const cancellationUpdate = persistence.updates + .filter((entry) => "resultJson" in entry.values) + .at(-1); + expect(cancellationUpdate?.values.resultJson).toMatchObject({ + durableReceipt: { operationId: "operation-1" }, + nativeCancellation: { + schema: "paperclip.native-cancellation.v1", + dispatchState: "acknowledged", + scope: "run", + dispatched: false, + intentAuditId: "native-cancellation-audit", + acknowledgementAuditId: "native-cancellation-ack-audit", + }, + }); + expect(state.persistActivity).toHaveBeenCalledWith( + persistence.tx, + expect.objectContaining({ + companyId: execution.binding.companyId, + issueId: execution.binding.issueId, + runId: execution.binding.runId, + }), + ); + expect(state.publishActivity).toHaveBeenCalledTimes(2); + }); + + it("recovers a post-dispatch persistence failure without cancelling the provider twice", async () => { + const persistence = cancellationDb({ failResultJsonUpdateAt: 2 }); + const running = executePaperclipNativeSession({ + db: leaseDb(), + execution, + runnerInstanceId: "runner", + }); + await vi.waitFor(() => expect(state.release).toBeTypeOf("function")); + + await expect( + cancelNativeSession(execution.binding.runId, "budget hard stop", { + db: persistence.db, + scope: "run", + }), + ).rejects.toThrow("post_dispatch_db_failure"); + expect(state.cancel).toHaveBeenCalledTimes(1); + expect(persistence.getResultJson()).toMatchObject({ + nativeCancellation: { + dispatchState: "pending", + dispatched: false, + intentAuditId: "native-cancellation-audit", + }, + }); + + await expect( + cancelNativeSession(execution.binding.runId, "budget hard stop", { + db: persistence.db, + scope: "run", + }), + ).resolves.toMatchObject({ + dispatched: true, + auditId: "native-cancellation-audit", + }); + expect(state.cancel).toHaveBeenCalledTimes(1); + expect(persistence.getResultJsonUpdateCount()).toBe(3); + expect(persistence.getResultJson()).toMatchObject({ + nativeCancellation: { + dispatchState: "acknowledged", + dispatched: true, + intentAuditId: "native-cancellation-audit", + acknowledgementAuditId: "native-cancellation-ack-audit", + }, + }); + expect( + state.persistActivity.mock.calls.filter( + ([, input]) => + (input as { action?: string }).action === + "native.cancellation_intent_recorded", + ), + ).toHaveLength(1); + const persistedActivities = state.persistActivity.mock.calls.length; + await expect( + cancelNativeSession(execution.binding.runId, "budget hard stop", { + db: persistence.db, + scope: "run", + }), + ).resolves.toMatchObject({ + dispatched: true, + auditId: "native-cancellation-audit", + }); + expect(state.cancel).toHaveBeenCalledTimes(1); + expect(persistence.getResultJsonUpdateCount()).toBe(3); + expect(state.persistActivity).toHaveBeenCalledTimes(persistedActivities); + + state.release?.(); + await running; + }); + + it("fails closed when the persisted native binding has no coordinator", async () => { + const persistence = cancellationDb({ coordinator: null }); + + await expect( + cancelNativeSession(execution.binding.runId, "budget hard stop", { + db: persistence.db, + scope: "run", + }), + ).rejects.toThrow("native_cancellation_coordinator_missing"); + expect(persistence.updates).toEqual([]); + expect(state.persistActivity).not.toHaveBeenCalled(); + }); +}); + +describe("native session execution lease fencing", () => { + it("renews only when the exact fenced owner remains current", async () => { + const returning = vi + .fn() + .mockResolvedValueOnce([{ runId: "run-lease" }]) + .mockResolvedValueOnce([]); + const where = vi.fn(() => ({ returning })); + const set = vi.fn(() => ({ where })); + const db = { update: vi.fn(() => ({ set })) } as unknown as Db; + const input = { + db, + runId: "run-lease", + companyId: "company-lease", + issueId: "issue-lease", + leaseOwner: "owner-lease", + attempt: 4, + leaseTtlMs: 60_000, + }; + + await expect( + renewNativeSessionExecutionLease(input), + ).resolves.toBeUndefined(); + await expect(renewNativeSessionExecutionLease(input)).rejects.toThrow( + "native_session_lease_lost", + ); + expect(returning).toHaveBeenCalledTimes(2); + }); + + it("does not reacquire a provider after a durable result exists", async () => { + state.execute.mockClear(); + state.createBackend.mockClear(); + state.createTransport.mockClear(); + + await expect( + executePaperclipNativeSession({ + db: leaseDb(execution, { + phase: "workspace_finalizing", + resultId: "native-result-1", + }), + execution, + runnerInstanceId: "runner", + }), + ).rejects.toThrow("native_result_pending_finalization"); + expect(state.execute).not.toHaveBeenCalled(); + expect(state.createBackend).not.toHaveBeenCalled(); + expect(state.createTransport).not.toHaveBeenCalled(); + }); + + it.each(["pending", "acknowledged"] as const)( + "does not reacquire a provider while durable cancellation is %s", + async (dispatchState) => { + state.execute.mockClear(); + state.createBackend.mockClear(); + state.createTransport.mockClear(); + + await expect( + executePaperclipNativeSession({ + db: leaseDb( + execution, + {}, + { + nativeCancellation: { + schema: "paperclip.native-cancellation.v1", + intentId: "native-cancellation:intent-1", + intentAuditId: "native-cancellation-audit", + companyId: execution.binding.companyId, + runId: execution.binding.runId, + issueId: execution.binding.issueId, + scope: "run", + reasonCode: "cancellation_run_only", + effects: ["release_run_resources"], + dispatchState, + dispatched: dispatchState === "acknowledged", + decisionId: null, + }, + }, + ), + execution, + runnerInstanceId: "runner", + }), + ).rejects.toThrow("native_cancellation_pending_recovery"); + expect(state.execute).not.toHaveBeenCalled(); + expect(state.createBackend).not.toHaveBeenCalled(); + expect(state.createTransport).not.toHaveBeenCalled(); + }, + ); +}); + +describe("native runtime request resolution", () => { + const capabilities = vi.fn(); + const snapshot = vi.fn(); + const resolveRuntimeRequest = vi.fn(); + + beforeEach(() => { + state.release = null; + capabilities.mockReset().mockResolvedValue({ + runtimeRequestResolution: true, + }); + snapshot.mockReset().mockResolvedValue({ activeTurnId: "provider-turn-1" }); + resolveRuntimeRequest.mockReset().mockResolvedValue(undefined); + state.execute.mockReset().mockImplementation(async (options) => { + options.onSession?.({ + capabilities, + snapshot, + resolveRuntimeRequest, + cancel: vi.fn(), + }); + await new Promise((resolve) => { + state.release = resolve; + }); + options.onSession?.(null); + return { + result: { summary: "completed" }, + terminal: { runTerminalState: "succeeded" }, + turnId: "provider-turn-1", + normalizedSessionId: "session", + providerSessionId: null, + driverKind: "test", + driverVersion: "1", + nativeEventCount: 1, + highestContiguousSourceSeq: 1, + }; + }); + }); + + it("revalidates lifecycle after provider reads and blocks stale dispatch", async () => { + const running = executePaperclipNativeSession({ + db: leaseDb(), + execution, + runnerInstanceId: "runner", + }); + await vi.waitFor(() => expect(state.release).toBeTypeOf("function")); + const authorizeBeforeDispatch = vi.fn(async () => { + expect(capabilities).toHaveBeenCalledTimes(1); + expect(snapshot).toHaveBeenCalledTimes(1); + throw new Error("runtime_request_no_longer_pending"); + }); + + await expect( + resolveNativeRuntimeRequest({ + runId: execution.binding.runId, + requestId: "runtime-request-1", + turnId: "provider-turn-1", + resolution: { action: "decline" }, + authorizeBeforeDispatch, + }), + ).rejects.toThrow("runtime_request_no_longer_pending"); + expect(authorizeBeforeDispatch).toHaveBeenCalledTimes(1); + expect(resolveRuntimeRequest).not.toHaveBeenCalled(); + + state.release?.(); + await running; + }); + + it("atomically joins duplicate responses and rejects a concurrent conflict", async () => { + const running = executePaperclipNativeSession({ + db: leaseDb(), + execution, + runnerInstanceId: "runner", + }); + await vi.waitFor(() => expect(state.release).toBeTypeOf("function")); + let releaseAuthorization!: () => void; + const authorization = new Promise((resolve) => { + releaseAuthorization = resolve; + }); + const authorizeBeforeDispatch = vi.fn(() => authorization); + const first = resolveNativeRuntimeRequest({ + runId: execution.binding.runId, + requestId: "runtime-request-concurrent", + turnId: "provider-turn-1", + resolution: { action: "decline" }, + authorizeBeforeDispatch, + }); + await vi.waitFor(() => + expect(authorizeBeforeDispatch).toHaveBeenCalledTimes(1), + ); + const duplicate = resolveNativeRuntimeRequest({ + runId: execution.binding.runId, + requestId: "runtime-request-concurrent", + turnId: "provider-turn-1", + resolution: { action: "decline" }, + authorizeBeforeDispatch, + }); + await vi.waitFor(() => expect(snapshot).toHaveBeenCalledTimes(2)); + + await expect( + resolveNativeRuntimeRequest({ + runId: execution.binding.runId, + requestId: "runtime-request-concurrent", + turnId: "provider-turn-1", + resolution: { action: "cancel" }, + authorizeBeforeDispatch, + }), + ).rejects.toMatchObject({ + code: "runtime_request_resolution_conflict", + }); + expect(authorizeBeforeDispatch).toHaveBeenCalledTimes(1); + expect(resolveRuntimeRequest).not.toHaveBeenCalled(); + + releaseAuthorization(); + const [firstResult, duplicateResult] = await Promise.all([ + first, + duplicate, + ]); + expect(duplicateResult.commandId).toBe(firstResult.commandId); + expect(resolveRuntimeRequest).toHaveBeenCalledTimes(1); + + state.release?.(); + await running; + }); + + it("clears completed response reservations when the session tears down", async () => { + const firstSession = executePaperclipNativeSession({ + db: leaseDb(), + execution, + runnerInstanceId: "runner", + }); + await vi.waitFor(() => expect(state.release).toBeTypeOf("function")); + const first = await resolveNativeRuntimeRequest({ + runId: execution.binding.runId, + requestId: "runtime-request-reused", + turnId: "provider-turn-1", + resolution: { action: "decline" }, + authorizeBeforeDispatch: vi.fn().mockResolvedValue(undefined), + }); + state.release?.(); + await firstSession; + + state.release = null; + const secondSession = executePaperclipNativeSession({ + db: leaseDb(), + execution, + runnerInstanceId: "runner", + }); + await vi.waitFor(() => expect(state.release).toBeTypeOf("function")); + const second = await resolveNativeRuntimeRequest({ + runId: execution.binding.runId, + requestId: "runtime-request-reused", + turnId: "provider-turn-1", + resolution: { action: "decline" }, + authorizeBeforeDispatch: vi.fn().mockResolvedValue(undefined), + }); + + expect(second.commandId).not.toBe(first.commandId); + expect(resolveRuntimeRequest).toHaveBeenCalledTimes(2); + (state.release as (() => void) | null)?.(); + await secondSession; + }); +}); + +describe("native session same-turn steering", () => { + const capabilities = vi.fn(); + const snapshot = vi.fn(); + const steer = vi.fn(); + + beforeEach(() => { + state.release = null; + capabilities.mockReset().mockResolvedValue({ steering: true }); + snapshot.mockReset().mockResolvedValue({ activeTurnId: "provider-turn-1" }); + steer.mockReset().mockResolvedValue(undefined); + state.execute.mockReset().mockImplementation(async (options) => { + options.onSession?.({ capabilities, snapshot, steer, cancel: vi.fn() }); + await new Promise((resolve) => { + state.release = resolve; + }); + options.onSession?.(null); + return { + result: { summary: "completed" }, + terminal: { runTerminalState: "succeeded" }, + turnId: "provider-turn-1", + normalizedSessionId: "session", + providerSessionId: null, + driverKind: "test", + driverVersion: "1", + nativeEventCount: 1, + highestContiguousSourceSeq: 1, + }; + }); + }); + + async function startActiveSession() { + const running = executePaperclipNativeSession({ + db: leaseDb(), + execution, + runnerInstanceId: "runner", + }); + await vi.waitFor(() => expect(state.release).toBeTypeOf("function")); + return { running }; + } + + it("correlates the queued comment with the active provider turn acknowledgement", async () => { + const { running } = await startActiveSession(); + + await expect( + getNativeSessionSteeringState(execution.binding.runId), + ).resolves.toEqual({ + disposition: "available", + activeTurnId: "provider-turn-1", + }); + await expect( + steerNativeSession({ + runId: execution.binding.runId, + message: "Check mobile overflow first.", + correlationId: "queued-comment-1", + }), + ).resolves.toEqual({ turnId: "provider-turn-1" }); + expect(steer).toHaveBeenCalledWith({ + turnId: "provider-turn-1", + message: { role: "user", text: "Check mobile overflow first." }, + correlationId: "queued-comment-1", + }); + + state.release?.(); + await running; + }); + + it.each([ + { + label: "unsupported provider", + prepare: () => capabilities.mockResolvedValue({ steering: false }), + code: "steering_unsupported", + }, + { + label: "stale turn", + prepare: () => snapshot.mockResolvedValue({ activeTurnId: null }), + code: "steering_stale_turn", + }, + { + label: "provider rejection", + prepare: () => steer.mockRejectedValue(new Error("request rejected")), + code: "steering_rejected", + }, + ])("keeps $label retryable with a stable code", async ({ prepare, code }) => { + prepare(); + const { running } = await startActiveSession(); + + const error = await steerNativeSession({ + runId: execution.binding.runId, + message: "Retryable steering", + correlationId: "queued-comment-error", + }).catch((value) => value); + expect(error).toBeInstanceOf(NativeSessionSteeringError); + expect(error.code).toBe(code); + + state.release?.(); + await running; + }); + + it("bounds the provider acknowledgement wait", async () => { + steer.mockReturnValue(new Promise(() => undefined)); + const { running } = await startActiveSession(); + + const error = await steerNativeSession({ + runId: execution.binding.runId, + message: "Do not wait forever", + correlationId: "queued-comment-timeout", + timeoutMs: 5, + }).catch((value) => value); + expect(error).toBeInstanceOf(NativeSessionSteeringError); + expect(error.code).toBe("steering_timeout"); + + state.release?.(); + await running; + }); +}); + +describe("native warm session supervision", () => { + it("reuses one session across distinct governed runs and closes it after idle expiry", async () => { + const close = vi.fn(async () => undefined); + const sharedSession = { close }; + const base = { + ...execution, + binding: { + ...execution.binding, + executionWorkspaceId: "workspace", + }, + workspace: { + cwd: "/tmp/warm-native", + repoUrl: null, + repoRef: null, + branchName: null, + }, + session: { + normalizedSessionId: "session-warm-native", + driverKind: "codex_app_server" as const, + protocolVersion: 1 as const, + lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 20 }, + }, + } as NativeExecutionInputV1; + const second = { + ...base, + binding: { ...base.binding, runId: "run-native-warm-second" }, + }; + const result = { + result: { summary: "completed" }, + terminal: { runTerminalState: "succeeded" }, + turnId: "turn", + normalizedSessionId: "session-warm-native", + providerSessionId: "provider-warm-native", + driverKind: "test", + driverVersion: "1", + nativeEventCount: 1, + highestContiguousSourceSeq: 1, + usage: null, + }; + state.execute + .mockReset() + .mockImplementationOnce(async (options) => { + expect(options.existingSession).toBeUndefined(); + options.onSession?.(sharedSession); + return result; + }) + .mockImplementationOnce(async (options) => { + expect(options.existingSession).toBe(sharedSession); + return result; + }); + + await executePaperclipNativeSession({ + db: leaseDb(base), + execution: base, + runnerInstanceId: "runner", + }); + await executePaperclipNativeSession({ + db: leaseDb(second), + execution: second, + runnerInstanceId: "runner", + }); + expect(close).not.toHaveBeenCalled(); + await vi.waitFor( + () => + expect(close).toHaveBeenCalledWith({ + reason: "warm native session idle timeout", + }), + { timeout: 500 }, + ); + }); + + it("does not replace a different company's warm session with the same normalized id", async () => { + const firstClose = vi.fn(async () => undefined); + const secondClose = vi.fn(async () => undefined); + const base = { + ...execution, + binding: { + ...execution.binding, + companyId: "company-warm-first", + runId: "run-warm-first", + executionWorkspaceId: "workspace", + }, + workspace: { + cwd: "/tmp/warm-native-company-isolation", + repoUrl: null, + repoRef: null, + branchName: null, + }, + session: { + normalizedSessionId: "shared-company-warm-session", + driverKind: "codex_app_server" as const, + protocolVersion: 1 as const, + lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 20 }, + }, + } as NativeExecutionInputV1; + const second = { + ...base, + binding: { + ...base.binding, + companyId: "company-warm-second", + runId: "run-warm-second", + }, + }; + const result = { + result: { summary: "completed" }, + terminal: { runTerminalState: "succeeded" }, + turnId: "turn", + normalizedSessionId: "shared-company-warm-session", + providerSessionId: "provider-warm-native", + driverKind: "test", + driverVersion: "1", + nativeEventCount: 1, + highestContiguousSourceSeq: 1, + usage: null, + }; + state.execute + .mockReset() + .mockImplementationOnce(async (options) => { + expect(options.existingSession).toBeUndefined(); + options.onSession?.({ close: firstClose }); + return result; + }) + .mockImplementationOnce(async (options) => { + expect(options.existingSession).toBeUndefined(); + options.onSession?.({ close: secondClose }); + return result; + }); + + await executePaperclipNativeSession({ + db: leaseDb(base), + execution: base, + runnerInstanceId: "runner-first", + }); + await executePaperclipNativeSession({ + db: leaseDb(second), + execution: second, + runnerInstanceId: "runner-second", + }); + await vi.waitFor(() => expect(firstClose).toHaveBeenCalled(), { + timeout: 500, + }); + await vi.waitFor(() => expect(secondClose).toHaveBeenCalled(), { + timeout: 500, + }); + expect(firstClose).toHaveBeenCalledWith({ + reason: "warm native session idle timeout", + }); + expect(secondClose).toHaveBeenCalledWith({ + reason: "warm native session idle timeout", + }); + }); + + it("replaces an idle warm provider session when its pinned permission mode changes", async () => { + const firstClose = vi.fn(async () => undefined); + const secondClose = vi.fn(async () => undefined); + const firstSession = { close: firstClose }; + const secondSession = { close: secondClose }; + const base = { + ...execution, + schema: "paperclip.native-execution-input.v4", + provider: { kind: "codex", model: null, approvalPolicy: "never" }, + binding: { + ...execution.binding, + runId: "run-permission-never", + executionWorkspaceId: "workspace", + }, + workspace: { + cwd: "/tmp/warm-native-permission", + repoUrl: null, + repoRef: null, + branchName: null, + }, + session: { + normalizedSessionId: "session-warm-permission", + driverKind: "codex_app_server" as const, + protocolVersion: 1 as const, + lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 20 }, + }, + runtimeContext: { aggregateDigest: "runtime-context" }, + } as unknown as NativeExecutionInputV1; + const lowered = { + ...base, + provider: { kind: "codex", model: null, approvalPolicy: "on-request" }, + binding: { ...base.binding, runId: "run-permission-on-request" }, + } as NativeExecutionInputV1; + const result = { + result: { summary: "completed" }, + terminal: { runTerminalState: "succeeded" }, + turnId: "turn", + normalizedSessionId: "session-warm-permission", + providerSessionId: "provider-warm-permission", + driverKind: "test", + driverVersion: "1", + nativeEventCount: 1, + highestContiguousSourceSeq: 1, + usage: null, + }; + state.execute + .mockReset() + .mockImplementationOnce(async (options) => { + expect(options.existingSession).toBeUndefined(); + options.onSession?.(firstSession); + return result; + }) + .mockImplementationOnce(async (options) => { + expect(options.existingSession).toBeUndefined(); + options.onSession?.(secondSession); + return result; + }); + + await executePaperclipNativeSession({ + db: leaseDb(base), + execution: base, + runnerInstanceId: "runner", + }); + await executePaperclipNativeSession({ + db: leaseDb(lowered), + execution: lowered, + runnerInstanceId: "runner", + }); + expect(firstClose).toHaveBeenCalledWith({ + reason: "warm native session configuration changed", + }); + await vi.waitFor( + () => + expect(secondClose).toHaveBeenCalledWith({ + reason: "warm native session idle timeout", + }), + { timeout: 500 }, + ); + }); +}); + +describe("native session bounded recovery", () => { + it("preserves stable provider and runner failure causes", () => { + expect( + nativeSessionFailureSourceCode( + new Error( + "provider_frame_too_large: harness stdout frame exceeded 4194304 bytes", + ), + ), + ).toBe("provider_frame_too_large"); + expect( + nativeSessionFailureSourceCode( + new Error( + "native_runner_process_exited: runnerd exited unexpectedly with code 1", + ), + ), + ).toBe("native_runner_process_exited"); + expect( + nativeSessionFailureSourceCode( + new Error("provider_transport_failed: invalid JSON-RPC"), + ), + ).toBe("provider_transport_failed"); + expect( + nativeSessionFailureSourceCode( + new Error( + "planning_mode_unsupported: installed Codex app-server did not confirm plan mode", + ), + ), + ).toBe("planning_mode_unsupported"); + expect( + nativeSessionFailureSourceCode( + new Error( + "native_event_replay_conflict: source sequence 41 contained different bytes", + ), + ), + ).toBe("native_event_replay_conflict"); + expect( + nativeSessionFailureSourceCode( + new Error( + "provider_process_exited: provider=codex stage=initialize exitCode=1", + ), + ), + ).toBe("provider_process_exited"); + expect( + nativeSessionFailureSourceCode( + new Error("provider_stdout_closed: provider=codex stage=initialize"), + ), + ).toBe("provider_stdout_closed"); + expect( + nativeSessionFailureSourceCode( + new Error( + "provider_process_status_failed: provider=codex stage=session.open", + ), + ), + ).toBe("provider_process_status_failed"); + expect( + nativeSessionFailureSourceCode( + new Error( + "provider_initialize_timeout: provider=codex stage=initialize", + ), + ), + ).toBe("provider_initialize_timeout"); + expect( + nativeSessionFailureSourceCode( + new Error( + "provider_initialize_protocol_error: provider=codex stage=initialize", + ), + ), + ).toBe("provider_initialize_protocol_error"); + expect( + nativeSessionFailureSourceCode( + new Error("provider_request_timeout: provider=codex stage=turn.start"), + ), + ).toBe("provider_request_timeout"); + }); + + it("retries the same run twice and stops at the third failed attempt", () => { + const now = new Date("2026-08-09T00:00:00.000Z"); + expect(nativeSessionFailureDisposition(1, now)).toEqual({ + phase: "retryable_failure", + failureCode: "native_session_interrupted", + nextAttemptAt: new Date("2026-08-09T00:00:30.000Z"), + }); + expect(nativeSessionFailureDisposition(2, now)).toEqual({ + phase: "retryable_failure", + failureCode: "native_session_interrupted", + nextAttemptAt: new Date("2026-08-09T00:00:30.000Z"), + }); + expect(nativeSessionFailureDisposition(3, now)).toEqual({ + phase: "terminal_failure", + failureCode: "native_session_retry_exhausted", + nextAttemptAt: null, + }); + expect( + nativeSessionFailureDisposition(1, now, "native_event_replay_conflict"), + ).toEqual({ + phase: "terminal_failure", + failureCode: "native_event_replay_conflict", + nextAttemptAt: null, + }); + }); + + it("escalates exhausted result-less sessions to board review instead of leaving the provider as its own owner", () => { + expect( + nativeSessionRecoveryProjection({ + phase: "retryable_failure", + failureCode: "native_session_interrupted", + agentId: "agent-low-capability", + }), + ).toEqual({ + exhausted: false, + issueStatus: null, + recoveryOwner: { kind: "agent", agentId: "agent-low-capability" }, + recoveryActionOwnerType: "agent", + recoveryActionOwnerAgentId: "agent-low-capability", + recoveryActionCause: "native_session_interrupted", + supersedeOnIdentityChange: true, + }); + expect( + nativeSessionRecoveryProjection({ + phase: "terminal_failure", + failureCode: "native_session_retry_exhausted", + agentId: "agent-low-capability", + }), + ).toEqual({ + exhausted: true, + issueStatus: "in_review", + recoveryOwner: { kind: "board" }, + recoveryActionOwnerType: "board", + recoveryActionOwnerAgentId: null, + recoveryActionCause: "native_session_retry_exhausted", + supersedeOnIdentityChange: true, + }); + }); +}); + +describe("native process ownership", () => { + it("forwards the app-server PID and process group through the production backend seam", async () => { + const processMetadata = { + pid: 42_001, + processGroupId: 42_001, + startedAt: "2026-08-18T18:00:00.000Z", + }; + const onSpawn = vi.fn(async () => undefined); + state.createBackend.mockClear(); + state.execute.mockReset().mockImplementation(async (options) => { + await options.backend.onSpawn(processMetadata); + return { + result: { summary: "completed" }, + terminal: { runTerminalState: "succeeded" }, + turnId: "turn", + normalizedSessionId: "session", + providerSessionId: null, + driverKind: "test", + driverVersion: "1", + nativeEventCount: 1, + highestContiguousSourceSeq: 1, + }; + }); + state.createBackend.mockImplementationOnce((_input, options) => ({ + kind: "test", + onSpawn: options.onSpawn, + })); + + await executePaperclipNativeSession({ + db: leaseDb(), + execution, + runnerInstanceId: "runner", + onSpawn, + }); + + expect(state.createBackend).toHaveBeenCalledWith( + execution, + expect.objectContaining({ + runnerInstanceId: "runner", + onSpawn, + }), + ); + expect(onSpawn).toHaveBeenCalledWith(processMetadata); + }); +}); + +describe("runnerd provider runtime wiring", () => { + it("reuses legacy unscoped state only for its exact durable run identity", async () => { + const stateBase = await mkdtemp(join(tmpdir(), "paperclip-legacy-runner-state-")); + const previousStateDirectory = process.env.PAPERCLIP_RUNNER_STATE_DIR; + process.env.PAPERCLIP_RUNNER_STATE_DIR = stateBase; + const legacyExecution = { + ...execution, + binding: { + ...execution.binding, + companyId: "company-legacy-state", + runId: "run-legacy-state", + }, + session: { + ...execution.session, + normalizedSessionId: "session-legacy-state", + }, + } as NativeExecutionInputV1; + const legacyRoot = join( + stateBase, + createHash("sha256").update("session-legacy-state").digest("hex"), + ); + try { + await mkdir(join(legacyRoot, "control-plane"), { recursive: true }); + await writeFile( + join(legacyRoot, "control-plane", "mock-core-state.json"), + JSON.stringify({ + identity: { + runId: "run-legacy-state", + normalizedSessionId: "session-legacy-state", + runnerInstanceId: "runner-legacy-state", + environmentLeaseId: "lease-legacy-state", + }, + }), + ); + state.createBackend.mockClear(); + await createRunnerdBackend({ + db: leaseDb(legacyExecution), + execution: legacyExecution, + runnerInstanceId: "runner-legacy-state", + }); + state.createTransport.mockClear(); + state.createBackend.mock.calls[0]![1].codexTransportFactory!(); + expect(state.createTransport.mock.calls[0]![0].stateDirectory).toBe( + legacyRoot, + ); + expect(state.createTransport.mock.calls[0]![0].runnerBinary).toBe( + "/tmp/paperclip-runnerd", + ); + expect(state.resolveRunnerBinary).toHaveBeenCalled(); + + const unrelatedExecution = { + ...legacyExecution, + binding: { + ...legacyExecution.binding, + companyId: "company-unrelated-state", + runId: "run-unrelated-state", + }, + } as NativeExecutionInputV1; + await createRunnerdBackend({ + db: leaseDb(unrelatedExecution), + execution: unrelatedExecution, + runnerInstanceId: "runner-unrelated-state", + }); + state.createBackend.mock.calls[1]![1].codexTransportFactory!(); + expect(state.createTransport.mock.calls[1]![0].stateDirectory).not.toBe( + legacyRoot, + ); + } finally { + if (previousStateDirectory === undefined) { + delete process.env.PAPERCLIP_RUNNER_STATE_DIR; + } else { + process.env.PAPERCLIP_RUNNER_STATE_DIR = previousStateDirectory; + } + await rm(stateBase, { recursive: true, force: true }); + } + }); + + it("isolates durable state and tool authority for equal session ids in different companies", async () => { + const scopedExecution = (companyId: string, runId: string) => + ({ + ...execution, + schema: "paperclip.native-execution-input.v4", + binding: { + ...execution.binding, + companyId, + runId, + executionWorkspaceId: "workspace", + }, + task: { + identifier: "DOT-ISOLATION", + title: "Isolation test", + description: null, + prompt: "Verify session isolation.", + workMode: "standard", + }, + workspace: { + cwd: "/tmp/native-session-isolation", + repoUrl: null, + repoRef: null, + branchName: null, + }, + session: { + normalizedSessionId: "shared-normalized-session", + driverKind: "codex_app_server", + protocolVersion: 1, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, + }, + provider: { kind: "codex", model: null, approvalPolicy: "never" }, + executionMode: "default", + planningContext: null, + interactionResponses: [], + credentialBindings: [], + runtimeContext: {}, + }) as unknown as NativeExecutionInputV1; + const firstExecution = scopedExecution("company-first", "run-first"); + const secondExecution = scopedExecution("company-second", "run-second"); + state.createBackend.mockClear(); + state.toolAuthorityExecute + .mockReset() + .mockImplementation((binding: Record) => + Promise.resolve({ runId: binding.runId }), + ); + + await createRunnerdBackend({ + db: leaseDb(firstExecution), + execution: firstExecution, + runnerInstanceId: "runner-first", + }); + await createRunnerdBackend({ + db: leaseDb(secondExecution), + execution: secondExecution, + runnerInstanceId: "runner-second", + }); + + const firstOptions = state.createBackend.mock.calls[0]![1]; + const secondOptions = state.createBackend.mock.calls[1]![1]; + state.createTransport.mockClear(); + firstOptions.codexTransportFactory!(); + secondOptions.codexTransportFactory!(); + expect(state.createTransport.mock.calls[0]![0].stateDirectory).not.toBe( + state.createTransport.mock.calls[1]![0].stateDirectory, + ); + await expect(firstOptions.dynamicToolHandler!({})).resolves.toEqual({ + runId: "run-first", + }); + await expect(secondOptions.dynamicToolHandler!({})).resolves.toEqual({ + runId: "run-second", + }); + }); + +}); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts new file mode 100644 index 0000000000..b2cbc5eb00 --- /dev/null +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -0,0 +1,3589 @@ +import { createHash, randomUUID } from "node:crypto"; +import { + chmodSync, + closeSync, + constants, + existsSync, + fstatSync, + lstatSync, + mkdirSync, + openSync, + readSync, + renameSync, + writeFileSync, +} from "node:fs"; +import { resolve } from "node:path"; +import type { + AdapterExecutionResult, + AdapterRuntimeEvent, +} from "../../adapters/index.js"; +import type { NativeFinalizationResult } from "@paperclipai/shared"; +import type { + HarnessRuntimeRequestResolution, + NativeExecutionInput, + NativeSession, + NativeSessionBackend, + PaperclipQuestionSet, + PersistedNativeSession, + PrpEvent, + PrpStructuredRunResult, +} from "../../vendor/paperclip-runner/index.js"; +import { + createNativeSessionBackend, + createRunnerdCodexTransport, + executeNativeSession, + parsePaperclipQuestionSet, +} from "../../vendor/paperclip-runner/index.js"; +import type { Db } from "@paperclipai/db"; +import { and, desc, eq, gt, inArray, or, sql } from "drizzle-orm"; +import { + documentRevisions, + heartbeatRunEvents, + heartbeatRuns, + issueDocuments, + issueThreadInteractions, + issues, + nativeRunFinalizations, +} from "@paperclipai/db"; +import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js"; +import { PaperclipRunnerToolAuthority } from "./paperclip-runner-tool-authority.js"; +import { registerRunnerPrpAuthority } from "../../realtime/runner-prp-ws.js"; +import { issueRecoveryActionService } from "../issue-recovery-actions.js"; +import { persistActivity, publishActivity } from "../activity-log.js"; +import { commitNativeStatusDecision } from "./status-decision-committer.js"; +import { resolvePaperclipInstanceRoot } from "../../home-paths.js"; +import { documentService } from "../documents.js"; +import { issueThreadInteractionService } from "../issue-thread-interactions.js"; +import { issueService } from "../issues.js"; +import { + NATIVE_STATUS_ARBITER_POLICY_VERSION, + type NativeAuthoritativeIssueStatus, + type NativeStatusDecision, +} from "./status-arbiter.js"; +import { HttpError } from "../../errors.js"; +import { resolvePaperclipRunnerBinary } from "./native-codex-runner.js"; +import { + createNativeRunTrace, + type NativeRunHistoricalSpan, + type NativeRunSpanScope, + type NativeRunTrace, +} from "./native-run-trace.js"; + +type ActiveNativeSession = { + session: NativeSession; + cancelRequested: boolean; +}; + +class NativeResultPendingFinalizationError extends Error { + constructor() { + super("native_result_pending_finalization"); + this.name = "NativeResultPendingFinalizationError"; + } +} + +export class NativeCancellationPendingRecoveryError extends Error { + constructor() { + super("native_cancellation_pending_recovery"); + this.name = "NativeCancellationPendingRecoveryError"; + } +} + +const activeNativeSessions = new Map(); +const NATIVE_DURABLE_IDENTITY_MAX_BYTES = 2 * 1024 * 1024; +const NATIVE_WARM_CHECKPOINT_MAX_BYTES = 8 * 1024 * 1024; +const NATIVE_SESSION_EXECUTION_LEASE_TTL_MS = 20 * 60_000; +const NATIVE_SESSION_EXECUTION_LEASE_RENEW_INTERVAL_MS = 5 * 60_000; +const NATIVE_SESSION_CANCELLATION_CLEANUP_GRACE_MS = 2_000; +const NATIVE_RUNTIME_REQUEST_RESOLUTION_CACHE_MAX = 256; +type NativeRuntimeRequestResolution = { + runId: string; + fingerprint: string; + commandId: string; + pending: Promise; + completedAt: number | null; +}; +const nativeRuntimeRequestResolutions = new Map< + string, + NativeRuntimeRequestResolution +>(); + +function pruneNativeRuntimeRequestResolutionCache(): void { + const completed = [...nativeRuntimeRequestResolutions.entries()] + .filter(([, resolution]) => resolution.completedAt !== null) + .sort( + ([, left], [, right]) => + (left.completedAt ?? 0) - (right.completedAt ?? 0), + ); + for ( + let index = 0; + index < completed.length - NATIVE_RUNTIME_REQUEST_RESOLUTION_CACHE_MAX; + index += 1 + ) { + nativeRuntimeRequestResolutions.delete(completed[index]![0]); + } +} + +function clearNativeRuntimeRequestResolutions(runId: string): void { + for (const [key, resolution] of nativeRuntimeRequestResolutions) { + if (resolution.runId === runId) { + nativeRuntimeRequestResolutions.delete(key); + } + } +} + +type WarmNativeSession = { + session: NativeSession; + configDigest: string; + busy: boolean; + idleTimer: ReturnType | null; + lastActivityAt: string; +}; + +const warmNativeSessions = new Map(); + +function readBoundedNativeFile( + path: string, + maxBytes: number, + errorCode: string, +): Buffer { + const descriptor = openSync( + path, + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + try { + const before = fstatSync(descriptor); + if (!before.isFile() || before.size > maxBytes) throw new Error(errorCode); + const output = Buffer.allocUnsafe(before.size); + let offset = 0; + while (offset < output.length) { + const bytesRead = readSync( + descriptor, + output, + offset, + output.length - offset, + offset, + ); + if (bytesRead === 0) break; + offset += bytesRead; + } + const after = fstatSync(descriptor); + if ( + offset !== before.size || + after.size !== before.size || + after.mtimeMs !== before.mtimeMs || + after.ino !== before.ino + ) { + throw new Error("native_state_file_changed"); + } + return output; + } finally { + closeSync(descriptor); + } +} + +const NATIVE_PROVIDER_HOST_ENV_KEYS = [ + "PATH", + "HOME", + "USER", + "LOGNAME", + "SHELL", + "TMPDIR", + "TEMP", + "TMP", + "CODEX_HOME", + "XDG_CONFIG_HOME", + "XDG_CACHE_HOME", + "XDG_DATA_HOME", + "SystemRoot", + "PATHEXT", +] as const; + +async function measureNativeRunnerSpan( + trace: NativeRunTrace | undefined, + name: string, + fn: () => Promise, + options: + | string + | { + parentName?: string; + attributes?: Record; + } = {}, +): Promise { + return trace + ? trace.measure( + name, + fn, + typeof options === "string" ? { parentName: options } : options, + ) + : fn(); +} + +/** + * Provider bootstrap needs a small amount of host process context even when + * the agent has no configured env. In particular, an empty environment makes + * a bare `codex` command unresolvable. Agent-configured values remain + * authoritative and may intentionally override the host defaults. + */ +export function buildNativeProviderEnvironment( + configured: NodeJS.ProcessEnv, + host: NodeJS.ProcessEnv = process.env, +): NodeJS.ProcessEnv { + const inherited = Object.fromEntries( + NATIVE_PROVIDER_HOST_ENV_KEYS.flatMap((key) => { + const value = host[key]; + return typeof value === "string" && value.length > 0 + ? [[key, value]] + : []; + }), + ); + return { ...inherited, ...configured }; +} + +type PlanSynchronization = { + eventId: string; + planId: string; + providerRevision: number; + status: + | "synchronized" + | "already_synchronized" + | "conflict" + | "invalid" + | "approval_failed"; + baseRevisionId: string | null; + digest: string; + documentRevision: number | null; + currentRevisionId: string | null; + confirmationId: string | null; +}; + +type RuntimeQuestionFallback = { + kind: "ask_user_questions"; + idempotencyKey: string; + sourceRunId: string; + title: string | null; + summary: string | null; + continuationPolicy: "wake_assignee"; + payload: { + version: 1; + title?: string; + submitLabel?: string; + supersedeOnUserComment: false; + runtimeRequestId: string; + questionSet: PaperclipQuestionSet; + questions: Array<{ + id: string; + prompt: string; + helpText?: string; + selectionMode: "single" | "multi"; + required: boolean; + options: Array<{ + id: string; + label: string; + description?: string; + freeText?: boolean; + }>; + }>; + }; +}; + +/** Translate non-replayable live-input expirations into one durable interaction. */ +export function runtimeQuestionFallbackFromEvent( + event: Pick, +): RuntimeQuestionFallback | null { + if (event.eventType !== "runtime_request.expired") return null; + const payload = record(event.payload); + if ( + !["durable_handoff", "provider_process_lost"].includes( + String(payload.reason), + ) || + payload.replayAllowed !== false + ) + return null; + const request = record(payload.request); + if ( + payload.requestKind !== "runtime" || + payload.requestType !== "input" || + request.schema !== "paperclip.runtime_request.v2" || + request.requestKind !== "runtime" || + request.type !== "input" || + typeof request.requestId !== "string" || + payload.requestId !== request.requestId || + typeof request.turnId !== "string" || + typeof request.itemId !== "string" + ) + return null; + let questionSet: PaperclipQuestionSet; + try { + questionSet = parsePaperclipQuestionSet(request.input); + } catch { + return null; + } + const questions = questionSet.questions.map((question) => ({ + id: question.id, + prompt: question.prompt, + ...(question.helpText ? { helpText: question.helpText } : {}), + selectionMode: + question.answerMode === "multi_select" + ? ("multi" as const) + : ("single" as const), + required: question.required, + options: + question.answerMode === "text" + ? [ + { + id: "__paperclip_text__", + label: + question.textValidation?.inputType === "integer" + ? "Enter an integer" + : question.textValidation?.inputType === "number" + ? "Enter a number" + : "Enter your answer", + freeText: true, + }, + ] + : (question.options ?? []).map((option) => ({ + id: option.id, + label: option.label, + ...(option.description ? { description: option.description } : {}), + })), + })); + return { + kind: "ask_user_questions", + idempotencyKey: `runtime-input-durable:v1:${event.runId}:${request.requestId}`, + sourceRunId: event.runId, + title: questionSet.title?.slice(0, 240) ?? null, + summary: questionSet.description?.slice(0, 1000) ?? null, + continuationPolicy: "wake_assignee", + payload: { + version: 1, + ...(questionSet.title ? { title: questionSet.title.slice(0, 240) } : {}), + ...(questionSet.submitLabel + ? { submitLabel: questionSet.submitLabel.slice(0, 120) } + : {}), + supersedeOnUserComment: false, + runtimeRequestId: request.requestId, + questionSet, + questions, + }, + }; +} + +/** + * Materialize the durable replacement for a non-replayable runtime question. + * + * The interaction service enforces the fallback's stable idempotency key, so + * this is safe both immediately after the event commit and while recovering an + * exact duplicate whose original post-commit callback did not finish. + */ +export async function materializeRuntimeQuestionFallback(input: { + db: Db; + binding: { + companyId: string; + issueId: string; + runId: string; + agentId: string; + }; + event: Pick; +}): Promise<{ + fallback: RuntimeQuestionFallback; + interaction: { id: string }; +} | null> { + const fallback = runtimeQuestionFallbackFromEvent(input.event); + if (!fallback) return null; + const interaction = await issueThreadInteractionService(input.db).create( + { + id: input.binding.issueId, + companyId: input.binding.companyId, + }, + fallback as never, + { + agentId: input.binding.agentId, + runId: input.binding.runId, + systemId: "native-runtime-question-handoff", + }, + ); + return { fallback, interaction }; +} + +export function runtimeInputLifecycleMetric( + event: Pick, +): { + outcome: + | "normalized" + | "rejected" + | "resolved" + | "expired" + | "durable_handoff" + | "provider_loss_handoff" + | "cancelled"; + adapter: string; + requestId: string | null; +} | null { + const payload = record(event.payload); + const request = record(payload.request); + if ( + event.eventType === "runtime_request.created" && + request.type === "input" + ) { + const origin = record(request.origin); + return { + outcome: "normalized", + adapter: typeof origin.adapter === "string" ? origin.adapter : "unknown", + requestId: + typeof request.requestId === "string" ? request.requestId : null, + }; + } + if ( + event.eventType === "harness.diagnostic" && + payload.code === "runtime_input_rejected" + ) { + return { + outcome: "rejected", + adapter: + typeof payload.adapter === "string" ? payload.adapter : "unknown", + requestId: null, + }; + } + const terminalOutcome = + event.eventType === "runtime_request.resolved" + ? "resolved" + : event.eventType === "runtime_request.expired" && + payload.reason === "durable_handoff" + ? "durable_handoff" + : event.eventType === "runtime_request.expired" && + payload.reason === "provider_process_lost" + ? "provider_loss_handoff" + : event.eventType === "runtime_request.expired" + ? "expired" + : event.eventType === "runtime_request.cancelled" + ? "cancelled" + : null; + const requestType = payload.requestType ?? request.type; + if (!terminalOutcome || requestType !== "input") return null; + const origin = record(request.origin); + return { + outcome: terminalOutcome, + adapter: + typeof payload.adapter === "string" + ? payload.adapter + : typeof origin.adapter === "string" + ? origin.adapter + : "unknown", + requestId: + typeof payload.requestId === "string" + ? payload.requestId + : typeof request.requestId === "string" + ? request.requestId + : null, + }; +} + +export function providerPlanMarkdown(payload: Record): string { + const completedMarkdown = + typeof payload.markdown === "string" ? payload.markdown.trim() : ""; + if (completedMarkdown) return completedMarkdown.slice(0, 256_000); + const explanation = + typeof payload.explanation === "string" ? payload.explanation.trim() : ""; + const steps = Array.isArray(payload.steps) ? payload.steps : []; + const lines = steps.slice(0, 256).flatMap((value) => { + if (!value || typeof value !== "object" || Array.isArray(value)) return []; + const step = value as Record; + const body = + typeof step.body === "string" ? step.body.trim().slice(0, 4_000) : ""; + if (!body) return []; + const status = step.status === "completed" ? "x" : " "; + const suffix = + step.status === "blocked" + ? " _(blocked)_" + : step.status === "in_progress" + ? " _(in progress)_" + : ""; + return [`- [${status}] ${body}${suffix}`]; + }); + return [explanation, lines.join("\n")] + .filter(Boolean) + .join("\n\n") + .slice(0, 256_000); +} + +export function semanticProviderPlanMarkdown( + result: Record, +): string | null { + const artifacts = Array.isArray(result.artifacts) ? result.artifacts : []; + for (const value of artifacts) { + const artifact = record(value); + if ( + artifact.kind !== "native_provider_plan" || + typeof artifact.ref !== "string" + ) + continue; + const match = artifact.ref.match( + /\s*([\s\S]*?)\s*<\/proposed_plan>/i, + ); + const completedMarkdown = match?.[1]?.trim(); + if (completedMarkdown) return completedMarkdown.slice(0, 256_000); + + const embedded = artifact.ref.match( + /^native-provider-plan:([^\n]+)\n([\s\S]+)$/i, + ); + if (embedded) { + const title = embedded[1]! + .replace(/^(?:DOT-\d+-)?/i, "") + .replace(/-v\d+$/i, "") + .replace(/-/g, " ") + .trim(); + const body = embedded[2]!.trim(); + if (title && body) { + return [`# ${title.charAt(0).toUpperCase()}${title.slice(1)}`, "", body] + .join("\n") + .slice(0, 256_000); + } + } + + if (/^\s*1\.\s+/.test(artifact.ref)) { + const numberedPlan = artifact.ref + .split(/\s+\|\s+(?=\d+\.\s+)/) + .join("\n") + .trim(); + if (numberedPlan) return `# Plan\n\n${numberedPlan}`.slice(0, 256_000); + } + + // Some qualified Codex builds use the artifact reference itself as a + // compact, human-readable plan. Accept only an explicitly numbered form; + // arbitrary opaque artifact references must never become plan documents. + const inlineNumbered = artifact.ref.trim(); + if (/\(1\)\s+.+\(2\)\s+/s.test(inlineNumbered)) { + const body = inlineNumbered + .replace(/^DOT-\d+\s+plan:\s*/i, "") + .replace(/^\(1\)\s*/, "1. ") + .replace(/;\s*\((\d+)\)\s*/g, "\n$1. ") + .trim(); + if (body) return `# Plan\n\n${body}`.slice(0, 256_000); + } + + const compact = + artifact.ref.match(/^native-provider-plan:([^#]+)#(.+)$/i) ?? + artifact.ref.match(/^native-plan:\/\/[^/]+\/([^#]+)#(.+)$/i); + if (!compact) continue; + const humanize = (slug: string) => + slug + .replace( + /\b(GET|POST|PUT|PATCH|DELETE)-([a-z0-9][a-z0-9-]*)/gi, + (_whole, method: string, path: string) => + `${method.toUpperCase()} /${path}`, + ) + .replace(/-/g, " ") + .replace(/\bjson\b/gi, "JSON") + .replace(/\bapi\b/gi, "API") + .replace(/\s+/g, " ") + .trim(); + const title = humanize(compact[1]!.replace(/-v\d+$/i, "")); + const steps = compact[2]!.split(";").flatMap((encoded) => { + const parsed = encoded.match(/^\d+-(.+)$/); + const sentence = humanize(parsed?.[1] ?? encoded); + return sentence + ? [sentence.charAt(0).toUpperCase() + sentence.slice(1)] + : []; + }); + if (!title || steps.length === 0) continue; + return [ + `# ${title.charAt(0).toUpperCase()}${title.slice(1)}`, + "", + ...steps.map((step, index) => `${index + 1}. ${step}`), + ] + .join("\n") + .slice(0, 256_000); + } + const hasNativePlanArtifact = artifacts.some( + (value) => record(value).kind === "native_provider_plan", + ); + const summary = + typeof result.summary === "string" ? result.summary.trim() : ""; + const summaryPlan = hasNativePlanArtifact + ? summary.match( + /(?:^|:\s*)(1\)\s+[\s\S]+;\s*2\)\s+[\s\S]+;\s*3\)\s+[\s\S]+)$/, + ) + : null; + if (summaryPlan) { + const body = summaryPlan[1]! + .replace(/^1\)\s*/, "1. ") + .replace(/;\s*(\d+)\)\s*/g, "\n$1. ") + .trim(); + if (body) return `# Plan\n\n${body}`.slice(0, 256_000); + } + return null; +} + +/** + * Convert a server-owned pending interaction into the semantic wait that a + * provider omitted. This is not a fabricated final response: it records that + * the current turn intentionally yielded to a durable governance surface. + */ +export function nativeGovernedWaitResult(input: { + interaction: { id: string; title: string | null; summary: string | null }; + completionContract: NativeExecutionInput["completionContract"]["contract"]; +}): PrpStructuredRunResult { + const interactionRef = `interaction:${input.interaction.id}`; + const label = + input.interaction.title?.trim() || + input.interaction.summary?.trim() || + "the requested response"; + return { + schema: "paperclip.run_result.v1", + reportedWorkDisposition: "yielded", + summary: `Waiting for ${label}.`, + completionClaim: { + contractRevision: input.completionContract.revision, + objectiveSatisfied: false, + criteria: input.completionContract.criteria.map((criterion) => ({ + criterionId: criterion.id, + status: "unknown", + evidenceRefs: [interactionRef], + })), + remainingWork: [ + { + description: "Resume after the durable interaction is resolved.", + blocksCompletion: true, + }, + ], + }, + evidence: [{ ref: interactionRef }], + verification: [], + attentionRequests: [], + artifacts: [{ kind: "issue_thread_interaction", ref: interactionRef }], + continuation: { + kind: "response_wake", + summary: + "Resume from the resolved interaction response without repeating prior work.", + idempotencyKey: `interaction-response:${input.interaction.id}`, + }, + }; +} + +/** + * Bridge an asynchronous durable-interaction lookup to the runner package's + * synchronous governed-wait boundary. Observations are single-use and bound + * to one exact source event so a delayed or replayed lookup cannot leak into a + * later provider event. + */ +export function createGovernedWaitEventObservation( + resolvePending: () => Promise, +) { + let generation = 0; + let observation: { + sourceInstanceId: string; + sourceEventId: string; + sourceSeq: number; + result: PrpStructuredRunResult; + } | null = null; + + return { + async observe(event: PrpEvent, eligible: boolean): Promise { + const currentGeneration = ++generation; + observation = null; + if (!eligible) return; + const result = await resolvePending(); + if (generation !== currentGeneration || result === null) return; + // If the interaction is answered after this read, parking remains the + // fail-closed outcome: the durable answer owns the response-wake path. + // Continuing provider work on a possibly stale authorization does not. + observation = { + sourceInstanceId: event.sourceInstanceId, + sourceEventId: event.sourceEventId, + sourceSeq: event.sourceSeq, + result, + }; + }, + consume(event: PrpEvent): PrpStructuredRunResult | null { + generation += 1; + const current = observation; + observation = null; + if ( + current === null || + current.sourceInstanceId !== event.sourceInstanceId || + current.sourceEventId !== event.sourceEventId || + current.sourceSeq !== event.sourceSeq + ) { + return null; + } + return current.result; + }, + }; +} + +/** + * Partial item-verdict responses deliberately leave their original durable + * interaction pending. They are already authority-checked before entering the + * closed native envelope, so that exact interaction may park the continuation + * run without requiring the model to recreate a second request. + */ +export function continuingPendingInteractionIds( + execution: NativeExecutionInput, +): string[] { + return execution.interactionResponses + .filter( + (response) => + response.kind === "request_item_verdicts" && + response.response.status === "pending", + ) + .map((response) => response.interactionId); +} + +export async function synchronizeCompletedProviderPlan(input: { + db: Db; + execution: NativeExecutionInput; + event: { + sourceEventId: string; + turnId?: string; + eventType: string; + payload: Record; + }; +}): Promise { + if ( + input.event.eventType !== "plan.updated" || + input.event.payload.complete !== true + ) + return null; + if ( + !("executionMode" in input.execution) || + input.execution.executionMode !== "plan" + ) + return null; + const planningContext = input.execution.planningContext; + if (!planningContext) return null; + const planId = + typeof input.event.payload.planId === "string" + ? input.event.payload.planId + : ""; + const providerRevision = Number.isSafeInteger(input.event.payload.revision) + ? Number(input.event.payload.revision) + : 0; + const body = providerPlanMarkdown(input.event.payload); + const digest = createHash("sha256").update(body).digest("hex"); + if (!planId || providerRevision < 1 || !body) { + return { + eventId: input.event.sourceEventId, + planId, + providerRevision, + status: "invalid", + baseRevisionId: planningContext.baseRevisionId, + digest, + documentRevision: null, + currentRevisionId: null, + confirmationId: null, + }; + } + const provenance = `runner-plan-sync:v2 run=${input.execution.binding.runId} turn=${input.event.turnId ?? "unknown"} provider=${input.execution.provider.kind} plan=${planId} revision=${providerRevision} digest=${digest}`; + const existingRevision = await input.db + .select({ + revisionNumber: documentRevisions.revisionNumber, + id: documentRevisions.id, + }) + .from(documentRevisions) + .innerJoin( + issueDocuments, + eq(issueDocuments.documentId, documentRevisions.documentId), + ) + .where( + and( + eq(issueDocuments.issueId, input.execution.binding.issueId), + eq(issueDocuments.key, "plan"), + eq(documentRevisions.changeSummary, provenance), + ), + ) + .orderBy(desc(documentRevisions.revisionNumber)) + .limit(1) + .then((rows) => rows[0] ?? null); + const documents = documentService(input.db); + let revision = existingRevision; + let status: PlanSynchronization["status"] = existingRevision + ? "already_synchronized" + : "synchronized"; + if (!revision) { + const latest = await documents.getIssueDocumentByKey( + input.execution.binding.issueId, + "plan", + ); + if (latest?.latestRevisionId && latest.body === body) { + const sameRunRevision = await input.db + .select({ + id: documentRevisions.id, + revisionNumber: documentRevisions.revisionNumber, + createdByRunId: documentRevisions.createdByRunId, + }) + .from(documentRevisions) + .where(eq(documentRevisions.id, latest.latestRevisionId)) + .limit(1) + .then((rows) => rows[0] ?? null); + if (sameRunRevision?.createdByRunId === input.execution.binding.runId) { + revision = sameRunRevision; + status = "already_synchronized"; + } + } + } + try { + if (!revision) { + const write = await documents.upsertIssueDocument({ + issueId: input.execution.binding.issueId, + key: "plan", + title: "Plan", + format: "markdown", + body, + baseRevisionId: planningContext.baseRevisionId, + changeSummary: provenance, + createdByAgentId: input.execution.binding.agentId, + createdByRunId: input.execution.binding.runId, + }); + revision = { + revisionNumber: write.document.latestRevisionNumber, + id: write.document.latestRevisionId!, + }; + } + } catch (error) { + if (!(error instanceof HttpError) || error.status !== 409) throw error; + const latest = await documents.getIssueDocumentByKey( + input.execution.binding.issueId, + "plan", + ); + return { + eventId: input.event.sourceEventId, + planId, + providerRevision, + status: "conflict", + baseRevisionId: planningContext.baseRevisionId, + digest, + documentRevision: latest?.latestRevisionNumber ?? null, + currentRevisionId: latest?.latestRevisionId ?? null, + confirmationId: null, + }; + } + const current = await documents.getIssueDocumentByKey( + input.execution.binding.issueId, + "plan", + ); + if (!current || !revision?.id || current.latestRevisionId !== revision.id) { + return { + eventId: input.event.sourceEventId, + planId, + providerRevision, + status: "conflict", + baseRevisionId: planningContext.baseRevisionId, + digest, + documentRevision: current?.latestRevisionNumber ?? null, + currentRevisionId: current?.latestRevisionId ?? null, + confirmationId: null, + }; + } + let confirmationId: string; + let confirmationPending = false; + try { + const confirmation = await issueThreadInteractionService(input.db).create( + { + id: input.execution.binding.issueId, + companyId: input.execution.binding.companyId, + }, + { + kind: "request_confirmation", + idempotencyKey: `runner-plan-approval:v1:${input.execution.binding.runId}:${planId}:${providerRevision}:${digest}`, + sourceRunId: input.execution.binding.runId, + title: `Review plan revision ${revision.revisionNumber}`, + summary: "Review the synchronized Paperclip plan.", + continuationPolicy: "wake_assignee", + payload: { + version: 1, + prompt: `Approve plan revision ${revision.revisionNumber}?`, + detailsMarkdown: + "The completed provider plan has been synchronized to the canonical Plan document.", + acceptLabel: "Approve plan", + rejectLabel: "Request changes", + rejectRequiresReason: true, + supersedeOnUserComment: false, + target: { + type: "issue_document", + issueId: input.execution.binding.issueId, + documentId: current.id, + key: "plan", + revisionId: revision.id, + revisionNumber: revision.revisionNumber, + label: `Plan v${revision.revisionNumber}`, + }, + }, + } as never, + { + agentId: input.execution.binding.agentId, + runId: input.execution.binding.runId, + }, + ); + confirmationId = confirmation.id; + confirmationPending = confirmation.status === "pending"; + } catch { + return { + eventId: input.event.sourceEventId, + planId, + providerRevision, + status: "approval_failed", + baseRevisionId: planningContext.baseRevisionId, + digest, + documentRevision: revision.revisionNumber, + currentRevisionId: revision.id, + confirmationId: null, + }; + } + if (confirmationPending) { + try { + const currentIssue = await input.db + .select({ status: issues.status }) + .from(issues) + .where(eq(issues.id, input.execution.binding.issueId)) + .limit(1) + .then((rows) => rows[0] ?? null); + if (currentIssue && currentIssue.status !== "in_review") { + await issueService(input.db).update(input.execution.binding.issueId, { + status: "in_review", + actorAgentId: input.execution.binding.agentId, + }); + } + } catch { + const settledIssue = await input.db + .select({ status: issues.status }) + .from(issues) + .where(eq(issues.id, input.execution.binding.issueId)) + .limit(1) + .then((rows) => rows[0] ?? null); + if (settledIssue?.status !== "in_review") { + return { + eventId: input.event.sourceEventId, + planId, + providerRevision, + status: "approval_failed", + baseRevisionId: planningContext.baseRevisionId, + digest, + documentRevision: revision.revisionNumber, + currentRevisionId: revision.id, + confirmationId, + }; + } + } + } + return { + eventId: input.event.sourceEventId, + planId, + providerRevision, + status, + baseRevisionId: planningContext.baseRevisionId, + digest, + documentRevision: revision.revisionNumber, + currentRevisionId: revision.id, + confirmationId, + }; +} + +function nativeSessionKey(execution: NativeExecutionInput): string { + return ( + execution.session.normalizedSessionId ?? + `session-${execution.binding.runId}` + ); +} + +function nativeSessionScopeKey(execution: NativeExecutionInput): string { + return JSON.stringify([ + execution.binding.companyId, + nativeSessionKey(execution), + ]); +} + +function runnerdStateBase(): string { + return process.env.PAPERCLIP_RUNNER_STATE_DIR ?? resolve( + resolvePaperclipInstanceRoot(), + "runtime", + "paperclip-runner", + "durable-sessions", + ); +} + +function scopedRunnerdStateRoot(execution: NativeExecutionInput): string { + return resolve( + runnerdStateBase(), + createHash("sha256") + .update(nativeSessionScopeKey(execution)) + .digest("hex"), + ); +} + +function legacyRunnerdStateRoot(execution: NativeExecutionInput): string { + return resolve( + runnerdStateBase(), + createHash("sha256").update(nativeSessionKey(execution)).digest("hex"), + ); +} + +function isSafeNativeStateDirectory(path: string): boolean { + if (!existsSync(path)) return false; + const stats = lstatSync(path); + return stats.isDirectory() && !stats.isSymbolicLink(); +} + +type RunnerdDurableIdentity = Record & { + runId: string; + normalizedSessionId: string; + runnerInstanceId: string; + environmentLeaseId: string; +}; + +function readRunnerdDurableIdentity( + root: string, +): Record | null { + if (!isSafeNativeStateDirectory(root)) return null; + const statePath = resolve(root, "control-plane", "mock-core-state.json"); + if (!existsSync(statePath)) return null; + try { + return record( + record( + JSON.parse( + readBoundedNativeFile( + statePath, + NATIVE_DURABLE_IDENTITY_MAX_BYTES, + "runner_durable_identity_too_large", + ).toString("utf8"), + ), + ).identity, + ); + } catch { + return null; + } +} + +function durableIdentityMatchesExecution( + identity: Record | null, + execution: NativeExecutionInput, +): identity is RunnerdDurableIdentity { + return Boolean( + identity && + identity.runId === execution.binding.runId && + identity.normalizedSessionId === nativeSessionKey(execution) && + typeof identity.runnerInstanceId === "string" && + identity.runnerInstanceId.length > 0 && + typeof identity.environmentLeaseId === "string" && + identity.environmentLeaseId.length > 0, + ); +} + +/** + * Pre-company-scope state is reused only when its durable identity is bound to + * this exact run and normalized session. Run ids are globally unique database + * keys, so another company cannot claim a legacy directory by choosing the + * same display/session id. New sessions always use the company-scoped root. + */ +function runnerdStateRoot(execution: NativeExecutionInput): string { + const scoped = scopedRunnerdStateRoot(execution); + if (existsSync(scoped)) { + if (!isSafeNativeStateDirectory(scoped)) { + throw new Error("runner_state_directory_unsafe"); + } + return scoped; + } + const legacy = legacyRunnerdStateRoot(execution); + return durableIdentityMatchesExecution( + readRunnerdDurableIdentity(legacy), + execution, + ) + ? legacy + : scoped; +} + +function loadRunnerdDurableBinding(execution: NativeExecutionInput): { + runnerInstanceId: string; + environmentLeaseId: string; +} | null { + const identity = readRunnerdDurableIdentity(runnerdStateRoot(execution)); + if (!durableIdentityMatchesExecution(identity, execution)) return null; + return { + runnerInstanceId: identity.runnerInstanceId, + environmentLeaseId: identity.environmentLeaseId, + }; +} + +function nativeSessionConfigDigest(execution: NativeExecutionInput): string { + const executionLocation = { + executionKind: "local_process", + workspaceId: execution.binding.executionWorkspaceId, + cwd: execution.workspace.cwd, + }; + return `sha256:${createHash("sha256") + .update( + JSON.stringify({ + companyId: execution.binding.companyId, + normalizedSessionId: nativeSessionKey(execution), + executionLocation, + provider: execution.provider, + driverKind: execution.session.driverKind, + lifecyclePolicy: execution.session.lifecyclePolicy, + executionMode: + "executionMode" in execution ? execution.executionMode : "default", + runtimeContextDigest: + "runtimeContext" in execution + ? execution.runtimeContext.aggregateDigest + : null, + }), + ) + .digest("hex")}`; +} + +function nativeSessionCheckpointDirectory(): string { + const directory = resolve( + resolvePaperclipInstanceRoot(), + "runtime", + "paperclip-runner", + "sessions", + ); + mkdirSync(directory, { recursive: true, mode: 0o700 }); + chmodSync(directory, 0o700); + return directory; +} + +function nativeSessionCheckpointPath(execution: NativeExecutionInput): string { + return resolve( + nativeSessionCheckpointDirectory(), + `${createHash("sha256") + .update(nativeSessionScopeKey(execution)) + .digest("hex")}.json`, + ); +} + +function legacyNativeSessionCheckpointPath( + execution: NativeExecutionInput, +): string { + return resolve( + nativeSessionCheckpointDirectory(), + `${createHash("sha256") + .update(nativeSessionKey(execution)) + .digest("hex")}.json`, + ); +} + +function persistWarmNativeCheckpoint( + execution: NativeExecutionInput, + configDigest: string, + snapshot: PersistedNativeSession, +): void { + const path = nativeSessionCheckpointPath(execution); + const temporary = `${path}.${process.pid}.${randomUUID()}.tmp`; + writeFileSync( + temporary, + JSON.stringify({ + schema: "paperclip.native-session-supervisor.v1", + configDigest, + updatedAt: new Date().toISOString(), + snapshot, + }), + { encoding: "utf8", mode: 0o600 }, + ); + renameSync(temporary, path); + chmodSync(path, 0o600); +} + +function loadWarmNativeCheckpoint( + execution: NativeExecutionInput, + configDigest: string, +): PersistedNativeSession | null { + const scopedPath = nativeSessionCheckpointPath(execution); + const path = existsSync(scopedPath) + ? scopedPath + : legacyNativeSessionCheckpointPath(execution); + if (!existsSync(path)) return null; + const envelope = JSON.parse( + readBoundedNativeFile( + path, + NATIVE_WARM_CHECKPOINT_MAX_BYTES, + "native_session_supervisor_checkpoint_too_large", + ).toString("utf8"), + ) as { + schema?: string; + configDigest?: string; + snapshot?: PersistedNativeSession; + }; + if ( + envelope.schema !== "paperclip.native-session-supervisor.v1" || + !envelope.snapshot + ) { + throw new Error("native_session_supervisor_checkpoint_mismatch"); + } + // A provider/model/runtime-context/permission change is an intentional + // incompatibility boundary. Leave the older checkpoint replayable by its + // original execution, but start a fresh provider session for this config. + if (envelope.configDigest !== configDigest) return null; + return { + ...envelope.snapshot, + identity: { + runId: execution.binding.runId, + sessionId: nativeSessionKey(execution), + companyId: execution.binding.companyId, + issueId: execution.binding.issueId, + agentId: execution.binding.agentId, + }, + semanticResult: null, + terminal: null, + activeTurnId: null, + terminalTurns: [], + pendingRuntimeRequests: [], + }; +} + +async function releaseWarmNativeSession( + sessionId: string, + idleTimeoutMs: number, + failed: boolean, +): Promise { + const entry = warmNativeSessions.get(sessionId); + if (!entry) return; + entry.busy = false; + entry.lastActivityAt = new Date().toISOString(); + if (entry.idleTimer !== null) clearTimeout(entry.idleTimer); + if (failed) { + warmNativeSessions.delete(sessionId); + await entry.session + .close({ reason: "warm native session failed" }) + .catch(() => undefined); + return; + } + entry.idleTimer = setTimeout(() => { + const current = warmNativeSessions.get(sessionId); + if (!current || current.busy) return; + warmNativeSessions.delete(sessionId); + void current.session.close({ reason: "warm native session idle timeout" }); + }, idleTimeoutMs); + entry.idleTimer.unref(); +} + +export function nativeSessionFailureDisposition( + attempt: number, + now = new Date(), + sourceFailureCode?: ReturnType, +) { + const permanentFailure = sourceFailureCode === "native_event_replay_conflict"; + const exhausted = permanentFailure || attempt >= 3; + return { + phase: exhausted + ? ("terminal_failure" as const) + : ("retryable_failure" as const), + failureCode: permanentFailure + ? sourceFailureCode! + : exhausted + ? ("native_session_retry_exhausted" as const) + : ("native_session_interrupted" as const), + nextAttemptAt: exhausted ? null : new Date(now.getTime() + 30_000), + }; +} + +export function nativeSessionRecoveryProjection(input: { + phase: "retryable_failure" | "terminal_failure"; + failureCode: string; + agentId: string; +}) { + const exhausted = input.phase === "terminal_failure"; + return { + exhausted, + issueStatus: exhausted ? ("in_review" as const) : null, + recoveryOwner: exhausted + ? { kind: "board" as const } + : { kind: "agent" as const, agentId: input.agentId }, + recoveryActionOwnerType: exhausted + ? ("board" as const) + : ("agent" as const), + recoveryActionOwnerAgentId: exhausted ? null : input.agentId, + recoveryActionCause: input.failureCode, + supersedeOnIdentityChange: true as const, + }; +} + +export function nativeSessionFailureSourceCode( + error: unknown, +): + | "provider_process_exited" + | "provider_stdout_closed" + | "provider_process_output_closed" + | "provider_process_status_failed" + | "provider_initialize_timeout" + | "provider_initialize_protocol_error" + | "provider_request_timeout" + | "provider_request_protocol_error" + | "provider_frame_too_large" + | "provider_transport_failed" + | "native_runner_process_exited" + | "planning_mode_unsupported" + | "native_event_replay_conflict" + | "native_session_interrupted" { + const message = error instanceof Error ? error.message : String(error); + if (/provider_process_exited/i.test(message)) { + return "provider_process_exited"; + } + if (/provider_stdout_closed/i.test(message)) { + return "provider_stdout_closed"; + } + if (/provider_process_output_closed/i.test(message)) { + return "provider_process_output_closed"; + } + if (/provider_process_status_failed/i.test(message)) { + return "provider_process_status_failed"; + } + if (/provider_initialize_timeout/i.test(message)) { + return "provider_initialize_timeout"; + } + if (/provider_initialize_protocol_error/i.test(message)) { + return "provider_initialize_protocol_error"; + } + if (/provider_request_timeout/i.test(message)) { + return "provider_request_timeout"; + } + if (/provider_request_protocol_error/i.test(message)) { + return "provider_request_protocol_error"; + } + if (/provider_frame_too_large|stdout frame exceeded/i.test(message)) { + return "provider_frame_too_large"; + } + if ( + /provider_transport_failed|invalid JSON-RPC|provider failed/i.test(message) + ) { + return "provider_transport_failed"; + } + if ( + /native_runner_process_exited|runnerd exited|runner process failed/i.test( + message, + ) + ) { + return "native_runner_process_exited"; + } + if (/planning_mode_unsupported/i.test(message)) { + return "planning_mode_unsupported"; + } + if (/native_event_replay_conflict/i.test(message)) { + return "native_event_replay_conflict"; + } + return "native_session_interrupted"; +} + +const PROVIDER_DURABLE_EVENT_TYPES = new Set([ + "harness.ready", + "session.started", + "session.resumed", + "session.updated", + "turn.started", + "provider.event", + "provider.rpc_result", +]); + +type NativeRecoveryMode = + "bootstrap_retry" | "exact_checkpoint_resume" | "ambiguous_state"; + +export async function nativeProviderRecoveryEvidence(input: { + db: Db; + runId: string; + sourceFailureCode: ReturnType; +}): Promise<{ + recoveryMode: NativeRecoveryMode; + providerSessionEstablished: boolean; + providerEventsExist: boolean; + checkpointExists: boolean; +}> { + const run = await input.db + .select({ runnerProfileJson: heartbeatRuns.runnerProfileJson }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, input.runId)) + .limit(1) + .then((rows) => rows[0] ?? null); + const checkpoint = record(run?.runnerProfileJson).sessionCheckpoint; + const checkpointRecord = record(checkpoint); + const checkpointExists = Object.keys(checkpointRecord).length > 0; + const providerSessionEstablished = + (typeof checkpointRecord.providerSessionId === "string" && + checkpointRecord.providerSessionId.length > 0) || + Object.keys(record(checkpointRecord.providerIdentity)).length > 0; + const durableEvents = await input.db + .select({ eventType: heartbeatRunEvents.eventType }) + .from(heartbeatRunEvents) + .where(eq(heartbeatRunEvents.runId, input.runId)); + const providerEventsExist = durableEvents.some((event) => + PROVIDER_DURABLE_EVENT_TYPES.has(event.eventType), + ); + if (checkpointExists && providerSessionEstablished) { + return { + recoveryMode: "exact_checkpoint_resume", + providerSessionEstablished: true, + providerEventsExist, + checkpointExists, + }; + } + const definitelyPreSession = new Set< + ReturnType + >([ + "provider_process_exited", + "provider_stdout_closed", + "provider_process_output_closed", + "provider_process_status_failed", + "provider_initialize_timeout", + "provider_initialize_protocol_error", + "provider_request_timeout", + "provider_request_protocol_error", + "native_runner_process_exited", + ]).has(input.sourceFailureCode); + if (!checkpointExists && !providerEventsExist && definitelyPreSession) { + return { + recoveryMode: "bootstrap_retry", + providerSessionEstablished: false, + providerEventsExist: false, + checkpointExists: false, + }; + } + return { + recoveryMode: "ambiguous_state", + providerSessionEstablished: + providerSessionEstablished || providerEventsExist, + providerEventsExist, + checkpointExists, + }; +} + +function record(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : {}; +} + +export type NativeSessionSteeringState = { + disposition: "available" | "unsupported" | "temporarily_unavailable"; + activeTurnId: string | null; +}; + +export class NativeSessionSteeringError extends Error { + constructor( + readonly code: + | "steering_unsupported" + | "steering_temporarily_unavailable" + | "steering_stale_turn" + | "steering_timeout" + | "steering_rejected", + message: string, + ) { + super(message); + this.name = "NativeSessionSteeringError"; + } +} + +export class NativeRuntimeRequestResolutionError extends Error { + constructor( + readonly code: + | "native_session_not_active" + | "runtime_request_resolution_unsupported" + | "runtime_request_stale_turn" + | "runtime_request_resolution_conflict", + message: string, + ) { + super(message); + this.name = "NativeRuntimeRequestResolutionError"; + } +} + +/** Resolve a provider runtime request on an in-process native backend. */ +export async function resolveNativeRuntimeRequest(input: { + runId: string; + requestId: string; + turnId: string; + resolution: HarnessRuntimeRequestResolution; + /** + * Revalidate the caller's durable lifecycle and authorization immediately + * before the provider mutation. Capability and snapshot reads above this + * edge are asynchronous, so route-level checks performed before entering + * this helper are not sufficient to authorize the eventual dispatch. + */ + authorizeBeforeDispatch: () => Promise; +}): Promise<{ commandId: string }> { + const active = activeNativeSessions.get(input.runId); + if (!active) { + throw new NativeRuntimeRequestResolutionError( + "native_session_not_active", + "The active native session is not attached.", + ); + } + const capabilities = await active.session.capabilities(); + if ( + !capabilities.runtimeRequestResolution || + active.session.resolveRuntimeRequest === undefined + ) { + throw new NativeRuntimeRequestResolutionError( + "runtime_request_resolution_unsupported", + "This native session does not resolve runtime requests in-process.", + ); + } + const snapshot = await active.session.snapshot(); + if (snapshot.activeTurnId !== input.turnId) { + throw new NativeRuntimeRequestResolutionError( + "runtime_request_stale_turn", + "The runtime request belongs to a turn that is no longer active.", + ); + } + + const key = `${input.runId}:${input.requestId}`; + const fingerprint = JSON.stringify({ + turnId: input.turnId, + resolution: input.resolution, + }); + const prior = nativeRuntimeRequestResolutions.get(key); + if (prior) { + if (prior.fingerprint !== fingerprint) { + throw new NativeRuntimeRequestResolutionError( + "runtime_request_resolution_conflict", + "A different response was already submitted for this runtime request.", + ); + } + await prior.pending; + return { commandId: prior.commandId }; + } + + const commandId = `native-runtime-response:${randomUUID()}`; + // Reserve the request key before yielding to authorization or provider I/O. + // This makes duplicate retries join one dispatch and makes a conflicting + // response fail closed even while the first authorization check is pending. + const pending = Promise.resolve().then(async () => { + await input.authorizeBeforeDispatch(); + if (activeNativeSessions.get(input.runId) !== active) { + throw new NativeRuntimeRequestResolutionError( + "native_session_not_active", + "The active native session changed before the response was dispatched.", + ); + } + await active.session.resolveRuntimeRequest!({ + requestId: input.requestId, + turnId: input.turnId, + resolution: input.resolution, + }); + }); + const resolution: NativeRuntimeRequestResolution = { + runId: input.runId, + fingerprint, + commandId, + pending, + completedAt: null, + }; + nativeRuntimeRequestResolutions.set(key, resolution); + try { + await pending; + resolution.completedAt = Date.now(); + pruneNativeRuntimeRequestResolutionCache(); + return { commandId }; + } catch (error) { + if (nativeRuntimeRequestResolutions.get(key) === resolution) { + nativeRuntimeRequestResolutions.delete(key); + } + throw error; + } +} + +export async function getNativeSessionSteeringState( + runId: string, +): Promise { + const active = activeNativeSessions.get(runId); + if (!active) + return { disposition: "temporarily_unavailable", activeTurnId: null }; + const capabilities = await active.session.capabilities(); + if (!capabilities.steering || !active.session.steer) { + return { disposition: "unsupported", activeTurnId: null }; + } + const snapshot = await active.session.snapshot(); + return { + disposition: snapshot.activeTurnId + ? "available" + : "temporarily_unavailable", + activeTurnId: snapshot.activeTurnId ?? null, + }; +} + +/** Dispatches a true same-turn steering message and resolves only after ack. */ +export async function steerNativeSession(input: { + runId: string; + message: string; + correlationId: string; + timeoutMs?: number; +}): Promise<{ turnId: string }> { + const active = activeNativeSessions.get(input.runId); + if (!active) { + throw new NativeSessionSteeringError( + "steering_temporarily_unavailable", + "The active native session is not attached.", + ); + } + const capabilities = await active.session.capabilities(); + if (!capabilities.steering || !active.session.steer) { + throw new NativeSessionSteeringError( + "steering_unsupported", + "This provider does not support same-turn steering.", + ); + } + const snapshot = await active.session.snapshot(); + const turnId = snapshot.activeTurnId ?? null; + if (!turnId) { + throw new NativeSessionSteeringError( + "steering_stale_turn", + "The target turn is no longer active.", + ); + } + + let timeout: ReturnType | null = null; + try { + await Promise.race([ + active.session.steer({ + turnId, + message: { role: "user", text: input.message }, + correlationId: input.correlationId, + }), + new Promise((_resolve, reject) => { + timeout = setTimeout( + () => + reject( + new NativeSessionSteeringError( + "steering_timeout", + "The provider did not acknowledge steering in time.", + ), + ), + input.timeoutMs ?? 10_000, + ); + }), + ]); + return { turnId }; + } catch (error) { + if (error instanceof NativeSessionSteeringError) throw error; + const message = error instanceof Error ? error.message : String(error); + if (/stale|terminal|active turn/i.test(message)) { + throw new NativeSessionSteeringError( + "steering_stale_turn", + "The target turn is no longer active.", + ); + } + if (/unsupported|unavailable|capability/i.test(message)) { + throw new NativeSessionSteeringError( + "steering_unsupported", + "This provider does not support same-turn steering.", + ); + } + throw new NativeSessionSteeringError( + "steering_rejected", + "The provider rejected the steering message.", + ); + } finally { + if (timeout) clearTimeout(timeout); + } +} + +export function cancelNativeSession( + runId: string, + reason: string, +): Promise; +export function cancelNativeSession( + runId: string, + reason: string, + options: { + db: Db; + scope?: "turn" | "run" | "issue"; + replacementAccepted?: boolean; + }, +): Promise<{ + dispatched: boolean; + decision: NativeStatusDecision | null; + decisionId: string | null; + auditId: string | null; +}>; +export async function cancelNativeSession( + runId: string, + reason: string, + options?: { + db: Db; + scope?: "turn" | "run" | "issue"; + replacementAccepted?: boolean; + }, +): Promise< + | boolean + | { + dispatched: boolean; + decision: NativeStatusDecision | null; + decisionId: string | null; + auditId: string | null; + } +> { + let decision: NativeStatusDecision | null = null; + let decisionContext: { + companyId: string; + issueId: string; + assessmentId: string | null; + priorStatus: string; + priorStatusVersion: number; + priorDecisionId: string | null; + coordinatorDecisionId: string | null; + agentId: string; + } | null = null; + if (options) { + const run = await options.db + .select({ + agentId: heartbeatRuns.agentId, + companyId: heartbeatRuns.companyId, + nativeIssueId: heartbeatRuns.nativeIssueId, + runtimeMode: heartbeatRuns.runtimeMode, + }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, runId)) + .limit(1) + .then((rows) => rows[0] ?? null); + if (run?.runtimeMode === "native") { + const issueId = run.nativeIssueId; + if (!issueId) throw new Error("native_cancellation_binding_missing"); + const issue = await options.db + .select({ + status: issues.status, + statusVersion: issues.statusVersion, + lastStatusDecisionId: issues.lastStatusDecisionId, + }) + .from(issues) + .where( + and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)), + ) + .limit(1) + .then((rows) => rows[0] ?? null); + if (!issue) throw new Error("native_cancellation_binding_missing"); + const coordinator = await options.db + .select({ + assessmentId: nativeRunFinalizations.assessmentId, + decisionId: nativeRunFinalizations.decisionId, + }) + .from(nativeRunFinalizations) + .where( + and( + eq(nativeRunFinalizations.runId, runId), + eq(nativeRunFinalizations.companyId, run.companyId), + eq(nativeRunFinalizations.issueId, issueId), + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null); + if (!coordinator) + throw new Error("native_cancellation_coordinator_missing"); + decision = resolveNativeCancellationStatus({ + scope: options.scope ?? "run", + priorIssueStatus: issue.status as NativeAuthoritativeIssueStatus, + agentId: run.agentId, + replacementAccepted: options.replacementAccepted, + }); + decisionContext = { + companyId: run.companyId, + issueId, + assessmentId: coordinator.assessmentId ?? null, + priorStatus: issue.status, + priorStatusVersion: Number(issue.statusVersion), + priorDecisionId: issue.lastStatusDecisionId, + coordinatorDecisionId: coordinator.decisionId ?? null, + agentId: run.agentId, + }; + } + } + let decisionId: string | null = null; + let auditId: string | null = null; + let cancellationIntentId: string | null = null; + let recoveringCancellationIntent = false; + let priorCoordinatorDecisionIdAtIntent: string | null = null; + if (options && decision && decisionContext) { + const cancellationDecision = decision; + const cancellationContext = decisionContext; + const effects = cancellationDecision.effects.map((effect) => effect.kind); + let intentPublication: Parameters[0] | null = null; + const intent = await options.db.transaction(async (tx) => { + const lockedRun = await tx + .select({ + agentId: heartbeatRuns.agentId, + companyId: heartbeatRuns.companyId, + nativeIssueId: heartbeatRuns.nativeIssueId, + resultJson: heartbeatRuns.resultJson, + runtimeMode: heartbeatRuns.runtimeMode, + }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, runId)) + .for("update") + .limit(1) + .then((rows) => rows[0] ?? null); + if ( + !lockedRun || + lockedRun.runtimeMode !== "native" || + lockedRun.companyId !== cancellationContext.companyId || + lockedRun.agentId !== cancellationContext.agentId || + lockedRun.nativeIssueId !== cancellationContext.issueId + ) { + throw new Error("native_cancellation_binding_changed"); + } + const coordinator = await tx + .select({ runId: nativeRunFinalizations.runId }) + .from(nativeRunFinalizations) + .where( + and( + eq(nativeRunFinalizations.runId, runId), + eq( + nativeRunFinalizations.companyId, + cancellationContext.companyId, + ), + eq(nativeRunFinalizations.issueId, cancellationContext.issueId), + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null); + if (!coordinator) + throw new Error("native_cancellation_coordinator_missing"); + + const resultJson = record(lockedRun.resultJson); + const existing = record(resultJson.nativeCancellation); + const existingIntentId = + typeof existing.intentId === "string" && existing.intentId.length > 0 + ? existing.intentId + : null; + if (existingIntentId) { + const matchingIntent = + existing.schema === "paperclip.native-cancellation.v1" && + existing.companyId === cancellationContext.companyId && + existing.runId === runId && + existing.issueId === cancellationContext.issueId && + existing.scope === (options.scope ?? "run") && + existing.reasonCode === cancellationDecision.reasonCode && + JSON.stringify(existing.effects) === JSON.stringify(effects); + if (!matchingIntent) + throw new Error("native_cancellation_intent_conflict"); + const existingAuditId = + typeof existing.intentAuditId === "string" && + existing.intentAuditId.length > 0 + ? existing.intentAuditId + : null; + if (!existingAuditId) + throw new Error("native_cancellation_intent_audit_missing"); + return { + intentId: existingIntentId, + auditId: existingAuditId, + acknowledged: existing.dispatchState === "acknowledged", + dispatched: existing.dispatched === true, + decisionId: + typeof existing.decisionId === "string" + ? existing.decisionId + : null, + priorCoordinatorDecisionId: + typeof existing.priorCoordinatorDecisionId === "string" + ? existing.priorCoordinatorDecisionId + : null, + existing: true, + }; + } + + const intentId = `native-cancellation:${randomUUID()}`; + const activity = await persistActivity(tx as unknown as Db, { + companyId: cancellationContext.companyId, + actorType: "system", + actorId: "native-session-cancellation", + action: "native.cancellation_intent_recorded", + entityType: "heartbeat_run", + entityId: runId, + agentId: cancellationContext.agentId, + runId, + issueId: cancellationContext.issueId, + details: { + intentId, + scope: options.scope ?? "run", + reasonCode: cancellationDecision.reasonCode, + effects, + }, + }); + const intentAuditId = activity.activity?.id ?? null; + if (!intentAuditId) + throw new Error("native_cancellation_intent_audit_missing"); + const written = await tx + .update(heartbeatRuns) + .set({ + resultJson: { + ...resultJson, + nativeCancellation: { + schema: "paperclip.native-cancellation.v1", + intentId, + intentAuditId, + companyId: cancellationContext.companyId, + runId, + issueId: cancellationContext.issueId, + scope: options.scope ?? "run", + reasonCode: cancellationDecision.reasonCode, + effects, + dispatchState: "pending", + dispatched: false, + decisionId: null, + priorCoordinatorDecisionId: + cancellationContext.coordinatorDecisionId, + recordedAt: new Date().toISOString(), + }, + }, + updatedAt: new Date(), + }) + .where( + and( + eq(heartbeatRuns.id, runId), + eq(heartbeatRuns.companyId, cancellationContext.companyId), + eq(heartbeatRuns.agentId, cancellationContext.agentId), + eq(heartbeatRuns.nativeIssueId, cancellationContext.issueId), + ), + ) + .returning({ id: heartbeatRuns.id }) + .then((rows) => rows[0] ?? null); + if (!written) throw new Error("native_cancellation_binding_changed"); + intentPublication = activity.publication; + return { + intentId, + auditId: intentAuditId, + acknowledged: false, + dispatched: false, + decisionId: null, + priorCoordinatorDecisionId: + cancellationContext.coordinatorDecisionId, + existing: false, + }; + }); + if (intentPublication) publishActivity(intentPublication); + cancellationIntentId = intent.intentId; + auditId = intent.auditId; + decisionId = intent.decisionId; + recoveringCancellationIntent = intent.existing; + priorCoordinatorDecisionIdAtIntent = intent.priorCoordinatorDecisionId; + if (intent.acknowledged) { + return { + dispatched: intent.dispatched, + decision, + decisionId, + auditId, + }; + } + } + const active = activeNativeSessions.get(runId); + let dispatched = false; + if (active) { + dispatched = true; + if (!active.cancelRequested) { + active.cancelRequested = true; + try { + if (active.session.cancel) { + const cancellationAbort = new AbortController(); + const cleanup = active.session.cancel({ + reason, + signal: cancellationAbort.signal, + }).cleanup; + let cleanupTimer: ReturnType | undefined; + const settled = await Promise.race([ + cleanup.then( + () => true, + () => true, + ), + new Promise((resolve) => { + cleanupTimer = setTimeout( + () => resolve(false), + NATIVE_SESSION_CANCELLATION_CLEANUP_GRACE_MS, + ); + }), + ]); + if (cleanupTimer) clearTimeout(cleanupTimer); + if (!settled) { + cancellationAbort.abort( + new Error("native session cancellation cleanup timed out"), + ); + void cleanup.catch(() => undefined); + } + } else if (active.session.interrupt) + await active.session.interrupt({ reason }); + } catch (error) { + active.cancelRequested = false; + throw error; + } + } + } + if (!options) return dispatched; + + if (decision && decisionContext) { + const cancellationDecision = decision; + const cancellationContext = decisionContext; + if (!cancellationIntentId || !auditId) + throw new Error("native_cancellation_intent_audit_missing"); + if ( + recoveringCancellationIntent && + cancellationContext.coordinatorDecisionId !== + priorCoordinatorDecisionIdAtIntent + ) { + decisionId ??= cancellationContext.coordinatorDecisionId; + } + if ( + cancellationContext.assessmentId && + cancellationDecision.reasonCode !== null && + !decisionId + ) { + const committed = await commitNativeStatusDecision({ + db: options.db, + companyId: cancellationContext.companyId, + issueId: cancellationContext.issueId, + runId, + assessmentId: cancellationContext.assessmentId, + priorStatus: cancellationContext.priorStatus, + priorStatusVersion: cancellationContext.priorStatusVersion, + priorDecisionId: cancellationContext.priorDecisionId, + decision: cancellationDecision, + }); + decisionId = committed.decision.id; + } + const acknowledgement = await options.db.transaction(async (tx) => { + const lockedRun = await tx + .select({ + agentId: heartbeatRuns.agentId, + companyId: heartbeatRuns.companyId, + nativeIssueId: heartbeatRuns.nativeIssueId, + resultJson: heartbeatRuns.resultJson, + runtimeMode: heartbeatRuns.runtimeMode, + }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, runId)) + .for("update") + .limit(1) + .then((rows) => rows[0] ?? null); + if ( + !lockedRun || + lockedRun.runtimeMode !== "native" || + lockedRun.companyId !== cancellationContext.companyId || + lockedRun.agentId !== cancellationContext.agentId || + lockedRun.nativeIssueId !== cancellationContext.issueId + ) { + throw new Error("native_cancellation_binding_changed"); + } + const coordinator = await tx + .select({ runId: nativeRunFinalizations.runId }) + .from(nativeRunFinalizations) + .where( + and( + eq(nativeRunFinalizations.runId, runId), + eq( + nativeRunFinalizations.companyId, + cancellationContext.companyId, + ), + eq(nativeRunFinalizations.issueId, cancellationContext.issueId), + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null); + if (!coordinator) + throw new Error("native_cancellation_coordinator_missing"); + + const resultJson = record(lockedRun.resultJson); + const intent = record(resultJson.nativeCancellation); + const matchingIntent = + intent.schema === "paperclip.native-cancellation.v1" && + intent.intentId === cancellationIntentId && + intent.intentAuditId === auditId && + intent.companyId === cancellationContext.companyId && + intent.runId === runId && + intent.issueId === cancellationContext.issueId; + if (!matchingIntent) + throw new Error("native_cancellation_intent_conflict"); + if (intent.dispatchState === "acknowledged") { + return { + publication: null, + decisionId: + typeof intent.decisionId === "string" + ? intent.decisionId + : decisionId, + }; + } + + if ( + options.replacementAccepted && + cancellationDecision.effects.some( + (effect) => effect.kind === "accept_replacement_turn", + ) + ) { + await tx + .update(heartbeatRuns) + .set({ + status: "running", + continuationAttempt: sql`${heartbeatRuns.continuationAttempt} + 1`, + nextAction: + "Accept a replacement native turn on the existing run.", + updatedAt: new Date(), + }) + .where( + and( + eq(heartbeatRuns.id, runId), + eq(heartbeatRuns.companyId, cancellationContext.companyId), + eq(heartbeatRuns.agentId, cancellationContext.agentId), + eq(heartbeatRuns.nativeIssueId, cancellationContext.issueId), + ), + ); + } + const activity = await persistActivity(tx as unknown as Db, { + companyId: cancellationContext.companyId, + actorType: "system", + actorId: "native-session-cancellation", + action: "native.cancellation_dispatch_acknowledged", + entityType: "heartbeat_run", + entityId: runId, + agentId: cancellationContext.agentId, + runId, + issueId: cancellationContext.issueId, + details: { + intentId: cancellationIntentId, + intentAuditId: auditId, + scope: options.scope ?? "run", + reasonCode: cancellationDecision.reasonCode, + effects: cancellationDecision.effects.map((effect) => effect.kind), + dispatched, + decisionId, + }, + }); + const acknowledgementAuditId = activity.activity?.id ?? null; + if (!acknowledgementAuditId) + throw new Error("native_cancellation_ack_audit_missing"); + const cancellationWrite = await tx + .update(heartbeatRuns) + .set({ + resultJson: { + ...resultJson, + nativeCancellation: { + ...intent, + dispatchState: "acknowledged", + dispatched, + decisionId, + acknowledgementAuditId, + acknowledgedAt: new Date().toISOString(), + }, + }, + updatedAt: new Date(), + }) + .where( + and( + eq(heartbeatRuns.id, runId), + eq(heartbeatRuns.companyId, cancellationContext.companyId), + eq(heartbeatRuns.agentId, cancellationContext.agentId), + eq(heartbeatRuns.nativeIssueId, cancellationContext.issueId), + ), + ) + .returning({ id: heartbeatRuns.id }) + .then((rows) => rows[0] ?? null); + if (!cancellationWrite) + throw new Error("native_cancellation_binding_changed"); + return { publication: activity.publication, decisionId }; + }); + decisionId = acknowledgement.decisionId; + if (acknowledgement.publication) + publishActivity(acknowledgement.publication); + } + return { dispatched, decision, decisionId, auditId }; +} + +/** Authenticated cancellation scope projected through the shared arbiter. */ +export function resolveNativeCancellationStatus(input: { + scope: "turn" | "run" | "issue"; + priorIssueStatus: NativeAuthoritativeIssueStatus; + agentId: string; + replacementAccepted?: boolean; +}): NativeStatusDecision { + if (input.scope === "turn") { + return { + policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, + statusAction: "preserve", + toStatus: input.priorIssueStatus, + reasonCode: input.replacementAccepted ? null : "cancellation_turn_only", + unblockDescriptor: null, + effects: [{ kind: "accept_replacement_turn" }], + }; + } + if (input.scope === "run") { + return { + policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, + statusAction: "preserve", + toStatus: input.priorIssueStatus, + reasonCode: "cancellation_run_only", + unblockDescriptor: null, + effects: [{ kind: "release_run_resources" }], + }; + } + return { + policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, + statusAction: "cancelled", + toStatus: "cancelled", + reasonCode: "cancellation_issue_authorized", + unblockDescriptor: null, + effects: [{ kind: "release_checkout" }, { kind: "cancel_continuations" }], + }; +} + +export async function renewNativeSessionExecutionLease(input: { + db: Db; + runId: string; + companyId: string; + issueId: string; + leaseOwner: string; + attempt: number; + leaseTtlMs?: number; +}): Promise { + const leaseTtlMs = + input.leaseTtlMs ?? NATIVE_SESSION_EXECUTION_LEASE_TTL_MS; + if ( + !Number.isInteger(leaseTtlMs) || + leaseTtlMs < 1_000 || + leaseTtlMs > NATIVE_SESSION_EXECUTION_LEASE_TTL_MS + ) { + throw new Error("native_session_lease_ttl_invalid"); + } + const [updated] = await input.db + .update(nativeRunFinalizations) + .set({ + leaseExpiresAt: sql`now() + (${leaseTtlMs} * interval '1 millisecond')`, + updatedAt: sql`now()`, + }) + .where( + and( + eq(nativeRunFinalizations.runId, input.runId), + eq(nativeRunFinalizations.companyId, input.companyId), + eq(nativeRunFinalizations.issueId, input.issueId), + eq(nativeRunFinalizations.leaseOwner, input.leaseOwner), + eq(nativeRunFinalizations.attempt, input.attempt), + gt(nativeRunFinalizations.leaseExpiresAt, sql`now()`), + ), + ) + .returning({ runId: nativeRunFinalizations.runId }); + if (!updated) throw new Error("native_session_lease_lost"); +} + +function startNativeSessionExecutionLeaseRenewal(input: { + db: Db; + runId: string; + companyId: string; + issueId: string; + leaseOwner: string; + attempt: number; +}): { stop: () => Promise } { + let leaseLost: Error | null = null; + let renewal = Promise.resolve(); + const renew = () => { + if (leaseLost) return; + renewal = renewal + .then(() => renewNativeSessionExecutionLease(input)) + .then(() => undefined) + .catch(async (error: unknown) => { + leaseLost = + error instanceof Error + ? error + : new Error("native_session_lease_lost"); + await cancelNativeSession( + input.runId, + "native session execution lease lost", + ).catch(() => undefined); + }); + }; + const timer = setInterval( + renew, + NATIVE_SESSION_EXECUTION_LEASE_RENEW_INTERVAL_MS, + ); + timer.unref?.(); + return { + stop: async () => { + clearInterval(timer); + await renewal; + if (leaseLost) throw leaseLost; + }, + }; +} + +export async function executePaperclipNativeSession(input: { + db: Db; + execution: NativeExecutionInput; + runnerInstanceId: string; + leaseOwner?: string; + onSpawn?: (meta: { + pid: number; + processGroupId: number | null; + startedAt: string; + }) => Promise; + /** Test seam at the provider boundary; production always uses the package Codex backend. */ + backend?: NativeSessionBackend; + useRunnerd?: boolean; + onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise; + onEvent?: (event: AdapterRuntimeEvent) => Promise; + preparationSpans?: NativeRunHistoricalSpan[]; + /** Resolved adapter env; the runner transport applies a provider allowlist before spawn. */ + runnerEnvironment?: NodeJS.ProcessEnv; + enqueueWakeup?: ( + agentId: string, + options: { + source: "assignment"; + triggerDetail: "system"; + reason: "issue_assigned"; + payload: Record; + idempotencyKey: string; + requestedByActorType: "agent"; + requestedByActorId: string; + contextSnapshot: Record; + }, + ) => Promise; +}): Promise { + if (input.execution.provider.kind !== "codex") { + throw new Error("paperclip_runner_provider_unsupported"); + } + const earliestPreparationStart = input.preparationSpans?.reduce( + (earliest, span) => Math.min(earliest, span.startedAtMs), + Date.now(), + ); + const trace = createNativeRunTrace({ + runId: input.execution.binding.runId, + startedAtMs: earliestPreparationStart, + onEvent: input.onEvent, + }); + const preparationSpans = input.preparationSpans ?? []; + const taskPrepareScope = trace.start("task.prepare", { + parentName: "task.run", + startedAtMs: earliestPreparationStart, + }); + const environmentSpans = preparationSpans.filter( + (span) => + span.name === "environment.acquire" || + span.name === "environment.workspace.realize", + ); + const environmentStartedAtMs = environmentSpans.reduce( + (earliest, span) => Math.min(earliest, span.startedAtMs), + Date.now(), + ); + const environmentEndedAtMs = environmentSpans.reduce( + (latest, span) => Math.max(latest, span.endedAtMs), + environmentStartedAtMs, + ); + const environmentScope = + environmentSpans.length > 0 + ? trace.start("environment.startup", { + parentName: "task.prepare", + startedAtMs: environmentStartedAtMs, + }) + : null; + for (const span of preparationSpans) { + const rootMilestone = + span.name === "heartbeat.queue" || span.name === "comment.to_run_created"; + await trace.record({ + ...span, + parentName: rootMilestone + ? "task.run" + : environmentSpans.includes(span) + ? "environment.startup" + : "task.prepare", + }); + } + if (environmentScope) { + await trace.end(environmentScope, { endedAtMs: environmentEndedAtMs }); + } + const durableRunnerBinding = input.useRunnerd + ? loadRunnerdDurableBinding(input.execution) + : null; + const effectiveRunnerInstanceId = + durableRunnerBinding?.runnerInstanceId ?? input.runnerInstanceId; + if (effectiveRunnerInstanceId !== input.runnerInstanceId) { + await input.db + .update(heartbeatRuns) + .set({ + runnerInstanceId: effectiveRunnerInstanceId, + updatedAt: new Date(), + }) + .where( + and( + eq(heartbeatRuns.id, input.execution.binding.runId), + eq(heartbeatRuns.companyId, input.execution.binding.companyId), + eq(heartbeatRuns.agentId, input.execution.binding.agentId), + ), + ); + } + const leaseOwner = + input.leaseOwner ?? `${effectiveRunnerInstanceId}:${randomUUID()}`; + let attempt: number; + try { + attempt = await trace.measure( + "native.coordinator.claim", + () => + input.db.transaction(async (tx) => { + const coordinator = await tx + .select() + .from(nativeRunFinalizations) + .where( + and( + eq(nativeRunFinalizations.runId, input.execution.binding.runId), + eq( + nativeRunFinalizations.companyId, + input.execution.binding.companyId, + ), + eq( + nativeRunFinalizations.issueId, + input.execution.binding.issueId, + ), + ), + ) + .for("update") + .limit(1) + .then((rows) => rows[0] ?? null); + if (!coordinator) + throw new Error("native_finalization_coordinator_missing"); + const leaseNow = new Date(); + const leaseExpiresAt = new Date( + leaseNow.getTime() + NATIVE_SESSION_EXECUTION_LEASE_TTL_MS, + ); + // A durable result means provider execution already completed. The + // recovery/finalization path must reconcile it; never reacquire a + // provider session and execute the turn a second time. + if (coordinator.resultId) + throw new NativeResultPendingFinalizationError(); + const boundRun = await tx + .select({ + agentId: heartbeatRuns.agentId, + companyId: heartbeatRuns.companyId, + nativeIssueId: heartbeatRuns.nativeIssueId, + resultJson: heartbeatRuns.resultJson, + runtimeMode: heartbeatRuns.runtimeMode, + }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, input.execution.binding.runId)) + .for("update") + .limit(1) + .then((rows) => rows[0] ?? null); + if ( + !boundRun || + boundRun.runtimeMode !== "native" || + boundRun.companyId !== input.execution.binding.companyId || + boundRun.agentId !== input.execution.binding.agentId || + boundRun.nativeIssueId !== input.execution.binding.issueId + ) { + throw new Error("native_execution_binding_changed"); + } + const cancellationIntent = record( + record(boundRun.resultJson).nativeCancellation, + ); + if ( + cancellationIntent.scope === "run" && + (cancellationIntent.dispatchState === "pending" || + cancellationIntent.dispatchState === "acknowledged") + ) { + const intentMatchesBinding = + cancellationIntent.schema === + "paperclip.native-cancellation.v1" && + cancellationIntent.companyId === + input.execution.binding.companyId && + cancellationIntent.runId === input.execution.binding.runId && + cancellationIntent.issueId === input.execution.binding.issueId; + if (!intentMatchesBinding) + throw new Error("native_cancellation_intent_conflict"); + // Run cancellation remains a claim fence after dispatch ack until + // the heartbeat cancellation path terminalizes the run. + throw new NativeCancellationPendingRecoveryError(); + } + if (["committed", "applied"].includes(coordinator.phase)) + throw new Error("native_run_already_committed"); + if ( + coordinator.leaseOwner && + coordinator.leaseOwner !== leaseOwner && + coordinator.leaseExpiresAt && + coordinator.leaseExpiresAt > leaseNow + ) + throw new Error("native_finalization_lease_busy"); + const claimed = await tx + .update(nativeRunFinalizations) + .set({ + phase: "observed", + attempt: coordinator.attempt + 1, + leaseOwner, + leaseExpiresAt, + failureCode: null, + failureDetail: null, + nextAttemptAt: null, + updatedAt: leaseNow, + }) + .where( + and( + eq(nativeRunFinalizations.runId, coordinator.runId), + eq( + nativeRunFinalizations.companyId, + input.execution.binding.companyId, + ), + eq( + nativeRunFinalizations.issueId, + input.execution.binding.issueId, + ), + eq(nativeRunFinalizations.attempt, coordinator.attempt), + eq(nativeRunFinalizations.phase, coordinator.phase), + ), + ) + .returning({ runId: nativeRunFinalizations.runId }) + .then((rows) => rows[0] ?? null); + if (!claimed) throw new Error("native_session_lease_lost"); + await tx + .update(heartbeatRuns) + .set({ + nativePhase: "observed", + nativePhaseUpdatedAt: leaseNow, + updatedAt: leaseNow, + }) + .where(eq(heartbeatRuns.id, coordinator.runId)); + return coordinator.attempt + 1; + }), + { parentName: "task.prepare" }, + ); + await trace.end(taskPrepareScope); + } catch (error) { + await trace.end(taskPrepareScope, { outcome: "failed" }); + await trace.finish("failed"); + throw error; + } + const controlPlaneInstanceId = `${effectiveRunnerInstanceId}:control`; + const planSynchronizations: PlanSynchronization[] = []; + const upsertPlanSynchronization = ( + synchronization: PlanSynchronization, + ): void => { + const existingIndex = planSynchronizations.findIndex( + (candidate) => candidate.eventId === synchronization.eventId, + ); + if (existingIndex >= 0) { + planSynchronizations[existingIndex] = synchronization; + return; + } + planSynchronizations.push(synchronization); + }; + const recordPlanSynchronization = async (event: { + sourceEventId: string; + turnId?: string; + eventType: string; + payload: Record; + }) => { + const synchronization = await synchronizeCompletedProviderPlan({ + db: input.db, + execution: input.execution, + event, + }); + if (!synchronization) return; + upsertPlanSynchronization(synchronization); + const activity = await persistActivity(input.db, { + companyId: input.execution.binding.companyId, + actorType: "agent", + actorId: input.execution.binding.agentId, + agentId: input.execution.binding.agentId, + runId: input.execution.binding.runId, + issueId: input.execution.binding.issueId, + action: "issue.document_updated", + entityType: "issue", + entityId: input.execution.binding.issueId, + details: { + key: "plan", + source: "native_plan_synchronization", + synchronization, + }, + }); + publishActivity(activity.publication); + if (input.onLog) + await input.onLog( + "stdout", + `${JSON.stringify({ type: "paperclip.plan.synchronization", synchronization })}\n`, + ); + }; + let nativeSessionExecuteStartedAtMs = Date.now(); + let sessionStartedAtMs: number | null = null; + let sessionStartupMode: "bootstrap" | "resume" | null = null; + let turnSubmittedAtMs: number | null = null; + let turnStartedAtMs: number | null = null; + let firstAgentEventRecorded = false; + let turnCompletedAtMs: number | null = null; + let runnerSessionStartupScope: NativeRunSpanScope | null = null; + let agentTurnScope: NativeRunSpanScope | null = null; + let taskSettleScope: NativeRunSpanScope | null = null; + const governedWaitObservation = createGovernedWaitEventObservation( + resolvePendingGovernedWait, + ); + const controlPlane = new PaperclipControlPlanePort( + input.db, + { + companyId: input.execution.binding.companyId, + issueId: input.execution.binding.issueId, + runId: input.execution.binding.runId, + agentId: input.execution.binding.agentId, + sessionId: nativeSessionKey(input.execution), + completionContractId: input.execution.completionContract.id, + completionContractSha256: input.execution.completionContract.sha256, + sourceInstanceId: effectiveRunnerInstanceId, + controlPlaneSourceInstanceId: controlPlaneInstanceId, + }, + { + onCommittedEvent: async (event) => { + const eventAtMs = Date.parse(event.emittedAt); + const milestoneAtMs = Number.isFinite(eventAtMs) + ? eventAtMs + : Date.now(); + if ( + event.eventType === "session.started" && + sessionStartedAtMs === null + ) { + sessionStartedAtMs = milestoneAtMs; + sessionStartupMode = "bootstrap"; + await trace.record({ + name: "runner.session.bootstrap", + parentName: "runner.session.startup", + startedAtMs: nativeSessionExecuteStartedAtMs, + endedAtMs: milestoneAtMs, + }); + } + if ( + event.eventType === "turn.submitted" && + turnSubmittedAtMs === null + ) { + turnSubmittedAtMs = milestoneAtMs; + // A recovered provider session does not emit session.started again. In + // that case the first durable turn.submitted event is the earliest + // transport-neutral proof that runnerd reattached to the exact + // provider session and is ready for work. Keep that startup time out of + // runner.turn.submit so cold-resume latency is visible as its own span. + if (sessionStartedAtMs === null) { + await trace.record({ + name: "runner.session.resume", + parentName: "runner.session.startup", + startedAtMs: nativeSessionExecuteStartedAtMs, + endedAtMs: milestoneAtMs, + attributes: { + provider: input.execution.provider.kind, + strategy: "exact_provider_session", + }, + }); + sessionStartedAtMs = milestoneAtMs; + sessionStartupMode = "resume"; + } + await trace.record({ + name: "runner.turn.submit", + parentName: "runner.session.startup", + startedAtMs: sessionStartedAtMs, + endedAtMs: milestoneAtMs, + }); + if (runnerSessionStartupScope) { + trace.annotate(runnerSessionStartupScope, { + mode: sessionStartupMode ?? "bootstrap", + }); + await trace.end(runnerSessionStartupScope, { + endedAtMs: milestoneAtMs, + }); + } + agentTurnScope = trace.start("agent.turn", { + parentName: "native.session.execute", + startedAtMs: milestoneAtMs, + attributes: { provider: input.execution.provider.kind }, + }); + trace.activate(agentTurnScope); + } + if (event.eventType === "turn.started" && turnStartedAtMs === null) { + turnStartedAtMs = milestoneAtMs; + await trace.record({ + name: "provider.turn.queue", + parentName: "agent.turn", + startedAtMs: turnSubmittedAtMs ?? milestoneAtMs, + endedAtMs: milestoneAtMs, + }); + } + if ( + !firstAgentEventRecorded && + turnStartedAtMs !== null && + (event.eventType === "item.started" || + event.eventType === "item.completed") + ) { + const payload = record(event.payload); + const kind = + typeof payload.kind === "string" ? payload.kind : "unknown"; + if ( + [ + "reasoning", + "agentMessage", + "toolCall", + "dynamicToolCall", + ].includes(kind) + ) { + firstAgentEventRecorded = true; + await trace.record({ + name: "provider.time_to_first_agent_event", + parentName: "agent.turn", + startedAtMs: turnStartedAtMs, + endedAtMs: milestoneAtMs, + attributes: { eventKind: kind }, + }); + } + } + if ( + [ + "turn.completed", + "turn.failed", + "turn.interrupted", + "turn.cancelled", + ].includes(event.eventType) && + turnCompletedAtMs === null + ) { + turnCompletedAtMs = milestoneAtMs; + if (!agentTurnScope) { + agentTurnScope = trace.start("agent.turn", { + parentName: "native.session.execute", + startedAtMs: + turnSubmittedAtMs ?? + turnStartedAtMs ?? + nativeSessionExecuteStartedAtMs, + attributes: { provider: input.execution.provider.kind }, + }); + trace.activate(agentTurnScope); + } + const outcome = + event.eventType === "turn.completed" ? "ok" : "failed"; + await trace.end(agentTurnScope, { + endedAtMs: milestoneAtMs, + outcome, + }); + taskSettleScope = trace.start("task.settle", { + parentName: "task.run", + startedAtMs: milestoneAtMs, + }); + trace.activate(taskSettleScope); + } + if (input.onLog) + await input.onLog( + "stdout", + `${JSON.stringify({ type: "paperclip.prp.event", event })}\n`, + ); + const inputMetric = runtimeInputLifecycleMetric(event); + if (inputMetric && input.onLog) { + await input.onLog( + "stdout", + `${JSON.stringify({ + type: "paperclip.runtime_input.metric", + ...inputMetric, + })}\n`, + ); + } + const questionFallback = await materializeRuntimeQuestionFallback({ + db: input.db, + binding: input.execution.binding, + event, + }); + if (questionFallback) { + if (input.onLog) { + const origin = record(record(record(event.payload).request).origin); + await input.onLog( + "stdout", + `${JSON.stringify({ + type: "paperclip.runtime_input.metric", + outcome: + record(event.payload).reason === "durable_handoff" + ? "durable_handoff_materialized" + : "provider_loss_materialized", + requestId: record(event.payload).requestId, + interactionId: questionFallback.interaction.id, + adapter: + typeof origin.adapter === "string" + ? origin.adapter + : "unknown", + })}\n`, + ); + } + } + await governedWaitObservation.observe( + event, + event.eventType === "item.completed" || questionFallback !== null, + ); + await recordPlanSynchronization( + event as { + sourceEventId: string; + turnId?: string; + eventType: string; + payload: Record; + }, + ); + }, + onDuplicateEvent: async (event) => { + // A crash can happen after the event commit but before its callback + // finishes. Recover only idempotent durable projections here; activity, + // publication, logging, trace, and metric effects remain committed-only. + const questionFallback = await materializeRuntimeQuestionFallback({ + db: input.db, + binding: input.execution.binding, + event, + }); + await governedWaitObservation.observe( + event, + event.eventType === "item.completed" || questionFallback !== null, + ); + const planSynchronization = await synchronizeCompletedProviderPlan({ + db: input.db, + execution: input.execution, + event: event as { + sourceEventId: string; + turnId?: string; + eventType: string; + payload: Record; + }, + }); + if (planSynchronization) { + upsertPlanSynchronization(planSynchronization); + } + }, + }, + ); + let native: Awaited>; + const lifecyclePolicy = input.execution.session?.lifecyclePolicy ?? { + mode: "per_turn" as const, + idleTimeoutMs: null, + }; + const warmSessionId = + lifecyclePolicy.mode === "warm" + ? nativeSessionScopeKey(input.execution) + : null; + const warmConfigDigest = + lifecyclePolicy.mode === "warm" + ? nativeSessionConfigDigest(input.execution) + : null; + let existingWarmSession: NativeSession | undefined; + let persistedWarmSession: PersistedNativeSession | null | undefined; + if (warmSessionId !== null && warmConfigDigest !== null) { + const entry = warmNativeSessions.get(warmSessionId); + if (entry) { + if (entry.configDigest !== warmConfigDigest) { + if (entry.busy) throw new Error("native_session_supervisor_busy"); + if (entry.idleTimer !== null) clearTimeout(entry.idleTimer); + warmNativeSessions.delete(warmSessionId); + await entry.session.close({ + reason: "warm native session configuration changed", + }); + persistedWarmSession = loadWarmNativeCheckpoint( + input.execution, + warmConfigDigest, + ); + } else { + if (entry.busy) throw new Error("native_session_supervisor_busy"); + entry.busy = true; + if (entry.idleTimer !== null) clearTimeout(entry.idleTimer); + entry.idleTimer = null; + existingWarmSession = entry.session; + } + } else { + persistedWarmSession = loadWarmNativeCheckpoint( + input.execution, + warmConfigDigest, + ); + } + } + async function resolvePendingGovernedWait() { + const continuingInteractionIds = continuingPendingInteractionIds( + input.execution, + ); + const interaction = await input.db + .select({ + id: issueThreadInteractions.id, + title: issueThreadInteractions.title, + summary: issueThreadInteractions.summary, + }) + .from(issueThreadInteractions) + .where( + and( + eq( + issueThreadInteractions.companyId, + input.execution.binding.companyId, + ), + eq(issueThreadInteractions.issueId, input.execution.binding.issueId), + or( + eq( + issueThreadInteractions.sourceRunId, + input.execution.binding.runId, + ), + ...(continuingInteractionIds.length > 0 + ? [inArray(issueThreadInteractions.id, continuingInteractionIds)] + : []), + ), + eq(issueThreadInteractions.status, "pending"), + ), + ) + .orderBy( + desc(issueThreadInteractions.createdAt), + desc(issueThreadInteractions.id), + ) + .limit(1) + .then((rows) => rows[0] ?? null); + return interaction + ? nativeGovernedWaitResult({ + interaction, + completionContract: input.execution.completionContract.contract, + }) + : null; + } + const runnerExecution = input.execution; + const leaseRenewal = startNativeSessionExecutionLeaseRenewal({ + db: input.db, + runId: input.execution.binding.runId, + companyId: input.execution.binding.companyId, + issueId: input.execution.binding.issueId, + leaseOwner, + attempt, + }); + try { + const runnerdBackend = + input.useRunnerd && input.backend === undefined + ? await createRunnerdBackend({ + ...input, + execution: runnerExecution, + runnerInstanceId: effectiveRunnerInstanceId, + durableEnvironmentLeaseId: durableRunnerBinding?.environmentLeaseId, + trace, + }) + : null; + nativeSessionExecuteStartedAtMs = Date.now(); + native = await trace.measure( + "native.session.execute", + async () => { + runnerSessionStartupScope = trace.start("runner.session.startup", { + parentName: "native.session.execute", + startedAtMs: nativeSessionExecuteStartedAtMs, + }); + trace.activate(runnerSessionStartupScope); + const result = await trace.run(runnerSessionStartupScope, () => + executeNativeSession({ + input: runnerExecution, + backend: + input.backend ?? + runnerdBackend ?? + createNativeSessionBackend(input.execution, { + runnerInstanceId: input.runnerInstanceId, + onSpawn: input.onSpawn, + }), + controlPlane, + runnerInstanceId: effectiveRunnerInstanceId, + controlPlaneInstanceId, + resolveGovernedWait: ({ event }) => + governedWaitObservation.consume(event), + resolveMissingResult: async ({ terminalEvent }) => { + // A model may correctly create a durable question/confirmation and + // then end its provider turn without also invoking paperclip_finish. + // Recover only completed turns with a pending interaction created by + // this exact run; unrelated or failed turns still fail closed. + if (terminalEvent.eventType !== "turn.completed") return null; + return resolvePendingGovernedWait(); + }, + existingSession: existingWarmSession, + persistedSession: persistedWarmSession, + keepSessionOpen: warmSessionId !== null, + onCheckpoint: + warmSessionId !== null && warmConfigDigest !== null + ? async (snapshot) => + persistWarmNativeCheckpoint( + input.execution, + warmConfigDigest, + snapshot, + ) + : undefined, + onContinuityBreak: async (continuity) => { + const atMs = Date.now(); + await trace.record({ + name: "provider.session.continuity_break", + parentName: "native.session.execute", + startedAtMs: atMs, + endedAtMs: atMs, + outcome: "failed", + attributes: { + reason: continuity.reason, + previousDriverSessionId: continuity.previousDriverSessionId, + previousProviderSessionId: + continuity.previousProviderSessionId ?? "unavailable", + replacementDriverSessionId: + continuity.replacementDriverSessionId, + replacementProviderSessionId: + continuity.replacementProviderSessionId ?? "unavailable", + }, + }); + await input.onLog?.( + "stderr", + `[paperclip-runner] provider session continuity break: exact resume failed (${continuity.reason}); old driver session=${continuity.previousDriverSessionId}, old provider session=${continuity.previousProviderSessionId ?? "unavailable"}, replacement driver session=${continuity.replacementDriverSessionId}, replacement provider session=${continuity.replacementProviderSessionId ?? "unavailable"}\n`, + ); + }, + onSession: (session) => { + if ( + session && + warmSessionId !== null && + warmConfigDigest !== null + ) { + const existing = warmNativeSessions.get(warmSessionId); + if (existing) existing.session = session; + else + warmNativeSessions.set(warmSessionId, { + session, + configDigest: warmConfigDigest, + busy: true, + idleTimer: null, + lastActivityAt: new Date().toISOString(), + }); + } + if (session) + activeNativeSessions.set(input.execution.binding.runId, { + session, + cancelRequested: false, + }); + else { + activeNativeSessions.delete(input.execution.binding.runId); + clearNativeRuntimeRequestResolutions( + input.execution.binding.runId, + ); + } + }, + }), + ); + await trace.end(runnerSessionStartupScope, { + outcome: + result.terminal.runTerminalState === "succeeded" ? "ok" : "failed", + }); + return result; + }, + { parentName: "task.run" }, + ); + await leaseRenewal.stop(); + await trace.record({ + name: "native.result.finalize", + parentName: "task.settle", + startedAtMs: turnCompletedAtMs ?? nativeSessionExecuteStartedAtMs, + endedAtMs: Date.now(), + }); + activeNativeSessions.delete(input.execution.binding.runId); + clearNativeRuntimeRequestResolutions(input.execution.binding.runId); + } catch (error) { + await leaseRenewal.stop().catch(() => undefined); + const failedAtMs = Date.now(); + if (runnerSessionStartupScope) { + await trace.end(runnerSessionStartupScope, { + endedAtMs: failedAtMs, + outcome: "failed", + }); + } + if (agentTurnScope) { + await trace.end(agentTurnScope, { + endedAtMs: failedAtMs, + outcome: "failed", + }); + } + if (!taskSettleScope) { + taskSettleScope = trace.start("task.settle", { + parentName: "task.run", + startedAtMs: failedAtMs, + }); + } + trace.activate(taskSettleScope); + activeNativeSessions.delete(input.execution.binding.runId); + clearNativeRuntimeRequestResolutions(input.execution.binding.runId); + if (warmSessionId !== null && lifecyclePolicy.mode === "warm") { + await releaseWarmNativeSession( + warmSessionId, + lifecyclePolicy.idleTimeoutMs, + true, + ); + } + if ( + error instanceof NativeResultPendingFinalizationError || + error instanceof NativeCancellationPendingRecoveryError + ) { + // This is not a provider failure and must not overwrite the durable + // result/coordinator state. The heartbeat boundary will either hand an + // already-materialized result to the finalizer or retain the durable + // cancellation intent for cancellation recovery. + if (taskSettleScope) { + await trace.end(taskSettleScope, { outcome: "ok" }); + } + await trace.finish("ok"); + throw error; + } + const now = new Date(); + const sourceFailureCode = nativeSessionFailureSourceCode(error); + const recoveryEvidence = await nativeProviderRecoveryEvidence({ + db: input.db, + runId: input.execution.binding.runId, + sourceFailureCode, + }); + const disposition = nativeSessionFailureDisposition( + attempt, + now, + sourceFailureCode, + ); + const phase = + recoveryEvidence.recoveryMode === "ambiguous_state" + ? ("terminal_failure" as const) + : disposition.phase; + const failureCode = + recoveryEvidence.recoveryMode === "ambiguous_state" + ? sourceFailureCode + : disposition.failureCode; + const nextAttemptAt = + recoveryEvidence.recoveryMode === "ambiguous_state" + ? null + : disposition.nextAttemptAt; + const recoveryProjection = nativeSessionRecoveryProjection({ + phase, + failureCode, + agentId: input.execution.binding.agentId, + }); + const { exhausted } = recoveryProjection; + const integrityFailure = + sourceFailureCode === "native_event_replay_conflict"; + const message = + error instanceof Error + ? error.message.slice(0, 2_000) + : String(error).slice(0, 2_000); + await input.db.transaction(async (tx) => { + const updated = await tx + .update(nativeRunFinalizations) + .set({ + phase, + leaseOwner: null, + leaseExpiresAt: null, + failureCode, + failureDetail: { + message, + originalFailureCode: sourceFailureCode, + recoveryMode: recoveryEvidence.recoveryMode, + providerSessionEstablished: + recoveryEvidence.providerSessionEstablished, + providerEventsExist: recoveryEvidence.providerEventsExist, + checkpointExists: recoveryEvidence.checkpointExists, + recoveryOwner: recoveryProjection.recoveryOwner, + nextAction: + recoveryEvidence.recoveryMode === "ambiguous_state" + ? "Inspect the original provider failure and durable events; state is ambiguous and a replacement provider session is forbidden." + : integrityFailure + ? "Inspect the persisted runner events and checkpoint for a source-sequence integrity conflict; automatic recovery is stopped." + : exhausted + ? "Inspect the persisted native session after its bounded resume budget was exhausted." + : recoveryEvidence.recoveryMode === "bootstrap_retry" + ? "Retry provider bootstrap on this same run; durable evidence proves no provider session or provider event was created." + : "Resume this same run from its exact persisted native provider checkpoint after the retry delay.", + }, + nextAttemptAt, + updatedAt: now, + }) + .where( + and( + eq(nativeRunFinalizations.runId, input.execution.binding.runId), + eq( + nativeRunFinalizations.companyId, + input.execution.binding.companyId, + ), + eq( + nativeRunFinalizations.issueId, + input.execution.binding.issueId, + ), + eq(nativeRunFinalizations.leaseOwner, leaseOwner), + eq(nativeRunFinalizations.attempt, attempt), + gt(nativeRunFinalizations.leaseExpiresAt, sql`now()`), + ), + ) + .returning({ runId: nativeRunFinalizations.runId }) + .then((rows) => rows[0] ?? null); + if (!updated) throw new Error("native_session_lease_lost"); + await tx + .update(heartbeatRuns) + .set({ + nativePhase: phase, + nativePhaseUpdatedAt: now, + error: message, + errorCode: sourceFailureCode, + updatedAt: now, + }) + .where(eq(heartbeatRuns.id, input.execution.binding.runId)); + if (recoveryProjection.issueStatus) { + await issueService(tx as unknown as Db).update( + input.execution.binding.issueId, + { status: recoveryProjection.issueStatus }, + tx, + ); + } + await issueRecoveryActionService(tx as unknown as Db).upsertSourceScoped({ + companyId: input.execution.binding.companyId, + sourceIssueId: input.execution.binding.issueId, + kind: "active_run_watchdog", + ownerType: recoveryProjection.recoveryActionOwnerType, + ownerAgentId: recoveryProjection.recoveryActionOwnerAgentId, + returnOwnerAgentId: input.execution.binding.agentId, + cause: recoveryProjection.recoveryActionCause, + fingerprint: createHash("sha256") + .update(`${input.execution.binding.runId}:${failureCode}`) + .digest("hex"), + evidence: { + runId: input.execution.binding.runId, + coordinatorAttempt: attempt, + sourceFailureCode, + recoveryDisposition: failureCode, + recoveryMode: recoveryEvidence.recoveryMode, + providerSessionEstablished: + recoveryEvidence.providerSessionEstablished, + }, + nextAction: + recoveryEvidence.recoveryMode === "ambiguous_state" + ? "Inspect the original provider failure and explicitly resolve the ambiguous session state; do not open a replacement provider session." + : integrityFailure + ? "Inspect the persisted runner event collision and explicitly repair or replace the run; automatic retries are disabled." + : exhausted + ? "Inspect the provider trace and explicitly choose a replacement run or provider configuration; automatic provider work is stopped." + : recoveryEvidence.recoveryMode === "bootstrap_retry" + ? "Retry bootstrap on the same run without manufacturing a provider checkpoint." + : "Resume the exact persisted native session on the same heartbeat run.", + wakePolicy: nextAttemptAt + ? { + kind: "resume_native_run", + runId: input.execution.binding.runId, + notBefore: nextAttemptAt.toISOString(), + } + : null, + maxAttempts: 3, + supersedeOnIdentityChange: recoveryProjection.supersedeOnIdentityChange, + }); + }); + if (taskSettleScope) { + await trace.end(taskSettleScope, { outcome: "failed" }); + } + await trace.finish("failed"); + throw error; + } + if ( + planSynchronizations.length === 0 && + "executionMode" in input.execution && + input.execution.executionMode === "plan" + ) { + const markdown = semanticProviderPlanMarkdown( + native.result as unknown as Record, + ); + if (markdown) { + const digest = createHash("sha256").update(markdown).digest("hex"); + await recordPlanSynchronization({ + sourceEventId: `semantic-plan:${input.execution.binding.runId}:${digest}`, + ...(native.turnId ? { turnId: native.turnId } : {}), + eventType: "plan.updated", + payload: { + schema: "paperclip.plan.updated.v1", + planId: `semantic:${native.turnId ?? input.execution.binding.runId}`, + revision: 1, + complete: true, + markdown, + source: "semantic_result_artifact", + }, + }); + } + } + const releaseNow = new Date(); + const released = await input.db + .update(nativeRunFinalizations) + .set({ + leaseOwner: null, + leaseExpiresAt: null, + updatedAt: releaseNow, + }) + .where( + and( + eq(nativeRunFinalizations.runId, input.execution.binding.runId), + eq( + nativeRunFinalizations.companyId, + input.execution.binding.companyId, + ), + eq( + nativeRunFinalizations.issueId, + input.execution.binding.issueId, + ), + eq(nativeRunFinalizations.leaseOwner, leaseOwner), + eq(nativeRunFinalizations.attempt, attempt), + gt(nativeRunFinalizations.leaseExpiresAt, sql`now()`), + ), + ) + .returning({ runId: nativeRunFinalizations.runId }) + .then((rows) => rows[0] ?? null); + if (!released) throw new Error("native_session_lease_lost"); + const finalization: NativeFinalizationResult = { + schema: "paperclip.native-finalization.v1", + runtimeMode: "native", + runId: input.execution.binding.runId, + issueId: input.execution.binding.issueId, + companyId: input.execution.binding.companyId, + result: native.result as unknown as Record, + terminal: native.terminal, + turnId: native.turnId, + sourceInstanceId: effectiveRunnerInstanceId, + normalizedSessionId: native.normalizedSessionId, + providerSessionId: native.providerSessionId, + driverKind: native.driverKind, + driverVersion: native.driverVersion, + nativeEventCount: native.nativeEventCount, + highestContiguousSourceSeq: native.highestContiguousSourceSeq, + workspaceFinalizeStatus: "pending", + }; + // A following run cannot attach until the prior run's durable finalization + // is committed. Provider completion alone is not an authority boundary. + if (warmSessionId !== null && lifecyclePolicy.mode === "warm") { + await releaseWarmNativeSession( + warmSessionId, + lifecyclePolicy.idleTimeoutMs, + false, + ); + } + const adapterResult: AdapterExecutionResult = { + exitCode: native.terminal.runTerminalState === "succeeded" ? 0 : 1, + signal: null, + timedOut: false, + errorMessage: + native.terminal.runTerminalState === "succeeded" + ? null + : `Native session ${native.terminal.runTerminalState}`, + resultJson: { + nativeResult: native.result as unknown as Record, + nativeTerminal: native.terminal as unknown as Record, + planSynchronizations, + }, + summary: native.result.summary, + sessionId: native.normalizedSessionId, + sessionDisplayId: native.providerSessionId ?? native.normalizedSessionId, + provider: "openai", + model: input.execution.provider.model, + usage: normalizeNativeUsage(native.usage), + costUsd: nativeUsageCostUsd(native.usage), + usageBasis: "per_run", + nativeFinalization: finalization, + }; + if (taskSettleScope) { + await trace.end(taskSettleScope, { + outcome: + native.terminal.runTerminalState === "succeeded" ? "ok" : "failed", + }); + } + await trace.finish( + native.terminal.runTerminalState === "succeeded" ? "ok" : "failed", + ); + return adapterResult; +} + +function numericUsageField( + usage: Record | null, + keys: string[], +): number | undefined { + if (!usage) return undefined; + for (const key of keys) { + const value = usage[key]; + if (typeof value === "number" && Number.isFinite(value) && value >= 0) + return value; + } + return undefined; +} + +function nativeUsageMeasurement(usage: Record) { + const nestedUsage = record(usage.usage); + const candidates = [ + record(usage.runDelta), + record(nestedUsage.runDelta), + record(usage.total), + record(nestedUsage.total), + record(usage.cumulative), + record(nestedUsage.cumulative), + nestedUsage, + usage, + ]; + return ( + candidates.find( + (candidate) => + numericUsageField(candidate, [ + "inputTokens", + "input", + "promptTokens", + "outputTokens", + "output", + "completionTokens", + ]) !== undefined, + ) ?? usage + ); +} + +export function nativeUsageCostUsd(usage: Record | null) { + if (!usage) return undefined; + const measurement = nativeUsageMeasurement(usage); + const direct = + numericUsageField(usage, [ + "providerCostUsd", + "cacheAdjustedCostUsd", + "costUsd", + ]) ?? + numericUsageField(measurement, [ + "providerCostUsd", + "cacheAdjustedCostUsd", + "costUsd", + ]); + if (direct !== undefined) return direct; + const cost = record(usage.cost); + const currency = + typeof cost.currency === "string" ? cost.currency.toUpperCase() : "USD"; + if (currency !== "USD") return undefined; + return numericUsageField(cost, ["amount", "total"]); +} + +export function normalizeNativeUsage(usage: Record | null) { + if (!usage) return undefined; + const measurement = nativeUsageMeasurement(usage); + const cache = record(measurement.cache); + const cachedInputTokens = + numericUsageField(measurement, [ + "cachedInputTokens", + "cacheReadInputTokens", + "cacheReadTokens", + "cachedReadTokens", + ]) ?? numericUsageField(cache, ["read"]); + return { + inputTokens: + numericUsageField(measurement, [ + "inputTokens", + "input", + "promptTokens", + ]) ?? 0, + outputTokens: + numericUsageField(measurement, [ + "outputTokens", + "output", + "completionTokens", + ]) ?? 0, + ...(cachedInputTokens === undefined ? {} : { cachedInputTokens }), + }; +} + +export async function createRunnerdBackend(input: { + db: Db; + execution: NativeExecutionInput; + runnerInstanceId: string; + durableEnvironmentLeaseId?: string; + onSpawn?: (meta: { + pid: number; + processGroupId: number | null; + startedAt: string; + }) => Promise; + runnerEnvironment?: NodeJS.ProcessEnv; + trace?: NativeRunTrace; + onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise; + enqueueWakeup?: ( + agentId: string, + options: { + source: "assignment"; + triggerDetail: "system"; + reason: "issue_assigned"; + payload: Record; + idempotencyKey: string; + requestedByActorType: "agent"; + requestedByActorId: string; + contextSnapshot: Record; + }, + ) => Promise; +}): Promise { + if (input.execution.provider.kind !== "codex") { + throw new Error("paperclip_runner_provider_unsupported"); + } + const authority = new PaperclipRunnerToolAuthority(input.db, { + companyId: input.execution.binding.companyId, + issueId: input.execution.binding.issueId, + runId: input.execution.binding.runId, + agentId: input.execution.binding.agentId, + normalizedSessionId: nativeSessionKey(input.execution), + workMode: input.execution.task.workMode, + enqueueWakeup: input.enqueueWakeup, + }); + const dynamicTools = await authority.definitions(); + const root = runnerdStateRoot(input.execution); + mkdirSync(root, { recursive: true, mode: 0o700 }); + const environment = input.runnerEnvironment ?? process.env; + + const archiveContinuityState = async () => { + const archiveRoot = resolve( + root, + "continuity-breaks", + `${Date.now()}-${randomUUID()}`, + ); + mkdirSync(archiveRoot, { recursive: true, mode: 0o700 }); + for (const name of ["control-plane", "runner", "codex-home"]) { + const source = resolve(root, name); + if (existsSync(source)) renameSync(source, resolve(archiveRoot, name)); + } + }; + + const backend = createNativeSessionBackend(input.execution, { + runnerInstanceId: input.runnerInstanceId, + onSpawn: input.onSpawn, + dynamicTools, + dynamicToolHandler: (call) => authority.execute(call), + codexTransportFactory: (recoveryContext) => + createRunnerdCodexTransport({ + runnerBinary: resolvePaperclipRunnerBinary(), + stateDirectory: root, + environment, + lifecyclePolicy: input.execution.session.lifecyclePolicy, + runtimeContext: + "runtimeContext" in input.execution + ? input.execution.runtimeContext + : null, + resumeDynamicTools: dynamicTools, + providerRecoveryPolicy: recoveryContext?.providerRecoveryPolicy, + prpIdentity: { + runnerInstanceId: input.runnerInstanceId, + environmentLeaseId: + input.durableEnvironmentLeaseId ?? + input.execution.binding.executionWorkspaceId, + runId: input.execution.binding.runId, + normalizedSessionId: nativeSessionKey(input.execution), + turnId: `turn-${input.execution.binding.runId}`, + itemId: `item-${input.execution.binding.runId}`, + }, + controlPlaneRegistration: async (controlPlaneAuthority) => { + const selectedAtMs = Date.now(); + await input.trace?.record({ + name: "runner.transport.selected", + parentName: "runner.transport.connect", + startedAtMs: selectedAtMs, + endedAtMs: selectedAtMs, + attributes: { + mode: "local_loopback", + connectionOwner: "runnerd", + }, + }); + await input.onLog?.( + "stderr", + "[paperclip-runner] transport mode=local_loopback state=connecting\n", + ); + const registration = await measureNativeRunnerSpan( + input.trace, + "runner.prp.route.register", + () => + registerRunnerPrpAuthority({ + companyId: input.execution.binding.companyId, + runId: input.execution.binding.runId, + authority: controlPlaneAuthority, + }), + { parentName: "runner.transport.connect" }, + ); + return { + ...registration, + startupFailureCode: "runner_local_connect_failed" as const, + }; + }, + }).transport, + }); + + return { + descriptor: () => backend.descriptor(), + openSession: (sessionInput) => backend.openSession(sessionInput), + recoverSession: (snapshot, options) => + backend.recoverSession + ? backend.recoverSession(snapshot, options) + : Promise.resolve({ + recovered: false, + reason: "driver does not support recovery", + }), + openReplacementSession: async (sessionInput) => { + await measureNativeRunnerSpan( + input.trace, + "provider.session.archive_before_replacement", + archiveContinuityState, + { parentName: "native.session.execute" }, + ); + return backend.openSession(sessionInput); + }, + } satisfies NativeSessionBackend; +} diff --git a/server/src/services/native-runtime/native-session-resume.test.ts b/server/src/services/native-runtime/native-session-resume.test.ts new file mode 100644 index 0000000000..8277a07282 --- /dev/null +++ b/server/src/services/native-runtime/native-session-resume.test.ts @@ -0,0 +1,280 @@ +import { describe, expect, it } from "vitest"; +import { canonicalNativeRuntimeContextDigest } from "../../vendor/paperclip-runner/index.js"; +import { buildNativeExecutionInput } from "./native-execution-input.js"; +import { rebindNativeSessionCheckpoint } from "./native-session-resume.js"; +import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js"; + +const companyId = "10000000-0000-4000-8000-000000000001"; +const issueId = "20000000-0000-4000-8000-000000000002"; +const agentId = "30000000-0000-4000-8000-000000000003"; +const previousRunId = "40000000-0000-4000-8000-000000000004"; +const currentRunId = "50000000-0000-4000-8000-000000000005"; +const normalizedSessionId = "60000000-0000-4000-8000-000000000006"; + +function execution(runId: string, cwd = "/workspace") { + return buildNativeExecutionInput({ + companyId, + runId, + issue: { id: issueId, identifier: "DOT-2", title: "Test", description: null, workMode: "standard" }, + taskPrompt: "Only the current turn", + agentId, + workspace: { id: runId, cwd, repoUrl: null, repoRef: null, branchName: null }, + normalizedSessionId, + completionContract: { + id: "70000000-0000-4000-8000-000000000007", + sha256: `sha256:${"a".repeat(64)}`, + schemaVersion: "paperclip.run-result.v1", + contract: { + revision: "1", + objective: "Test session resumption", + criteria: [{ id: "objective", requirement: "Answer the current turn" }], + }, + }, + runtimeContext: nativeRuntimeContextFixture(), + }); +} + +function planningExecution(runId: string, revisionId: string) { + return buildNativeExecutionInput({ + companyId, + runId, + issue: { id: issueId, identifier: "DOT-2", title: "Test", description: null, workMode: "planning" }, + taskPrompt: "Revise the plan", + agentId, + workspace: { id: runId, cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null }, + normalizedSessionId, + executionMode: "plan", + planningContext: { + documentId: "80000000-0000-4000-8000-000000000008", + baseRevisionId: revisionId, + baseRevisionNumber: revisionId.endsWith("9") ? 9 : 8, + markdown: `Plan at ${revisionId}`, + sha256: `sha256:${"b".repeat(64)}`, + reviewContext: {}, + }, + completionContract: execution(runId).completionContract, + runtimeContext: nativeRuntimeContextFixture(), + }); +} + +function previousRun(overrides: Record = {}) { + return { + id: previousRunId, + companyId, + agentId, + nativeSessionId: normalizedSessionId, + runnerProfileJson: { + nativeExecutionInput: execution(previousRunId), + sessionCheckpoint: { + backendKind: "runner", + driverKind: "codex_app_server", + sessionId: "provider-thread-123", + providerSessionId: "provider-thread-123", + cursor: "42", + identity: { runId: previousRunId, sessionId: normalizedSessionId, companyId, issueId, agentId }, + semanticResult: { schema: "paperclip.run-result.v1", reportedWorkDisposition: "done", summary: "old" }, + terminal: { schema: "paperclip.prp.terminal.v1", turnTerminalState: "completed", runTerminalState: "succeeded", reportedWorkDisposition: "done" }, + activeTurnId: "old-turn", + terminalTurns: [{ turnId: "old-turn", state: "completed" }], + pendingRuntimeRequests: [{ requestId: "old-request" }], + lineage: [{ threadId: "provider-thread-123" }], + }, + ...overrides, + }, + }; +} + +describe("rebindNativeSessionCheckpoint", () => { + it("retains provider identity but clears prior turn and event state", () => { + const rebound = rebindNativeSessionCheckpoint({ + previousRun: previousRun(), + currentExecution: execution(currentRunId), + }); + expect(rebound).toMatchObject({ + sessionId: "provider-thread-123", + providerSessionId: "provider-thread-123", + driverKind: "codex_app_server", + cursor: null, + semanticResult: null, + terminal: null, + activeTurnId: null, + terminalTurns: [], + pendingRuntimeRequests: [], + providerRecoveryPolicy: "allow_replacement_after_resume_failure", + identity: { runId: currentRunId, sessionId: normalizedSessionId, companyId, issueId, agentId }, + }); + }); + + it("allows a provider replacement only after a durable response wake", () => { + const source = previousRun(); + const profile = source.runnerProfileJson as Record; + const checkpoint = profile.sessionCheckpoint as Record; + checkpoint.semanticResult = { + schema: "paperclip.run_result.v1", + reportedWorkDisposition: "yielded", + summary: "Waiting for a response.", + continuation: { + kind: "response_wake", + idempotencyKey: "interaction-response:one", + }, + }; + + expect( + rebindNativeSessionCheckpoint({ + previousRun: source, + currentExecution: execution(currentRunId), + }), + ).toMatchObject({ + providerRecoveryPolicy: "allow_replacement_after_governed_wait", + semanticResult: null, + activeTurnId: null, + }); + }); + + it("refuses to resume when the workspace changes", () => { + expect(rebindNativeSessionCheckpoint({ + previousRun: previousRun(), + currentExecution: execution(currentRunId, "/different-workspace"), + })).toBeNull(); + }); + + it("rotates when assigned context changes but permits a fresh run-scoped MCP binding", () => { + const reboundCredential = execution(currentRunId); + reboundCredential.runtimeContext.mcp.bindingId = "native-mcp:fresh-run"; + expect(rebindNativeSessionCheckpoint({ + previousRun: previousRun(), + currentExecution: reboundCredential, + })).not.toBeNull(); + + const changedAssignment = execution(currentRunId); + const withoutAggregate = { + prompt: changedAssignment.runtimeContext.prompt, + instructions: changedAssignment.runtimeContext.instructions, + skills: changedAssignment.runtimeContext.skills, + mcp: { + assignmentSetId: `sha256:${"1".repeat(64)}`, + digest: "1".repeat(64), + bindingId: "native-mcp:fresh-run", + }, + }; + changedAssignment.runtimeContext = { + ...withoutAggregate, + aggregateDigest: canonicalNativeRuntimeContextDigest(withoutAggregate), + }; + expect(rebindNativeSessionCheckpoint({ + previousRun: previousRun(), + currentExecution: changedAssignment, + })).toBeNull(); + }); + + it("refuses a checkpoint whose prior-run binding was rewritten", () => { + const source = previousRun(); + const profile = source.runnerProfileJson as Record; + const checkpoint = profile.sessionCheckpoint as Record; + checkpoint.identity = { ...(checkpoint.identity as Record), runId: currentRunId }; + expect(rebindNativeSessionCheckpoint({ previousRun: source, currentExecution: execution(currentRunId) })).toBeNull(); + }); + + it("reuses plan mode across canonical revisions but never across a mode change", () => { + const source = previousRun({ nativeExecutionInput: planningExecution(previousRunId, "revision-8") }); + expect(rebindNativeSessionCheckpoint({ + previousRun: source, + currentExecution: planningExecution(currentRunId, "revision-9"), + })).not.toBeNull(); + expect(rebindNativeSessionCheckpoint({ + previousRun: source, + currentExecution: execution(currentRunId), + })).toBeNull(); + }); +}); + +describe("buildNativeExecutionInput wake projection", () => { + it("writes native v4 and pins the complete Codex configuration", () => { + const common = { + companyId, + runId: currentRunId, + issue: { id: issueId, identifier: "DOT-4", title: "Permissions", description: null, workMode: "standard" }, + taskPrompt: "Verify permissions", + agentId, + workspace: { id: currentRunId, cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null }, + normalizedSessionId, + completionContract: execution(currentRunId).completionContract, + runtimeContext: nativeRuntimeContextFixture(), + } as const; + const codex = buildNativeExecutionInput({ + ...common, + codexApprovalPolicy: "on-request", + }); + + expect(codex).toMatchObject({ + schema: "paperclip.native-execution-input.v4", + provider: { kind: "codex", approvalPolicy: "on-request" }, + }); + expect(JSON.stringify(codex)) + .not.toMatch(/OPENAI_API_KEY|ANTHROPIC_API_KEY|AWS_SECRET_ACCESS_KEY|PAPERCLIP_API_KEY/); + }); + + it("places child completion summaries in the closed provider prompt", () => { + const input = buildNativeExecutionInput({ + companyId, + runId: currentRunId, + issue: { + id: issueId, + identifier: "DOT-146", + title: "Finish after child handoff", + description: "Use the child result.", + workMode: "standard", + }, + taskPrompt: "Paperclip task context:\n- Issue: DOT-146", + wakePayload: { + reason: "issue_children_completed", + issue: { + id: issueId, + identifier: "DOT-146", + title: "Finish after child handoff", + description: "Use the child result.", + status: "in_progress", + priority: "medium", + workMode: "standard", + }, + childIssueSummaries: [{ + id: "child-147", + identifier: "DOT-147", + title: "Build utility", + status: "done", + summary: "Created three files and passed 7/7 tests.", + }], + childIssueSummaryTruncated: false, + checkedOutByHarness: true, + }, + resumedSession: true, + agentId, + workspace: { + id: currentRunId, + cwd: "/workspace", + repoUrl: null, + repoRef: null, + branchName: null, + }, + normalizedSessionId, + completionContract: { + id: "70000000-0000-4000-8000-000000000007", + sha256: `sha256:${"a".repeat(64)}`, + schemaVersion: "paperclip.run-result.v1", + contract: { + revision: "1", + objective: "Finish after the child", + criteria: [{ id: "objective", requirement: "Report the child result" }], + }, + }, + runtimeContext: nativeRuntimeContextFixture(), + }); + + expect(input.task.prompt).toContain("## Paperclip Resume Delta"); + expect(input.task.prompt).toContain("reason: issue_children_completed"); + expect(input.task.prompt).toContain("DOT-147 Build utility (done)"); + expect(input.task.prompt).toContain("Created three files and passed 7/7 tests."); + expect(input.task.prompt).toContain("Paperclip task context:\n- Issue: DOT-146"); + expect(input.task.prompt).not.toContain("Use the child result."); + }); +}); diff --git a/server/src/services/native-runtime/native-session-resume.ts b/server/src/services/native-runtime/native-session-resume.ts new file mode 100644 index 0000000000..32194703e6 --- /dev/null +++ b/server/src/services/native-runtime/native-session-resume.ts @@ -0,0 +1,105 @@ +import type { + NativeExecutionInput, + PersistedNativeSession, +} from "../../vendor/paperclip-runner/index.js"; +import { parseNativeExecutionInput } from "../../vendor/paperclip-runner/index.js"; + +function record(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? value as Record + : {}; +} + +export function isNativeSessionId(value: unknown): value is string { + return typeof value === "string" + && /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(value); +} + +function sameProvider( + previous: NativeExecutionInput["provider"], + current: NativeExecutionInput["provider"], +): boolean { + return JSON.stringify(previous) === JSON.stringify(current); +} + +/** + * Rebind a completed prior run's provider checkpoint to a new heartbeat run. + * The provider/driver session identity is retained, while every per-turn and + * per-event field is reset so the new run starts one clean turn via resume. + */ +export function rebindNativeSessionCheckpoint(input: { + previousRun: { + id: string; + companyId: string; + agentId: string; + nativeSessionId: string | null; + runnerProfileJson: unknown; + }; + currentExecution: NativeExecutionInput; +}): PersistedNativeSession | null { + const previousProfile = record(input.previousRun.runnerProfileJson); + const rawCheckpoint = record(previousProfile.sessionCheckpoint); + const checkpointIdentity = record(rawCheckpoint.identity); + const current = input.currentExecution; + const normalizedSessionId = current.session.normalizedSessionId; + if ( + !isNativeSessionId(normalizedSessionId) + || input.previousRun.companyId !== current.binding.companyId + || input.previousRun.agentId !== current.binding.agentId + || input.previousRun.nativeSessionId !== normalizedSessionId + || typeof rawCheckpoint.sessionId !== "string" + || checkpointIdentity.runId !== input.previousRun.id + || checkpointIdentity.companyId !== current.binding.companyId + || checkpointIdentity.issueId !== current.binding.issueId + || checkpointIdentity.agentId !== current.binding.agentId + || checkpointIdentity.sessionId !== normalizedSessionId + ) return null; + + let previousExecution: NativeExecutionInput; + try { + previousExecution = parseNativeExecutionInput(previousProfile.nativeExecutionInput); + } catch { + return null; + } + if ( + previousExecution.binding.runId !== input.previousRun.id + || previousExecution.binding.companyId !== current.binding.companyId + || previousExecution.binding.issueId !== current.binding.issueId + || previousExecution.binding.agentId !== current.binding.agentId + || previousExecution.session.normalizedSessionId !== normalizedSessionId + || previousExecution.session.driverKind !== current.session.driverKind + || previousExecution.workspace.cwd !== current.workspace.cwd + || ("executionMode" in previousExecution ? previousExecution.executionMode : "default") + !== ("executionMode" in current ? current.executionMode : "default") + || !sameProvider(previousExecution.provider, current.provider) + || previousExecution.schema !== current.schema + || ("runtimeContext" in previousExecution && "runtimeContext" in current + && previousExecution.runtimeContext.aggregateDigest !== current.runtimeContext.aggregateDigest) + ) return null; + + const priorSemanticResult = record(rawCheckpoint.semanticResult); + const priorContinuation = record(priorSemanticResult.continuation); + const providerRecoveryPolicy = + priorSemanticResult.reportedWorkDisposition === "yielded" + && priorContinuation.kind === "response_wake" + ? "allow_replacement_after_governed_wait" as const + : "allow_replacement_after_resume_failure" as const; + + return { + ...(structuredClone(rawCheckpoint) as unknown as PersistedNativeSession), + identity: { + runId: current.binding.runId, + sessionId: normalizedSessionId, + companyId: current.binding.companyId, + issueId: current.binding.issueId, + agentId: current.binding.agentId, + }, + cursor: null, + semanticResult: null, + terminal: null, + activeTurnId: null, + terminalTurns: [], + pendingRuntimeRequests: [], + providerRecoveryPolicy, + }; +} diff --git a/server/src/services/native-runtime/paperclip-control-plane-port.test.ts b/server/src/services/native-runtime/paperclip-control-plane-port.test.ts index 2c9fa8d177..99853d98b2 100644 --- a/server/src/services/native-runtime/paperclip-control-plane-port.test.ts +++ b/server/src/services/native-runtime/paperclip-control-plane-port.test.ts @@ -39,6 +39,7 @@ import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js"; import { finalizeNativeRun } from "./native-run-finalizer.js"; import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js"; import { issueThreadInteractionService } from "../issue-thread-interactions.js"; +import { materializeRuntimeQuestionFallback } from "./native-session-executor.js"; describe("PaperclipControlPlanePort conformance", () => { let temporary: Awaited> | null = null; @@ -278,17 +279,30 @@ describe("PaperclipControlPlanePort conformance", () => { it("runs the unchanged package conformance suite against Paperclip persistence", async () => { const identity = CONTROL_PLANE_CONFORMANCE_OPEN.identity; - const port = new PaperclipControlPlanePort(db, { - companyId: identity.companyId, - issueId: identity.issueId, - runId: identity.runId, - agentId: identity.agentId, - sessionId: identity.sessionId, - completionContractId: contractId, - completionContractSha256: contractSha, - sourceInstanceId: conformanceRunnerId, - controlPlaneSourceInstanceId: "control-conformance", - }); + const committedEventIds: string[] = []; + const duplicateEventIds: string[] = []; + const port = new PaperclipControlPlanePort( + db, + { + companyId: identity.companyId, + issueId: identity.issueId, + runId: identity.runId, + agentId: identity.agentId, + sessionId: identity.sessionId, + completionContractId: contractId, + completionContractSha256: contractSha, + sourceInstanceId: conformanceRunnerId, + controlPlaneSourceInstanceId: "control-conformance", + }, + { + onCommittedEvent: async (event) => { + committedEventIds.push(event.sourceEventId); + }, + onDuplicateEvent: async (event) => { + duplicateEventIds.push(event.sourceEventId); + }, + }, + ); await expect(runControlPlanePortConformance({ port })).resolves.toEqual({ eventCount: 3, highestContiguousSourceSeq: 3, @@ -300,6 +314,14 @@ describe("PaperclipControlPlanePort conformance", () => { replayBindingRejected: true, resultMutationRejected: true, }); + expect(committedEventIds).toEqual([ + "00000000-0000-4000-8000-000000000005:event:1", + "00000000-0000-4000-8000-000000000005:event:3", + "00000000-0000-4000-8000-000000000005:event:2", + ]); + expect(duplicateEventIds).toEqual([ + "00000000-0000-4000-8000-000000000005:event:2", + ]); await expect(db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, identity.runId))).resolves.toHaveLength(1); await finalizeNativeRun({ db, @@ -324,6 +346,155 @@ describe("PaperclipControlPlanePort conformance", () => { ); }); + it("recovers a runtime question when the event commits before its callback", async () => { + const identity = CONTROL_PLANE_CONFORMANCE_OPEN.identity; + const issueId = "40000000-0000-4000-8000-000000000041"; + const runId = "41000000-0000-4000-8000-000000000041"; + const sessionId = "42000000-0000-4000-8000-000000000041"; + const runnerInstanceId = "43000000-0000-4000-8000-000000000041"; + const localContractId = "44000000-0000-4000-8000-000000000041"; + const contractSha256 = "runtime-question-recovery-contract"; + await db.insert(issues).values({ + id: issueId, + companyId: identity.companyId, + title: "Recover a committed runtime question", + status: "in_progress", + assigneeAgentId: identity.agentId, + workMode: "standard", + }); + await db.insert(completionContracts).values({ + id: localContractId, + companyId: identity.companyId, + issueId, + revision: 1, + schemaVersion: "paperclip.completion-contract.v1", + policyVersion: "phase6-v1", + risk: "standard", + completionAuthority: "server_arbiter", + incompleteCriteriaPolicy: "preserve_non_terminal", + contractJson: { + revision: "phase6-v1", + objective: "Recover a committed runtime question", + criteria: [{ id: "objective", requirement: "Recover the question" }], + }, + canonicalSha256: contractSha256, + createdByActorType: "system", + createdByActorId: "test", + }); + await db.insert(heartbeatRuns).values({ + id: runId, + companyId: identity.companyId, + agentId: identity.agentId, + status: "running", + runtimeMode: "native", + nativeIssueId: issueId, + nativeSessionId: sessionId, + runnerInstanceId, + completionContractId: localContractId, + completionContractSha256: contractSha256, + contextSnapshot: { issueId }, + }); + + const binding = { + companyId: identity.companyId, + issueId, + runId, + agentId: identity.agentId, + sessionId, + completionContractId: localContractId, + completionContractSha256: contractSha256, + sourceInstanceId: runnerInstanceId, + controlPlaneSourceInstanceId: "runtime-question-recovery-control", + }; + const questionEvent: PrpEvent = { + schema: "paperclip.prp.event.v1", + sourceEventId: "runtime-question-recovery:1", + sourceSeq: 1, + sourceInstanceId: runnerInstanceId, + sourceKind: "runner", + runId, + normalizedSessionId: sessionId, + turnId: "runtime-question-recovery-turn", + eventType: "runtime_request.expired", + schemaVersion: 1, + priority: 0, + emittedAt: "2026-08-09T03:00:00.000Z", + payload: { + requestId: "runtime-question-recovery-request", + requestKind: "runtime", + requestType: "input", + reason: "provider_process_lost", + replayAllowed: false, + request: { + schema: "paperclip.runtime_request.v2", + requestKind: "runtime", + requestId: "runtime-question-recovery-request", + type: "input", + status: "pending", + prompt: "Choose a recovery option", + turnId: "runtime-question-recovery-turn", + itemId: "runtime-question-recovery-item", + input: { + schema: "paperclip.question_set.v1", + title: "Choose a recovery option", + questions: [ + { + id: "recovery-option", + prompt: "Which option should recovery use?", + required: true, + answerMode: "single_select", + options: [ + { id: "safe", label: "Safe recovery" }, + { id: "fast", label: "Fast recovery" }, + ], + }, + ], + }, + }, + }, + }; + const port = new PaperclipControlPlanePort(db, binding, { + onCommittedEvent: async () => { + throw new Error("simulated_post_commit_crash"); + }, + onDuplicateEvent: async (event) => { + await materializeRuntimeQuestionFallback({ db, binding, event }); + }, + }); + await port.openRun({ + identity: { ...identity, issueId, runId, sessionId }, + backendKind: "mock", + sourceInstanceId: runnerInstanceId, + }); + + await expect(port.appendEvent(questionEvent)).rejects.toThrow( + "simulated_post_commit_crash", + ); + await expect( + db + .select() + .from(issueThreadInteractions) + .where(eq(issueThreadInteractions.issueId, issueId)), + ).resolves.toEqual([]); + + await expect(port.appendEvent(questionEvent)).resolves.toMatchObject({ + disposition: "duplicate", + }); + await expect( + db + .select() + .from(issueThreadInteractions) + .where(eq(issueThreadInteractions.issueId, issueId)), + ).resolves.toEqual([ + expect.objectContaining({ + kind: "ask_user_questions", + status: "pending", + idempotencyKey: `runtime-input-durable:v1:${runId}:runtime-question-recovery-request`, + sourceRunId: runId, + }), + ]); + }); + it("completes one selected Paperclip task through the public package session contract", async () => { const identity = CONTROL_PLANE_CONFORMANCE_OPEN.identity; const sessionId = taskSessionId; diff --git a/server/src/services/native-runtime/paperclip-control-plane-port.ts b/server/src/services/native-runtime/paperclip-control-plane-port.ts index ebe26a9281..85e4693af2 100644 --- a/server/src/services/native-runtime/paperclip-control-plane-port.ts +++ b/server/src/services/native-runtime/paperclip-control-plane-port.ts @@ -69,13 +69,20 @@ export class PaperclipControlPlanePort implements ControlPlanePort { readonly #binding: PaperclipControlPlaneBinding; #sessionId: string | null = null; readonly #onCommittedEvent?: (event: PrpEvent) => Promise; + readonly #onDuplicateEvent?: (event: PrpEvent) => Promise; - constructor(db: Db, binding: PaperclipControlPlaneBinding, options: { - onCommittedEvent?: (event: PrpEvent) => Promise; - } = {}) { + constructor( + db: Db, + binding: PaperclipControlPlaneBinding, + options: { + onCommittedEvent?: (event: PrpEvent) => Promise; + onDuplicateEvent?: (event: PrpEvent) => Promise; + } = {}, + ) { this.#db = db; this.#binding = structuredClone(binding); this.#onCommittedEvent = options.onCommittedEvent; + this.#onDuplicateEvent = options.onDuplicateEvent; } #matchesPersistedBinding(run: typeof heartbeatRuns.$inferSelect): boolean { @@ -197,7 +204,15 @@ export class PaperclipControlPlanePort implements ControlPlanePort { canonicalPayload: event as unknown as Record, }, }); - if (persisted.disposition === "committed") await this.#onCommittedEvent?.(event); + if (persisted.disposition === "committed") { + await this.#onCommittedEvent?.(event); + } else { + // A recovered runner may replay the event whose durable side effects + // parked the prior attempt. Do not repeat those effects, but let the + // embedding runtime refresh observational state before appendEvent + // returns to its synchronous governed-wait boundary. + await this.#onDuplicateEvent?.(event); + } return { cursor: persisted.row.seq, highestContiguousSourceSeq: persisted.highestContiguousSourceSeq, diff --git a/server/src/services/native-runtime/paperclip-runner-real-server.integration.test.ts b/server/src/services/native-runtime/paperclip-runner-real-server.integration.test.ts new file mode 100644 index 0000000000..0e09455398 --- /dev/null +++ b/server/src/services/native-runtime/paperclip-runner-real-server.integration.test.ts @@ -0,0 +1,221 @@ +import { existsSync } from "node:fs"; +import { mkdtemp, rm } from "node:fs/promises"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { resolve } from "node:path"; + +import { eq } from "drizzle-orm"; +import { agents, companies, createDb, heartbeatRuns, issues } from "@paperclipai/db"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; + +import { + createRunnerdCodexTransport, + defaultCapabilityRunnerdBinary, +} from "../../vendor/paperclip-runner/index.js"; +import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; +import { + registerRunnerPrpAuthority, + runnerPrpWebSocketInternals, + setupRunnerPrpWebSocketServer, +} from "../../realtime/runner-prp-ws.js"; +import { PaperclipRunnerToolAuthority } from "./paperclip-runner-tool-authority.js"; + +const fakeCodexAppServer = resolve( + import.meta.dirname, + "../../../../packages/paperclip-runner/runner/target/debug/fake-codex-app-server", +); +const runnerBinariesAvailable = + existsSync(defaultCapabilityRunnerdBinary()) && existsSync(fakeCodexAppServer); +const runnerBinaryIt = runnerBinariesAvailable ? it : it.skip; + +describe("paperclip-runner real server vertical slice", () => { + let temporary: Awaited>; + const companyId = "00000000-0000-4000-8000-000000000701"; + const agentId = "00000000-0000-4000-8000-000000000702"; + const issueId = "00000000-0000-4000-8000-000000000703"; + const runId = "00000000-0000-4000-8000-000000000704"; + const resumedRunId = "00000000-0000-4000-8000-000000000705"; + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("paperclip-runner-real-server-"); + }); + + afterAll(async () => { + runnerPrpWebSocketInternals.resetForTests(); + await temporary.cleanup(); + }); + + runnerBinaryIt("runs Rust runnerd through Paperclip PRP and reads the real bound task", async () => { + const db = createDb(temporary.connectionString); + await db.insert(companies).values({ id: companyId, name: "Real runner slice", issuePrefix: "RRS" }); + await db.insert(agents).values({ + id: agentId, + companyId, + name: "Real runner agent", + adapterType: "paperclip_runner", + adapterConfig: { provider: "codex" }, + runtimeConfig: {}, + status: "active", + }); + await db.insert(issues).values({ + id: issueId, + companyId, + identifier: "RRS-1", + title: "Read me through the real control plane", + status: "in_progress", + workMode: "standard", + assigneeAgentId: agentId, + }); + await db.insert(heartbeatRuns).values({ + id: runId, + companyId, + agentId, + status: "running", + runtimeMode: "native", + nativeIssueId: issueId, + invocationSource: "assignment", + triggerDetail: "system", + contextSnapshot: { issueId }, + }); + await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId)); + + const authority = new PaperclipRunnerToolAuthority(db, { companyId, agentId, issueId, runId }); + const server = createServer(); + await new Promise((resolveListen) => server.listen(0, "127.0.0.1", resolveListen)); + const address = server.address(); + if (address === null || typeof address === "string") throw new Error("Expected a TCP listener."); + setupRunnerPrpWebSocketServer(server, { + apiUrl: `http://127.0.0.1:${address.port}`, + }); + const stateDirectory = await mkdtemp(resolve(tmpdir(), "paperclip-runner-real-resume-")); + const bundle = createRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodexAppServer, + codexArgs: [], + stateDirectory, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, + prpIdentity: { + runnerInstanceId: "runner-real-server", + environmentLeaseId: "lease-real-server", + runId, + normalizedSessionId: "session-real-server", + turnId: "turn-real-server", + itemId: "item-real-server", + }, + controlPlaneRegistration: (prp) => registerRunnerPrpAuthority({ companyId, runId, authority: prp }), + }); + const observedResults: unknown[] = []; + bundle.transport.setServerRequestHandler(async (request) => { + const params = request.params as Record; + const result = await authority.execute({ + tool: String(params.tool), + callId: String(params.callId), + arguments: params.arguments, + }); + observedResults.push(result); + return { + success: true, + contentItems: [{ type: "inputText", text: JSON.stringify({ ok: true, result }) }], + }; + }); + + try { + await bundle.transport.request("initialize", {}); + await bundle.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: await authority.definitions(), + }); + await bundle.transport.request("turn/start", { + input: [{ type: "text", text: "Read your assigned task context." }], + }); + for await (const notification of bundle.transport.notifications()) { + if (notification.method === "turn/completed") break; + } + expect(observedResults).toHaveLength(1); + expect(observedResults[0]).toMatchObject({ + activeTask: { id: issueId, identifier: "RRS-1", title: "Read me through the real control plane" }, + actor: { id: agentId }, + run: { id: runId }, + }); + expect(bundle.evidence().diagnostics).toContain("runnerd authenticated to the durable PRP control plane"); + + await bundle.transport.close(); + await db.insert(heartbeatRuns).values({ + id: resumedRunId, + companyId, + agentId, + status: "running", + runtimeMode: "native", + nativeIssueId: issueId, + invocationSource: "assignment", + triggerDetail: "system", + contextSnapshot: { issueId }, + }); + await db.update(issues).set({ executionRunId: resumedRunId }).where(eq(issues.id, issueId)); + const resumedAuthority = new PaperclipRunnerToolAuthority(db, { + companyId, + agentId, + issueId, + runId: resumedRunId, + }); + const restored = createRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodexAppServer, + codexArgs: [], + stateDirectory, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, + resumeDynamicTools: await resumedAuthority.definitions(), + prpIdentity: { + runnerInstanceId: "runner-real-server", + environmentLeaseId: "lease-real-server", + runId: resumedRunId, + normalizedSessionId: "session-real-server", + turnId: "turn-real-server-resumed", + itemId: "item-real-server-resumed", + }, + controlPlaneRegistration: (prp) => registerRunnerPrpAuthority({ + companyId, + runId: resumedRunId, + authority: prp, + }), + }); + restored.transport.setServerRequestHandler(async (request) => { + const params = request.params as Record; + const result = await resumedAuthority.execute({ + tool: String(params.tool), + callId: String(params.callId), + arguments: params.arguments, + }); + observedResults.push(result); + return { + success: true, + contentItems: [{ type: "inputText", text: JSON.stringify({ ok: true, result }) }], + }; + }); + try { + await restored.transport.request("thread/read", {}); + await restored.transport.request("turn/start", { + input: [{ type: "text", text: "Read the same task in a resumed process." }], + }); + for await (const notification of restored.transport.notifications()) { + if (notification.method === "turn/completed") break; + } + expect(observedResults).toHaveLength(2); + expect(observedResults[1]).toMatchObject({ + activeTask: { id: issueId, identifier: "RRS-1" }, + run: { id: resumedRunId }, + }); + expect(restored.evidence().diagnostics).toContain( + "runnerd restored its durable PRP session and provider thread", + ); + } finally { + await restored.transport.close(); + } + } finally { + await bundle.transport.close(); + await rm(stateDirectory, { recursive: true, force: true }); + server.closeAllConnections(); + server.close(); + } + }, 30_000); +}); diff --git a/server/src/services/native-runtime/paperclip-runner-tool-authority.test.ts b/server/src/services/native-runtime/paperclip-runner-tool-authority.test.ts new file mode 100644 index 0000000000..459db62012 --- /dev/null +++ b/server/src/services/native-runtime/paperclip-runner-tool-authority.test.ts @@ -0,0 +1,606 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { eq } from "drizzle-orm"; +import { activityLog, agents, approvals, companies, createDb, documents, heartbeatRuns, issueApprovals, issueComments, issueThreadInteractions, issues } from "@paperclipai/db"; +import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; +import { documentService } from "../documents.js"; +import { issueService } from "../issues.js"; +import { PaperclipRunnerToolAuthority } from "./paperclip-runner-tool-authority.js"; + +describe("PaperclipRunnerToolAuthority", () => { + let temporary: Awaited> | null = null; + let db: ReturnType; + const companyId = "00000000-0000-4000-8000-000000000101"; + const agentId = "00000000-0000-4000-8000-000000000102"; + const issueId = "00000000-0000-4000-8000-000000000103"; + const runId = "00000000-0000-4000-8000-000000000104"; + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("paperclip-runner-tools-"); + db = createDb(temporary.connectionString); + await db.insert(companies).values({ + id: companyId, + name: "Runner tools", + issuePrefix: "RNT", + issueCounter: 1, + }); + await db.insert(agents).values({ + id: agentId, + companyId, + name: "Runner agent", + adapterType: "paperclip_runner", + adapterConfig: { provider: "codex", apiKey: "must-not-leak" }, + runtimeConfig: { token: "must-not-leak" }, + status: "active", + }); + await db.insert(issues).values({ + id: issueId, + companyId, + issueNumber: 1, + identifier: "RNT-1", + title: "Exercise real runner tools", + status: "in_progress", + workMode: "standard", + assigneeAgentId: agentId, + }); + await db.insert(heartbeatRuns).values({ + id: runId, + companyId, + agentId, + status: "running", + runtimeMode: "native", + nativeIssueId: issueId, + invocationSource: "assignment", + triggerDetail: "system", + contextSnapshot: { issueId }, + }); + await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId)); + }); + + afterAll(async () => { + await temporary?.cleanup(); + }); + + it("advertises only real bindings and reads the bound task", async () => { + const authority = new PaperclipRunnerToolAuthority(db, { companyId, agentId, issueId, runId }); + expect(authority.definitions()).toHaveLength(16); + expect(authority.definitions().map((tool) => tool.name)).toEqual(expect.arrayContaining([ + "get_task_context", "get_task_history", "search_tasks", "report_progress", + "request_human_input", + "create_task", "set_dependencies", + "list_documents", "read_document", "list_document_revisions", "write_document", + "list_agents", "get_agent", "list_approvals", "get_approval", "get_approval_context", + ])); + const context = await authority.execute({ tool: "get_task_context", callId: "context", arguments: {} }); + expect(context).toMatchObject({ + activeTask: { id: issueId, identifier: "RNT-1" }, + actor: { id: agentId }, + }); + expect(JSON.stringify(context)).not.toContain("must-not-leak"); + await expect(authority.execute({ tool: "finish_task", callId: "hidden", arguments: {} })) + .rejects.toThrow("paperclip_runner_tool_not_advertised"); + }); + + it("advertises structured human input in ask mode", () => { + const authority = new PaperclipRunnerToolAuthority(db, { + companyId, + agentId, + issueId, + runId, + workMode: "ask", + }); + expect(authority.definitions().map((tool) => tool.name)).toContain("request_human_input"); + expect(authority.definitions().map((tool) => tool.name)).not.toContain("create_task"); + expect(authority.definitions().map((tool) => tool.name)).not.toContain("set_dependencies"); + }); + + it("does not project a foreign-company task through approval context", async () => { + const foreignCompanyId = "00000000-0000-4000-8000-000000000211"; + const foreignIssueId = "00000000-0000-4000-8000-000000000212"; + const approvalId = "00000000-0000-4000-8000-000000000213"; + await db.insert(companies).values({ + id: foreignCompanyId, + name: "Foreign approval company", + issuePrefix: "FAC", + issueCounter: 1, + }); + await db.insert(issues).values({ + id: foreignIssueId, + companyId: foreignCompanyId, + issueNumber: 1, + identifier: "FAC-1", + title: "Must not cross the approval boundary", + status: "todo", + }); + await db.insert(approvals).values({ + id: approvalId, + companyId, + type: "runner_review", + status: "pending", + payload: {}, + }); + // The schema deliberately stores companyId independently on the link. A + // corrupt or historical cross-tenant link must still fail closed at read. + await db.insert(issueApprovals).values({ + companyId, + approvalId, + issueId: foreignIssueId, + linkedByAgentId: agentId, + }); + + const authority = new PaperclipRunnerToolAuthority(db, { + companyId, + agentId, + issueId, + runId, + }); + await expect(authority.execute({ + tool: "get_approval_context", + callId: "foreign-approval-context", + arguments: { approvalId }, + })).resolves.toMatchObject({ approval: { id: approvalId }, tasks: [] }); + }); + + it("does not advertise delegation tools during pre-acceptance planning", () => { + const authority = new PaperclipRunnerToolAuthority(db, { + companyId, + agentId, + issueId, + runId, + workMode: "planning", + }); + expect(authority.definitions().map((tool) => tool.name)).not.toContain("create_task"); + expect(authority.definitions().map((tool) => tool.name)).not.toContain("set_dependencies"); + }); + + it("writes progress through the real issue service and replays idempotently", async () => { + const authority = new PaperclipRunnerToolAuthority(db, { companyId, agentId, issueId, runId }); + const call = { + tool: "report_progress", + callId: "progress", + arguments: { body: "Runner progress", idempotencyKey: "progress-1" }, + }; + const first = await authority.execute(call); + const replay = await authority.execute({ ...call, callId: "progress-replay" }); + expect(replay).toEqual(first); + expect(await db.select().from(issueComments).where(eq(issueComments.issueId, issueId))) + .toHaveLength(1); + const progressActivity = await db.select().from(activityLog).where(eq(activityLog.entityId, issueId)); + expect(progressActivity).toHaveLength(1); + expect(progressActivity[0]).toMatchObject({ + action: "issue.comment_added", + actorType: "agent", + actorId: agentId, + agentId, + runId, + entityType: "issue", + entityId: issueId, + details: expect.objectContaining({ + bodySnippet: "Runner progress", + identifier: "RNT-1", + issueTitle: "Exercise real runner tools", + source: "paperclip_runner_protocol", + }), + }); + await expect(authority.execute({ + ...call, + arguments: { body: "Changed", idempotencyKey: "progress-1" }, + })).rejects.toThrow("paperclip_runner_tool_idempotency_conflict"); + }); + + it("creates checkbox interactions through the real interaction service", async () => { + const authority = new PaperclipRunnerToolAuthority(db, { companyId, agentId, issueId, runId }); + const call = { + tool: "request_human_input", + callId: "ask-checkbox", + arguments: { + idempotencyKey: "favorite-animals", + interactionKind: "checkbox", + title: "Favorite zoo animals", + prompt: "Which zoo animals are your favorites?", + continuationPolicy: "wake_assignee", + payload: { + options: [ + { id: "giraffes", label: "Giraffes" }, + { id: "lions", label: "Lions" }, + ], + }, + }, + }; + const first = await authority.execute(call); + await expect(authority.execute({ ...call, callId: "ask-checkbox-replay" })).resolves.toEqual(first); + expect(first).toMatchObject({ + interaction: { kind: "request_checkbox_confirmation", status: "pending" }, + }); + expect(await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.issueId, issueId))) + .toHaveLength(1); + expect((await db.select().from(activityLog).where(eq(activityLog.entityId, issueId))) + .filter((entry) => entry.action === "issue.thread_interaction_created")) + .toHaveLength(1); + await expect(authority.execute({ + ...call, + callId: "ask-checkbox-conflict", + arguments: { + ...call.arguments, + prompt: "Use the same key for a different prompt.", + }, + })).rejects.toThrow("paperclip_runner_tool_idempotency_conflict"); + }); + + it("writes a real revisioned document and replays the mutation receipt", async () => { + const authority = new PaperclipRunnerToolAuthority(db, { companyId, agentId, issueId, runId }); + const call = { + tool: "write_document", + callId: "write-plan", + arguments: { + idempotencyKey: "write-plan-1", + key: "plan", + title: "Execution plan", + body: "Use the real document service.", + // Provider bridges may serialize nullable string inputs as the literal + // "null". The protocol boundary treats that as document creation. + baseRevisionId: "null", + changeSummary: "Initial plan", + }, + }; + const first = await authority.execute(call); + const replay = await authority.execute({ ...call, callId: "write-plan-replay" }); + expect(replay).toEqual(first); + expect(first).toMatchObject({ + disposition: "applied", + created: true, + document: { key: "plan", body: "Use the real document service." }, + }); + expect(await db.select().from(documents).where(eq(documents.companyId, companyId))).toHaveLength(1); + const documentActivity = await db.select().from(activityLog).where(eq(activityLog.entityId, issueId)); + expect(documentActivity.filter((entry) => entry.action === "issue.document_created")).toEqual([ + expect.objectContaining({ + actorType: "agent", + actorId: agentId, + agentId, + runId, + entityType: "issue", + details: expect.objectContaining({ + key: "plan", + source: "paperclip_runner_protocol", + }), + }), + ]); + await expect(authority.execute({ + ...call, + arguments: { ...call.arguments, body: "Conflicting retry." }, + })).rejects.toThrow("paperclip_runner_tool_idempotency_conflict"); + }); + + it("returns the exact accepted plan revision in task context", async () => { + const plan = await documentService(db).getIssueDocumentByKey(issueId, "plan"); + expect(plan).not.toBeNull(); + const authority = new PaperclipRunnerToolAuthority(db, { companyId, agentId, issueId, runId }); + const requested = await authority.execute({ + tool: "request_human_input", + callId: "approve-plan", + arguments: { + idempotencyKey: `confirmation:${issueId}:plan:${plan!.latestRevisionId}`, + interactionKind: "confirmation", + title: "Approve the plan", + prompt: "Approve this exact plan revision?", + payload: { + target: { + type: "issue_document", + issueId, + documentId: plan!.id, + key: "plan", + revisionId: plan!.latestRevisionId, + revisionNumber: plan!.latestRevisionNumber, + }, + }, + targetRevisionId: plan!.latestRevisionId, + continuationPolicy: "wake_assignee_on_accept", + }, + }); + expect(requested).toMatchObject({ + interaction: { + kind: "request_confirmation", + status: "pending", + payload: { + target: { + type: "issue_document", + issueId, + key: "plan", + revisionId: plan!.latestRevisionId, + }, + }, + }, + }); + await db.update(issueThreadInteractions).set({ + status: "accepted", + resolvedByUserId: "test-user", + resolvedAt: new Date(), + result: { outcome: "accepted" } as never, + }).where(eq(issueThreadInteractions.id, (requested as { interaction: { id: string } }).interaction.id)); + await db.update(heartbeatRuns).set({ + contextSnapshot: { + issueId, + workspaceRefreshReason: "accepted_plan_confirmation", + planReviewInteraction: { + acceptedTargetRevision: { + issueId, + documentId: plan!.id, + key: "plan", + revisionId: plan!.latestRevisionId, + revisionNumber: plan!.latestRevisionNumber, + }, + }, + }, + }).where(eq(heartbeatRuns.id, runId)); + + await expect(authority.execute({ tool: "get_task_context", callId: "accepted-context", arguments: {} })) + .resolves.toMatchObject({ + acceptedPlan: { + documentId: plan!.id, + revisionId: plan!.latestRevisionId, + revisionNumber: plan!.latestRevisionNumber, + markdown: "Use the real document service.", + }, + }); + }); + + it("creates ordinary children, preserves blockers, and deduplicates across runs", async () => { + const wakes: Array<{ agentId: string; options: Record }> = []; + const authority = new PaperclipRunnerToolAuthority(db, { + companyId, + agentId, + issueId, + runId, + workMode: "standard", + enqueueWakeup: async (wakeAgentId, options) => { + wakes.push({ agentId: wakeAgentId, options }); + return null; + }, + }); + expect(authority.definitions().map((tool) => tool.name)).toContain("create_task"); + + const prerequisite = await authority.execute({ + tool: "create_task", + callId: "create-prerequisite", + arguments: { + idempotencyKey: "ordinary-prerequisite", + title: "Prepare delegated input", + description: "A self-contained prerequisite delegated from the active task.", + }, + }); + + expect(prerequisite).toMatchObject({ + disposition: "applied", + task: { + parentId: issueId, + status: "todo", + assigneeActorId: agentId, + }, + }); + expect(wakes).toHaveLength(1); + expect(wakes[0]).toMatchObject({ + agentId, + options: { + reason: "issue_assigned", + payload: { parentIssueId: issueId }, + }, + }); + const prerequisiteId = (prerequisite as { task: { id: string } }).task.id; + const dependent = await authority.execute({ + tool: "create_task", + callId: "create-dependent", + arguments: { + idempotencyKey: "ordinary-dependent", + title: "Use delegated input", + blockedByTaskIds: [prerequisiteId], + }, + }); + expect(dependent).toMatchObject({ + disposition: "applied", + scheduledWakeIds: [], + task: { parentId: issueId, status: "blocked", assigneeActorId: agentId }, + }); + expect(wakes).toHaveLength(1); + await expect(issueService(db).getRelationSummaries(issueId)).resolves.toMatchObject({ + blockedBy: [], + }); + + await authority.execute({ + tool: "set_dependencies", + callId: "wait-for-prerequisite", + arguments: { + idempotencyKey: "source-waits-for-prerequisite", + blockedByTaskIds: [prerequisiteId], + }, + }); + await expect(issueService(db).getRelationSummaries(issueId)).resolves.toMatchObject({ + blockedBy: [expect.objectContaining({ id: prerequisiteId })], + }); + + await issueService(db).update(prerequisiteId, { + status: "done", + actorAgentId: agentId, + }); + await expect(authority.execute({ + tool: "create_task", + callId: "create-dependency-ready-child", + arguments: { + idempotencyKey: "ordinary-ready-dependent", + title: "Start after completed delegated input", + blockedByTaskIds: [prerequisiteId], + }, + })).resolves.toMatchObject({ + disposition: "applied", + task: { parentId: issueId, status: "todo", assigneeActorId: agentId }, + scheduledWakeIds: [expect.any(String)], + }); + expect(wakes).toHaveLength(2); + + const nextRunId = "00000000-0000-4000-8000-000000000106"; + await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, runId)); + await db.insert(heartbeatRuns).values({ + id: nextRunId, + companyId, + agentId, + status: "running", + runtimeMode: "native", + nativeIssueId: issueId, + invocationSource: "automation", + triggerDetail: "system", + contextSnapshot: { issueId }, + }); + await db.update(issues).set({ executionRunId: nextRunId }).where(eq(issues.id, issueId)); + const retryWakes: Array = []; + const retryAuthority = new PaperclipRunnerToolAuthority(db, { + companyId, + agentId, + issueId, + runId: nextRunId, + workMode: "standard", + enqueueWakeup: async (_wakeAgentId, options) => { + retryWakes.push(options); + return null; + }, + }); + await expect(retryAuthority.execute({ + tool: "create_task", + callId: "cross-run-retry", + arguments: { + idempotencyKey: "ordinary-prerequisite", + title: "Prepare delegated input", + description: "A self-contained prerequisite delegated from the active task.", + }, + })).resolves.toMatchObject({ disposition: "duplicate", task: { id: prerequisiteId } }); + await expect(retryAuthority.execute({ + tool: "create_task", + callId: "cross-run-conflicting-retry", + arguments: { + idempotencyKey: "ordinary-prerequisite", + title: "Conflicting title for the same caller key", + }, + })).rejects.toThrow("paperclip_runner_tool_idempotency_conflict"); + + const foreignCompanyId = "00000000-0000-4000-8000-000000000201"; + const foreignAgentId = "00000000-0000-4000-8000-000000000202"; + const foreignIssueId = "00000000-0000-4000-8000-000000000203"; + await db.insert(companies).values({ + id: foreignCompanyId, + name: "Foreign company", + issuePrefix: "FGN", + issueCounter: 1, + }); + await db.insert(agents).values({ + id: foreignAgentId, + companyId: foreignCompanyId, + name: "Foreign agent", + adapterType: "paperclip_runner", + adapterConfig: { provider: "codex" }, + runtimeConfig: {}, + status: "active", + }); + await db.insert(issues).values({ + id: foreignIssueId, + companyId: foreignCompanyId, + issueNumber: 1, + identifier: "FGN-1", + title: "Foreign blocker", + status: "todo", + }); + await expect(retryAuthority.execute({ + tool: "create_task", + callId: "foreign-assignee", + arguments: { + idempotencyKey: "foreign-assignee", + title: "Invalid foreign assignment", + assigneeActorId: foreignAgentId, + }, + })).rejects.toThrow("paperclip_runner_agent_not_found"); + await expect(retryAuthority.execute({ + tool: "create_task", + callId: "foreign-blocker", + arguments: { + idempotencyKey: "foreign-blocker", + title: "Invalid foreign blocker", + blockedByTaskIds: [foreignIssueId], + }, + })).rejects.toThrow(); + expect(retryWakes).toHaveLength(0); + expect(await db.select().from(issues).where(eq(issues.parentId, issueId))).toHaveLength(3); + }); + + it("rejects mutations after reassignment, run replacement, or terminalization", async () => { + const guardedIssueId = "00000000-0000-4000-8000-000000000107"; + const guardedRunId = "00000000-0000-4000-8000-000000000108"; + const guardedReplacementRunId = "00000000-0000-4000-8000-000000000109"; + await db.insert(issues).values({ + id: guardedIssueId, + companyId, + issueNumber: 999, + identifier: "RNT-999", + title: "Guard mutation authorization", + status: "in_progress", + workMode: "standard", + assigneeAgentId: agentId, + }); + await db.insert(heartbeatRuns).values({ + id: guardedRunId, + companyId, + agentId, + status: "running", + runtimeMode: "native", + nativeIssueId: guardedIssueId, + invocationSource: "assignment", + triggerDetail: "system", + contextSnapshot: { issueId: guardedIssueId }, + }); + await db.insert(heartbeatRuns).values({ + id: guardedReplacementRunId, + companyId, + agentId, + status: "running", + runtimeMode: "native", + nativeIssueId: guardedIssueId, + invocationSource: "assignment", + triggerDetail: "system", + contextSnapshot: { issueId: guardedIssueId }, + }); + await db.update(issues).set({ executionRunId: guardedRunId }).where(eq(issues.id, guardedIssueId)); + const authority = new PaperclipRunnerToolAuthority(db, { + companyId, + agentId, + issueId: guardedIssueId, + runId: guardedRunId, + }); + const mutation = { + tool: "report_progress", + callId: "guarded-progress", + arguments: { body: "Must remain authorized", idempotencyKey: "guarded-progress" }, + }; + + await db.update(issues).set({ assigneeAgentId: null }).where(eq(issues.id, guardedIssueId)); + await expect(authority.execute(mutation)) + .rejects.toThrow("paperclip_runner_tool_binding_not_authorized"); + + await db.update(issues).set({ + assigneeAgentId: agentId, + executionRunId: guardedReplacementRunId, + }).where(eq(issues.id, guardedIssueId)); + await expect(authority.execute({ ...mutation, callId: "replaced-run" })) + .rejects.toThrow("paperclip_runner_tool_binding_not_authorized"); + + await db.update(issues).set({ executionRunId: guardedRunId }).where(eq(issues.id, guardedIssueId)); + await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, guardedRunId)); + await expect(authority.execute({ ...mutation, callId: "terminal-run" })) + .rejects.toThrow("paperclip_runner_tool_binding_not_authorized"); + + expect(await db.select().from(issueComments).where(eq(issueComments.issueId, guardedIssueId))) + .toHaveLength(0); + }); + + it("fails closed once the run is no longer active", async () => { + await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, runId)); + const authority = new PaperclipRunnerToolAuthority(db, { companyId, agentId, issueId, runId }); + await expect(authority.execute({ tool: "get_task_context", callId: "late", arguments: {} })) + .rejects.toThrow("paperclip_runner_tool_binding_not_authorized"); + }); +}); diff --git a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts new file mode 100644 index 0000000000..1f43499bba --- /dev/null +++ b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts @@ -0,0 +1,757 @@ +import { createHash } from "node:crypto"; +import { and, desc, eq, isNull } from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { + agents, + documentRevisions, + heartbeatRuns, + issueApprovals, + issueComments, + issueDocuments, + issues, + issueThreadInteractions, +} from "@paperclipai/db"; +import { CAPABILITY_SEMANTIC_TOOL_CATALOG } from "../../vendor/paperclip-runner/index.js"; +import { agentService } from "../agents.js"; +import { approvalService } from "../approvals.js"; +import { documentService } from "../documents.js"; +import { issueService } from "../issues.js"; +import { issueThreadInteractionService } from "../issue-thread-interactions.js"; +import { persistActivity, publishActivity } from "../activity-log.js"; + +const IMPLEMENTED_OPERATIONS = new Set([ + "get_task_context", "get_task_history", "search_tasks", "report_progress", + "request_human_input", + "create_task", "set_dependencies", + "list_documents", "read_document", "list_document_revisions", "write_document", + "list_agents", "get_agent", "list_approvals", "get_approval", "get_approval_context", +]); + +type Binding = { + companyId: string; + issueId: string; + runId: string; + agentId: string; + normalizedSessionId?: string; + workMode?: "standard" | "planning" | "ask"; + enqueueWakeup?: (agentId: string, options: { + source: "assignment"; + triggerDetail: "system"; + reason: "issue_assigned"; + payload: Record; + idempotencyKey: string; + requestedByActorType: "agent"; + requestedByActorId: string; + contextSnapshot: Record; + }) => Promise; +}; + +type ToolReceipt = { + operationId: string; + input: unknown; + result: unknown; +}; + +function record(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? value as Record + : {}; +} + +function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (typeof value === "object" && value !== null) { + const object = value as Record; + return `{${Object.keys(object).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(object[key])}`).join(",")}}`; + } + return JSON.stringify(value) ?? "null"; +} + +export class PaperclipRunnerToolAuthority { + constructor(readonly db: Db, readonly binding: Binding) {} + + definitions(): Array> { + const workMode = this.binding.workMode ?? "standard"; + return CAPABILITY_SEMANTIC_TOOL_CATALOG + .filter((descriptor) => + IMPLEMENTED_OPERATIONS.has(descriptor.operationId) + && descriptor.allowedModes.includes(workMode) + ) + .map((descriptor) => ({ + name: descriptor.operationId, + description: descriptor.description, + inputSchema: descriptor.inputSchema, + })); + } + + async execute(call: { tool: string; callId: string; arguments: unknown }): Promise { + if (!IMPLEMENTED_OPERATIONS.has(call.tool)) throw new Error("paperclip_runner_tool_not_advertised"); + const context = await this.#boundContext(); + const descriptor = CAPABILITY_SEMANTIC_TOOL_CATALOG.find((candidate) => candidate.operationId === call.tool); + if (!descriptor || !descriptor.allowedModes.includes( + context.issue.workMode as "standard" | "planning" | "ask", + )) { + throw new Error("paperclip_runner_tool_mode_denied"); + } + const input = record(call.arguments); + switch (call.tool) { + case "get_task_context": return { + company: { id: this.binding.companyId }, + actor: redactedActor(context.actor), + activeTask: redactedTask(context.issue), + run: { + id: this.binding.runId, + status: context.run.status, + invocationSource: context.run.invocationSource, + }, + acceptedPlan: await this.#acceptedPlan(context.run.contextSnapshot), + }; + case "get_task_history": { + const limit = boundedLimit(input.limit); + const comments = await this.db.select({ + id: issueComments.id, + body: issueComments.body, + authorAgentId: issueComments.authorAgentId, + authorUserId: issueComments.authorUserId, + createdAt: issueComments.createdAt, + }).from(issueComments) + .where(and( + eq(issueComments.companyId, this.binding.companyId), + eq(issueComments.issueId, this.binding.issueId), + isNull(issueComments.deletedAt), + )) + .orderBy(desc(issueComments.createdAt)) + .limit(limit); + return { comments: comments.reverse() }; + } + case "search_tasks": { + const tasks = await issueService(this.db).list(this.binding.companyId); + const query = typeof input.query === "string" ? input.query.toLowerCase() : ""; + const statuses = Array.isArray(input.statuses) ? new Set(input.statuses.filter((value): value is string => typeof value === "string")) : null; + return { tasks: tasks.filter((task) => + (!query || `${task.identifier} ${task.title} ${task.description ?? ""}`.toLowerCase().includes(query)) + && (!statuses || statuses.size === 0 || statuses.has(task.status)) + ).slice(0, boundedLimit(input.limit)).map(redactedTask) }; + } + case "list_documents": + return { documents: await documentService(this.db).listIssueDocuments(this.binding.issueId) }; + case "read_document": { + const document = await documentService(this.db).getIssueDocumentByKey(this.binding.issueId, requiredString(input.key)); + if (!document) throw new Error("paperclip_runner_document_not_found"); + return { document }; + } + case "list_document_revisions": + return { revisions: await documentService(this.db).listIssueDocumentRevisions(this.binding.issueId, requiredString(input.key)) }; + case "write_document": return this.#writeDocument(input); + case "list_agents": + return { actors: (await agentService(this.db).list(this.binding.companyId)).map(redactedActor) }; + case "get_agent": { + const actor = await agentService(this.db).getById(requiredString(input.actorId)); + if (!actor || actor.companyId !== this.binding.companyId) throw new Error("paperclip_runner_agent_not_found"); + return { actor: redactedActor(actor) }; + } + case "list_approvals": + return { approvals: await approvalService(this.db).list(this.binding.companyId) }; + case "get_approval": { + const approval = await this.#approval(requiredString(input.approvalId)); + return { approval }; + } + case "get_approval_context": { + const approval = await this.#approval(requiredString(input.approvalId)); + const tasks = await this.db.select({ issue: issues }).from(issueApprovals) + .innerJoin(issues, eq(issues.id, issueApprovals.issueId)) + .where(and( + eq(issueApprovals.approvalId, approval.id), + eq(issueApprovals.companyId, this.binding.companyId), + eq(issues.companyId, this.binding.companyId), + )); + return { approval, tasks: tasks.map((row) => row.issue) }; + } + case "report_progress": return this.#reportProgress(input); + case "request_human_input": return this.#requestHumanInput(input); + case "create_task": return this.#createTask(input); + case "set_dependencies": return this.#setDependencies(input); + default: throw new Error("paperclip_runner_tool_not_bound"); + } + } + + async #approval(id: string) { + const approval = await approvalService(this.db).getById(id); + if (!approval || approval.companyId !== this.binding.companyId) throw new Error("paperclip_runner_approval_not_found"); + return approval; + } + + async #boundContext() { + const [row] = await this.db.select({ issue: issues, actor: agents, run: heartbeatRuns }) + .from(heartbeatRuns) + .innerJoin(issues, eq(issues.id, this.binding.issueId)) + .innerJoin(agents, eq(agents.id, this.binding.agentId)) + .where(and( + eq(heartbeatRuns.id, this.binding.runId), + eq(heartbeatRuns.companyId, this.binding.companyId), + eq(heartbeatRuns.agentId, this.binding.agentId), + eq(heartbeatRuns.nativeIssueId, this.binding.issueId), + eq(issues.companyId, this.binding.companyId), + eq(issues.assigneeAgentId, this.binding.agentId), + eq(issues.executionRunId, this.binding.runId), + eq(agents.companyId, this.binding.companyId), + )) + .limit(1); + if ( + !row + || row.run.runtimeMode !== "native" + || row.run.status !== "running" + || ["paused", "terminated", "pending_approval", "error"].includes(row.actor.status) + ) { + throw new Error("paperclip_runner_tool_binding_not_authorized"); + } + return row; + } + + async #reportProgress(input: Record): Promise { + const body = typeof input.body === "string" ? input.body.trim() : ""; + const idempotencyKey = typeof input.idempotencyKey === "string" ? input.idempotencyKey.trim() : ""; + if (!body || !idempotencyKey) throw new Error("paperclip_runner_tool_input_invalid"); + let publication: Awaited>["publication"] | null = null; + const result = await this.#withMutationReceipt( + "report_progress", + idempotencyKey, + input, + async (tx, context) => { + const comment = await issueService(tx).addComment( + this.binding.issueId, + body, + { agentId: this.binding.agentId, runId: this.binding.runId }, + { authorizationReason: "paperclip_runner_protocol" }, + tx, + ); + const result = { commentId: comment.id, issueId: this.binding.issueId, disposition: "applied" }; + const activity = await persistActivity(tx, { + companyId: this.binding.companyId, + actorType: "agent", + actorId: this.binding.agentId, + agentId: this.binding.agentId, + runId: this.binding.runId, + issueId: this.binding.issueId, + action: "issue.comment_added", + entityType: "issue", + entityId: this.binding.issueId, + details: { + commentId: comment.id, + bodySnippet: comment.body.slice(0, 120), + identifier: context.issue.identifier, + issueTitle: context.issue.title, + authorizationReason: "paperclip_runner_protocol", + source: "paperclip_runner_protocol", + }, + }); + publication = activity.publication; + return result; + }, + ); + if (publication) publishActivity(publication); + return result; + } + + async #writeDocument(input: Record): Promise { + const idempotencyKey = requiredString(input.idempotencyKey); + let publication: Awaited>["publication"] | null = null; + const result = await this.#withMutationReceipt("write_document", idempotencyKey, input, async (tx) => { + const write = await documentService(tx).upsertIssueDocument({ + issueId: this.binding.issueId, + key: requiredString(input.key), + title: requiredString(input.title), + format: "markdown", + body: requiredString(input.body), + baseRevisionId: nullableProviderId(input.baseRevisionId), + changeSummary: input.changeSummary === null || input.changeSummary === undefined + ? null + : requiredString(input.changeSummary), + createdByAgentId: this.binding.agentId, + createdByRunId: this.binding.runId, + }); + const activity = await persistActivity(tx, { + companyId: this.binding.companyId, + actorType: "agent", + actorId: this.binding.agentId, + agentId: this.binding.agentId, + runId: this.binding.runId, + issueId: this.binding.issueId, + action: write.created ? "issue.document_created" : "issue.document_updated", + entityType: "issue", + entityId: this.binding.issueId, + details: { + key: write.document.key, + documentId: write.document.id, + title: write.document.title, + format: write.document.format, + revisionNumber: write.document.latestRevisionNumber, + source: "paperclip_runner_protocol", + }, + }); + publication = activity.publication; + return { + disposition: "applied", + created: write.created, + document: write.document, + }; + }); + if (publication) publishActivity(publication); + return result; + } + + async #createTask(input: Record): Promise { + const idempotencyKey = requiredString(input.idempotencyKey); + const assigneeAgentId = input.assigneeActorId === null || input.assigneeActorId === undefined + ? this.binding.agentId + : requiredString(input.assigneeActorId); + const assignee = await agentService(this.db).getById(assigneeAgentId); + if (!assignee || assignee.companyId !== this.binding.companyId) { + throw new Error("paperclip_runner_agent_not_found"); + } + const priority = input.priority === "critical" || input.priority === "high" + || input.priority === "medium" || input.priority === "low" + ? input.priority + : "medium"; + const blockedByIssueIds = Array.isArray(input.blockedByTaskIds) + ? input.blockedByTaskIds.map(requiredString) + : []; + const durableIdempotencyKey = + `paperclip-runner:create-task:${this.binding.issueId}:${idempotencyKey}`; + const inputFingerprint = createHash("sha256") + .update(canonicalJson(input)) + .digest("hex"); + const result = await this.#withMutationReceipt("create_task", idempotencyKey, input, async (tx) => { + const existingChild = await tx.select().from(issues).where(and( + eq(issues.companyId, this.binding.companyId), + eq(issues.parentId, this.binding.issueId), + eq(issues.originId, durableIdempotencyKey), + )).limit(1).then((rows) => rows[0] ?? null); + if (existingChild) { + if (existingChild.originFingerprint !== inputFingerprint) { + throw new Error("paperclip_runner_tool_idempotency_conflict"); + } + return { + commandId: `create-task:${existingChild.id}`, + disposition: "duplicate", + stateRevision: existingChild.statusVersion, + entityRefs: [existingChild.id], + scheduledWakeIds: [], + task: { + id: existingChild.id, + identifier: existingChild.identifier, + parentId: existingChild.parentId, + status: existingChild.status, + assigneeActorId: existingChild.assigneeAgentId, + }, + }; + } + let deduplicated = false; + const created = await issueService(tx).createChild(this.binding.issueId, { + title: requiredString(input.title), + description: input.description === null || input.description === undefined + ? null + : requiredString(input.description), + status: blockedByIssueIds.length > 0 ? "blocked" : "todo", + workMode: "standard", + priority, + assigneeAgentId, + blockedByIssueIds, + blockParentUntilDone: false, + createdByAgentId: this.binding.agentId, + originKind: "manual", + originId: durableIdempotencyKey, + originFingerprint: inputFingerprint, + actorAgentId: this.binding.agentId, + actorRunId: this.binding.runId, + idempotencyKey: durableIdempotencyKey, + onDeduplicated: () => { deduplicated = true; }, + }); + const child = created.issue; + if (deduplicated && child.originFingerprint !== inputFingerprint) { + throw new Error("paperclip_runner_tool_idempotency_conflict"); + } + let childStatus = child.status; + let childStatusVersion = child.statusVersion; + if (child.status === "blocked" && blockedByIssueIds.length > 0) { + const readiness = await issueService(tx).getDependencyReadiness(child.id, tx); + if (readiness.isDependencyReady) { + const readyChild = await issueService(tx).update(child.id, { + status: "todo", + actorAgentId: this.binding.agentId, + }, tx); + if (readyChild) { + childStatus = readyChild.status; + childStatusVersion = readyChild.statusVersion; + } + } + } + const wakeId = `created-child:${child.id}`; + const shouldWake = !deduplicated && childStatus === "todo" && Boolean(child.assigneeAgentId); + return { + commandId: `create-task:${child.id}`, + disposition: deduplicated ? "duplicate" : "applied", + stateRevision: childStatusVersion, + entityRefs: [child.id], + scheduledWakeIds: shouldWake ? [wakeId] : [], + task: { + id: child.id, + identifier: child.identifier, + parentId: child.parentId, + status: childStatus, + assigneeActorId: child.assigneeAgentId, + }, + }; + }) as Record; + + const task = record(result.task); + const childId = requiredString(task.id); + const scheduledWakeIds = Array.isArray(result.scheduledWakeIds) + ? result.scheduledWakeIds.filter((value): value is string => typeof value === "string") + : []; + const assignedAgentId = typeof task.assigneeActorId === "string" + ? task.assigneeActorId + : null; + if (this.binding.enqueueWakeup && assignedAgentId && scheduledWakeIds.length > 0) { + await this.binding.enqueueWakeup(assignedAgentId, { + source: "assignment", + triggerDetail: "system", + reason: "issue_assigned", + payload: { + issueId: childId, + mutation: "create_child", + parentIssueId: this.binding.issueId, + }, + idempotencyKey: scheduledWakeIds[0]!, + requestedByActorType: "agent", + requestedByActorId: this.binding.agentId, + contextSnapshot: { + issueId: childId, + source: "paperclip_runner.create_task", + parentIssueId: this.binding.issueId, + }, + }); + } + return result; + } + + async #setDependencies(input: Record): Promise { + const idempotencyKey = requiredString(input.idempotencyKey); + if (!Array.isArray(input.blockedByTaskIds)) { + throw new Error("paperclip_runner_tool_input_invalid"); + } + const blockedByIssueIds = input.blockedByTaskIds.map(requiredString); + return this.#withMutationReceipt("set_dependencies", idempotencyKey, input, async (tx) => { + const updated = await issueService(tx).update(this.binding.issueId, { + blockedByIssueIds, + actorAgentId: this.binding.agentId, + }, tx); + if (!updated) throw new Error("paperclip_runner_task_not_found"); + return { + commandId: `set-dependencies:${updated.id}:${updated.statusVersion}`, + disposition: "applied", + stateRevision: updated.statusVersion, + entityRefs: [updated.id, ...blockedByIssueIds], + scheduledWakeIds: [], + }; + }); + } + + async #acceptedPlan(contextSnapshot: unknown): Promise<{ + documentId: string; + revisionId: string; + revisionNumber: number; + markdown: string; + } | null> { + const acceptedTarget = record( + record(record(contextSnapshot).planReviewInteraction).acceptedTargetRevision, + ); + let revisionId = typeof acceptedTarget.revisionId === "string" + ? acceptedTarget.revisionId + : null; + if (!revisionId) revisionId = await this.#latestAcceptedPlanRevisionId(); + if (!revisionId) return null; + + const [revision] = await this.db.select({ + documentId: documentRevisions.documentId, + revisionId: documentRevisions.id, + revisionNumber: documentRevisions.revisionNumber, + markdown: documentRevisions.body, + }) + .from(documentRevisions) + .innerJoin(issueDocuments, and( + eq(issueDocuments.documentId, documentRevisions.documentId), + eq(issueDocuments.companyId, this.binding.companyId), + eq(issueDocuments.issueId, this.binding.issueId), + eq(issueDocuments.key, "plan"), + )) + .where(and( + eq(documentRevisions.id, revisionId), + eq(documentRevisions.companyId, this.binding.companyId), + )) + .limit(1); + return revision ?? null; + } + + async #latestAcceptedPlanRevisionId(): Promise { + const rows = await this.db.select({ payload: issueThreadInteractions.payload }) + .from(issueThreadInteractions) + .where(and( + eq(issueThreadInteractions.companyId, this.binding.companyId), + eq(issueThreadInteractions.issueId, this.binding.issueId), + eq(issueThreadInteractions.kind, "request_confirmation"), + eq(issueThreadInteractions.status, "accepted"), + )) + .orderBy(desc(issueThreadInteractions.resolvedAt), desc(issueThreadInteractions.createdAt)); + for (const row of rows) { + const target = record(record(row.payload).target); + if ( + target.type === "issue_document" + && (target.issueId === undefined || target.issueId === this.binding.issueId) + && target.key === "plan" + && typeof target.revisionId === "string" + && target.revisionId.length > 0 + ) return target.revisionId; + } + return null; + } + + async #withMutationReceipt( + operationId: string, + idempotencyKey: string, + input: Record, + effect: (tx: Db, context: { + run: typeof heartbeatRuns.$inferSelect; + issue: typeof issues.$inferSelect; + actor: typeof agents.$inferSelect; + }) => Promise, + ): Promise { + return this.db.transaction(async (tx) => { + const context = await this.#lockAuthorizedMutationContext(tx as unknown as Db); + const resultJson = record(context.run.resultJson); + const receipts = record(resultJson.semanticToolReceipts); + const prior = receipts[idempotencyKey] as ToolReceipt | undefined; + if (prior !== undefined) { + if (prior.operationId !== operationId || canonicalJson(prior.input) !== canonicalJson(input)) { + throw new Error("paperclip_runner_tool_idempotency_conflict"); + } + return prior.result; + } + const result = JSON.parse(JSON.stringify( + await effect(tx as unknown as Db, context), + )) as unknown; + receipts[idempotencyKey] = { operationId, input, result } satisfies ToolReceipt; + await tx.update(heartbeatRuns).set({ + resultJson: { ...resultJson, semanticToolReceipts: receipts }, + updatedAt: new Date(), + }).where(eq(heartbeatRuns.id, this.binding.runId)); + return result; + }); + } + + async #lockAuthorizedMutationContext(tx: Db): Promise<{ + run: typeof heartbeatRuns.$inferSelect; + issue: typeof issues.$inferSelect; + actor: typeof agents.$inferSelect; + }> { + // Authorization for writes is intentionally re-read only after the + // transaction starts. Locking the run and issue in the same statement + // closes the gap between the discovery-time check and the mutation: a + // reassignment, replacement run, or terminal transition must commit either + // before this check (and be rejected) or after this transaction completes. + const [context] = await tx + .select({ run: heartbeatRuns, issue: issues, actor: agents }) + .from(heartbeatRuns) + .innerJoin(issues, eq(issues.id, this.binding.issueId)) + .innerJoin(agents, eq(agents.id, this.binding.agentId)) + .where(and( + eq(heartbeatRuns.id, this.binding.runId), + eq(heartbeatRuns.companyId, this.binding.companyId), + eq(heartbeatRuns.agentId, this.binding.agentId), + eq(heartbeatRuns.nativeIssueId, this.binding.issueId), + eq(issues.companyId, this.binding.companyId), + eq(issues.assigneeAgentId, this.binding.agentId), + eq(issues.executionRunId, this.binding.runId), + eq(agents.companyId, this.binding.companyId), + )) + .for("update") + .limit(1); + if ( + !context + || context.run.runtimeMode !== "native" + || context.run.status !== "running" + || context.run.companyId !== this.binding.companyId + || context.run.agentId !== this.binding.agentId + || context.run.nativeIssueId !== this.binding.issueId + || context.issue.companyId !== this.binding.companyId + || context.issue.assigneeAgentId !== this.binding.agentId + || context.issue.executionRunId !== this.binding.runId + || context.actor.companyId !== this.binding.companyId + || ["paused", "terminated", "pending_approval", "error"].includes(context.actor.status) + ) { + throw new Error("paperclip_runner_tool_binding_not_authorized"); + } + return context; + } + + async #requestHumanInput(input: Record): Promise { + const interactionKind = requiredString(input.interactionKind); + const interactionKinds = { + confirmation: "request_confirmation", + checkbox: "request_checkbox_confirmation", + questions: "ask_user_questions", + suggest_tasks: "suggest_tasks", + item_verdicts: "request_item_verdicts", + } as const; + const kind = interactionKinds[ + interactionKind as keyof typeof interactionKinds + ]; + if (!kind) throw new Error("paperclip_runner_interaction_kind_invalid"); + const prompt = requiredString(input.prompt); + const idempotencyKey = requiredString(input.idempotencyKey); + let publication: Awaited>["publication"] | null = null; + const result = await this.#withMutationReceipt( + "request_human_input", + idempotencyKey, + input, + async (tx, context) => { + const suppliedPayload = record(input.payload); + const targetRevisionId = nullableProviderId(input.targetRevisionId); + const suppliedTarget = record(suppliedPayload.target); + const inferredPlanningTarget = targetRevisionId !== null + && suppliedPayload.target === undefined + && kind === "request_confirmation" + && context.issue.workMode === "planning" + ? { + type: "issue_document", + issueId: context.issue.id, + key: "plan", + revisionId: targetRevisionId, + } + : null; + if (targetRevisionId !== null && suppliedPayload.target === undefined && inferredPlanningTarget === null) { + throw new Error("paperclip_runner_interaction_target_incomplete"); + } + const normalizedPayload = inferredPlanningTarget !== null + ? { ...suppliedPayload, target: inferredPlanningTarget } + : suppliedTarget.type === "issue_document" + ? { + ...suppliedPayload, + target: { + ...suppliedTarget, + issueId: suppliedTarget.issueId ?? context.issue.id, + revisionId: suppliedTarget.revisionId ?? targetRevisionId, + }, + } + : suppliedPayload; + const interaction = await issueThreadInteractionService(tx).create(context.issue, { + kind, + idempotencyKey, + sourceRunId: this.binding.runId, + title: requiredString(input.title), + summary: prompt, + continuationPolicy: requiredString(input.continuationPolicy), + payload: { + ...normalizedPayload, + version: 1, + prompt, + ...(kind === "request_confirmation" ? { + detailsMarkdown: normalizedPayload.detailsMarkdown ?? "", + acceptLabel: normalizedPayload.acceptLabel ?? "Confirm", + rejectLabel: normalizedPayload.rejectLabel ?? "Request changes", + rejectRequiresReason: normalizedPayload.rejectRequiresReason ?? false, + supersedeOnUserComment: normalizedPayload.supersedeOnUserComment ?? true, + } : {}), + }, + } as never, { agentId: this.binding.agentId, userId: null }); + const activity = await persistActivity(tx, { + companyId: this.binding.companyId, + actorType: "agent", + actorId: this.binding.agentId, + agentId: this.binding.agentId, + runId: this.binding.runId, + issueId: this.binding.issueId, + action: "issue.thread_interaction_created", + entityType: "issue", + entityId: this.binding.issueId, + details: { + interactionId: interaction.id, + interactionKind: interaction.kind, + interactionStatus: interaction.status, + continuationPolicy: interaction.continuationPolicy, + source: "paperclip_runner_protocol", + }, + }); + publication = activity.publication; + return { interaction, disposition: "applied" }; + }, + ); + if (publication) publishActivity(publication); + return result; + } +} + +function requiredString(value: unknown): string { + if (typeof value !== "string" || value.trim() === "") throw new Error("paperclip_runner_tool_input_invalid"); + return value.trim(); +} + +/** + * Some native tool transports cannot faithfully express a nullable string in + * their provider-facing schema and send the JSON null sentinel as a string. + * Normalize only the well-known empty/null sentinels at the control-plane + * boundary; real revision ids remain untouched and optimistic concurrency is + * still enforced by the document service. + */ +function nullableProviderId(value: unknown): string | null { + if (value === null || value === undefined) return null; + const normalized = requiredString(value); + return normalized === "null" || normalized === "undefined" ? null : normalized; +} + +function boundedLimit(value: unknown): number { + return typeof value === "number" && Number.isInteger(value) + ? Math.max(1, Math.min(value, 100)) + : 50; +} + +function redactedActor(actor: { + id: string; + companyId: string; + name: string; + role: string; + title?: string | null; + status: string; + reportsTo?: string | null; + capabilities?: string | null; +}) { + return { + id: actor.id, + companyId: actor.companyId, + name: actor.name, + role: actor.role, + title: actor.title ?? null, + status: actor.status, + reportsTo: actor.reportsTo ?? null, + capabilities: actor.capabilities ?? null, + }; +} + +function redactedTask(task: typeof issues.$inferSelect) { + return { + id: task.id, + companyId: task.companyId, + identifier: task.identifier, + title: task.title, + description: task.description, + status: task.status, + statusVersion: task.statusVersion, + priority: task.priority, + workMode: task.workMode, + assigneeAgentId: task.assigneeAgentId, + executionRunId: task.executionRunId, + parentId: task.parentId, + projectId: task.projectId, + goalId: task.goalId, + }; +} diff --git a/server/src/services/native-runtime/runtime-mode.test.ts b/server/src/services/native-runtime/runtime-mode.test.ts index f954a05fd6..376aa6f4af 100644 --- a/server/src/services/native-runtime/runtime-mode.test.ts +++ b/server/src/services/native-runtime/runtime-mode.test.ts @@ -3,10 +3,236 @@ import { describe, expect, it } from "vitest"; import { BUILTIN_ADAPTER_TYPES } from "../../adapters/builtin-adapter-types.js"; import { NativeRunnerSelectionError, + NativeRuntimeEligibilityError, + resolveHeartbeatNativeRuntimeMode, resolveHeartbeatRuntimeMode, + resolveNativeRuntimeMode, } from "./runtime-mode.js"; -const base = { +const eligible = { + enabled: true, + runtimeConfig: {}, + adapterConfig: { provider: "codex" }, + agent: { status: "running", adapterType: "paperclip_runner" }, + issue: { id: "issue", workMode: "standard" }, + target: { kind: "local" }, + workspaceId: "workspace", +} as const; + +describe("resolveNativeRuntimeMode", () => { + it("keeps every direct built-in adapter outside native arbitration", () => { + for (const adapterType of BUILTIN_ADAPTER_TYPES) { + if (adapterType === "paperclip_runner") continue; + expect(resolveNativeRuntimeMode({ + ...eligible, + enabled: false, + runtimeConfig: { + nativeRunner: { + mode: "native", + backend: "codex_app_server", + protocolVersion: 1, + }, + }, + agent: { ...eligible.agent, adapterType }, + })).toEqual({ + kind: "legacy", + resolverVersion: "phase6-v1", + reason: "direct_adapter", + }); + } + }); + + it("rejects a fresh Paperclip Runner start while the rollout flag is disabled", () => { + expect(() => resolveNativeRuntimeMode({ + ...eligible, + enabled: false, + })).toThrow(expect.objectContaining({ + code: "paperclip_runner_rollout_disabled", + })); + }); + + it("rejects unknown Paperclip Runner providers", () => { + expect(() => resolveNativeRuntimeMode({ + ...eligible, + runtimeConfig: {}, + adapterConfig: { provider: "claude" }, + agent: { ...eligible.agent, adapterType: "paperclip_runner" }, + })).toThrow(expect.objectContaining({ + code: "paperclip_runner_provider_unsupported", + })); + }); + + it("rejects fresh OpenCode and ACPX starts until their app profiles are activated", () => { + expect(() => resolveNativeRuntimeMode({ + ...eligible, + adapterConfig: { provider: "opencode", model: "openrouter/deepseek/deepseek-v4-flash-0731" }, + })).toThrow(expect.objectContaining({ + code: "paperclip_runner_provider_unsupported", + })); + expect(() => resolveNativeRuntimeMode({ + ...eligible, + adapterConfig: { provider: "acpx", acpxAgent: "claude", model: "claude-sonnet-5" }, + })).toThrow(expect.objectContaining({ + code: "paperclip_runner_provider_unsupported", + })); + }); + + it("preserves legacy as the default and as the kill-switch behavior", () => { + const direct = { + ...eligible, + agent: { ...eligible.agent, adapterType: "codex_local" }, + runtimeConfig: { nativeRunner: { mode: "native", backend: "codex_app_server", protocolVersion: 1 } }, + }; + expect(resolveNativeRuntimeMode(direct)).toEqual(expect.objectContaining({ + kind: "legacy", + reason: "direct_adapter", + })); + expect(resolveNativeRuntimeMode({ ...direct, enabled: false })).toEqual(expect.objectContaining({ + kind: "legacy", + reason: "direct_adapter", + })); + }); + + it("selects native only for an eligible explicit profile", () => { + expect(resolveNativeRuntimeMode(eligible)).toEqual(expect.objectContaining({ + kind: "native", + reason: "eligible_opt_in", + })); + }); + + it("keeps a persisted active run native while the global flag rejects a fresh runner start", () => { + const disabled = { ...eligible, enabled: false }; + expect(resolveHeartbeatNativeRuntimeMode({ + ...disabled, + persisted: { + runtimeMode: "native", + runtimeModeReason: "eligible_opt_in", + runtimeModeResolvedAt: new Date(), + }, + })).toEqual(expect.objectContaining({ + kind: "native", + reason: "eligible_opt_in", + authorityDecision: expect.objectContaining({ reasonCode: "live_continuation_registered" }), + })); + expect(() => resolveHeartbeatNativeRuntimeMode({ + ...disabled, + persisted: { runtimeMode: null, runtimeModeReason: null, runtimeModeResolvedAt: null }, + })).toThrow(expect.objectContaining({ + code: "paperclip_runner_rollout_disabled", + })); + }); + + it.each(["paused", "terminated", "pending_approval"])( + "refuses persisted native recovery for a %s agent", + (status) => { + expect(() => resolveHeartbeatNativeRuntimeMode({ + ...eligible, + enabled: false, + agent: { ...eligible.agent, status }, + persisted: { + runtimeMode: "native", + runtimeModeReason: "eligible_opt_in", + runtimeModeResolvedAt: new Date(), + driverKind: "codex_app_server", + }, + })).toThrow(expect.objectContaining({ + code: "paperclip_runner_agent_ineligible", + })); + }, + ); + + it("fails closed for an unknown persisted driver", () => { + expect(() => resolveHeartbeatNativeRuntimeMode({ + ...eligible, + enabled: false, + persisted: { + runtimeMode: "native", + runtimeModeReason: "eligible_opt_in", + runtimeModeResolvedAt: new Date(), + driverKind: "unknown_driver", + }, + })).toThrow(expect.objectContaining({ + code: "paperclip_runner_driver_unsupported", + })); + }); + + it("does not recover a native run through a direct adapter", () => { + expect(() => resolveHeartbeatNativeRuntimeMode({ + ...eligible, + enabled: false, + agent: { ...eligible.agent, adapterType: "codex_local" }, + persisted: { + runtimeMode: "native", + runtimeModeReason: "eligible_opt_in", + runtimeModeResolvedAt: new Date(), + driverKind: "codex_app_server", + }, + })).toThrow(expect.objectContaining({ + code: "paperclip_runner_adapter_binding_mismatch", + })); + }); + + it("rejects an explicit native profile outside the approved boundary", () => { + expect(resolveNativeRuntimeMode({ ...eligible, agent: { ...eligible.agent, adapterType: "claude_local" } })) + .toEqual(expect.objectContaining({ kind: "legacy", reason: "direct_adapter" })); + expect(() => resolveNativeRuntimeMode({ ...eligible, issue: { id: "issue", workMode: "skill_test" } })) + .toThrow(NativeRuntimeEligibilityError); + }); + + it("rejects remote targets for fresh paperclip_runner starts", () => { + expect(() => resolveNativeRuntimeMode({ + ...eligible, + target: { kind: "remote" }, + runtimeConfig: {}, + adapterConfig: { provider: "codex" }, + agent: { ...eligible.agent, adapterType: "paperclip_runner" }, + })).toThrow(expect.objectContaining({ + code: "paperclip_runner_environment_unsupported", + })); + }); + + it("allows paperclip_runner to use a transient local workspace for projectless issues", () => { + expect(resolveNativeRuntimeMode({ + ...eligible, + workspaceId: null, + agent: { ...eligible.agent, adapterType: "paperclip_runner" }, + runtimeConfig: {}, + adapterConfig: { provider: "codex" }, + })).toEqual(expect.objectContaining({ kind: "native" })); + }); + + it("admits planning only through paperclip_runner", () => { + expect(resolveNativeRuntimeMode({ + ...eligible, + issue: { id: "plan-issue", workMode: "planning" }, + runtimeConfig: {}, + adapterConfig: { provider: "codex" }, + agent: { ...eligible.agent, adapterType: "paperclip_runner" }, + })).toMatchObject({ kind: "native", profile: { backend: "codex_app_server" } }); + expect(resolveNativeRuntimeMode({ + ...eligible, + issue: { id: "plan-issue", workMode: "planning" }, + agent: { ...eligible.agent, adapterType: "codex_local" }, + })).toEqual(expect.objectContaining({ kind: "legacy", reason: "direct_adapter" })); + }); + + it("admits ask mode through paperclip_runner while preserving the legacy native boundary", () => { + expect(resolveNativeRuntimeMode({ + ...eligible, + issue: { id: "ask-issue", workMode: "ask" }, + runtimeConfig: {}, + adapterConfig: { provider: "codex" }, + agent: { ...eligible.agent, adapterType: "paperclip_runner" }, + })).toMatchObject({ kind: "native", profile: { backend: "codex_app_server" } }); + expect(resolveNativeRuntimeMode({ + ...eligible, + issue: { id: "ask-issue", workMode: "ask" }, + agent: { ...eligible.agent, adapterType: "codex_local" }, + })).toEqual(expect.objectContaining({ kind: "legacy", reason: "direct_adapter" })); + }); +}); + +const compatibilityInput = { persisted: { runtimeMode: "legacy", runtimeModeResolvedAt: null }, enabled: true, adapterConfig: { provider: "codex" }, @@ -15,11 +241,14 @@ const base = { executionTarget: { kind: "local" }, } as const; -describe("resolveHeartbeatRuntimeMode", () => { +describe("resolveHeartbeatRuntimeMode compatibility", () => { it("keeps every direct built-in adapter on the legacy path", () => { for (const adapterType of BUILTIN_ADAPTER_TYPES) { if (adapterType === "paperclip_runner") continue; - expect(resolveHeartbeatRuntimeMode({ ...base, adapterType })).toEqual({ + expect(resolveHeartbeatRuntimeMode({ + ...compatibilityInput, + adapterType, + })).toEqual({ kind: "legacy", resolverVersion: "paperclip-runner-v1", reason: "direct_adapter", @@ -27,36 +256,18 @@ describe("resolveHeartbeatRuntimeMode", () => { } }); - it("fails closed for fresh runner starts while the flag is off", () => { - expect(() => resolveHeartbeatRuntimeMode({ - ...base, - enabled: false, - adapterType: "paperclip_runner", - })).toThrowError(expect.objectContaining({ - code: "paperclip_runner_rollout_disabled", - }) as NativeRunnerSelectionError); - }); - - it("selects only Codex on a local target", () => { + it("preserves the original public result and error contracts", () => { expect(resolveHeartbeatRuntimeMode({ - ...base, + ...compatibilityInput, adapterType: "paperclip_runner", - })).toMatchObject({ kind: "native", provider: "codex" }); - expect(() => resolveHeartbeatRuntimeMode({ - ...base, - adapterType: "paperclip_runner", - adapterConfig: { provider: "opencode" }, - })).toThrow(/only the Codex provider/); - expect(() => resolveHeartbeatRuntimeMode({ - ...base, - adapterType: "paperclip_runner", - executionTarget: { kind: "remote" }, - })).toThrow(/local execution environment/); - }); - - it("recovers a persisted native run after the flag changes", () => { + })).toEqual({ + kind: "native", + resolverVersion: "paperclip-runner-v1", + reason: "explicit_paperclip_runner", + provider: "codex", + }); expect(resolveHeartbeatRuntimeMode({ - ...base, + ...compatibilityInput, enabled: false, adapterType: "paperclip_runner", persisted: { runtimeMode: "native", runtimeModeResolvedAt: new Date() }, @@ -66,5 +277,10 @@ describe("resolveHeartbeatRuntimeMode", () => { reason: "persisted_native_selection", provider: "codex", }); + expect(() => resolveHeartbeatRuntimeMode({ + ...compatibilityInput, + enabled: false, + adapterType: "paperclip_runner", + })).toThrow(NativeRunnerSelectionError); }); }); diff --git a/server/src/services/native-runtime/runtime-mode.ts b/server/src/services/native-runtime/runtime-mode.ts index 0ce02a4503..aa1741fe8d 100644 --- a/server/src/services/native-runtime/runtime-mode.ts +++ b/server/src/services/native-runtime/runtime-mode.ts @@ -1,5 +1,19 @@ +import { + NATIVE_STATUS_ARBITER_POLICY_VERSION, + type NativeAuthoritativeIssueStatus, + type NativeStatusDecision, +} from "./status-arbiter.js"; + +/** + * Public compatibility resolver version. This value is persisted by the + * original heartbeat selection seam and must remain stable for existing runs + * and downstream importers. + */ export const NATIVE_RUNTIME_RESOLVER_VERSION = "paperclip-runner-v1" as const; +/** Resolver version for the richer native runtime profile used by runnerd. */ +export const NATIVE_RUNTIME_PROFILE_RESOLVER_VERSION = "phase6-v1" as const; + export type HeartbeatRuntimeResolution = | { kind: "legacy"; @@ -13,6 +27,25 @@ export type HeartbeatRuntimeResolution = provider: "codex"; }; +export type NativeRuntimeResolution = + | { + kind: "legacy"; + resolverVersion: typeof NATIVE_RUNTIME_PROFILE_RESOLVER_VERSION; + reason: string; + authorityDecision?: NativeStatusDecision; + } + | { + kind: "native"; + resolverVersion: typeof NATIVE_RUNTIME_PROFILE_RESOLVER_VERSION; + reason: "eligible_opt_in"; + profile: { + mode: "native"; + backend: "codex_app_server"; + protocolVersion: 1; + }; + authorityDecision: NativeStatusDecision; + }; + export class NativeRunnerSelectionError extends Error { constructor(readonly code: string, message: string) { super(message); @@ -20,13 +53,117 @@ export class NativeRunnerSelectionError extends Error { } } -function record(value: unknown): Record { - return typeof value === "object" && value !== null && !Array.isArray(value) - ? value as Record - : {}; +export class NativeRuntimeEligibilityError extends NativeRunnerSelectionError { + constructor( + code: string, + reason?: string, + ) { + super(code, reason ?? `Native runner profile is ineligible: ${code}`); + this.name = "NativeRuntimeEligibilityError"; + } } -/** Resolve a run once. Persisted selections do not consult a later flag change. */ +function ineligible( + code: string, + reason: string, +): NativeRuntimeEligibilityError { + return new NativeRuntimeEligibilityError( + code, + reason, + ); +} + +export function resolveNativeRuntimeMode(input: { + enabled: boolean; + runtimeConfig: unknown; + adapterConfig?: unknown; + agent: { id?: string; status: string; adapterType: string | null }; + issue: { id: string; workMode: string; executionWorkspaceId?: string | null } | null; + target: { kind?: string } | null | undefined; + workspaceId: string | null; +}): NativeRuntimeResolution { + const runnerAdapterSelected = input.agent.adapterType === "paperclip_runner"; + // Fresh direct-adapter runs never enter the native control plane, even if an + // obsolete runtimeConfig.nativeRunner value is still present. Persisted + // native runs are handled by resolveHeartbeatNativeRuntimeMode above this + // fresh-selection seam so they remain recoverable after rollout changes. + if (!runnerAdapterSelected) { + return { + kind: "legacy", + resolverVersion: NATIVE_RUNTIME_PROFILE_RESOLVER_VERSION, + reason: "direct_adapter", + }; + } + if (!input.enabled) { + throw ineligible( + "paperclip_runner_rollout_disabled", + "Paperclip Runner is experimental and disabled on this instance.", + ); + } + const adapterConfig = input.adapterConfig; + const runnerProvider = + typeof adapterConfig === "object" + && adapterConfig !== null + && !Array.isArray(adapterConfig) + ? (adapterConfig as Record).provider ?? "codex" + : "codex"; + if (runnerProvider !== "codex") { + throw ineligible( + "paperclip_runner_provider_unsupported", + "Paperclip Runner currently supports only the Codex provider.", + ); + } + if ( + input.agent.adapterType !== "paperclip_runner" + || input.agent.status !== "active" && input.agent.status !== "running" + ) { + throw ineligible( + "paperclip_runner_agent_ineligible", + "Paperclip Runner requires an active agent.", + ); + } + const allowedWorkModes = ["standard", "planning", "ask"]; + if (!input.issue || !allowedWorkModes.includes(input.issue.workMode)) { + throw ineligible( + "paperclip_runner_issue_ineligible", + "Paperclip Runner requires a standard, planning, or ask task.", + ); + } + if (!input.target || input.target.kind !== "local") { + throw ineligible( + "paperclip_runner_environment_unsupported", + "Paperclip Runner currently requires a local execution environment.", + ); + } + const rollout = resolveNativeMigrationStatus({ + facts: { applicationEnabled: true }, + priorIssueStatus: "in_progress", + agentId: input.agent.id ?? "00000000-0000-4000-8000-000000000000", + }); + if (!rollout.effects.some((effect) => effect.kind === "record_mode_native")) { + throw ineligible( + "paperclip_runner_rollout_policy_rejected", + "Native rollout policy did not select native mode.", + ); + } + return { + kind: "native", + resolverVersion: NATIVE_RUNTIME_PROFILE_RESOLVER_VERSION, + reason: "eligible_opt_in", + profile: { + mode: "native", + backend: "codex_app_server", + protocolVersion: 1, + }, + authorityDecision: rollout, + }; +} + +/** + * Backward-compatible heartbeat selection API. New runnerd code consumes the + * richer profile from resolveHeartbeatNativeRuntimeMode; this public seam + * keeps the original result shape, reason codes, and resolver version. + */ export function resolveHeartbeatRuntimeMode(input: { persisted: { runtimeMode: string | null; @@ -55,44 +192,35 @@ export function resolveHeartbeatRuntimeMode(input: { }; } - if (input.adapterType !== "paperclip_runner") { + let resolution: NativeRuntimeResolution; + try { + resolution = resolveNativeRuntimeMode({ + enabled: input.enabled, + runtimeConfig: {}, + adapterConfig: input.adapterConfig, + agent: { + status: input.agentStatus, + adapterType: input.adapterType, + }, + issue: input.issue + ? { id: "heartbeat-runtime-selection", workMode: input.issue.workMode } + : null, + target: input.executionTarget, + workspaceId: null, + }); + } catch (error) { + if (error instanceof NativeRuntimeEligibilityError) { + throw new NativeRunnerSelectionError(error.code, error.message); + } + throw error; + } + if (resolution.kind === "legacy") { return { kind: "legacy", resolverVersion: NATIVE_RUNTIME_RESOLVER_VERSION, reason: "direct_adapter", }; } - if (!input.enabled) { - throw new NativeRunnerSelectionError( - "paperclip_runner_rollout_disabled", - "Paperclip Runner is experimental and disabled on this instance.", - ); - } - const provider = record(input.adapterConfig).provider ?? "codex"; - if (provider !== "codex") { - throw new NativeRunnerSelectionError( - "paperclip_runner_provider_unsupported", - "Paperclip Runner currently supports only the Codex provider.", - ); - } - if (!input.issue || !["standard", "planning", "ask"].includes(input.issue.workMode)) { - throw new NativeRunnerSelectionError( - "paperclip_runner_issue_ineligible", - "Paperclip Runner requires a standard, planning, or ask task.", - ); - } - if (!input.executionTarget || input.executionTarget.kind !== "local") { - throw new NativeRunnerSelectionError( - "paperclip_runner_environment_unsupported", - "Paperclip Runner currently requires a local execution environment.", - ); - } - if (!["active", "running"].includes(input.agentStatus)) { - throw new NativeRunnerSelectionError( - "paperclip_runner_agent_ineligible", - "Paperclip Runner requires an active agent.", - ); - } return { kind: "native", resolverVersion: NATIVE_RUNTIME_RESOLVER_VERSION, @@ -100,3 +228,281 @@ export function resolveHeartbeatRuntimeMode(input: { provider: "codex", }; } + +/** + * Production heartbeat selection seam. A resolved run keeps its persisted + * mode across configuration changes; only a fresh unresolved run consults the + * current global flag and agent profile. + */ +export function resolveHeartbeatNativeRuntimeMode(input: { + persisted: { + runtimeMode: string | null; + runtimeModeReason: string | null; + runtimeModeResolvedAt: Date | null; + driverKind?: string | null; + }; + enabled: boolean; + runtimeConfig: unknown; + adapterConfig?: unknown; + agent: { id?: string; status: string; adapterType: string | null }; + issue: { id: string; workMode: string; executionWorkspaceId?: string | null } | null; + target: { kind?: string } | null | undefined; + workspaceId: string | null; +}): NativeRuntimeResolution { + if (input.persisted.runtimeModeResolvedAt) { + if (input.persisted.runtimeMode === "native") { + if (input.agent.adapterType !== "paperclip_runner") { + throw ineligible( + "paperclip_runner_adapter_binding_mismatch", + "A persisted native run must remain bound to the Paperclip Runner adapter.", + ); + } + if ( + input.agent.status !== "active" && + input.agent.status !== "running" + ) { + throw ineligible( + "paperclip_runner_agent_ineligible", + "A persisted Paperclip Runner run cannot recover through a non-invokable agent.", + ); + } + const driverKind = input.persisted.driverKind; + const backend = driverKind === null + || driverKind === undefined + || driverKind === "codex" + || driverKind === "codex_app_server" + ? "codex_app_server" + : null; + if (!backend) { + throw ineligible( + "paperclip_runner_driver_unsupported", + `Persisted Paperclip Runner driver is unsupported: ${driverKind}`, + ); + } + return { + kind: "native", + resolverVersion: NATIVE_RUNTIME_PROFILE_RESOLVER_VERSION, + reason: "eligible_opt_in", + profile: { + mode: "native", + backend, + protocolVersion: 1, + }, + authorityDecision: resolveNativeMigrationStatus({ + facts: input.enabled + ? { applicationEnabled: true } + : { killSwitchActiveForNewRuns: true }, + priorIssueStatus: "in_progress", + agentId: input.agent.id ?? "00000000-0000-4000-8000-000000000000", + }), + }; + } + return { + kind: "legacy", + resolverVersion: NATIVE_RUNTIME_PROFILE_RESOLVER_VERSION, + reason: input.persisted.runtimeModeReason ?? "persisted_legacy_selection", + }; + } + return resolveNativeRuntimeMode(input); +} + +/** Production read-model facts used by compatibility and mixed-ledger views. */ +export function inspectNativeCompatibilityState(input: { + resolution: NativeRuntimeResolution; + nativeRecordCount: number; + decisionCount: number; + issueStatus: string; + statusVersion: number; + persistedEffectKinds: string[]; +}) { + const effects = input.persistedEffectKinds.length > 0 + ? [...input.persistedEffectKinds] + : input.resolution.kind === "legacy" + ? ["legacy_existing_behavior"] + : input.nativeRecordCount === 0 && input.statusVersion === 0 + ? ["initialize_status_version_zero"] + : []; + return { + mode: input.resolution.kind, + native: input.nativeRecordCount > 0, + hasNativeDecisionLineage: input.decisionCount > 0, + issueStatus: input.issueStatus, + statusVersion: input.statusVersion, + statusAction: input.resolution.kind === "legacy" ? "legacy_finalizer" : "preserve", + reasonCode: null, + effects, + } as const; +} + +/** Expand-only migration evidence; it never mutates or synthesizes history. */ +export function inspectNativeMigrationState(input: { + resolution: NativeRuntimeResolution; + nativeRecordCount: number; + decisionCount: number; + issueStatusBefore: string; + issueStatusAfter: string; + statusVersion: number; + hasPendingReview: boolean; +}) { + const effects = input.resolution.kind === "legacy" + ? input.issueStatusBefore === "done" + ? ["retain_legacy_mode", "retain_audit_lineage"] + : ["return_native_false"] + : input.nativeRecordCount === 0 && input.hasPendingReview && input.statusVersion > 0 + ? ["increment_status_version_once", "bind_reviewer"] + : input.nativeRecordCount === 0 + ? ["expand_schema", "status_version_default_zero"] + : []; + return { + mode: input.resolution.kind, + native: input.nativeRecordCount > 0, + hasSyntheticHistory: input.nativeRecordCount === 0 && input.decisionCount > 0, + statusPreserved: input.issueStatusBefore === input.issueStatusAfter, + statusVersion: input.statusVersion, + statusAction: input.resolution.kind === "legacy" ? "legacy_finalizer" + : input.hasPendingReview ? input.issueStatusAfter : "preserve", + reasonCode: null, + effects, + } as const; +} + +export type NativeCompatibilityFacts = { + invalidNativeFinalization?: boolean; + terminalResumeAuthorized?: boolean; + shadowApplicationDisabled?: boolean; + mixedLedger?: boolean; + statusWriterAdvancedVersion?: boolean; +}; + +export function resolveNativeCompatibilityStatus(input: { + facts: NativeCompatibilityFacts; + priorIssueStatus: NativeAuthoritativeIssueStatus; + agentId: string; +}): NativeStatusDecision { + const preserve = (reasonCode: string, effects: NativeStatusDecision["effects"]): NativeStatusDecision => ({ + policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, + statusAction: "preserve", + toStatus: input.priorIssueStatus, + reasonCode, + unblockDescriptor: null, + effects, + }); + if (input.facts.invalidNativeFinalization) { + return preserve("native_finalization_invalid", [{ + kind: "record_finalization_error", + cause: "native_finalization_invalid", + nextAction: "Repair the persisted native result.", + agentId: input.agentId, + }]); + } + if (input.facts.terminalResumeAuthorized) { + return { + policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, + statusAction: "in_progress", + toStatus: "in_progress", + reasonCode: "authorized_resume", + unblockDescriptor: null, + effects: [{ + kind: "enqueue_continuation", + continuationKind: "same_agent", + summary: "Resume the terminal issue through the authorized compatibility path.", + idempotencyKey: "native-compatibility:authorized-resume", + agentId: input.agentId, + }], + }; + } + if (input.facts.shadowApplicationDisabled) { + return preserve("completion_contract_satisfied", [{ kind: "record_shadow_decision" }]); + } + if (input.facts.mixedLedger) { + return preserve("completion_contract_satisfied", [{ kind: "render_four_layers" }]); + } + if (input.facts.statusWriterAdvancedVersion) { + return preserve("arbitration_conflict_reloaded", [ + { kind: "increment_status_version" }, + { kind: "schedule_reconciliation" }, + ]); + } + throw new Error("native_compatibility_facts_invalid"); +} + +export type NativeMigrationFacts = { + shadowMaterialization?: boolean; + classifiedDivergence?: boolean; + applicationEnabled?: boolean; + policyPinned?: boolean; + killSwitchActiveForNewRuns?: boolean; +}; + +export function resolveNativeMigrationStatus(input: { + facts: NativeMigrationFacts; + priorIssueStatus: NativeAuthoritativeIssueStatus; + agentId: string; +}): NativeStatusDecision { + const preserve = (reasonCode: string, effects: NativeStatusDecision["effects"]): NativeStatusDecision => ({ + policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, + statusAction: "preserve", + toStatus: input.priorIssueStatus, + reasonCode, + unblockDescriptor: null, + effects, + }); + if (input.facts.shadowMaterialization) { + return preserve("completion_contract_satisfied", [ + { kind: "materialize_contract" }, + { kind: "record_shadow_decision" }, + ]); + } + if (input.facts.classifiedDivergence) { + return preserve("completion_evidence_incomplete", [{ kind: "record_mode_labeled_divergence" }]); + } + if (input.facts.killSwitchActiveForNewRuns) { + return { + policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, + statusAction: "in_progress", + toStatus: "in_progress", + reasonCode: "live_continuation_registered", + unblockDescriptor: null, + effects: [ + { + kind: "enqueue_continuation", + continuationKind: "same_agent", + summary: "Finish the already-active run in native mode.", + idempotencyKey: "native-migration:kill-switch-active-run", + agentId: input.agentId, + }, + { kind: "finish_as_native" }, + ], + }; + } + if (input.facts.policyPinned) { + return { + policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, + statusAction: "done", + toStatus: "done", + reasonCode: "completion_contract_satisfied", + unblockDescriptor: null, + effects: [{ kind: "record_mode_native" }, { kind: "record_policy_version" }], + }; + } + if (input.facts.applicationEnabled) { + return { + policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, + statusAction: "in_progress", + toStatus: "in_progress", + reasonCode: "live_continuation_registered", + unblockDescriptor: null, + effects: [ + { + kind: "enqueue_continuation", + continuationKind: "same_agent", + summary: "Continue the allowlisted native run.", + idempotencyKey: "native-migration:application-enabled", + agentId: input.agentId, + }, + { kind: "record_mode_native" }, + ], + }; + } + throw new Error("native_migration_facts_invalid"); +} diff --git a/server/src/services/plugin-worker-manager.ts b/server/src/services/plugin-worker-manager.ts index 9d2d8b8ab3..459ceb585e 100644 --- a/server/src/services/plugin-worker-manager.ts +++ b/server/src/services/plugin-worker-manager.ts @@ -1765,6 +1765,7 @@ export function createPluginWorkerHandle( interface HeldDuplexExitEvent { workerSessionId: string; exitCode: number | null; + transportClosed?: boolean; } interface DuplexChannelRoute { @@ -2169,7 +2170,11 @@ export function createPluginWorkerHandle( // Normalize the exit to the narrow duplex-event schema. A replaced exit // simply overwrites the earlier held exit. const exitCode = typeof params.exitCode === "number" ? params.exitCode : null; - route.preBindExit = { workerSessionId, exitCode }; + route.preBindExit = { + workerSessionId, + exitCode, + ...(params.transportClosed === true ? { transportClosed: true } : {}), + }; return; } // A data event. Validate and normalize it to the narrow duplex-event schema @@ -2237,6 +2242,7 @@ export function createPluginWorkerHandle( hostRouteId: route.hostRouteId, workerSessionId: heldExit.workerSessionId, exitCode: heldExit.exitCode, + ...(heldExit.transportClosed === true ? { transportClosed: true } : {}), }, }); } diff --git a/server/src/vendor/paperclip-runner/index.ts b/server/src/vendor/paperclip-runner/index.ts index 496025a310..a8c56cddab 100644 --- a/server/src/vendor/paperclip-runner/index.ts +++ b/server/src/vendor/paperclip-runner/index.ts @@ -26,13 +26,10 @@ export type { ControlPlanePort, HarnessRuntimeRequestKind, HarnessRuntimeRequestResolution, - NativeAcpxAgent, - NativeAcpxPermissionMode, NativeCodexApprovalPolicy, NativeExecutionInput, NativeExecutionInputV4, NativeInteractionResponseEnvelope, - NativeOpenCodePermissionMode, NativePlanningContext, NativeRunEvent, NativeRunResult, @@ -64,6 +61,8 @@ const runner = await import(sourceUrl.href) as RunnerModule; export const DurablePrpControlPlane = runner.DurablePrpControlPlane; export const PaperclipSemanticDispatcher = runner.PaperclipSemanticDispatcher; +export const CAPABILITY_SEMANTIC_TOOL_CATALOG = + runner.CAPABILITY_SEMANTIC_TOOL_CATALOG; export const HarnessRuntimeRequestResolutionError = runner.HarnessRuntimeRequestResolutionError; export const NATIVE_RUNTIME_ASSET_SCHEMA = runner.NATIVE_RUNTIME_ASSET_SCHEMA; @@ -75,6 +74,12 @@ export const canonicalNativeRuntimeContextDigest = export const createNativeSessionBackend = runner.createNativeSessionBackend; export const createPaperclipRunnerAuthorizedToolSet = runner.createPaperclipRunnerAuthorizedToolSet; +export const createRunnerdCodexTransport: ( + options?: import("@paperclipai/paperclip-runner").RunnerdCodexTransportOptions, +) => import("@paperclipai/paperclip-runner").RunnerdCodexTransport = + runner.createRunnerdCodexTransport; +export const defaultCapabilityRunnerdBinary = + runner.defaultCapabilityRunnerdBinary; export const executeNativeSession = runner.executeNativeSession; export const nativeRuntimePromptDigest = runner.nativeRuntimePromptDigest; export const normalizePrpResultSignals = runner.normalizePrpResultSignals; diff --git a/server/src/vendor/paperclip-runner/testing.ts b/server/src/vendor/paperclip-runner/testing.ts index 176fec90ad..e889df919d 100644 --- a/server/src/vendor/paperclip-runner/testing.ts +++ b/server/src/vendor/paperclip-runner/testing.ts @@ -8,17 +8,50 @@ */ type RunnerTestingModule = typeof import("@paperclipai/paperclip-runner/testing"); +export type { + CapabilityCommandEnvelope, + CapabilityCommandOutcome, + CapabilityCommandResult, + CapabilityFixtureState, + CapabilityRunContext, + CapabilitySemanticCommand, + SemanticConformanceAdapter, + SemanticConformanceObservation, + SemanticConformanceVector, +} from "@paperclipai/paperclip-runner/testing"; + const sourceUrl = new URL( "../../../../packages/paperclip-runner/src/testing.ts", import.meta.url, ); const runnerTesting = await import(sourceUrl.href) as RunnerTestingModule; +export const CAPABILITY_HIGH_RISK_SEMANTIC_VECTORS: + RunnerTestingModule["CAPABILITY_HIGH_RISK_SEMANTIC_VECTORS"] = + runnerTesting.CAPABILITY_HIGH_RISK_SEMANTIC_VECTORS; +export const CAPABILITY_SEMANTIC_CONFORMANCE_IDS: + RunnerTestingModule["CAPABILITY_SEMANTIC_CONFORMANCE_IDS"] = + runnerTesting.CAPABILITY_SEMANTIC_CONFORMANCE_IDS; +export const CapabilityMockSemanticConformanceAdapter: + RunnerTestingModule["CapabilityMockSemanticConformanceAdapter"] = + runnerTesting.CapabilityMockSemanticConformanceAdapter; +export const CapabilitySemanticDispatcher: + RunnerTestingModule["CapabilitySemanticDispatcher"] = + runnerTesting.CapabilitySemanticDispatcher; export const CONTROL_PLANE_CONFORMANCE_OPEN = runnerTesting.CONTROL_PLANE_CONFORMANCE_OPEN; export const CONTROL_PLANE_CONFORMANCE_RESULT = runnerTesting.CONTROL_PLANE_CONFORMANCE_RESULT; export const CONTROL_PLANE_CONFORMANCE_TERMINAL = runnerTesting.CONTROL_PLANE_CONFORMANCE_TERMINAL; +export const createCapabilityFixtureState: + RunnerTestingModule["createCapabilityFixtureState"] = + runnerTesting.createCapabilityFixtureState; +export const normalizeCapabilitySemanticObservation: + RunnerTestingModule["normalizeCapabilitySemanticObservation"] = + runnerTesting.normalizeCapabilitySemanticObservation; export const runControlPlanePortConformance = runnerTesting.runControlPlanePortConformance; +export const runSemanticConformanceKit: + RunnerTestingModule["runSemanticConformanceKit"] = + runnerTesting.runSemanticConformanceKit;