diff --git a/doc/run-log-events.md b/doc/run-log-events.md index 86ee8499bd..fa98161673 100644 --- a/doc/run-log-events.md +++ b/doc/run-log-events.md @@ -113,3 +113,20 @@ The sandbox duplex transport also writes one run-log event as one of its three sinks. See the [Sandbox Duplex Transport Instrumentation](observability.md#sandbox-duplex-transport-instrumentation) section in the Observability contract. + +## Native Process Rotation + +`native.session.process_rotation` records a controller-initiated close of a +settled warm session before opening the next run. Its system-stream payload +contains `reason` (`run_scoped_github_capability` or `configuration_changed`), +`previousRunId` (nullable), `runId`, `companyId`, `agentId`, `nativeSessionId`, +and `runnerInstanceId`. It contains no token, environment, path, or credential +value. The event records rotation intent after the prior process closes; the +new run must still succeed to establish successful continuation. + +A subsequent run uses a fresh process for its own GitHub capability while +preserving the durable conversation. Warm qualification accepts a changed +process fingerprint only with a matching system rotation event for that exact +run transition and a process start inside the new run. Unexpected restarts, +configuration changes, and conversation, runner-instance or sandbox changes +remain failures. Same-process warm reuse remains required without a rotation. diff --git a/doc/sandbox-work-folders.md b/doc/sandbox-work-folders.md index 3772c0c1a1..da0a0e6742 100644 --- a/doc/sandbox-work-folders.md +++ b/doc/sandbox-work-folders.md @@ -372,3 +372,11 @@ appending and after writing. Missing or changed prior content fails without repair; repeating the first turn cannot truncate existing work. This avoids model-generated byte-count arithmetic while retaining independent per-turn persistence checks and the same-sandbox, same-provider continuity requirements. + +Native continuation retains its sandbox, runner-instance identity, provider +conversation, and files across runs. The existing GitHub identity contract +rotates the provider process for each run-scoped capability. Qualification +requires an explicit controller rotation event for that exact transition before +accepting a changed PID/process fingerprint; other process changes still fail. +See [GitHub execution identity](execution-github-identity.md) and the +[run-log contract](run-log-events.md#native-process-rotation). diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 2993743449..28f387acd8 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -3521,6 +3521,7 @@ describe("native warm session supervision", () => { process.env.PAPERCLIP_RUNNER_STATE_DIR = stateBase; process.env.PAPERCLIP_HOME = stateBase; const firstClose = vi.fn(async () => undefined); + const onEvent = vi.fn(async () => undefined); const firstSession = { close: firstClose }; const first = { ...execution, @@ -3639,6 +3640,7 @@ describe("native warm session supervision", () => { } as unknown as Db; await executePaperclipNativeSession({ db: continuationDb, + onEvent, execution: second, runnerEnvironment: useBroker ? { PAPERCLIP_GITHUB_BROKER_TOKEN: "second-run-capability" } : undefined, runnerInstanceId: "runner-runnerd-warm", @@ -3646,9 +3648,28 @@ describe("native warm session supervision", () => { runnerExecutionTarget: remoteTarget, }); if (useBroker) { + expect(onEvent).toHaveBeenCalledWith({ + eventType: "native.session.process_rotation", + stream: "system", + level: "info", + message: "Native process rotated for the next run", + payload: { + reason: "run_scoped_github_capability", + previousRunId: first.binding.runId, + runId: second.binding.runId, + companyId: second.binding.companyId, + agentId: second.binding.agentId, + nativeSessionId: second.session.normalizedSessionId, + runnerInstanceId: "runner-runnerd-warm", + }, + }); + expect(JSON.stringify(onEvent.mock.calls)).not.toContain("run-capability"); expect(firstClose).toHaveBeenCalledOnce(); expect(firstClose).toHaveBeenCalledWith({ reason: "warm native session configuration changed" }); } else { + expect(onEvent).not.toHaveBeenCalledWith(expect.objectContaining({ + eventType: "native.session.process_rotation", + })); expect(firstClose).not.toHaveBeenCalled(); await vi.waitFor( () => diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index f5ab407cef..d47352264b 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -4339,6 +4339,22 @@ async function executePaperclipNativeSessionWithinScope( input.execution, warmConfigDigest, ); + await input.onEvent?.({ + eventType: "native.session.process_rotation", + stream: "system", + level: "info", + message: "Native process rotated for the next run", + payload: { + reason: entry.configDigest === warmConfigDigest && credentialRunChanged + ? "run_scoped_github_capability" : "configuration_changed", + previousRunId: entry.credentialRunId ?? null, + runId: input.execution.binding.runId, + companyId: input.execution.binding.companyId, + agentId: input.execution.binding.agentId, + nativeSessionId: nativeSessionKey(input.execution), + runnerInstanceId: input.runnerInstanceId, + }, + }); } else { if (entry.busy) throw new Error("native_session_supervisor_busy"); entry.busy = true; diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 379cdcd214..11d61d8296 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -19,7 +19,10 @@ import { providerSessionContinuityFailures, } from "./run-observations.js"; import { resolveRunnerE2ESource } from "./source.js"; -import { readWarmWorkspaceFile } from "./warm-workspace.js"; +import { + nativeWarmProcessFailures, + readWarmWorkspaceFile, +} from "./warm-workspace.js"; import { isPublicRunnerScreenshotRoute, PUBLIC_RUNNER_SCREENSHOT_MARKER, @@ -143,6 +146,7 @@ interface IssueDocumentRecord { } interface RunEventRecord { seq?: number; + stream?: string | null; eventType?: string; payload?: Record | null; sourceInstanceId?: string | null; @@ -1980,6 +1984,9 @@ for (const execution of executions) { ); } if (execution.profile.generation === "native") { + invariantFailures.push( + ...nativeWarmProcessFailures(selectedRuns, runEventsByRun), + ); const stableIdentityFields: Array<{ label: string; values: unknown[]; @@ -2000,16 +2007,6 @@ for (const execution of executions) { label: "provider session", values: selectedRuns.map((candidate) => candidate.sessionIdAfter), }, - { - label: "runner pid", - values: selectedRuns.map((candidate) => candidate.processPid), - }, - { - label: "runner process fingerprint", - values: selectedRuns.map( - (candidate) => candidate.processStartedAt, - ), - }, ]; for (const { label, values } of stableIdentityFields) { if ( diff --git a/tests/runner-e2e/warm-workspace.test.ts b/tests/runner-e2e/warm-workspace.test.ts index e7674a4e03..24e5e888d2 100644 --- a/tests/runner-e2e/warm-workspace.test.ts +++ b/tests/runner-e2e/warm-workspace.test.ts @@ -5,7 +5,10 @@ import path from "node:path"; import { promisify } from "node:util"; import { afterEach, describe, expect, it, vi } from "vitest"; import { daytonaWarmContinuityTask } from "./catalog.js"; -import { readWarmWorkspaceFile } from "./warm-workspace.js"; +import { + nativeWarmProcessFailures, + readWarmWorkspaceFile, +} from "./warm-workspace.js"; const temporaryDirectories: string[] = []; afterEach(async () => { @@ -291,3 +294,106 @@ describe("warm workspace persistence observation", () => { expect(input.api.request.get).not.toHaveBeenCalled(); }); }); + +function processFixture() { + const runs = [0, 1, 2].map((index) => ({ + id: `run-${index}`, + companyId: "company", + agentId: "agent", + nativeSessionId: "native-session", + runnerInstanceId: "runner-instance", + processPid: 100 + index, + processStartedAt: `2026-09-09T04:0${index}:01Z`, + startedAt: `2026-09-09T04:0${index}:00Z`, + finishedAt: `2026-09-09T04:0${index}:10Z`, + })); + const groups = runs.map((run, index) => ({ + runId: run.id, + events: + index === 0 + ? [] + : [ + { + eventType: "native.session.process_rotation", + stream: "system", + payload: { + reason: "run_scoped_github_capability", + previousRunId: runs[index - 1]!.id, + runId: run.id, + companyId: run.companyId, + agentId: run.agentId, + nativeSessionId: run.nativeSessionId, + runnerInstanceId: run.runnerInstanceId, + }, + }, + ], + })); + return { runs, groups }; +} + +describe("native warm process continuity", () => { + it("requires matching controller events for each run-capability rotation", () => { + const { runs, groups } = processFixture(); + expect(nativeWarmProcessFailures(runs, groups)).toEqual([]); + }); + + it("retains the same-process requirement without credential rotation", () => { + const { runs, groups } = processFixture(); + for (const run of runs) { + run.processPid = runs[0]!.processPid; + run.processStartedAt = runs[0]!.processStartedAt; + } + for (const group of groups) group.events = []; + expect(nativeWarmProcessFailures(runs, groups)).toEqual([]); + runs[1]!.processPid += 1; + expect(nativeWarmProcessFailures(runs, groups)).not.toEqual([]); + }); + + it.each([ + "reason", + "previousRunId", + "runId", + "companyId", + "agentId", + "nativeSessionId", + "runnerInstanceId", + ] as const)("rejects an unrelated rotation %s", (field) => { + const { runs, groups } = processFixture(); + groups[1]!.events[0]!.payload[field] = "unrelated"; + expect(nativeWarmProcessFailures(runs, groups)).toHaveLength(1); + }); + + it.each(["missing", "stdout", "duplicate", "configuration_changed"])( + "rejects %s rotation evidence", + (kind) => { + const { runs, groups } = processFixture(); + const events = groups[1]!.events; + if (kind === "missing") events.pop(); + if (kind === "stdout") events[0]!.stream = "stdout"; + if (kind === "duplicate") events.push(events[0]!); + if (kind === "configuration_changed") events[0]!.payload.reason = kind; + expect(nativeWarmProcessFailures(runs, groups)).toHaveLength(1); + }, + ); + + it.each(["invalid", "2026-09-09T04:00:59Z", "2026-09-09T04:01:11Z"])( + "rejects a process start outside the new run: %s", + (time) => { + const { runs, groups } = processFixture(); + runs[1]!.processStartedAt = time; + expect(nativeWarmProcessFailures(runs, groups).length).toBeGreaterThan(0); + }, + ); + + it("rejects a claimed rotation that kept the previous process", () => { + const { runs, groups } = processFixture(); + runs[1]!.processPid = runs[0]!.processPid; + runs[1]!.processStartedAt = runs[0]!.processStartedAt; + expect(nativeWarmProcessFailures(runs, groups)).toHaveLength(1); + }); + + it("rejects a missing event stream even for stable processes", () => { + const { runs } = processFixture(); + expect(nativeWarmProcessFailures(runs, [])).toHaveLength(3); + }); +}); diff --git a/tests/runner-e2e/warm-workspace.ts b/tests/runner-e2e/warm-workspace.ts index 31b49a6c5c..b8355f780b 100644 --- a/tests/runner-e2e/warm-workspace.ts +++ b/tests/runner-e2e/warm-workspace.ts @@ -98,3 +98,84 @@ export async function readWarmWorkspaceFile(input: { ); return { source: "task-cache", content: await response.text() }; } + +/** Stable warm processes or an explicitly recorded run-capability rotation. */ +export function nativeWarmProcessFailures( + runs: readonly { + id: string; + companyId: string; + agentId: string; + nativeSessionId?: string | null; + runnerInstanceId?: string | null; + processPid?: number | null; + processStartedAt?: string | null; + startedAt?: string | null; + finishedAt?: string | null; + }[], + eventGroups: readonly { + runId: string; + events: readonly { + eventType?: string; + stream?: string | null; + payload?: Record | null; + }[]; + }[], +): string[] { + const failures: string[] = []; + for (const [index, run] of runs.entries()) { + if ( + !Number.isSafeInteger(run.processPid) || + run.processPid! <= 0 || + !Number.isFinite(Date.parse(run.processStartedAt ?? "")) + ) { + failures.push(`missing native process identity for warm run ${run.id}`); + continue; + } + const group = eventGroups.find((group) => group.runId === run.id); + if (!group) { + failures.push(`missing native process events for warm run ${run.id}`); + continue; + } + const rotations = group.events.filter( + (event) => event.eventType === "native.session.process_rotation", + ); + const previous = runs[index - 1]; + const sameProcess = + !previous || + (previous.processPid === run.processPid && + previous.processStartedAt === run.processStartedAt); + if (sameProcess) { + if (rotations.length) + failures.push( + `unexpected native process rotation for warm run ${run.id}`, + ); + continue; + } + const event = rotations[0]; + const payload = event?.payload; + const started = Date.parse(run.startedAt ?? ""); + const finished = Date.parse(run.finishedAt ?? ""); + const processStarted = Date.parse(run.processStartedAt!); + if ( + rotations.length !== 1 || + event?.stream !== "system" || + payload?.reason !== "run_scoped_github_capability" || + payload.previousRunId !== previous.id || + payload.runId !== run.id || + payload.companyId !== run.companyId || + payload.agentId !== run.agentId || + payload.nativeSessionId !== run.nativeSessionId || + payload.runnerInstanceId !== run.runnerInstanceId || + !Number.isFinite(started) || + !Number.isFinite(finished) || + processStarted < started || + processStarted > finished || + processStarted <= Date.parse(previous.processStartedAt ?? "") + ) { + failures.push( + `native warm process changed without a matching run-scoped credential rotation for ${run.id}`, + ); + } + } + return failures; +}