test: verify explicit native credential process rotations
Record controller-owned rotation events and require an exact run transition before accepting a new process fingerprint. Preserve stable sandbox, runner and provider-session checks. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
parent
a24d331c41
commit
1ca638736a
|
|
@ -113,3 +113,20 @@ The sandbox duplex transport also writes one run-log event as one of its three
|
|||
sinks. See the
|
||||
[Sandbox Duplex Transport Instrumentation](observability.md#sandbox-duplex-transport-instrumentation)
|
||||
section in the Observability contract.
|
||||
|
||||
## Native Process Rotation
|
||||
|
||||
`native.session.process_rotation` records a controller-initiated close of a
|
||||
settled warm session before opening the next run. Its system-stream payload
|
||||
contains `reason` (`run_scoped_github_capability` or `configuration_changed`),
|
||||
`previousRunId` (nullable), `runId`, `companyId`, `agentId`, `nativeSessionId`,
|
||||
and `runnerInstanceId`. It contains no token, environment, path, or credential
|
||||
value. The event records rotation intent after the prior process closes; the
|
||||
new run must still succeed to establish successful continuation.
|
||||
|
||||
A subsequent run uses a fresh process for its own GitHub capability while
|
||||
preserving the durable conversation. Warm qualification accepts a changed
|
||||
process fingerprint only with a matching system rotation event for that exact
|
||||
run transition and a process start inside the new run. Unexpected restarts,
|
||||
configuration changes, and conversation, runner-instance or sandbox changes
|
||||
remain failures. Same-process warm reuse remains required without a rotation.
|
||||
|
|
|
|||
|
|
@ -372,3 +372,11 @@ appending and after writing. Missing or changed prior content fails without
|
|||
repair; repeating the first turn cannot truncate existing work. This avoids
|
||||
model-generated byte-count arithmetic while retaining independent per-turn
|
||||
persistence checks and the same-sandbox, same-provider continuity requirements.
|
||||
|
||||
Native continuation retains its sandbox, runner-instance identity, provider
|
||||
conversation, and files across runs. The existing GitHub identity contract
|
||||
rotates the provider process for each run-scoped capability. Qualification
|
||||
requires an explicit controller rotation event for that exact transition before
|
||||
accepting a changed PID/process fingerprint; other process changes still fail.
|
||||
See [GitHub execution identity](execution-github-identity.md) and the
|
||||
[run-log contract](run-log-events.md#native-process-rotation).
|
||||
|
|
|
|||
|
|
@ -3521,6 +3521,7 @@ describe("native warm session supervision", () => {
|
|||
process.env.PAPERCLIP_RUNNER_STATE_DIR = stateBase;
|
||||
process.env.PAPERCLIP_HOME = stateBase;
|
||||
const firstClose = vi.fn(async () => undefined);
|
||||
const onEvent = vi.fn(async () => undefined);
|
||||
const firstSession = { close: firstClose };
|
||||
const first = {
|
||||
...execution,
|
||||
|
|
@ -3639,6 +3640,7 @@ describe("native warm session supervision", () => {
|
|||
} as unknown as Db;
|
||||
await executePaperclipNativeSession({
|
||||
db: continuationDb,
|
||||
onEvent,
|
||||
execution: second,
|
||||
runnerEnvironment: useBroker ? { PAPERCLIP_GITHUB_BROKER_TOKEN: "second-run-capability" } : undefined,
|
||||
runnerInstanceId: "runner-runnerd-warm",
|
||||
|
|
@ -3646,9 +3648,28 @@ describe("native warm session supervision", () => {
|
|||
runnerExecutionTarget: remoteTarget,
|
||||
});
|
||||
if (useBroker) {
|
||||
expect(onEvent).toHaveBeenCalledWith({
|
||||
eventType: "native.session.process_rotation",
|
||||
stream: "system",
|
||||
level: "info",
|
||||
message: "Native process rotated for the next run",
|
||||
payload: {
|
||||
reason: "run_scoped_github_capability",
|
||||
previousRunId: first.binding.runId,
|
||||
runId: second.binding.runId,
|
||||
companyId: second.binding.companyId,
|
||||
agentId: second.binding.agentId,
|
||||
nativeSessionId: second.session.normalizedSessionId,
|
||||
runnerInstanceId: "runner-runnerd-warm",
|
||||
},
|
||||
});
|
||||
expect(JSON.stringify(onEvent.mock.calls)).not.toContain("run-capability");
|
||||
expect(firstClose).toHaveBeenCalledOnce();
|
||||
expect(firstClose).toHaveBeenCalledWith({ reason: "warm native session configuration changed" });
|
||||
} else {
|
||||
expect(onEvent).not.toHaveBeenCalledWith(expect.objectContaining({
|
||||
eventType: "native.session.process_rotation",
|
||||
}));
|
||||
expect(firstClose).not.toHaveBeenCalled();
|
||||
await vi.waitFor(
|
||||
() =>
|
||||
|
|
|
|||
|
|
@ -4339,6 +4339,22 @@ async function executePaperclipNativeSessionWithinScope(
|
|||
input.execution,
|
||||
warmConfigDigest,
|
||||
);
|
||||
await input.onEvent?.({
|
||||
eventType: "native.session.process_rotation",
|
||||
stream: "system",
|
||||
level: "info",
|
||||
message: "Native process rotated for the next run",
|
||||
payload: {
|
||||
reason: entry.configDigest === warmConfigDigest && credentialRunChanged
|
||||
? "run_scoped_github_capability" : "configuration_changed",
|
||||
previousRunId: entry.credentialRunId ?? null,
|
||||
runId: input.execution.binding.runId,
|
||||
companyId: input.execution.binding.companyId,
|
||||
agentId: input.execution.binding.agentId,
|
||||
nativeSessionId: nativeSessionKey(input.execution),
|
||||
runnerInstanceId: input.runnerInstanceId,
|
||||
},
|
||||
});
|
||||
} else {
|
||||
if (entry.busy) throw new Error("native_session_supervisor_busy");
|
||||
entry.busy = true;
|
||||
|
|
|
|||
|
|
@ -19,7 +19,10 @@ import {
|
|||
providerSessionContinuityFailures,
|
||||
} from "./run-observations.js";
|
||||
import { resolveRunnerE2ESource } from "./source.js";
|
||||
import { readWarmWorkspaceFile } from "./warm-workspace.js";
|
||||
import {
|
||||
nativeWarmProcessFailures,
|
||||
readWarmWorkspaceFile,
|
||||
} from "./warm-workspace.js";
|
||||
import {
|
||||
isPublicRunnerScreenshotRoute,
|
||||
PUBLIC_RUNNER_SCREENSHOT_MARKER,
|
||||
|
|
@ -143,6 +146,7 @@ interface IssueDocumentRecord {
|
|||
}
|
||||
interface RunEventRecord {
|
||||
seq?: number;
|
||||
stream?: string | null;
|
||||
eventType?: string;
|
||||
payload?: Record<string, unknown> | null;
|
||||
sourceInstanceId?: string | null;
|
||||
|
|
@ -1980,6 +1984,9 @@ for (const execution of executions) {
|
|||
);
|
||||
}
|
||||
if (execution.profile.generation === "native") {
|
||||
invariantFailures.push(
|
||||
...nativeWarmProcessFailures(selectedRuns, runEventsByRun),
|
||||
);
|
||||
const stableIdentityFields: Array<{
|
||||
label: string;
|
||||
values: unknown[];
|
||||
|
|
@ -2000,16 +2007,6 @@ for (const execution of executions) {
|
|||
label: "provider session",
|
||||
values: selectedRuns.map((candidate) => candidate.sessionIdAfter),
|
||||
},
|
||||
{
|
||||
label: "runner pid",
|
||||
values: selectedRuns.map((candidate) => candidate.processPid),
|
||||
},
|
||||
{
|
||||
label: "runner process fingerprint",
|
||||
values: selectedRuns.map(
|
||||
(candidate) => candidate.processStartedAt,
|
||||
),
|
||||
},
|
||||
];
|
||||
for (const { label, values } of stableIdentityFields) {
|
||||
if (
|
||||
|
|
|
|||
|
|
@ -5,7 +5,10 @@ import path from "node:path";
|
|||
import { promisify } from "node:util";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { daytonaWarmContinuityTask } from "./catalog.js";
|
||||
import { readWarmWorkspaceFile } from "./warm-workspace.js";
|
||||
import {
|
||||
nativeWarmProcessFailures,
|
||||
readWarmWorkspaceFile,
|
||||
} from "./warm-workspace.js";
|
||||
|
||||
const temporaryDirectories: string[] = [];
|
||||
afterEach(async () => {
|
||||
|
|
@ -291,3 +294,106 @@ describe("warm workspace persistence observation", () => {
|
|||
expect(input.api.request.get).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
function processFixture() {
|
||||
const runs = [0, 1, 2].map((index) => ({
|
||||
id: `run-${index}`,
|
||||
companyId: "company",
|
||||
agentId: "agent",
|
||||
nativeSessionId: "native-session",
|
||||
runnerInstanceId: "runner-instance",
|
||||
processPid: 100 + index,
|
||||
processStartedAt: `2026-09-09T04:0${index}:01Z`,
|
||||
startedAt: `2026-09-09T04:0${index}:00Z`,
|
||||
finishedAt: `2026-09-09T04:0${index}:10Z`,
|
||||
}));
|
||||
const groups = runs.map((run, index) => ({
|
||||
runId: run.id,
|
||||
events:
|
||||
index === 0
|
||||
? []
|
||||
: [
|
||||
{
|
||||
eventType: "native.session.process_rotation",
|
||||
stream: "system",
|
||||
payload: {
|
||||
reason: "run_scoped_github_capability",
|
||||
previousRunId: runs[index - 1]!.id,
|
||||
runId: run.id,
|
||||
companyId: run.companyId,
|
||||
agentId: run.agentId,
|
||||
nativeSessionId: run.nativeSessionId,
|
||||
runnerInstanceId: run.runnerInstanceId,
|
||||
},
|
||||
},
|
||||
],
|
||||
}));
|
||||
return { runs, groups };
|
||||
}
|
||||
|
||||
describe("native warm process continuity", () => {
|
||||
it("requires matching controller events for each run-capability rotation", () => {
|
||||
const { runs, groups } = processFixture();
|
||||
expect(nativeWarmProcessFailures(runs, groups)).toEqual([]);
|
||||
});
|
||||
|
||||
it("retains the same-process requirement without credential rotation", () => {
|
||||
const { runs, groups } = processFixture();
|
||||
for (const run of runs) {
|
||||
run.processPid = runs[0]!.processPid;
|
||||
run.processStartedAt = runs[0]!.processStartedAt;
|
||||
}
|
||||
for (const group of groups) group.events = [];
|
||||
expect(nativeWarmProcessFailures(runs, groups)).toEqual([]);
|
||||
runs[1]!.processPid += 1;
|
||||
expect(nativeWarmProcessFailures(runs, groups)).not.toEqual([]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
"reason",
|
||||
"previousRunId",
|
||||
"runId",
|
||||
"companyId",
|
||||
"agentId",
|
||||
"nativeSessionId",
|
||||
"runnerInstanceId",
|
||||
] as const)("rejects an unrelated rotation %s", (field) => {
|
||||
const { runs, groups } = processFixture();
|
||||
groups[1]!.events[0]!.payload[field] = "unrelated";
|
||||
expect(nativeWarmProcessFailures(runs, groups)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it.each(["missing", "stdout", "duplicate", "configuration_changed"])(
|
||||
"rejects %s rotation evidence",
|
||||
(kind) => {
|
||||
const { runs, groups } = processFixture();
|
||||
const events = groups[1]!.events;
|
||||
if (kind === "missing") events.pop();
|
||||
if (kind === "stdout") events[0]!.stream = "stdout";
|
||||
if (kind === "duplicate") events.push(events[0]!);
|
||||
if (kind === "configuration_changed") events[0]!.payload.reason = kind;
|
||||
expect(nativeWarmProcessFailures(runs, groups)).toHaveLength(1);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["invalid", "2026-09-09T04:00:59Z", "2026-09-09T04:01:11Z"])(
|
||||
"rejects a process start outside the new run: %s",
|
||||
(time) => {
|
||||
const { runs, groups } = processFixture();
|
||||
runs[1]!.processStartedAt = time;
|
||||
expect(nativeWarmProcessFailures(runs, groups).length).toBeGreaterThan(0);
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects a claimed rotation that kept the previous process", () => {
|
||||
const { runs, groups } = processFixture();
|
||||
runs[1]!.processPid = runs[0]!.processPid;
|
||||
runs[1]!.processStartedAt = runs[0]!.processStartedAt;
|
||||
expect(nativeWarmProcessFailures(runs, groups)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("rejects a missing event stream even for stable processes", () => {
|
||||
const { runs } = processFixture();
|
||||
expect(nativeWarmProcessFailures(runs, [])).toHaveLength(3);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -98,3 +98,84 @@ export async function readWarmWorkspaceFile(input: {
|
|||
);
|
||||
return { source: "task-cache", content: await response.text() };
|
||||
}
|
||||
|
||||
/** Stable warm processes or an explicitly recorded run-capability rotation. */
|
||||
export function nativeWarmProcessFailures(
|
||||
runs: readonly {
|
||||
id: string;
|
||||
companyId: string;
|
||||
agentId: string;
|
||||
nativeSessionId?: string | null;
|
||||
runnerInstanceId?: string | null;
|
||||
processPid?: number | null;
|
||||
processStartedAt?: string | null;
|
||||
startedAt?: string | null;
|
||||
finishedAt?: string | null;
|
||||
}[],
|
||||
eventGroups: readonly {
|
||||
runId: string;
|
||||
events: readonly {
|
||||
eventType?: string;
|
||||
stream?: string | null;
|
||||
payload?: Record<string, unknown> | null;
|
||||
}[];
|
||||
}[],
|
||||
): string[] {
|
||||
const failures: string[] = [];
|
||||
for (const [index, run] of runs.entries()) {
|
||||
if (
|
||||
!Number.isSafeInteger(run.processPid) ||
|
||||
run.processPid! <= 0 ||
|
||||
!Number.isFinite(Date.parse(run.processStartedAt ?? ""))
|
||||
) {
|
||||
failures.push(`missing native process identity for warm run ${run.id}`);
|
||||
continue;
|
||||
}
|
||||
const group = eventGroups.find((group) => group.runId === run.id);
|
||||
if (!group) {
|
||||
failures.push(`missing native process events for warm run ${run.id}`);
|
||||
continue;
|
||||
}
|
||||
const rotations = group.events.filter(
|
||||
(event) => event.eventType === "native.session.process_rotation",
|
||||
);
|
||||
const previous = runs[index - 1];
|
||||
const sameProcess =
|
||||
!previous ||
|
||||
(previous.processPid === run.processPid &&
|
||||
previous.processStartedAt === run.processStartedAt);
|
||||
if (sameProcess) {
|
||||
if (rotations.length)
|
||||
failures.push(
|
||||
`unexpected native process rotation for warm run ${run.id}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const event = rotations[0];
|
||||
const payload = event?.payload;
|
||||
const started = Date.parse(run.startedAt ?? "");
|
||||
const finished = Date.parse(run.finishedAt ?? "");
|
||||
const processStarted = Date.parse(run.processStartedAt!);
|
||||
if (
|
||||
rotations.length !== 1 ||
|
||||
event?.stream !== "system" ||
|
||||
payload?.reason !== "run_scoped_github_capability" ||
|
||||
payload.previousRunId !== previous.id ||
|
||||
payload.runId !== run.id ||
|
||||
payload.companyId !== run.companyId ||
|
||||
payload.agentId !== run.agentId ||
|
||||
payload.nativeSessionId !== run.nativeSessionId ||
|
||||
payload.runnerInstanceId !== run.runnerInstanceId ||
|
||||
!Number.isFinite(started) ||
|
||||
!Number.isFinite(finished) ||
|
||||
processStarted < started ||
|
||||
processStarted > finished ||
|
||||
processStarted <= Date.parse(previous.processStartedAt ?? "")
|
||||
) {
|
||||
failures.push(
|
||||
`native warm process changed without a matching run-scoped credential rotation for ${run.id}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return failures;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue