fix: preserve sandbox credentials and Git state across restarts
Park idle native sandbox sessions before app shutdown, adopt retained legacy workspaces without restaging Git, and carry explicitly bound GitHub access through both OpenCode launch filters. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta
parent
816045d643
commit
c7b68b52b0
|
|
@ -71,6 +71,11 @@ startup recovers the retained workspace from the matching task, project, agent,
|
|||
responsible user, and sandbox environment. Local execution, explicit workspace
|
||||
choices, and tasks that have entered scoped persistence do not use this fallback.
|
||||
The normal workspace freshness and provider identity checks still apply.
|
||||
After a validated legacy resume, both runner generations adopt the existing
|
||||
sandbox working copy without uploading a replacement Git directory or host
|
||||
overlay. This preserves its index and repository-local state. The legacy
|
||||
outbound merge still saves working files to the host. A missing native sync
|
||||
stamp on a pre-change lease does not make the host copy authoritative.
|
||||
|
||||
Acceptance must resume representative pre-upgrade legacy and native tasks with
|
||||
committed, staged, unstaged, and untracked work, verify their original paths and
|
||||
|
|
@ -93,6 +98,10 @@ inside ES-module repositories can still execute Git and GitHub CLI commands. Nat
|
|||
through both runnerd’s Rust sidecar filter and the ACPX JavaScript launch filter,
|
||||
including the explicitly controller-projected GitHub broker environment. ACPX does not inherit ambient host GitHub credentials or
|
||||
shell startup hooks. CLI configuration remains in its private runtime directories.
|
||||
Native OpenCode preserves the same explicit GitHub binding through both its
|
||||
runner proxy and provider process. It does not inherit repository credentials
|
||||
or shell startup hooks from the host; provider diagnostics redact the projected
|
||||
capabilities and credential configuration values.
|
||||
Warm sandbox task bindings persist independently of the experimental isolated
|
||||
workspace setting. Only the active host run can establish that binding; the
|
||||
setting still controls user-configurable worktree operations.
|
||||
|
|
@ -310,3 +319,11 @@ The configured origin, commit identity, artifact integrity, migration coverage,
|
|||
dependency lockfile and tenant readiness are checked before acceptance. Preview
|
||||
artifacts cannot become the fleet default. Record both the build workflow and
|
||||
the resulting object identities with the acceptance evidence.
|
||||
|
||||
On graceful app shutdown, idle native sandbox sessions are parked and checkpointed
|
||||
before application services and the database close. The drain runs with bounded
|
||||
concurrency and a 30-second deadline; failed or timed-out checkpoints are logged
|
||||
as incomplete. Active native turns keep their existing restart/reattach behavior.
|
||||
Local and SSH session shutdown behavior is unchanged. A hard process kill cannot
|
||||
guarantee a provider-session checkpoint; scoped files and repository durability
|
||||
remain limited to the last successfully published work-folder checkpoint.
|
||||
|
|
|
|||
|
|
@ -195,6 +195,8 @@ export interface AdapterSandboxExecutionTarget extends AdapterExecutionTargetWor
|
|||
readonly reusableLeaseConfigured?: boolean;
|
||||
/** Host-observed provenance for this exact sandbox acquisition. */
|
||||
readonly sandboxLeaseAcquisition?: SandboxLeaseAcquisition | null;
|
||||
/** Host-validated resume of an old task: retain its authoritative working copy. */
|
||||
readonly legacyWorkspaceResume?: boolean;
|
||||
shellCommand?: "bash" | "sh" | null;
|
||||
environmentId?: string | null;
|
||||
leaseId?: string | null;
|
||||
|
|
@ -1474,7 +1476,8 @@ export async function prepareAdapterExecutionTargetRuntime(input: {
|
|||
workspaceLocalDir: input.workspaceLocalDir,
|
||||
workspaceRemoteDir: input.workspaceRemoteDir,
|
||||
syncWorkspace: target.workFolderHome ? false : input.syncWorkspace,
|
||||
workspaceInboundMode: input.workspaceInboundMode,
|
||||
workspaceInboundMode: input.workspaceInboundMode
|
||||
?? (target.legacyWorkspaceResume ? "adopt_remote" : undefined),
|
||||
workspaceDurableSeed: input.workspaceDurableSeed,
|
||||
workspaceBaseline: input.workspaceBaseline,
|
||||
workspaceGitSnapshot: input.workspaceGitSnapshot,
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ import {
|
|||
type StartupTracer,
|
||||
} from "./acpx-engine/startup-timing.js";
|
||||
import type { RunProcessResult } from "./server-utils.js";
|
||||
import { prepareAdapterExecutionTargetRuntime } from "./execution-target.js";
|
||||
|
||||
function toArrayBuffer(bytes: Buffer): ArrayBuffer {
|
||||
return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer;
|
||||
|
|
@ -402,6 +403,51 @@ describe("sandbox managed runtime", () => {
|
|||
).resolves.toBe("remote finalized\n");
|
||||
});
|
||||
|
||||
it.each([undefined, "adopt_remote"] as const)("preserves an old task's Git index and local state through a resumed target (%s)", async (workspaceInboundMode) => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-legacy-git-resume-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const remoteDir = path.join(rootDir, "remote");
|
||||
const localDir = path.join(rootDir, "host");
|
||||
await initGitRepo(remoteDir);
|
||||
await git(rootDir, ["clone", remoteDir, localDir]);
|
||||
await writeFile(path.join(remoteDir, "unpublished.txt"), "remote commit\n");
|
||||
await git(remoteDir, ["add", "unpublished.txt"]);
|
||||
await git(remoteDir, ["commit", "-qm", "unpublished"]);
|
||||
const head = await git(remoteDir, ["rev-parse", "HEAD"]);
|
||||
await writeFile(path.join(remoteDir, "README.md"), "staged\n");
|
||||
await git(remoteDir, ["add", "README.md"]);
|
||||
await writeFile(path.join(remoteDir, "README.md"), "unstaged\n");
|
||||
await writeFile(path.join(remoteDir, "untracked.txt"), "untracked\n");
|
||||
await fsPromises.appendFile(path.join(remoteDir, ".git", "info", "exclude"), "\n.cache-marker\n");
|
||||
await writeFile(path.join(remoteDir, ".cache-marker"), "retained cache\n");
|
||||
await git(remoteDir, ["config", "paperclip.retained", "yes"]);
|
||||
const prepared = await prepareAdapterExecutionTargetRuntime({
|
||||
runId: "legacy-resume", adapterKey: "test", workspaceLocalDir: localDir,
|
||||
target: { kind: "remote", transport: "sandbox", remoteCwd: remoteDir,
|
||||
providerKey: "test", leaseId: "retained-lease", legacyWorkspaceResume: true,
|
||||
runner: makeInlineSpawnRunner() },
|
||||
workspaceInboundMode,
|
||||
...(workspaceInboundMode ? { workspaceDurableSeed: {
|
||||
workspaceArchivePath: path.join(rootDir, "seed", "workspace.tar"),
|
||||
gitArchivePath: path.join(rootDir, "seed", "git.tar"),
|
||||
} } : {}),
|
||||
});
|
||||
const verifyRemote = async () => {
|
||||
expect(await git(remoteDir, ["rev-parse", "HEAD"])).toBe(head);
|
||||
expect(await git(remoteDir, ["show", ":README.md"])).toBe("staged");
|
||||
expect(await readFile(path.join(remoteDir, "README.md"), "utf8")).toBe("unstaged\n");
|
||||
expect(await readFile(path.join(remoteDir, "untracked.txt"), "utf8")).toBe("untracked\n");
|
||||
expect(await git(remoteDir, ["config", "paperclip.retained"])).toBe("yes");
|
||||
expect(await git(remoteDir, ["check-ignore", ".cache-marker"])).toBe(".cache-marker");
|
||||
expect(await readFile(path.join(remoteDir, ".cache-marker"), "utf8")).toBe("retained cache\n");
|
||||
};
|
||||
await verifyRemote();
|
||||
await prepared.restoreWorkspace();
|
||||
await verifyRemote();
|
||||
expect(await readFile(path.join(localDir, "README.md"), "utf8")).toBe("unstaged\n");
|
||||
expect(await git(localDir, ["rev-parse", "HEAD"])).toBe(head);
|
||||
});
|
||||
|
||||
it("reconstructs a replacement workspace from the exact durable pre-turn seed", async () => {
|
||||
const rootDir = await mkdtemp(
|
||||
path.join(os.tmpdir(), "paperclip-sandbox-durable-seed-"),
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import {
|
|||
} from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { afterAll, afterEach, describe, expect, it } from "vitest";
|
||||
import { afterAll, afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import {
|
||||
NATIVE_RUNTIME_ASSET_SCHEMA,
|
||||
|
|
@ -101,6 +101,30 @@ afterAll(async () => {
|
|||
});
|
||||
|
||||
describe("OpenCodeServerDriver", () => {
|
||||
it.each([undefined, { PATH: process.env.PATH }])("does not inherit ambient GitHub access with environment %j", async (environment) => {
|
||||
await chmod(fixture, 0o755);
|
||||
const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-no-ambient-"));
|
||||
const workspace = join(root, "workspace");
|
||||
await mkdir(workspace);
|
||||
roots.push(root);
|
||||
vi.stubEnv("PAPERCLIP_GITHUB_BROKER_TOKEN", "ambient-host-secret");
|
||||
vi.stubEnv("BASH_ENV", "/ambient/shell-hook");
|
||||
const driver = new OpenCodeServerDriver({
|
||||
model: "openrouter/deepseek/deepseek-v4-flash-0731",
|
||||
runtimeDirectory: root, command: fixture, environment,
|
||||
});
|
||||
let session;
|
||||
try {
|
||||
session = await driver.openSession({runId: "run-no-ambient", normalizedSessionId: "no-ambient", workingDirectory: workspace});
|
||||
const child = JSON.parse(await readFile(join(root, "no-ambient", "data", "fake-environment.json"), "utf8"));
|
||||
expect(child.keys).not.toContain("PAPERCLIP_GITHUB_BROKER_TOKEN");
|
||||
expect(child.keys).not.toContain("BASH_ENV");
|
||||
} finally {
|
||||
await session?.close({reason: "test"});
|
||||
vi.unstubAllEnvs();
|
||||
}
|
||||
});
|
||||
|
||||
it("advertises within-turn plans as unsupported", async () => {
|
||||
const driver = new OpenCodeServerDriver({
|
||||
model: "openrouter/deepseek/deepseek-v4-flash-0731",
|
||||
|
|
@ -604,6 +628,14 @@ describe("OpenCodeServerDriver", () => {
|
|||
OPENROUTER_API_KEY: "test-openrouter-key",
|
||||
PAPERCLIP_API_KEY: "must-not-leak",
|
||||
UNRELATED_SECRET: "must-not-leak",
|
||||
PAPERCLIP_GITHUB_BROKER_TOKEN: "fixture-github-capability",
|
||||
PAPERCLIP_GITHUB_BRIDGE_TOKEN: "fixture-github-bridge",
|
||||
PAPERCLIP_GITHUB_BROKER_URL: "http://127.0.0.1:3456",
|
||||
BASH_ENV: "/runtime/github-launcher/profile.sh",
|
||||
GIT_CONFIG_COUNT: "1",
|
||||
GIT_CONFIG_KEY_0: "credential.helper",
|
||||
GIT_CONFIG_VALUE_0: "",
|
||||
GIT_CONFIG_KEY_1: "must-not-reach-provider",
|
||||
PAPERCLIP_PROVIDER_TRACE_PATH: tracePath,
|
||||
PAPERCLIP_PROVIDER_TRACE_MAX_BYTES: String(64 * 1024 * 1024),
|
||||
},
|
||||
|
|
@ -750,6 +782,12 @@ describe("OpenCodeServerDriver", () => {
|
|||
expect(environment.keys).not.toContain("PAPERCLIP_API_KEY");
|
||||
expect(environment.keys).not.toContain("UNRELATED_SECRET");
|
||||
expect(environment.keys).not.toContain("PAPERCLIP_PROVIDER_TRACE_PATH");
|
||||
expect(environment.keys).toEqual(expect.arrayContaining([
|
||||
"PAPERCLIP_GITHUB_BROKER_TOKEN", "PAPERCLIP_GITHUB_BRIDGE_TOKEN",
|
||||
"PAPERCLIP_GITHUB_BROKER_URL", "BASH_ENV", "GIT_CONFIG_COUNT",
|
||||
"GIT_CONFIG_KEY_0", "GIT_CONFIG_VALUE_0",
|
||||
]));
|
||||
expect(environment.keys).not.toContain("GIT_CONFIG_KEY_1");
|
||||
expect(environment.projectConfigDisabled).toBe("true");
|
||||
expect(mcpEvidence.tools).toEqual(
|
||||
expect.arrayContaining(["paperclip_finish", "paperclip_block"]),
|
||||
|
|
@ -1792,7 +1830,7 @@ describe("OpenCodeServerDriver", () => {
|
|||
const exitingFixture = join(root, "exit-before-health.mjs");
|
||||
await writeFile(
|
||||
exitingFixture,
|
||||
"#!/usr/bin/env node\nprocess.stderr.write(`credential=${process.env.OPENROUTER_API_KEY}\\nauthorization=super-secret-opencode-token\\n`);\nprocess.exit(17);\n",
|
||||
"#!/usr/bin/env node\nprocess.stderr.write(`credential=${process.env.OPENROUTER_API_KEY}\\nauthorization=super-secret-opencode-token\\n${process.env.PAPERCLIP_GITHUB_BROKER_TOKEN}\\n${process.env.PAPERCLIP_GITHUB_BRIDGE_TOKEN}\\n${process.env.GIT_CONFIG_VALUE_0}\\n`);\nprocess.exit(17);\n",
|
||||
{ mode: 0o755 },
|
||||
);
|
||||
const driver = new OpenCodeServerDriver({
|
||||
|
|
@ -1802,6 +1840,11 @@ describe("OpenCodeServerDriver", () => {
|
|||
environment: {
|
||||
PATH: process.env.PATH,
|
||||
OPENROUTER_API_KEY: "fixture-key",
|
||||
PAPERCLIP_GITHUB_BROKER_TOKEN: "fixture-github-capability",
|
||||
PAPERCLIP_GITHUB_BRIDGE_TOKEN: "fixture-github-bridge",
|
||||
GIT_CONFIG_COUNT: "1",
|
||||
GIT_CONFIG_KEY_0: "http.extraHeader",
|
||||
GIT_CONFIG_VALUE_0: "fixture-github-header",
|
||||
},
|
||||
});
|
||||
const error = await driver
|
||||
|
|
@ -1820,5 +1863,8 @@ describe("OpenCodeServerDriver", () => {
|
|||
expect(error).toContain("[REDACTED]");
|
||||
expect(error).not.toContain("fixture-key");
|
||||
expect(error).not.toContain("super-secret-opencode-token");
|
||||
expect(error).not.toContain("fixture-github-capability");
|
||||
expect(error).not.toContain("fixture-github-bridge");
|
||||
expect(error).not.toContain("fixture-github-header");
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { externalWorkFolderEnvironment } from "../../work-folder-environment.js";
|
||||
import { githubCredentialEnvironment } from "../../github-credential-environment.js";
|
||||
import { spawn, type ChildProcess } from "node:child_process";
|
||||
import { randomBytes } from "node:crypto";
|
||||
import {
|
||||
|
|
@ -251,9 +252,7 @@ export class OpenCodeServerDriver implements HarnessDriver {
|
|||
} catch (error) {
|
||||
return {
|
||||
recovered: false,
|
||||
reason: redact(String(error), [
|
||||
this.#options.environment?.OPENROUTER_API_KEY,
|
||||
]),
|
||||
reason: redact(String(error), openCodeSensitiveValues(this.#options.environment)),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -1858,12 +1857,12 @@ async function startRuntime(input: {
|
|||
const assignedMcp = nativeMcpLaunchBinding(
|
||||
input.options.environment ?? process.env,
|
||||
);
|
||||
const sensitiveValues = [password, ...openCodeSensitiveValues(input.options.environment)];
|
||||
input.trace?.addSensitiveValues([
|
||||
password,
|
||||
...sensitiveValues,
|
||||
authHeader,
|
||||
bridge.secret,
|
||||
assignedMcp?.token,
|
||||
input.options.environment?.OPENROUTER_API_KEY,
|
||||
]);
|
||||
const instructionRoot =
|
||||
input.options.runtimeContext?.instructions.bundle.rootPath;
|
||||
|
|
@ -1933,6 +1932,9 @@ async function startRuntime(input: {
|
|||
{
|
||||
HOME: isolatedHome,
|
||||
...externalWorkFolderEnvironment(input.options.environment ?? {}),
|
||||
...(input.options.environment
|
||||
? githubCredentialEnvironment(input.options.environment)
|
||||
: {}),
|
||||
XDG_CONFIG_HOME: configHome,
|
||||
XDG_DATA_HOME: dataHome,
|
||||
XDG_CACHE_HOME: cacheHome,
|
||||
|
|
@ -1969,10 +1971,7 @@ async function startRuntime(input: {
|
|||
let diagnostics = "";
|
||||
child.stderr?.on("data", (chunk) => {
|
||||
const raw = Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk));
|
||||
const redactedDiagnostic = redact(raw.toString("utf8"), [
|
||||
password,
|
||||
input.options.environment?.OPENROUTER_API_KEY,
|
||||
]);
|
||||
const redactedDiagnostic = redact(raw.toString("utf8"), sensitiveValues);
|
||||
diagnostics = `${diagnostics}${redactedDiagnostic}`.slice(-8_192);
|
||||
const frameId = input.trace?.frame({
|
||||
direction: "provider_stderr",
|
||||
|
|
@ -2035,10 +2034,7 @@ async function startRuntime(input: {
|
|||
process: child,
|
||||
bridge,
|
||||
trace: input.trace,
|
||||
sensitiveValues: [
|
||||
password,
|
||||
input.options.environment?.OPENROUTER_API_KEY,
|
||||
].filter((value): value is string => Boolean(value)),
|
||||
sensitiveValues,
|
||||
close: async (closeInput = {}) => {
|
||||
await bridge.close().catch(() => {});
|
||||
if (child.exitCode === null && child.signalCode === null && child.pid) {
|
||||
|
|
@ -2537,6 +2533,16 @@ function safeTraceRulePath(value: string): string {
|
|||
.replaceAll("/", "_")
|
||||
.slice(0, 120);
|
||||
}
|
||||
function openCodeSensitiveValues(environment: NodeJS.ProcessEnv | undefined): string[] {
|
||||
const github = environment ? githubCredentialEnvironment(environment) : {};
|
||||
return [
|
||||
environment?.OPENROUTER_API_KEY,
|
||||
...Object.entries(github)
|
||||
.filter(([key]) => key.endsWith("TOKEN") || /^GIT_CONFIG_VALUE_\d+$/.test(key))
|
||||
.map(([, value]) => value),
|
||||
].filter((value): value is string => Boolean(value));
|
||||
}
|
||||
|
||||
function redact(
|
||||
value: string,
|
||||
sensitiveValues: readonly (string | undefined)[] = [],
|
||||
|
|
|
|||
|
|
@ -1070,6 +1070,39 @@ it.each(["codex", "opencode", "acpx"] as const)("keeps the natural sandbox home
|
|||
expect(() => createRunnerdCodexAppServerArgs({ environment, codexHome: `${home}/.codex` })).not.toThrow();
|
||||
});
|
||||
|
||||
it("preserves only explicit GitHub bindings through the OpenCode runner boundary", () => {
|
||||
vi.stubEnv("PAPERCLIP_GITHUB_BROKER_TOKEN", "ambient-host-secret");
|
||||
vi.stubEnv("BASH_ENV", "/ambient/shell-hook");
|
||||
const projected = {
|
||||
PAPERCLIP_GITHUB_BROKER_TOKEN: "controller-run-capability",
|
||||
PAPERCLIP_GITHUB_BRIDGE_TOKEN: "controller-bridge-capability",
|
||||
PAPERCLIP_GITHUB_BROKER_URL: "http://127.0.0.1:3456",
|
||||
BASH_ENV: "/runtime/github-launcher/profile.sh",
|
||||
GIT_CONFIG_COUNT: "1",
|
||||
GIT_CONFIG_KEY_0: "credential.helper",
|
||||
GIT_CONFIG_VALUE_0: "",
|
||||
};
|
||||
const launch = (environment: NodeJS.ProcessEnv | undefined) => createCapabilityRunnerdProviderEnvironment({
|
||||
provider: "opencode",
|
||||
options: { provider: "opencode", environment },
|
||||
identity: { runnerInstanceId: "runner", environmentLeaseId: "lease", runId: "run", normalizedSessionId: "session", turnId: "turn", itemId: "item" },
|
||||
codexHome: "/isolated/home", runtimeContextPath: "/runtime/context.json", hasRuntimeContext: false,
|
||||
});
|
||||
try {
|
||||
const environment = launch({ ...projected, GIT_CONFIG_KEY_1: "unbounded", PAPERCLIP_API_KEY: "control-plane-secret", DATABASE_URL: "host-secret" });
|
||||
expect(environment).toMatchObject(projected);
|
||||
expect(environment).not.toHaveProperty("GIT_CONFIG_KEY_1");
|
||||
expect(environment).not.toHaveProperty("PAPERCLIP_API_KEY");
|
||||
expect(environment).not.toHaveProperty("DATABASE_URL");
|
||||
for (const input of [undefined, {}]) {
|
||||
expect(launch(input)).not.toHaveProperty("PAPERCLIP_GITHUB_BROKER_TOKEN");
|
||||
expect(launch(input)).not.toHaveProperty("BASH_ENV");
|
||||
}
|
||||
} finally {
|
||||
vi.unstubAllEnvs();
|
||||
}
|
||||
});
|
||||
|
||||
it("denies the isolated Codex home without denying a remote execution workspace", () => {
|
||||
const args = createRunnerdCodexAppServerArgs({
|
||||
environment: {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { externalWorkFolderEnvironment } from "../work-folder-environment.js";
|
||||
import { githubCredentialEnvironment } from "../github-credential-environment.js";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import {
|
||||
|
|
@ -1901,7 +1902,7 @@ function createSanitizedOpenCodeRunnerEnvironment(
|
|||
source: NodeJS.ProcessEnv | undefined,
|
||||
): NodeJS.ProcessEnv {
|
||||
const candidate = { ...process.env, ...source };
|
||||
return Object.fromEntries(
|
||||
const environment = Object.fromEntries(
|
||||
Object.entries(candidate).filter(
|
||||
([key, value]) =>
|
||||
typeof value === "string" &&
|
||||
|
|
@ -1909,6 +1910,9 @@ function createSanitizedOpenCodeRunnerEnvironment(
|
|||
/^LC_[A-Z0-9_]{1,32}$/.test(key)),
|
||||
),
|
||||
);
|
||||
// Repository access comes only from the explicit controller projection,
|
||||
// never from credentials or shell hooks in the runner host's environment.
|
||||
return { ...environment, ...(source ? githubCredentialEnvironment(source) : {}) };
|
||||
}
|
||||
|
||||
export function resolveSourceCodexHome(
|
||||
|
|
|
|||
|
|
@ -219,7 +219,13 @@ describe("resolveEnvironmentExecutionTarget effective capability snapshot", () =
|
|||
expect(target.reusableLeaseConfigured).toBe(true);
|
||||
});
|
||||
|
||||
it("carries host-owned sandbox acquisition provenance without persisting provider ids in metadata", async () => {
|
||||
it.each([
|
||||
{ layout: "legacy", outcome: "resumed", adopt: true },
|
||||
{ layout: "legacy", outcome: "created", adopt: false },
|
||||
{ layout: "legacy", outcome: "replacement", adopt: false },
|
||||
{ layout: "scoped", outcome: "resumed", adopt: false },
|
||||
{ layout: undefined, outcome: "resumed", adopt: false },
|
||||
])("carries host-owned acquisition and legacy adoption for $layout/$outcome", async ({layout, outcome, adopt}) => {
|
||||
mockResolveEnvironmentDriverConfigForRuntime.mockResolvedValue({
|
||||
driver: "sandbox",
|
||||
config: { provider: "daytona", reuseLease: true, timeoutMs: 30_000 },
|
||||
|
|
@ -235,13 +241,14 @@ describe("resolveEnvironmentExecutionTarget effective capability snapshot", () =
|
|||
leaseId: "lease-row-1",
|
||||
leaseMetadata: {
|
||||
remoteCwd: "/work",
|
||||
workFolderLayout: "legacy",
|
||||
sandboxLeaseAcquisition: { outcome: "resumed" },
|
||||
},
|
||||
lease: {
|
||||
id: "lease-row-1",
|
||||
providerLeaseId: "daytona-sandbox-1",
|
||||
leasePolicy: "reuse_by_environment",
|
||||
metadata: { sandboxLeaseAcquisition: { outcome: "resumed" } },
|
||||
metadata: { workFolderLayout: layout, sandboxLeaseAcquisition: { outcome } },
|
||||
} as never,
|
||||
environmentRuntime: {
|
||||
supportsSync: () => false,
|
||||
|
|
@ -252,9 +259,10 @@ describe("resolveEnvironmentExecutionTarget effective capability snapshot", () =
|
|||
throw new Error("expected a sandbox target");
|
||||
}
|
||||
expect(target.sandboxLeaseAcquisition).toEqual({
|
||||
outcome: "resumed",
|
||||
outcome,
|
||||
providerLeaseId: "daytona-sandbox-1",
|
||||
});
|
||||
expect(target.legacyWorkspaceResume).toBe(adopt);
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -94,6 +94,17 @@ describe("native workspace sync durable metadata", () => {
|
|||
).toThrow("native_workspace_sync_unexpected_existing_descriptor");
|
||||
});
|
||||
|
||||
it("adopts only a validated legacy resume when an older release has no sync stamp", () => {
|
||||
for (const acquisition of ["created", "replacement", null] as const) {
|
||||
expect(classifyNativeWorkspaceInbound({kind: "new_run", acquisition,
|
||||
hasPriorStamp: false, legacyWorkspaceResume: true})).toBe("host_current");
|
||||
}
|
||||
for (const hasPriorStamp of [true, false]) {
|
||||
expect(classifyNativeWorkspaceInbound({kind: "new_run", acquisition: "resumed",
|
||||
hasPriorStamp, legacyWorkspaceResume: true})).toBe("adopt_remote");
|
||||
}
|
||||
});
|
||||
|
||||
it("reads backward-compatible references and the resource disposition", () => {
|
||||
const base = {
|
||||
schema: "paperclip.native-workspace-sync/v1",
|
||||
|
|
|
|||
|
|
@ -113,6 +113,7 @@ import {
|
|||
} from "./shutdown.js";
|
||||
import { initializeCloudRuntimeIdentity } from "./services/cloud-runtime-identity.js";
|
||||
import { systemdNotify } from "./services/systemd-notify.js";
|
||||
import { closeIdleSandboxNativeSessionsForShutdown } from "./services/native-runtime/native-session-executor.js";
|
||||
import { flushInFlightRunLogMirrors } from "./services/run-log-store.js";
|
||||
import {
|
||||
createEmbeddedPostgresSupervisor,
|
||||
|
|
@ -1900,6 +1901,9 @@ export async function startServer(): Promise<StartedServer> {
|
|||
await finalizeServerShutdown({
|
||||
signal,
|
||||
shutdownAppServices: appShutdown,
|
||||
closeIdleSandboxSessions: () => closeIdleSandboxNativeSessionsForShutdown({
|
||||
reason: `server shutdown (${signal})`,
|
||||
}),
|
||||
stopEmbeddedPostgres,
|
||||
shutdownInstrumentation,
|
||||
shutdownSentry,
|
||||
|
|
|
|||
|
|
@ -366,6 +366,10 @@ export async function resolveEnvironmentExecutionTarget(input: {
|
|||
const syncLease = () => target.workFolderHome
|
||||
? { ...input.lease!, metadata: { ...input.lease!.metadata, remoteCwd: target.workFolderHome } }
|
||||
: input.lease!;
|
||||
const acquisition = sandboxLeaseAcquisitionFromMetadata(
|
||||
input.lease?.metadata?.sandboxLeaseAcquisition,
|
||||
input.lease?.providerLeaseId,
|
||||
);
|
||||
const target: AdapterSandboxExecutionTarget = {
|
||||
kind: "remote",
|
||||
transport: "sandbox",
|
||||
|
|
@ -388,10 +392,9 @@ export async function resolveEnvironmentExecutionTarget(input: {
|
|||
? { mode: "per_turn", idleTimeoutMs: null }
|
||||
: null,
|
||||
reusableLeaseConfigured: parsed.config.reuseLease === true,
|
||||
sandboxLeaseAcquisition: sandboxLeaseAcquisitionFromMetadata(
|
||||
input.lease?.metadata?.sandboxLeaseAcquisition,
|
||||
input.lease?.providerLeaseId,
|
||||
),
|
||||
sandboxLeaseAcquisition: acquisition,
|
||||
legacyWorkspaceResume: acquisition?.outcome === "resumed"
|
||||
&& input.lease?.metadata?.workFolderLayout === "legacy",
|
||||
// Attach the host duplex observability recorder next to the runner. The bridge
|
||||
// binds it to the fixed observability surface. Absent keeps the no-op
|
||||
// default, so the surface stays inert on a run with no injected recorder.
|
||||
|
|
|
|||
|
|
@ -171,6 +171,7 @@ import {
|
|||
buildNativeHarnessBackupManifest,
|
||||
cancelNativeSession,
|
||||
closeWarmNativeSessionsForEnvironment,
|
||||
closeIdleSandboxNativeSessionsForShutdown,
|
||||
createGovernedWaitEventObservation,
|
||||
createRemoteRunnerProcessLauncher,
|
||||
createRunnerdBackend,
|
||||
|
|
@ -3073,6 +3074,73 @@ describe("native session same-turn steering", () => {
|
|||
});
|
||||
|
||||
describe("native warm session supervision", () => {
|
||||
it.each([
|
||||
{ transport: "sandbox", busy: false, fail: false },
|
||||
{ transport: "sandbox", busy: true, fail: false },
|
||||
{ transport: "sandbox", busy: false, fail: true },
|
||||
{ transport: "ssh", busy: false, fail: false },
|
||||
{ transport: "local", busy: false, fail: false },
|
||||
] as const)("shutdown parks only idle sandbox sessions: $transport busy=$busy fail=$fail", async ({ transport, busy, fail }) => {
|
||||
const id = `shutdown-${transport}-${busy}-${fail}`;
|
||||
const close = vi.fn(async () => {
|
||||
if (fail) throw new Error("checkpoint unavailable");
|
||||
});
|
||||
let release!: () => void;
|
||||
const held = new Promise<void>((resolve) => { release = resolve; });
|
||||
const warmExecution = {
|
||||
...execution,
|
||||
binding: { ...execution.binding, runId: id, executionWorkspaceId: id },
|
||||
session: {
|
||||
...execution.session,
|
||||
normalizedSessionId: id,
|
||||
lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 100 },
|
||||
},
|
||||
} as NativeExecutionInputV1;
|
||||
state.execute.mockReset().mockImplementationOnce(async (options) => {
|
||||
options.onSession?.({ close });
|
||||
if (busy) await held;
|
||||
return {
|
||||
result: { summary: "completed" },
|
||||
terminal: { runTerminalState: "succeeded" },
|
||||
turnId: id, normalizedSessionId: id, providerSessionId: id,
|
||||
driverKind: "test", driverVersion: "1", nativeEventCount: 1,
|
||||
highestContiguousSourceSeq: 1, usage: null,
|
||||
};
|
||||
});
|
||||
const running = executePaperclipNativeSession({
|
||||
db: leaseDb(warmExecution), execution: warmExecution, runnerInstanceId: id,
|
||||
runnerExecutionTarget: transport === "local" ? undefined : transport === "sandbox" ? {
|
||||
kind: "remote", transport: "sandbox", environmentId: id, remoteCwd: `/tmp/${id}`,
|
||||
} : {
|
||||
kind: "remote", transport: "ssh", environmentId: id, remoteCwd: `/tmp/${id}`,
|
||||
spec: {
|
||||
host: "runner.internal", port: 22, username: "runner",
|
||||
remoteWorkspacePath: `/tmp/${id}`, remoteCwd: `/tmp/${id}`,
|
||||
privateKey: null, knownHosts: null, strictHostKeyChecking: true,
|
||||
},
|
||||
},
|
||||
});
|
||||
await vi.waitFor(() => expect(state.execute).toHaveBeenCalledOnce());
|
||||
if (!busy) await running;
|
||||
try {
|
||||
const result = await closeIdleSandboxNativeSessionsForShutdown({ reason: "server shutdown" });
|
||||
if (transport === "sandbox" && !busy) {
|
||||
expect(close).toHaveBeenCalledExactlyOnceWith({ reason: "server shutdown" });
|
||||
expect(result[fail ? "failed" : "closed"]).toBeGreaterThanOrEqual(1);
|
||||
await closeIdleSandboxNativeSessionsForShutdown({ reason: "duplicate shutdown" });
|
||||
expect(close).toHaveBeenCalledOnce();
|
||||
} else {
|
||||
expect(close).not.toHaveBeenCalled();
|
||||
if (busy) expect(result.busy).toBeGreaterThanOrEqual(1);
|
||||
}
|
||||
} finally {
|
||||
release();
|
||||
await running;
|
||||
// Unselected sessions retain their existing idle-close behavior.
|
||||
await vi.waitFor(() => expect(close).toHaveBeenCalled(), { timeout: 1_000 });
|
||||
}
|
||||
});
|
||||
|
||||
it("closes an idle warm session before its remote environment is destroyed", async () => {
|
||||
const close = vi.fn(async () => undefined);
|
||||
const warmExecution = {
|
||||
|
|
|
|||
|
|
@ -272,6 +272,7 @@ type WarmNativeSession = {
|
|||
configDigest: string;
|
||||
companyId: string;
|
||||
environmentId: string | null;
|
||||
sandbox: boolean;
|
||||
busy: boolean;
|
||||
idleTimer: ReturnType<typeof setTimeout> | null;
|
||||
lastActivityAt: string;
|
||||
|
|
@ -315,6 +316,37 @@ export async function closeWarmNativeSessionsForEnvironment(input: {
|
|||
return { closed, busy, failed };
|
||||
}
|
||||
|
||||
/** Park idle sandbox sessions while their host still has database and transport
|
||||
* access. Active turns retain their existing restart/reattach contract. */
|
||||
export async function closeIdleSandboxNativeSessionsForShutdown(input: {
|
||||
reason: string;
|
||||
}): Promise<{ closed: number; busy: number; failed: number }> {
|
||||
const result = { closed: 0, busy: 0, failed: 0 };
|
||||
const pending: WarmNativeSession[] = [];
|
||||
for (const [sessionId, entry] of warmNativeSessions) {
|
||||
if (!entry.sandbox) continue;
|
||||
if (entry.busy) {
|
||||
result.busy += 1;
|
||||
continue;
|
||||
}
|
||||
if (entry.idleTimer !== null) clearTimeout(entry.idleTimer);
|
||||
// Fence every selected owner before the first asynchronous close.
|
||||
warmNativeSessions.delete(sessionId);
|
||||
pending.push(entry);
|
||||
}
|
||||
await Promise.all(Array.from({ length: Math.min(4, pending.length) }, async () => {
|
||||
for (let entry = pending.shift(); entry; entry = pending.shift()) {
|
||||
try {
|
||||
await entry.session.close({ reason: input.reason });
|
||||
result.closed += 1;
|
||||
} catch {
|
||||
result.failed += 1;
|
||||
}
|
||||
}
|
||||
}));
|
||||
return result;
|
||||
}
|
||||
|
||||
function readBoundedNativeFile(
|
||||
path: string,
|
||||
maxBytes: number,
|
||||
|
|
@ -4528,6 +4560,9 @@ async function executePaperclipNativeSessionWithinScope(
|
|||
companyId: input.execution.binding.companyId,
|
||||
environmentId:
|
||||
input.runnerExecutionTarget?.environmentId ?? null,
|
||||
sandbox:
|
||||
input.runnerExecutionTarget?.kind === "remote" &&
|
||||
input.runnerExecutionTarget.transport === "sandbox",
|
||||
busy: true,
|
||||
idleTimer: null,
|
||||
lastActivityAt: new Date().toISOString(),
|
||||
|
|
|
|||
|
|
@ -97,6 +97,7 @@ export type NativeWorkspaceInboundEvidence =
|
|||
kind: "new_run";
|
||||
acquisition: "created" | "resumed" | "replacement" | null;
|
||||
hasPriorStamp: boolean;
|
||||
legacyWorkspaceResume?: boolean;
|
||||
};
|
||||
|
||||
export function classifyNativeWorkspaceInbound(
|
||||
|
|
@ -108,7 +109,7 @@ export function classifyNativeWorkspaceInbound(
|
|||
}
|
||||
return evidence.sameProviderLease ? "adopt_remote" : "durable_seed";
|
||||
}
|
||||
return evidence.acquisition === "resumed" && evidence.hasPriorStamp
|
||||
return evidence.acquisition === "resumed" && (evidence.hasPriorStamp || evidence.legacyWorkspaceResume)
|
||||
? "adopt_remote"
|
||||
: "host_current";
|
||||
}
|
||||
|
|
@ -823,6 +824,7 @@ export async function prepareNativeWorkspaceSync(input: {
|
|||
kind: "new_run",
|
||||
acquisition,
|
||||
hasPriorStamp: priorStamp !== null,
|
||||
legacyWorkspaceResume: target.legacyWorkspaceResume,
|
||||
});
|
||||
runtime = await prepareRuntime({
|
||||
runId: input.runId,
|
||||
|
|
@ -838,7 +840,7 @@ export async function prepareNativeWorkspaceSync(input: {
|
|||
if (!currentSnapshot) {
|
||||
throw new Error("native_workspace_sync_snapshot_missing");
|
||||
}
|
||||
if (acquisition === "resumed" && priorStamp) {
|
||||
if (acquisition === "resumed" && priorStamp && !target.legacyWorkspaceResume) {
|
||||
const currentHostSha256 = directorySnapshotSha256(
|
||||
currentSnapshot.baseline,
|
||||
);
|
||||
|
|
|
|||
|
|
@ -22,6 +22,47 @@ function stubLogger() {
|
|||
}
|
||||
|
||||
describe("finalizeServerShutdown", () => {
|
||||
it("awaits idle sandbox checkpoints before app and database teardown", async () => {
|
||||
const release = deferred<{ closed: number; busy: number; failed: number }>();
|
||||
const closeIdleSandboxSessions = vi.fn(() => release.promise);
|
||||
const shutdownAppServices = vi.fn(async () => undefined);
|
||||
const stopEmbeddedPostgres = vi.fn(async () => undefined);
|
||||
const pending = finalizeServerShutdown({
|
||||
signal: "SIGTERM", closeIdleSandboxSessions, shutdownAppServices, stopEmbeddedPostgres,
|
||||
shutdownInstrumentation: async () => undefined,
|
||||
shutdownSentry: async () => undefined, log: stubLogger(),
|
||||
});
|
||||
await vi.waitFor(() => expect(closeIdleSandboxSessions).toHaveBeenCalledOnce());
|
||||
expect(shutdownAppServices).not.toHaveBeenCalled();
|
||||
expect(stopEmbeddedPostgres).not.toHaveBeenCalled();
|
||||
release.resolve({ closed: 2, busy: 1, failed: 0 });
|
||||
await pending;
|
||||
expect(stopEmbeddedPostgres).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each(["timeout", "rejected", "incomplete"] as const)("reports %s idle checkpoint without blocking shutdown forever", async (failure) => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const log = stubLogger();
|
||||
const stopEmbeddedPostgres = vi.fn(async () => undefined);
|
||||
const pending = finalizeServerShutdown({
|
||||
signal: "SIGTERM", sandboxSessionTimeoutMs: 250,
|
||||
closeIdleSandboxSessions: async () => {
|
||||
if (failure === "timeout") return new Promise(() => undefined);
|
||||
if (failure === "rejected") throw new Error("transport disconnected");
|
||||
return { closed: 0, busy: 0, failed: 1 };
|
||||
},
|
||||
shutdownAppServices: undefined, stopEmbeddedPostgres,
|
||||
shutdownInstrumentation: async () => undefined,
|
||||
shutdownSentry: async () => undefined, log,
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(250);
|
||||
await pending;
|
||||
expect(log.error).toHaveBeenCalledOnce();
|
||||
expect(stopEmbeddedPostgres).toHaveBeenCalledOnce();
|
||||
} finally { vi.useRealTimers(); }
|
||||
});
|
||||
|
||||
it("awaits the setup-token cleanup before the database stop and the process exit", async () => {
|
||||
const order: string[] = [];
|
||||
// The held promise models the setup-token session cancellation and its
|
||||
|
|
|
|||
|
|
@ -52,6 +52,8 @@ export async function drainRunExecutionFinalizersForShutdown(input: {
|
|||
export async function finalizeServerShutdown(input: {
|
||||
signal: "SIGINT" | "SIGTERM";
|
||||
shutdownAppServices: (() => Promise<void>) | undefined;
|
||||
closeIdleSandboxSessions?: () => Promise<{ closed: number; busy: number; failed: number }>;
|
||||
sandboxSessionTimeoutMs?: number;
|
||||
stopEmbeddedPostgres: (() => Promise<void>) | null;
|
||||
shutdownInstrumentation: () => Promise<void>;
|
||||
shutdownSentry: () => Promise<void>;
|
||||
|
|
@ -59,6 +61,32 @@ export async function finalizeServerShutdown(input: {
|
|||
}): Promise<void> {
|
||||
const { signal } = input;
|
||||
|
||||
if (input.closeIdleSandboxSessions) {
|
||||
let timer: NodeJS.Timeout | null = null;
|
||||
const timeoutMs = input.sandboxSessionTimeoutMs ?? 30_000;
|
||||
try {
|
||||
const result = await Promise.race([
|
||||
input.closeIdleSandboxSessions(),
|
||||
new Promise<null>((resolve) => {
|
||||
timer = setTimeout(() => resolve(null), timeoutMs);
|
||||
timer.unref?.();
|
||||
}),
|
||||
]);
|
||||
if (result === null || result.failed > 0) {
|
||||
input.log.error(
|
||||
{ signal, timeoutMs, result },
|
||||
"Idle sandbox native session checkpoint incomplete during shutdown",
|
||||
);
|
||||
} else {
|
||||
input.log.info({ signal, result }, "Idle sandbox native sessions parked for shutdown");
|
||||
}
|
||||
} catch (err) {
|
||||
input.log.error({ signal, err }, "Idle sandbox native session shutdown failed");
|
||||
} finally {
|
||||
if (timer) clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
// Await the application service cleanup, so a live setup-token login session
|
||||
// releases its sandbox lease before the database and the provider stop. A
|
||||
// rejected cleanup stays durable for the reaper; it does not block the exit.
|
||||
|
|
|
|||
Loading…
Reference in New Issue