diff --git a/doc/sandbox-work-folders.md b/doc/sandbox-work-folders.md index e10bdf69ff..771e888586 100644 --- a/doc/sandbox-work-folders.md +++ b/doc/sandbox-work-folders.md @@ -71,6 +71,11 @@ startup recovers the retained workspace from the matching task, project, agent, responsible user, and sandbox environment. Local execution, explicit workspace choices, and tasks that have entered scoped persistence do not use this fallback. The normal workspace freshness and provider identity checks still apply. +After a validated legacy resume, both runner generations adopt the existing +sandbox working copy without uploading a replacement Git directory or host +overlay. This preserves its index and repository-local state. The legacy +outbound merge still saves working files to the host. A missing native sync +stamp on a pre-change lease does not make the host copy authoritative. Acceptance must resume representative pre-upgrade legacy and native tasks with committed, staged, unstaged, and untracked work, verify their original paths and @@ -93,6 +98,10 @@ inside ES-module repositories can still execute Git and GitHub CLI commands. Nat through both runnerd’s Rust sidecar filter and the ACPX JavaScript launch filter, including the explicitly controller-projected GitHub broker environment. ACPX does not inherit ambient host GitHub credentials or shell startup hooks. CLI configuration remains in its private runtime directories. +Native OpenCode preserves the same explicit GitHub binding through both its +runner proxy and provider process. It does not inherit repository credentials +or shell startup hooks from the host; provider diagnostics redact the projected +capabilities and credential configuration values. Warm sandbox task bindings persist independently of the experimental isolated workspace setting. Only the active host run can establish that binding; the setting still controls user-configurable worktree operations. @@ -310,3 +319,11 @@ The configured origin, commit identity, artifact integrity, migration coverage, dependency lockfile and tenant readiness are checked before acceptance. Preview artifacts cannot become the fleet default. Record both the build workflow and the resulting object identities with the acceptance evidence. + +On graceful app shutdown, idle native sandbox sessions are parked and checkpointed +before application services and the database close. The drain runs with bounded +concurrency and a 30-second deadline; failed or timed-out checkpoints are logged +as incomplete. Active native turns keep their existing restart/reattach behavior. +Local and SSH session shutdown behavior is unchanged. A hard process kill cannot +guarantee a provider-session checkpoint; scoped files and repository durability +remain limited to the last successfully published work-folder checkpoint. diff --git a/packages/adapter-utils/src/execution-target.ts b/packages/adapter-utils/src/execution-target.ts index 1ae721da9d..6f7628e324 100644 --- a/packages/adapter-utils/src/execution-target.ts +++ b/packages/adapter-utils/src/execution-target.ts @@ -195,6 +195,8 @@ export interface AdapterSandboxExecutionTarget extends AdapterExecutionTargetWor readonly reusableLeaseConfigured?: boolean; /** Host-observed provenance for this exact sandbox acquisition. */ readonly sandboxLeaseAcquisition?: SandboxLeaseAcquisition | null; + /** Host-validated resume of an old task: retain its authoritative working copy. */ + readonly legacyWorkspaceResume?: boolean; shellCommand?: "bash" | "sh" | null; environmentId?: string | null; leaseId?: string | null; @@ -1474,7 +1476,8 @@ export async function prepareAdapterExecutionTargetRuntime(input: { workspaceLocalDir: input.workspaceLocalDir, workspaceRemoteDir: input.workspaceRemoteDir, syncWorkspace: target.workFolderHome ? false : input.syncWorkspace, - workspaceInboundMode: input.workspaceInboundMode, + workspaceInboundMode: input.workspaceInboundMode + ?? (target.legacyWorkspaceResume ? "adopt_remote" : undefined), workspaceDurableSeed: input.workspaceDurableSeed, workspaceBaseline: input.workspaceBaseline, workspaceGitSnapshot: input.workspaceGitSnapshot, diff --git a/packages/adapter-utils/src/sandbox-managed-runtime.test.ts b/packages/adapter-utils/src/sandbox-managed-runtime.test.ts index 387ac215df..182bf4de21 100644 --- a/packages/adapter-utils/src/sandbox-managed-runtime.test.ts +++ b/packages/adapter-utils/src/sandbox-managed-runtime.test.ts @@ -42,6 +42,7 @@ import { type StartupTracer, } from "./acpx-engine/startup-timing.js"; import type { RunProcessResult } from "./server-utils.js"; +import { prepareAdapterExecutionTargetRuntime } from "./execution-target.js"; function toArrayBuffer(bytes: Buffer): ArrayBuffer { return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer; @@ -402,6 +403,51 @@ describe("sandbox managed runtime", () => { ).resolves.toBe("remote finalized\n"); }); + it.each([undefined, "adopt_remote"] as const)("preserves an old task's Git index and local state through a resumed target (%s)", async (workspaceInboundMode) => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-legacy-git-resume-")); + cleanupDirs.push(rootDir); + const remoteDir = path.join(rootDir, "remote"); + const localDir = path.join(rootDir, "host"); + await initGitRepo(remoteDir); + await git(rootDir, ["clone", remoteDir, localDir]); + await writeFile(path.join(remoteDir, "unpublished.txt"), "remote commit\n"); + await git(remoteDir, ["add", "unpublished.txt"]); + await git(remoteDir, ["commit", "-qm", "unpublished"]); + const head = await git(remoteDir, ["rev-parse", "HEAD"]); + await writeFile(path.join(remoteDir, "README.md"), "staged\n"); + await git(remoteDir, ["add", "README.md"]); + await writeFile(path.join(remoteDir, "README.md"), "unstaged\n"); + await writeFile(path.join(remoteDir, "untracked.txt"), "untracked\n"); + await fsPromises.appendFile(path.join(remoteDir, ".git", "info", "exclude"), "\n.cache-marker\n"); + await writeFile(path.join(remoteDir, ".cache-marker"), "retained cache\n"); + await git(remoteDir, ["config", "paperclip.retained", "yes"]); + const prepared = await prepareAdapterExecutionTargetRuntime({ + runId: "legacy-resume", adapterKey: "test", workspaceLocalDir: localDir, + target: { kind: "remote", transport: "sandbox", remoteCwd: remoteDir, + providerKey: "test", leaseId: "retained-lease", legacyWorkspaceResume: true, + runner: makeInlineSpawnRunner() }, + workspaceInboundMode, + ...(workspaceInboundMode ? { workspaceDurableSeed: { + workspaceArchivePath: path.join(rootDir, "seed", "workspace.tar"), + gitArchivePath: path.join(rootDir, "seed", "git.tar"), + } } : {}), + }); + const verifyRemote = async () => { + expect(await git(remoteDir, ["rev-parse", "HEAD"])).toBe(head); + expect(await git(remoteDir, ["show", ":README.md"])).toBe("staged"); + expect(await readFile(path.join(remoteDir, "README.md"), "utf8")).toBe("unstaged\n"); + expect(await readFile(path.join(remoteDir, "untracked.txt"), "utf8")).toBe("untracked\n"); + expect(await git(remoteDir, ["config", "paperclip.retained"])).toBe("yes"); + expect(await git(remoteDir, ["check-ignore", ".cache-marker"])).toBe(".cache-marker"); + expect(await readFile(path.join(remoteDir, ".cache-marker"), "utf8")).toBe("retained cache\n"); + }; + await verifyRemote(); + await prepared.restoreWorkspace(); + await verifyRemote(); + expect(await readFile(path.join(localDir, "README.md"), "utf8")).toBe("unstaged\n"); + expect(await git(localDir, ["rev-parse", "HEAD"])).toBe(head); + }); + it("reconstructs a replacement workspace from the exact durable pre-turn seed", async () => { const rootDir = await mkdtemp( path.join(os.tmpdir(), "paperclip-sandbox-durable-seed-"), diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts index b3452ab6c8..17c425e976 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts @@ -11,7 +11,7 @@ import { } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; -import { afterAll, afterEach, describe, expect, it } from "vitest"; +import { afterAll, afterEach, describe, expect, it, vi } from "vitest"; import { NATIVE_RUNTIME_ASSET_SCHEMA, @@ -101,6 +101,30 @@ afterAll(async () => { }); describe("OpenCodeServerDriver", () => { + it.each([undefined, { PATH: process.env.PATH }])("does not inherit ambient GitHub access with environment %j", async (environment) => { + await chmod(fixture, 0o755); + const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-no-ambient-")); + const workspace = join(root, "workspace"); + await mkdir(workspace); + roots.push(root); + vi.stubEnv("PAPERCLIP_GITHUB_BROKER_TOKEN", "ambient-host-secret"); + vi.stubEnv("BASH_ENV", "/ambient/shell-hook"); + const driver = new OpenCodeServerDriver({ + model: "openrouter/deepseek/deepseek-v4-flash-0731", + runtimeDirectory: root, command: fixture, environment, + }); + let session; + try { + session = await driver.openSession({runId: "run-no-ambient", normalizedSessionId: "no-ambient", workingDirectory: workspace}); + const child = JSON.parse(await readFile(join(root, "no-ambient", "data", "fake-environment.json"), "utf8")); + expect(child.keys).not.toContain("PAPERCLIP_GITHUB_BROKER_TOKEN"); + expect(child.keys).not.toContain("BASH_ENV"); + } finally { + await session?.close({reason: "test"}); + vi.unstubAllEnvs(); + } + }); + it("advertises within-turn plans as unsupported", async () => { const driver = new OpenCodeServerDriver({ model: "openrouter/deepseek/deepseek-v4-flash-0731", @@ -604,6 +628,14 @@ describe("OpenCodeServerDriver", () => { OPENROUTER_API_KEY: "test-openrouter-key", PAPERCLIP_API_KEY: "must-not-leak", UNRELATED_SECRET: "must-not-leak", + PAPERCLIP_GITHUB_BROKER_TOKEN: "fixture-github-capability", + PAPERCLIP_GITHUB_BRIDGE_TOKEN: "fixture-github-bridge", + PAPERCLIP_GITHUB_BROKER_URL: "http://127.0.0.1:3456", + BASH_ENV: "/runtime/github-launcher/profile.sh", + GIT_CONFIG_COUNT: "1", + GIT_CONFIG_KEY_0: "credential.helper", + GIT_CONFIG_VALUE_0: "", + GIT_CONFIG_KEY_1: "must-not-reach-provider", PAPERCLIP_PROVIDER_TRACE_PATH: tracePath, PAPERCLIP_PROVIDER_TRACE_MAX_BYTES: String(64 * 1024 * 1024), }, @@ -750,6 +782,12 @@ describe("OpenCodeServerDriver", () => { expect(environment.keys).not.toContain("PAPERCLIP_API_KEY"); expect(environment.keys).not.toContain("UNRELATED_SECRET"); expect(environment.keys).not.toContain("PAPERCLIP_PROVIDER_TRACE_PATH"); + expect(environment.keys).toEqual(expect.arrayContaining([ + "PAPERCLIP_GITHUB_BROKER_TOKEN", "PAPERCLIP_GITHUB_BRIDGE_TOKEN", + "PAPERCLIP_GITHUB_BROKER_URL", "BASH_ENV", "GIT_CONFIG_COUNT", + "GIT_CONFIG_KEY_0", "GIT_CONFIG_VALUE_0", + ])); + expect(environment.keys).not.toContain("GIT_CONFIG_KEY_1"); expect(environment.projectConfigDisabled).toBe("true"); expect(mcpEvidence.tools).toEqual( expect.arrayContaining(["paperclip_finish", "paperclip_block"]), @@ -1792,7 +1830,7 @@ describe("OpenCodeServerDriver", () => { const exitingFixture = join(root, "exit-before-health.mjs"); await writeFile( exitingFixture, - "#!/usr/bin/env node\nprocess.stderr.write(`credential=${process.env.OPENROUTER_API_KEY}\\nauthorization=super-secret-opencode-token\\n`);\nprocess.exit(17);\n", + "#!/usr/bin/env node\nprocess.stderr.write(`credential=${process.env.OPENROUTER_API_KEY}\\nauthorization=super-secret-opencode-token\\n${process.env.PAPERCLIP_GITHUB_BROKER_TOKEN}\\n${process.env.PAPERCLIP_GITHUB_BRIDGE_TOKEN}\\n${process.env.GIT_CONFIG_VALUE_0}\\n`);\nprocess.exit(17);\n", { mode: 0o755 }, ); const driver = new OpenCodeServerDriver({ @@ -1802,6 +1840,11 @@ describe("OpenCodeServerDriver", () => { environment: { PATH: process.env.PATH, OPENROUTER_API_KEY: "fixture-key", + PAPERCLIP_GITHUB_BROKER_TOKEN: "fixture-github-capability", + PAPERCLIP_GITHUB_BRIDGE_TOKEN: "fixture-github-bridge", + GIT_CONFIG_COUNT: "1", + GIT_CONFIG_KEY_0: "http.extraHeader", + GIT_CONFIG_VALUE_0: "fixture-github-header", }, }); const error = await driver @@ -1820,5 +1863,8 @@ describe("OpenCodeServerDriver", () => { expect(error).toContain("[REDACTED]"); expect(error).not.toContain("fixture-key"); expect(error).not.toContain("super-secret-opencode-token"); + expect(error).not.toContain("fixture-github-capability"); + expect(error).not.toContain("fixture-github-bridge"); + expect(error).not.toContain("fixture-github-header"); }); }); diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts index edff57a181..df48dc1416 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts @@ -1,4 +1,5 @@ import { externalWorkFolderEnvironment } from "../../work-folder-environment.js"; +import { githubCredentialEnvironment } from "../../github-credential-environment.js"; import { spawn, type ChildProcess } from "node:child_process"; import { randomBytes } from "node:crypto"; import { @@ -251,9 +252,7 @@ export class OpenCodeServerDriver implements HarnessDriver { } catch (error) { return { recovered: false, - reason: redact(String(error), [ - this.#options.environment?.OPENROUTER_API_KEY, - ]), + reason: redact(String(error), openCodeSensitiveValues(this.#options.environment)), }; } } @@ -1858,12 +1857,12 @@ async function startRuntime(input: { const assignedMcp = nativeMcpLaunchBinding( input.options.environment ?? process.env, ); + const sensitiveValues = [password, ...openCodeSensitiveValues(input.options.environment)]; input.trace?.addSensitiveValues([ - password, + ...sensitiveValues, authHeader, bridge.secret, assignedMcp?.token, - input.options.environment?.OPENROUTER_API_KEY, ]); const instructionRoot = input.options.runtimeContext?.instructions.bundle.rootPath; @@ -1933,6 +1932,9 @@ async function startRuntime(input: { { HOME: isolatedHome, ...externalWorkFolderEnvironment(input.options.environment ?? {}), + ...(input.options.environment + ? githubCredentialEnvironment(input.options.environment) + : {}), XDG_CONFIG_HOME: configHome, XDG_DATA_HOME: dataHome, XDG_CACHE_HOME: cacheHome, @@ -1969,10 +1971,7 @@ async function startRuntime(input: { let diagnostics = ""; child.stderr?.on("data", (chunk) => { const raw = Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)); - const redactedDiagnostic = redact(raw.toString("utf8"), [ - password, - input.options.environment?.OPENROUTER_API_KEY, - ]); + const redactedDiagnostic = redact(raw.toString("utf8"), sensitiveValues); diagnostics = `${diagnostics}${redactedDiagnostic}`.slice(-8_192); const frameId = input.trace?.frame({ direction: "provider_stderr", @@ -2035,10 +2034,7 @@ async function startRuntime(input: { process: child, bridge, trace: input.trace, - sensitiveValues: [ - password, - input.options.environment?.OPENROUTER_API_KEY, - ].filter((value): value is string => Boolean(value)), + sensitiveValues, close: async (closeInput = {}) => { await bridge.close().catch(() => {}); if (child.exitCode === null && child.signalCode === null && child.pid) { @@ -2537,6 +2533,16 @@ function safeTraceRulePath(value: string): string { .replaceAll("/", "_") .slice(0, 120); } +function openCodeSensitiveValues(environment: NodeJS.ProcessEnv | undefined): string[] { + const github = environment ? githubCredentialEnvironment(environment) : {}; + return [ + environment?.OPENROUTER_API_KEY, + ...Object.entries(github) + .filter(([key]) => key.endsWith("TOKEN") || /^GIT_CONFIG_VALUE_\d+$/.test(key)) + .map(([, value]) => value), + ].filter((value): value is string => Boolean(value)); +} + function redact( value: string, sensitiveValues: readonly (string | undefined)[] = [], diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index 50577a44c6..714c101067 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -1070,6 +1070,39 @@ it.each(["codex", "opencode", "acpx"] as const)("keeps the natural sandbox home expect(() => createRunnerdCodexAppServerArgs({ environment, codexHome: `${home}/.codex` })).not.toThrow(); }); +it("preserves only explicit GitHub bindings through the OpenCode runner boundary", () => { + vi.stubEnv("PAPERCLIP_GITHUB_BROKER_TOKEN", "ambient-host-secret"); + vi.stubEnv("BASH_ENV", "/ambient/shell-hook"); + const projected = { + PAPERCLIP_GITHUB_BROKER_TOKEN: "controller-run-capability", + PAPERCLIP_GITHUB_BRIDGE_TOKEN: "controller-bridge-capability", + PAPERCLIP_GITHUB_BROKER_URL: "http://127.0.0.1:3456", + BASH_ENV: "/runtime/github-launcher/profile.sh", + GIT_CONFIG_COUNT: "1", + GIT_CONFIG_KEY_0: "credential.helper", + GIT_CONFIG_VALUE_0: "", + }; + const launch = (environment: NodeJS.ProcessEnv | undefined) => createCapabilityRunnerdProviderEnvironment({ + provider: "opencode", + options: { provider: "opencode", environment }, + identity: { runnerInstanceId: "runner", environmentLeaseId: "lease", runId: "run", normalizedSessionId: "session", turnId: "turn", itemId: "item" }, + codexHome: "/isolated/home", runtimeContextPath: "/runtime/context.json", hasRuntimeContext: false, + }); + try { + const environment = launch({ ...projected, GIT_CONFIG_KEY_1: "unbounded", PAPERCLIP_API_KEY: "control-plane-secret", DATABASE_URL: "host-secret" }); + expect(environment).toMatchObject(projected); + expect(environment).not.toHaveProperty("GIT_CONFIG_KEY_1"); + expect(environment).not.toHaveProperty("PAPERCLIP_API_KEY"); + expect(environment).not.toHaveProperty("DATABASE_URL"); + for (const input of [undefined, {}]) { + expect(launch(input)).not.toHaveProperty("PAPERCLIP_GITHUB_BROKER_TOKEN"); + expect(launch(input)).not.toHaveProperty("BASH_ENV"); + } + } finally { + vi.unstubAllEnvs(); + } +}); + it("denies the isolated Codex home without denying a remote execution workspace", () => { const args = createRunnerdCodexAppServerArgs({ environment: { diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index 7c756129c9..f26452c489 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -1,4 +1,5 @@ import { externalWorkFolderEnvironment } from "../work-folder-environment.js"; +import { githubCredentialEnvironment } from "../github-credential-environment.js"; import { execFileSync } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; import { @@ -1901,7 +1902,7 @@ function createSanitizedOpenCodeRunnerEnvironment( source: NodeJS.ProcessEnv | undefined, ): NodeJS.ProcessEnv { const candidate = { ...process.env, ...source }; - return Object.fromEntries( + const environment = Object.fromEntries( Object.entries(candidate).filter( ([key, value]) => typeof value === "string" && @@ -1909,6 +1910,9 @@ function createSanitizedOpenCodeRunnerEnvironment( /^LC_[A-Z0-9_]{1,32}$/.test(key)), ), ); + // Repository access comes only from the explicit controller projection, + // never from credentials or shell hooks in the runner host's environment. + return { ...environment, ...(source ? githubCredentialEnvironment(source) : {}) }; } export function resolveSourceCodexHome( diff --git a/server/src/__tests__/environment-execution-target-capabilities.test.ts b/server/src/__tests__/environment-execution-target-capabilities.test.ts index 3d7313f7a9..b6d9609401 100644 --- a/server/src/__tests__/environment-execution-target-capabilities.test.ts +++ b/server/src/__tests__/environment-execution-target-capabilities.test.ts @@ -219,7 +219,13 @@ describe("resolveEnvironmentExecutionTarget effective capability snapshot", () = expect(target.reusableLeaseConfigured).toBe(true); }); - it("carries host-owned sandbox acquisition provenance without persisting provider ids in metadata", async () => { + it.each([ + { layout: "legacy", outcome: "resumed", adopt: true }, + { layout: "legacy", outcome: "created", adopt: false }, + { layout: "legacy", outcome: "replacement", adopt: false }, + { layout: "scoped", outcome: "resumed", adopt: false }, + { layout: undefined, outcome: "resumed", adopt: false }, + ])("carries host-owned acquisition and legacy adoption for $layout/$outcome", async ({layout, outcome, adopt}) => { mockResolveEnvironmentDriverConfigForRuntime.mockResolvedValue({ driver: "sandbox", config: { provider: "daytona", reuseLease: true, timeoutMs: 30_000 }, @@ -235,13 +241,14 @@ describe("resolveEnvironmentExecutionTarget effective capability snapshot", () = leaseId: "lease-row-1", leaseMetadata: { remoteCwd: "/work", + workFolderLayout: "legacy", sandboxLeaseAcquisition: { outcome: "resumed" }, }, lease: { id: "lease-row-1", providerLeaseId: "daytona-sandbox-1", leasePolicy: "reuse_by_environment", - metadata: { sandboxLeaseAcquisition: { outcome: "resumed" } }, + metadata: { workFolderLayout: layout, sandboxLeaseAcquisition: { outcome } }, } as never, environmentRuntime: { supportsSync: () => false, @@ -252,9 +259,10 @@ describe("resolveEnvironmentExecutionTarget effective capability snapshot", () = throw new Error("expected a sandbox target"); } expect(target.sandboxLeaseAcquisition).toEqual({ - outcome: "resumed", + outcome, providerLeaseId: "daytona-sandbox-1", }); + expect(target.legacyWorkspaceResume).toBe(adopt); }); }); diff --git a/server/src/__tests__/native-workspace-sync.test.ts b/server/src/__tests__/native-workspace-sync.test.ts index c4598fc708..fc4cec8f6b 100644 --- a/server/src/__tests__/native-workspace-sync.test.ts +++ b/server/src/__tests__/native-workspace-sync.test.ts @@ -94,6 +94,17 @@ describe("native workspace sync durable metadata", () => { ).toThrow("native_workspace_sync_unexpected_existing_descriptor"); }); + it("adopts only a validated legacy resume when an older release has no sync stamp", () => { + for (const acquisition of ["created", "replacement", null] as const) { + expect(classifyNativeWorkspaceInbound({kind: "new_run", acquisition, + hasPriorStamp: false, legacyWorkspaceResume: true})).toBe("host_current"); + } + for (const hasPriorStamp of [true, false]) { + expect(classifyNativeWorkspaceInbound({kind: "new_run", acquisition: "resumed", + hasPriorStamp, legacyWorkspaceResume: true})).toBe("adopt_remote"); + } + }); + it("reads backward-compatible references and the resource disposition", () => { const base = { schema: "paperclip.native-workspace-sync/v1", diff --git a/server/src/index.ts b/server/src/index.ts index 9f80e5f706..b04b3d3a20 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -113,6 +113,7 @@ import { } from "./shutdown.js"; import { initializeCloudRuntimeIdentity } from "./services/cloud-runtime-identity.js"; import { systemdNotify } from "./services/systemd-notify.js"; +import { closeIdleSandboxNativeSessionsForShutdown } from "./services/native-runtime/native-session-executor.js"; import { flushInFlightRunLogMirrors } from "./services/run-log-store.js"; import { createEmbeddedPostgresSupervisor, @@ -1900,6 +1901,9 @@ export async function startServer(): Promise { await finalizeServerShutdown({ signal, shutdownAppServices: appShutdown, + closeIdleSandboxSessions: () => closeIdleSandboxNativeSessionsForShutdown({ + reason: `server shutdown (${signal})`, + }), stopEmbeddedPostgres, shutdownInstrumentation, shutdownSentry, diff --git a/server/src/services/environment-execution-target.ts b/server/src/services/environment-execution-target.ts index 281f772aaa..a4d7c75dec 100644 --- a/server/src/services/environment-execution-target.ts +++ b/server/src/services/environment-execution-target.ts @@ -366,6 +366,10 @@ export async function resolveEnvironmentExecutionTarget(input: { const syncLease = () => target.workFolderHome ? { ...input.lease!, metadata: { ...input.lease!.metadata, remoteCwd: target.workFolderHome } } : input.lease!; + const acquisition = sandboxLeaseAcquisitionFromMetadata( + input.lease?.metadata?.sandboxLeaseAcquisition, + input.lease?.providerLeaseId, + ); const target: AdapterSandboxExecutionTarget = { kind: "remote", transport: "sandbox", @@ -388,10 +392,9 @@ export async function resolveEnvironmentExecutionTarget(input: { ? { mode: "per_turn", idleTimeoutMs: null } : null, reusableLeaseConfigured: parsed.config.reuseLease === true, - sandboxLeaseAcquisition: sandboxLeaseAcquisitionFromMetadata( - input.lease?.metadata?.sandboxLeaseAcquisition, - input.lease?.providerLeaseId, - ), + sandboxLeaseAcquisition: acquisition, + legacyWorkspaceResume: acquisition?.outcome === "resumed" + && input.lease?.metadata?.workFolderLayout === "legacy", // Attach the host duplex observability recorder next to the runner. The bridge // binds it to the fixed observability surface. Absent keeps the no-op // default, so the surface stays inert on a run with no injected recorder. diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 9076b1506c..5f706f9539 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -171,6 +171,7 @@ import { buildNativeHarnessBackupManifest, cancelNativeSession, closeWarmNativeSessionsForEnvironment, + closeIdleSandboxNativeSessionsForShutdown, createGovernedWaitEventObservation, createRemoteRunnerProcessLauncher, createRunnerdBackend, @@ -3073,6 +3074,73 @@ describe("native session same-turn steering", () => { }); describe("native warm session supervision", () => { + it.each([ + { transport: "sandbox", busy: false, fail: false }, + { transport: "sandbox", busy: true, fail: false }, + { transport: "sandbox", busy: false, fail: true }, + { transport: "ssh", busy: false, fail: false }, + { transport: "local", busy: false, fail: false }, + ] as const)("shutdown parks only idle sandbox sessions: $transport busy=$busy fail=$fail", async ({ transport, busy, fail }) => { + const id = `shutdown-${transport}-${busy}-${fail}`; + const close = vi.fn(async () => { + if (fail) throw new Error("checkpoint unavailable"); + }); + let release!: () => void; + const held = new Promise((resolve) => { release = resolve; }); + const warmExecution = { + ...execution, + binding: { ...execution.binding, runId: id, executionWorkspaceId: id }, + session: { + ...execution.session, + normalizedSessionId: id, + lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 100 }, + }, + } as NativeExecutionInputV1; + state.execute.mockReset().mockImplementationOnce(async (options) => { + options.onSession?.({ close }); + if (busy) await held; + return { + result: { summary: "completed" }, + terminal: { runTerminalState: "succeeded" }, + turnId: id, normalizedSessionId: id, providerSessionId: id, + driverKind: "test", driverVersion: "1", nativeEventCount: 1, + highestContiguousSourceSeq: 1, usage: null, + }; + }); + const running = executePaperclipNativeSession({ + db: leaseDb(warmExecution), execution: warmExecution, runnerInstanceId: id, + runnerExecutionTarget: transport === "local" ? undefined : transport === "sandbox" ? { + kind: "remote", transport: "sandbox", environmentId: id, remoteCwd: `/tmp/${id}`, + } : { + kind: "remote", transport: "ssh", environmentId: id, remoteCwd: `/tmp/${id}`, + spec: { + host: "runner.internal", port: 22, username: "runner", + remoteWorkspacePath: `/tmp/${id}`, remoteCwd: `/tmp/${id}`, + privateKey: null, knownHosts: null, strictHostKeyChecking: true, + }, + }, + }); + await vi.waitFor(() => expect(state.execute).toHaveBeenCalledOnce()); + if (!busy) await running; + try { + const result = await closeIdleSandboxNativeSessionsForShutdown({ reason: "server shutdown" }); + if (transport === "sandbox" && !busy) { + expect(close).toHaveBeenCalledExactlyOnceWith({ reason: "server shutdown" }); + expect(result[fail ? "failed" : "closed"]).toBeGreaterThanOrEqual(1); + await closeIdleSandboxNativeSessionsForShutdown({ reason: "duplicate shutdown" }); + expect(close).toHaveBeenCalledOnce(); + } else { + expect(close).not.toHaveBeenCalled(); + if (busy) expect(result.busy).toBeGreaterThanOrEqual(1); + } + } finally { + release(); + await running; + // Unselected sessions retain their existing idle-close behavior. + await vi.waitFor(() => expect(close).toHaveBeenCalled(), { timeout: 1_000 }); + } + }); + it("closes an idle warm session before its remote environment is destroyed", async () => { const close = vi.fn(async () => undefined); const warmExecution = { diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 19fcc20884..382c1b21a3 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -272,6 +272,7 @@ type WarmNativeSession = { configDigest: string; companyId: string; environmentId: string | null; + sandbox: boolean; busy: boolean; idleTimer: ReturnType | null; lastActivityAt: string; @@ -315,6 +316,37 @@ export async function closeWarmNativeSessionsForEnvironment(input: { return { closed, busy, failed }; } +/** Park idle sandbox sessions while their host still has database and transport + * access. Active turns retain their existing restart/reattach contract. */ +export async function closeIdleSandboxNativeSessionsForShutdown(input: { + reason: string; +}): Promise<{ closed: number; busy: number; failed: number }> { + const result = { closed: 0, busy: 0, failed: 0 }; + const pending: WarmNativeSession[] = []; + for (const [sessionId, entry] of warmNativeSessions) { + if (!entry.sandbox) continue; + if (entry.busy) { + result.busy += 1; + continue; + } + if (entry.idleTimer !== null) clearTimeout(entry.idleTimer); + // Fence every selected owner before the first asynchronous close. + warmNativeSessions.delete(sessionId); + pending.push(entry); + } + await Promise.all(Array.from({ length: Math.min(4, pending.length) }, async () => { + for (let entry = pending.shift(); entry; entry = pending.shift()) { + try { + await entry.session.close({ reason: input.reason }); + result.closed += 1; + } catch { + result.failed += 1; + } + } + })); + return result; +} + function readBoundedNativeFile( path: string, maxBytes: number, @@ -4528,6 +4560,9 @@ async function executePaperclipNativeSessionWithinScope( companyId: input.execution.binding.companyId, environmentId: input.runnerExecutionTarget?.environmentId ?? null, + sandbox: + input.runnerExecutionTarget?.kind === "remote" && + input.runnerExecutionTarget.transport === "sandbox", busy: true, idleTimer: null, lastActivityAt: new Date().toISOString(), diff --git a/server/src/services/native-runtime/native-workspace-sync.ts b/server/src/services/native-runtime/native-workspace-sync.ts index 2933cf9212..562508fabb 100644 --- a/server/src/services/native-runtime/native-workspace-sync.ts +++ b/server/src/services/native-runtime/native-workspace-sync.ts @@ -97,6 +97,7 @@ export type NativeWorkspaceInboundEvidence = kind: "new_run"; acquisition: "created" | "resumed" | "replacement" | null; hasPriorStamp: boolean; + legacyWorkspaceResume?: boolean; }; export function classifyNativeWorkspaceInbound( @@ -108,7 +109,7 @@ export function classifyNativeWorkspaceInbound( } return evidence.sameProviderLease ? "adopt_remote" : "durable_seed"; } - return evidence.acquisition === "resumed" && evidence.hasPriorStamp + return evidence.acquisition === "resumed" && (evidence.hasPriorStamp || evidence.legacyWorkspaceResume) ? "adopt_remote" : "host_current"; } @@ -823,6 +824,7 @@ export async function prepareNativeWorkspaceSync(input: { kind: "new_run", acquisition, hasPriorStamp: priorStamp !== null, + legacyWorkspaceResume: target.legacyWorkspaceResume, }); runtime = await prepareRuntime({ runId: input.runId, @@ -838,7 +840,7 @@ export async function prepareNativeWorkspaceSync(input: { if (!currentSnapshot) { throw new Error("native_workspace_sync_snapshot_missing"); } - if (acquisition === "resumed" && priorStamp) { + if (acquisition === "resumed" && priorStamp && !target.legacyWorkspaceResume) { const currentHostSha256 = directorySnapshotSha256( currentSnapshot.baseline, ); diff --git a/server/src/shutdown.test.ts b/server/src/shutdown.test.ts index c3500b3a88..f3e8f44e7a 100644 --- a/server/src/shutdown.test.ts +++ b/server/src/shutdown.test.ts @@ -22,6 +22,47 @@ function stubLogger() { } describe("finalizeServerShutdown", () => { + it("awaits idle sandbox checkpoints before app and database teardown", async () => { + const release = deferred<{ closed: number; busy: number; failed: number }>(); + const closeIdleSandboxSessions = vi.fn(() => release.promise); + const shutdownAppServices = vi.fn(async () => undefined); + const stopEmbeddedPostgres = vi.fn(async () => undefined); + const pending = finalizeServerShutdown({ + signal: "SIGTERM", closeIdleSandboxSessions, shutdownAppServices, stopEmbeddedPostgres, + shutdownInstrumentation: async () => undefined, + shutdownSentry: async () => undefined, log: stubLogger(), + }); + await vi.waitFor(() => expect(closeIdleSandboxSessions).toHaveBeenCalledOnce()); + expect(shutdownAppServices).not.toHaveBeenCalled(); + expect(stopEmbeddedPostgres).not.toHaveBeenCalled(); + release.resolve({ closed: 2, busy: 1, failed: 0 }); + await pending; + expect(stopEmbeddedPostgres).toHaveBeenCalledOnce(); + }); + + it.each(["timeout", "rejected", "incomplete"] as const)("reports %s idle checkpoint without blocking shutdown forever", async (failure) => { + vi.useFakeTimers(); + try { + const log = stubLogger(); + const stopEmbeddedPostgres = vi.fn(async () => undefined); + const pending = finalizeServerShutdown({ + signal: "SIGTERM", sandboxSessionTimeoutMs: 250, + closeIdleSandboxSessions: async () => { + if (failure === "timeout") return new Promise(() => undefined); + if (failure === "rejected") throw new Error("transport disconnected"); + return { closed: 0, busy: 0, failed: 1 }; + }, + shutdownAppServices: undefined, stopEmbeddedPostgres, + shutdownInstrumentation: async () => undefined, + shutdownSentry: async () => undefined, log, + }); + await vi.advanceTimersByTimeAsync(250); + await pending; + expect(log.error).toHaveBeenCalledOnce(); + expect(stopEmbeddedPostgres).toHaveBeenCalledOnce(); + } finally { vi.useRealTimers(); } + }); + it("awaits the setup-token cleanup before the database stop and the process exit", async () => { const order: string[] = []; // The held promise models the setup-token session cancellation and its diff --git a/server/src/shutdown.ts b/server/src/shutdown.ts index adffd4dc6d..f7b2d7c7de 100644 --- a/server/src/shutdown.ts +++ b/server/src/shutdown.ts @@ -52,6 +52,8 @@ export async function drainRunExecutionFinalizersForShutdown(input: { export async function finalizeServerShutdown(input: { signal: "SIGINT" | "SIGTERM"; shutdownAppServices: (() => Promise) | undefined; + closeIdleSandboxSessions?: () => Promise<{ closed: number; busy: number; failed: number }>; + sandboxSessionTimeoutMs?: number; stopEmbeddedPostgres: (() => Promise) | null; shutdownInstrumentation: () => Promise; shutdownSentry: () => Promise; @@ -59,6 +61,32 @@ export async function finalizeServerShutdown(input: { }): Promise { const { signal } = input; + if (input.closeIdleSandboxSessions) { + let timer: NodeJS.Timeout | null = null; + const timeoutMs = input.sandboxSessionTimeoutMs ?? 30_000; + try { + const result = await Promise.race([ + input.closeIdleSandboxSessions(), + new Promise((resolve) => { + timer = setTimeout(() => resolve(null), timeoutMs); + timer.unref?.(); + }), + ]); + if (result === null || result.failed > 0) { + input.log.error( + { signal, timeoutMs, result }, + "Idle sandbox native session checkpoint incomplete during shutdown", + ); + } else { + input.log.info({ signal, result }, "Idle sandbox native sessions parked for shutdown"); + } + } catch (err) { + input.log.error({ signal, err }, "Idle sandbox native session shutdown failed"); + } finally { + if (timer) clearTimeout(timer); + } + } + // Await the application service cleanup, so a live setup-token login session // releases its sandbox lease before the database and the provider stop. A // rejected cleanup stays durable for the reaper; it does not block the exit.