Carry verified staging integration fixes into remote recovery

Co-Authored-By: Paperclip <noreply@paperclip.ing> * codex/work-folders-staging-hardening-refresh: Align staging runtime packaging and lifecycle with current master Preserve the retained identity read limit during runtime upgrade
2026-09-11 14:32:46 -05:00 · 2026-09-11 14:32:46 -05:00 · f4b28e23eb
parent f42cde8bcf d2ecec3359
commit f4b28e23eb
10 changed files with 1047 additions and 839 deletions
--- a/4
+++ b/4
@ -296,7 +296,7 @@ COPY packages ./packages
 COPY server/package.json ./server/package.json
 COPY ui/package.json ./ui/package.json
 COPY cli/package.json ./cli/package.json
-ARG PAPERCLIP_RUNNER_LOCK_SHA256=84409576c7cbd2bec50b535c6df6acf3691bdec7c7697e6c50b2fb834b56f203
+ARG PAPERCLIP_RUNNER_LOCK_SHA256=21aa3df50da53c338660b9f5f2611e9d46c896ad05c0e7a9976185ebce340f52
 RUN printf '%s  pnpm-lock.yaml\n' "${PAPERCLIP_RUNNER_LOCK_SHA256}" > /tmp/provider-lock.sha256 \
    && sha256sum -c /tmp/provider-lock.sha256 \
    && pnpm install --frozen-lockfile --filter '@paperclipai/paperclip-runner...'
@ -306,7 +306,7 @@ RUN test -n "${PAPERCLIP_BUILD_COMMIT}" \
  && PAPERCLIP_RUNNER_SOURCE_REVISION="${PAPERCLIP_BUILD_COMMIT}" \
    node packages/paperclip-runner/scripts/assemble-provider-pack.mjs /provider-pack \
  && node packages/paperclip-runner/scripts/verify-pi-provider-launch.mjs /provider-pack \
-  && PATH=/provider-pack/node_modules/.bin:$PATH sh -ec 'for cli in node acpx claude-agent-acp codex-acp pi-acp pi claude codex opencode; do test -x "/provider-pack/node_modules/.bin/$cli"; done; test "$(acpx --version)" = "0.13.1"; test "$(claude-agent-acp --version)" = "0.70.0"; test "$(codex-acp --version)" = "@agentclientprotocol/codex-acp 1.6.2"' \
+  && PATH=/provider-pack/node_modules/.bin:$PATH sh -ec 'for cli in node acpx claude-agent-acp codex-acp pi-acp pi claude codex opencode; do test -x "/provider-pack/node_modules/.bin/$cli"; done; test "$(acpx --version)" = "0.13.1"; test "$(claude-agent-acp --version)" = "0.73.0"; test "$(codex-acp --version)" = "@agentclientprotocol/codex-acp 1.6.2"' \
    && node --input-type=module -e "import {verifyProviderPack} from './packages/paperclip-runner/scripts/provider-pack-integrity.mjs'; verifyProviderPack('/provider-pack', {revision: '${PAPERCLIP_BUILD_COMMIT}', lockSha256: '${PAPERCLIP_RUNNER_LOCK_SHA256}'});"

 FROM production AS cloud
--- a/docker/daytona-runner/Dockerfile
+++ b/docker/daytona-runner/Dockerfile
@ -29,7 +29,7 @@ COPY cli/package.json ./cli/package.json
 # The complete resolved lock (including transitive integrity hashes) is reviewed.
 # Reject registry-time drift BEFORE installing packages or running lifecycle code.
 # Refresh this digest together with source/provider dependency changes.
-ARG PAPERCLIP_RUNNER_LOCK_SHA256=84409576c7cbd2bec50b535c6df6acf3691bdec7c7697e6c50b2fb834b56f203
+ARG PAPERCLIP_RUNNER_LOCK_SHA256=21aa3df50da53c338660b9f5f2611e9d46c896ad05c0e7a9976185ebce340f52
 RUN printf '%s  pnpm-lock.yaml\n' "${PAPERCLIP_RUNNER_LOCK_SHA256}" > /tmp/provider-lock.sha256 \
    && sha256sum -c /tmp/provider-lock.sha256 \
    && pnpm install --frozen-lockfile --filter '@paperclipai/paperclip-runner...'
@ -39,7 +39,7 @@ RUN pnpm --filter @paperclipai/paperclip-runner build:typescript \
    && PAPERCLIP_RUNNER_SOURCE_REVISION="${PAPERCLIP_RUNNER_SOURCE_REVISION}" \
      node packages/paperclip-runner/scripts/assemble-provider-pack.mjs /provider-pack \
    && node packages/paperclip-runner/scripts/verify-pi-provider-launch.mjs /provider-pack \
-    && PATH=/provider-pack/node_modules/.bin:$PATH sh -ec 'for cli in node acpx claude-agent-acp codex-acp pi-acp pi claude codex opencode; do test -x "/provider-pack/node_modules/.bin/$cli"; done; test "$(acpx --version)" = "0.13.1"; test "$(claude-agent-acp --version)" = "0.70.0"; test "$(codex-acp --version)" = "@agentclientprotocol/codex-acp 1.6.2"' \
+    && PATH=/provider-pack/node_modules/.bin:$PATH sh -ec 'for cli in node acpx claude-agent-acp codex-acp pi-acp pi claude codex opencode; do test -x "/provider-pack/node_modules/.bin/$cli"; done; test "$(acpx --version)" = "0.13.1"; test "$(claude-agent-acp --version)" = "0.73.0"; test "$(codex-acp --version)" = "@agentclientprotocol/codex-acp 1.6.2"' \
    && node --input-type=module -e "import {verifyProviderPack} from './packages/paperclip-runner/scripts/provider-pack-integrity.mjs'; verifyProviderPack('/provider-pack', {revision: '${PAPERCLIP_RUNNER_SOURCE_REVISION}', lockSha256: '${PAPERCLIP_RUNNER_LOCK_SHA256}'});"

 # The trusted qualification entry exports the identical canonical provider stage.
--- a/docker/daytona-runner/provider-dependencies.lock.yaml
+++ b/docker/daytona-runner/provider-dependencies.lock.yaml
--- a/packages/adapter-utils/src/execution-target-sandbox.test.ts
+++ b/packages/adapter-utils/src/execution-target-sandbox.test.ts
@ -177,7 +177,9 @@ describe("sandbox adapter execution targets", () => {
        kind: "remote", transport: "sandbox", providerKey: "test", environmentId: "env-1",
        leaseId: "lease-1", remoteCwd: root, timeoutMs: 30_000, runner: { execute },
      };
-      return { runId: "launcher-retry", target, cwd: root, env: {} };
+      // Pin this local fixture PATH so these tests inject failures into writes,
+      // independently of the execution-target PATH discovery tests.
+      return { runId: "launcher-retry", target, cwd: root, env: { PATH: "/usr/bin:/bin" } };
    }

    it("hash-skips an accepted upload after its provider reply is lost", async () => {
@ -3600,7 +3602,7 @@ process.stdin.resume();setTimeout(()=>process.exit(2),20000);`);
          "x-paperclip-github-capability": "current-github-capability",
          "content-type": "application/json",
        },
-        body: "{}",
+        body: Buffer.from("{}"),
      });
      expect(credentials.status).toBe(200);
      expect(api.requests[1]).toMatchObject({
@ -3609,7 +3611,7 @@ process.stdin.resume();setTimeout(()=>process.exit(2),20000);`);
        auth: "Bearer real-run-jwt",
        runId: "run-http2",
        headers: { "x-paperclip-github-capability": "current-github-capability" },
-        body: "{}",
+        body: Buffer.from("{}"),
      });
    } finally {
      sessionRef.current?.close();
--- a/packages/adapter-utils/src/github-launcher.ts
+++ b/packages/adapter-utils/src/github-launcher.ts
@ -141,7 +141,10 @@ async function main() {
      }
      }
    } else { diagnostic('capability_missing'); }
-    } catch (error) { diagnostic(error && error.credentialCategory === 'denied' ? 'capability_rejected' : 'broker_transport_unavailable'); }
+    } catch (error) {
+      if (program === 'gh') throw error;
+      diagnostic(error && error.credentialCategory === 'denied' ? 'capability_rejected' : 'broker_transport_unavailable');
+    }
  }
  // Only this invocation and its children inherit the captured credential.
  // Its Git children use the real binary, so steering cannot split a gh operation.
@ -156,7 +159,12 @@ async function main() {
  child.once('error', () => { process.stderr.write('Paperclip: GitHub command could not start.\n'); process.exitCode = 1; });
  child.once('exit', (code, signal) => { process.exitCode = code === null ? 128 : code; });
 }
-main().catch(() => { process.stderr.write('Paperclip: GitHub launcher_setup_failed.\n'); process.exitCode = 1; });
+main().catch((error) => {
+  const category = ['timeout', 'unavailable', 'denied', 'invalidresponse'].includes(error && error.credentialCategory)
+    ? error.credentialCategory : 'unavailable';
+  process.stderr.write('Paperclip: GitHub credential context unavailable (' + category + '); retry this operation.\n');
+  process.exitCode = 1;
+});
 `;
 }

--- a/packages/paperclip-runner/scripts/assemble-provider-pack.mjs
+++ b/packages/paperclip-runner/scripts/assemble-provider-pack.mjs
@ -242,7 +242,7 @@ try {
      codex: "0.153.4",
      opencode: "1.18.29",
      acpx: "0.13.1",
-      claudeAcp: "0.70.0",
+      claudeAcp: "0.73.0",
      codexAcp: "1.6.2",
      pi: "0.84.2",
      piAcp: "0.0.33",
@ -262,7 +262,7 @@ try {
      claude:
        "sha256:9d73d1f0f121fb96cc8badb28c22d5bff02d8582eb2e40360a81c189e1b9422a",
      codex:
-        "sha256:7a923b3829884d3cabcc9659d22cace3f86813e7bfffc90974b10140a45bc400",
+        "sha256:c4538599d1ab767db5dff50934f13bb5ba313a59d9c4a83e993fac4617ea63d3",
    },
    artifacts: {
      nodeCommand: {
--- a/packages/paperclip-runner/scripts/build-provider-pack.test.mjs
+++ b/packages/paperclip-runner/scripts/build-provider-pack.test.mjs
@ -185,7 +185,7 @@ test("both canonical stages exercise pinned CLI versions and verify full tree af
    assert(body);
    const pi = body.indexOf("verify-pi-provider-launch.mjs /provider-pack"), versions = body.indexOf('test "$(acpx --version)" = "0.13.1"'), verification = body.indexOf("verifyProviderPack('/provider-pack'");
    assert(pi >= 0 && versions > pi && verification > versions);
-    assert(body.includes('test "$(claude-agent-acp --version)" = "0.70.0"'));
+    assert(body.includes('test "$(claude-agent-acp --version)" = "0.73.0"'));
    assert(body.includes('test "$(codex-acp --version)" = "@agentclientprotocol/codex-acp 1.6.2"'));
    assert(!body.includes("chmod -R a+rX /provider-pack"), "Modes must be normalized before binding the inventory");
  }
--- a/packages/paperclip-runner/src/drivers/acpx/installation-integrity.ts
+++ b/packages/paperclip-runner/src/drivers/acpx/installation-integrity.ts
@ -1325,6 +1325,14 @@ function commandLease(
  privateSnapshot: AcpxPrivateSnapshot | null,
 ): VerifiedAcpxCommandLease {
  let consumed = false;
+  let closed = false;
+  let activeChildren = 0;
+  // A spawned child still needs the verified private snapshot while its
+  // bootstrap loads. Closing the reusable lease prevents new launches, but
+  // must not unlink bytes already handed to an existing child.
+  const closeSnapshotIfIdle = async (): Promise<void> => {
+    if (activeChildren === 0 && (closed || !reusable)) await privateSnapshot?.close();
+  };
  let directoriesReleased = false;
  const releaseDirectories = async (): Promise<void> => {
    if (directoriesReleased) return;
@ -1341,11 +1349,12 @@ function commandLease(
    void releaseDirectories().catch(() => undefined);
  };
  const close = async (): Promise<void> => {
-    if (consumed) return;
+    if (closed) return;
+    closed = true;
    consumed = true;
    verifiedBytes.fill(0);
    await releaseDirectories();
-    await privateSnapshot?.close();
+    await closeSnapshotIfIdle();
  };
  return {
    spawn(
@ -1528,11 +1537,17 @@ function commandLease(
        void privateSnapshot?.close();
        throw error;
      }
-      if (!reusable) {
-      releaseDirectoriesBestEffort();
-      child.once("exit", () => { void privateSnapshot?.close(); });
-      child.once("error", () => { void privateSnapshot?.close(); });
-      }
+      activeChildren++;
+      let childSettled = false;
+      const settleChild = (): void => {
+        if (childSettled) return;
+        childSettled = true;
+        activeChildren--;
+        void closeSnapshotIfIdle().catch(() => undefined);
+      };
+      child.once("exit", settleChild);
+      child.once("error", settleChild);
+      if (!reusable) releaseDirectoriesBestEffort();
      const sourceInput = child.stdio[COMMAND_SOURCE_FD] as Writable | null;
      if (sourceInput === null) {
        consumed = true;
--- a/server/src/services/heartbeat.ts
+++ b/server/src/services/heartbeat.ts
@ -22026,8 +22026,9 @@ export function heartbeatService(
          // recovery existed. Only an entirely unused replacement row may
          // inherit its source checkpoint; any process/provider evidence on the
          // replacement makes the ownership ambiguous and therefore ineligible.
+          const legacyRetrySourceRunId = run.retryOfRunId;
          const legacyRetrySource =
-            !sandboxWorkFolders?.identityChanged && run.retryOfRunId && !isFailedChatRunRetry
+            !sandboxWorkFolders?.identityChanged && legacyRetrySourceRunId && !isFailedChatRunRetry
              ? await measureSandboxOperation("heartbeat.db.select.from.where.limit.then", { operationIndex: 121 }, async () => (db
                  .select({
                    id: heartbeatRuns.id,
@ -22042,7 +22043,7 @@ export function heartbeatService(
                  .from(heartbeatRuns)
                  .where(
                    and(
-                      eq(heartbeatRuns.id, run.retryOfRunId),
+                      eq(heartbeatRuns.id, legacyRetrySourceRunId),
                      eq(heartbeatRuns.companyId, agent.companyId),
                      eq(heartbeatRuns.agentId, agent.id),
                    ),
@ -22926,8 +22927,9 @@ export function heartbeatService(
            // A hard restart replays the heartbeat context, not a new user
            // action. Do not repeat a completed create/replace/edit (which
            // could reactivate or clear a goal that finished while detached).
+            const goalControlRequestId = sessionGoalControl?.requestId;
            const completedGoalControl =
-              sessionGoalControl !== null &&
+              goalControlRequestId !== undefined &&
              taskKey !== null &&
              (await measureSandboxOperation("heartbeat.is_runner_goal_action_completed", { operationIndex: 152 }, async () => (isRunnerGoalActionCompleted(
                db,
@ -22936,7 +22938,7 @@ export function heartbeatService(
                  agentId: agent.id,
                  issueId: taskKey,
                },
-                sessionGoalControl.requestId,
+                goalControlRequestId,
              ))));
            if (completedGoalControl) sessionGoalControl = null;
            const nativeDispatchAtMs = Date.now();
@ -23935,6 +23937,7 @@ export function heartbeatService(
                  issueId,
                  runId: livenessRun.id,
                })));
+              const resolvedText = resolved.text;
              const comment = await measureSandboxOperation("heartbeat.issues_svc.add_comment", { operationIndex: 210 }, async () => (issuesSvc.addComment(
                issueId,
                resolvedText,
@ -24825,7 +24828,8 @@ export function heartbeatService(
          latestRun &&
          isHeartbeatRunTerminalStatus(latestRun.status)
        ) {
-          await measureSandboxOperation("heartbeat.cleanup_git_hub_operation_launchers.catch", { operationIndex: 284 }, async () => (cleanupGitHubOperationLaunchers(githubLauncherLocation).catch(
+          const completedLauncherLocation = githubLauncherLocation;
+          await measureSandboxOperation("heartbeat.cleanup_git_hub_operation_launchers.catch", { operationIndex: 284 }, async () => (cleanupGitHubOperationLaunchers(completedLauncherLocation).catch(
            (err) => {
              logger.warn(
                { err, runId: run.id },
--- a/server/src/services/native-runtime/native-session-executor.ts
+++ b/server/src/services/native-runtime/native-session-executor.ts
@ -214,6 +214,7 @@ const MAX_REMOTE_CHECKPOINT_ENTRIES = 20_000;
 // runner transport's 64 MiB control-plane state bound; ordinary tool output
 // can exceed 2 MiB without invalidating the session identity.
 const NATIVE_CONTROL_PLANE_STATE_MAX_BYTES = 64 * 1024 * 1024;
+const NATIVE_DURABLE_IDENTITY_MAX_BYTES = 2 * 1024 * 1024;
 const NATIVE_RUNNER_STATE_MAX_BYTES = 16 * 1024 * 1024;
 const NATIVE_WARM_CHECKPOINT_MAX_BYTES = 8 * 1024 * 1024;
 const CODEX_HOME_NON_PERSISTENT_ENTRIES = [