From 2862e1848452d578a27054b66c433e0abc2315aa Mon Sep 17 00:00:00 2001 From: Nicky Leach Date: Tue, 25 Aug 2026 08:47:43 -0700 Subject: [PATCH] refactor(adapter-utils): remove the retired duplex_v1 sandbox bridge transport (#12171) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The adapter utilities provide sandbox transport paths for agent execution > - The retired `duplex_v1` path remains in host, gateway, and test code after `http2_v1` replaced it > - Retired transport code adds maintenance cost and leaves an unsafe fallback for unknown gateway modes > - This pull request removes the retired path, moves shared `http2_v1` contracts to a leaf module, and closes mode dispatch to a fixed allowlist > - The benefit is a smaller transport surface and explicit failure for unsupported modes ## Linked Issues or Issue Description Refs #12120 The `http2_v1` transport replaced `duplex_v1`, but the retired broker, gateway, constants, and tests remain in the adapter utilities. An unknown bridge mode can also fall through to the queue gateway when a queue directory exists. This change removes the retired code and rejects unsupported modes before gateway selection. ## What Changed - Delete the host `duplex_v1` broker and its transport-only tests. - Delete the in-sandbox duplex gateway and retired mode constants. - Move shared `http2_v1` symbols into `bridge-transport-contract.ts`. - Update the remaining importers and repair their focused tests. - Validate bridge modes against `http2_v1` and `queue_v1` before queue lookup. - Keep `queue_v1`, `duplex-frame-codec.ts`, and duplex telemetry dimensions unchanged. ## Verification - [x] `npx tsc --noEmit -p packages/adapter-utils` passes. - [x] `npx vitest run packages/adapter-utils/src` passes: 48 files and 968 tests pass, with 4 pre-existing platform skips. - [x] Full CI is green on this pull request. - [x] Greptile review is complete and every finding is resolved. ## Risks The change removes an internal transport that no host path selects. The main risk is an overlooked import or test dependency. Targeted typecheck and tests cover the adapter utility package. Full CI must confirm workspace-wide compatibility. ## Model Used Anthropic Claude Sonnet 5 assisted with the implementation, as recorded in the commit. The commit does not record a context-window size or reasoning mode. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge Co-authored-by: Paperclip --- .../src/acpx-engine/execute.test.ts | 125 +- .../adapter-utils/src/acpx-engine/execute.ts | 2 +- .../src/bridge-transport-contract.ts | 61 + .../duplex-bridge-broker-byte-ledger.test.ts | 384 ----- .../src/duplex-bridge-broker.test.ts | 1039 ------------ .../adapter-utils/src/duplex-bridge-broker.ts | 1457 ----------------- .../src/duplex-bridge-local-e2e.test.ts | 734 --------- .../adapter-utils/src/duplex-frame-codec.ts | 6 +- .../src/execution-target-sandbox.test.ts | 1128 +------------ .../adapter-utils/src/execution-target.ts | 2 +- .../src/sandbox-callback-bridge.test.ts | 65 + .../src/sandbox-callback-bridge.ts | 431 +---- 12 files changed, 276 insertions(+), 5158 deletions(-) create mode 100644 packages/adapter-utils/src/bridge-transport-contract.ts delete mode 100644 packages/adapter-utils/src/duplex-bridge-broker-byte-ledger.test.ts delete mode 100644 packages/adapter-utils/src/duplex-bridge-broker.test.ts delete mode 100644 packages/adapter-utils/src/duplex-bridge-broker.ts delete mode 100644 packages/adapter-utils/src/duplex-bridge-local-e2e.test.ts diff --git a/packages/adapter-utils/src/acpx-engine/execute.test.ts b/packages/adapter-utils/src/acpx-engine/execute.test.ts index 217e4f582d..8e0d712b09 100644 --- a/packages/adapter-utils/src/acpx-engine/execute.test.ts +++ b/packages/adapter-utils/src/acpx-engine/execute.test.ts @@ -36,8 +36,6 @@ import { type AcpxEngineExecutorOptions, } from "./execute.js"; import { runChildProcess } from "../server-utils.js"; -import { createDuplexBridgeBroker } from "../duplex-bridge-broker.js"; -import type { CommandManagedDuplexChannel } from "../command-managed-runtime.js"; import { getActiveStepContext, runWithRuntimeParent, @@ -5734,56 +5732,55 @@ describe("ACPX engine run lifecycle corrections (F3: one teardown error policy)" }); }); -describe("ACPX engine sandbox duplex run-disposition seam (fail-closed)", () => { +describe("ACPX engine sandbox bridge run-disposition seam (fail-closed)", () => { beforeEach(() => { vi.clearAllMocks(); }); - // A minimal in-memory duplex channel. The test drives a channel exit to latch a - // real loss in a real broker, so the seam reads a real run disposition. - function createFakeDuplexChannel() { - let exitListener: ((exit: { exitCode: number | null }) => void) | null = null; - const channel: CommandManagedDuplexChannel = { - write(): void {}, - onData(): void {}, - onExit(listener: (exit: { exitCode: number | null }) => void): void { - exitListener = listener; - }, - stop(): void {}, - close(): Promise { - return Promise.resolve(); - }, - }; - return { - channel, - emitExit: (exit: { exitCode: number | null }) => exitListener?.(exit), - }; - } - - // Wrap a started real broker in a paperclip bridge handle. The handle exposes - // the same run-disposition surface the sandbox bridge exposes, so the seam runs - // against the real latch and the real orderly-completion mark. - async function bridgeOverBroker(fake: ReturnType) { - const broker = await createDuplexBridgeBroker({ - channel: fake.channel, - forwardRequest: async () => ({ status: 200 }), + /** + * A minimal run-disposition latch. It reproduces the same ordering rule the + * bridge transport applies: the first ordered loss or orderly completion + * latches the terminal disposition, and a later call never overrides it. + * The test drives the latch directly, with no channel and no live process. + */ + function createFakeBridgeHandle() { + let lossOrdered = false; + let lossReason: string | null = null; + let completionOrdered = false; + const readDisposition = () => ({ failed: lossOrdered, lossReason }); + const markOrderlyCompletion = vi.fn(() => { + if (completionOrdered || lossOrdered) return; + completionOrdered = true; + }); + const settleRunDisposition = vi.fn(() => { + markOrderlyCompletion(); + return readDisposition(); }); - broker.start(); - const markOrderlyCompletion = vi.fn(() => broker.markOrderlyCompletion()); - const settleRunDisposition = vi.fn(() => broker.settleRunDisposition()); const stop = vi.fn(async () => {}); const handle = { env: { PAPERCLIP_API_URL: "http://127.0.0.1:1", PAPERCLIP_API_KEY: "bridge-token", - PAPERCLIP_API_BRIDGE_MODE: "duplex_v1", + PAPERCLIP_API_BRIDGE_MODE: "http2_v1", }, - readRunDisposition: () => broker.runDisposition, + readRunDisposition: () => readDisposition(), settleRunDisposition, markOrderlyCompletion, stop, }; - return { broker, handle, markOrderlyCompletion, settleRunDisposition }; + return { + handle, + markOrderlyCompletion, + settleRunDisposition, + readDisposition, + // Record the first ordered loss. A loss ordered after a completion, or a + // second loss, is a no-op — the same rule the real transport applies. + emitLoss: (reason: string) => { + if (lossOrdered || completionOrdered) return; + lossOrdered = true; + lossReason = reason; + }, + }; } // A runtime whose one turn completes cleanly. The `beforeResult` hook runs at @@ -5888,14 +5885,13 @@ describe("ACPX engine sandbox duplex run-disposition seam (fail-closed)", () => } as never); } - it("fails a completed run when the duplex channel was lost before the completion", async () => { + it("fails a completed run when the bridge channel was lost before the completion", async () => { const sandbox = await setupRemoteSandbox(); - const fake = createFakeDuplexChannel(); - const { broker, handle, settleRunDisposition } = await bridgeOverBroker(fake); + const fake = createFakeBridgeHandle(); // Latch the loss before the ACP terminal resolves. - const runtime = runtimeWithControlledResult(() => fake.emitExit({ exitCode: 1 })); + const runtime = runtimeWithControlledResult(() => fake.emitLoss("provider_exit")); - const result = await runRemote(handle, runtime, sandbox); + const result = await runRemote(fake.handle, runtime, sandbox); // The lost channel overrides the nominally completed terminal to a failure. expect(result.exitCode).not.toBe(0); @@ -5905,65 +5901,62 @@ describe("ACPX engine sandbox duplex run-disposition seam (fail-closed)", () => expect(result.resultJson).toMatchObject({ status: "failed" }); // The seam read the disposition through the atomic settle step, and the // latched loss kept the failure, so no orderly completion ordered. - expect(settleRunDisposition).toHaveBeenCalledTimes(1); - expect(broker.runDisposition.failed).toBe(true); + expect(fake.settleRunDisposition).toHaveBeenCalledTimes(1); + expect(fake.readDisposition().failed).toBe(true); }); it("keeps a completed run a success when the channel stays live, and a later teardown loss is benign", async () => { const sandbox = await setupRemoteSandbox(); - const fake = createFakeDuplexChannel(); - const { broker, handle, settleRunDisposition } = await bridgeOverBroker(fake); + const fake = createFakeBridgeHandle(); // No loss before the completion. const runtime = runtimeWithControlledResult(); - const result = await runRemote(handle, runtime, sandbox); + const result = await runRemote(fake.handle, runtime, sandbox); expect(result.exitCode).toBe(0); expect(result.errorCode ?? null).toBeNull(); // The atomic settle step marked the orderly completion for the // success-eligible terminal. - expect(settleRunDisposition).toHaveBeenCalledTimes(1); + expect(fake.settleRunDisposition).toHaveBeenCalledTimes(1); // A teardown loss ordered after the orderly completion is a normal teardown, // so the run disposition stays a success. - fake.emitExit({ exitCode: 0 }); - expect(broker.runDisposition.failed).toBe(false); + fake.emitLoss("provider_exit"); + expect(fake.readDisposition().failed).toBe(false); }); it("does not let a later completion or activity clear the loss latch", async () => { const sandbox = await setupRemoteSandbox(); - const fake = createFakeDuplexChannel(); - const { broker, handle } = await bridgeOverBroker(fake); + const fake = createFakeBridgeHandle(); // Latch the loss before the ACP terminal resolves. - const runtime = runtimeWithControlledResult(() => fake.emitExit({ exitCode: 1 })); + const runtime = runtimeWithControlledResult(() => fake.emitLoss("provider_exit")); - const result = await runRemote(handle, runtime, sandbox); + const result = await runRemote(fake.handle, runtime, sandbox); expect(result.exitCode).not.toBe(0); expect(result.errorCode).toBe("duplex_channel_lost"); // A later orderly-completion mark and further channel activity cannot clear // the latched loss. - broker.markOrderlyCompletion(); - fake.emitExit({ exitCode: 0 }); - expect(broker.runDisposition.failed).toBe(true); - expect(broker.runDisposition.lossReason).toBe("provider_exit"); + fake.markOrderlyCompletion(); + fake.emitLoss("provider_exit"); + expect(fake.readDisposition().failed).toBe(true); + expect(fake.readDisposition().lossReason).toBe("provider_exit"); }); it("marks an orderly completion on a failed terminal so the teardown loss emits no false loss", async () => { const sandbox = await setupRemoteSandbox(); - const fake = createFakeDuplexChannel(); - const { broker, handle, markOrderlyCompletion } = await bridgeOverBroker(fake); + const fake = createFakeBridgeHandle(); // The turn fails, and no channel loss ordered before the finalization. const runtime = runtimeWithFailedResult(); - const result = await runRemote(handle, runtime, sandbox); + const result = await runRemote(fake.handle, runtime, sandbox); - // The failed terminal stays a failure, but not a duplex loss. + // The failed terminal stays a failure, but not a bridge-channel loss. expect(result.exitCode).not.toBe(0); expect(result.errorCode).not.toBe("duplex_channel_lost"); // The non-success-eligible terminal marked the orderly completion, so the - // teardown channel_exit orders after the mark and does not latch a loss. - expect(markOrderlyCompletion).toHaveBeenCalledTimes(1); - fake.emitExit({ exitCode: 0 }); - expect(broker.runDisposition.failed).toBe(false); + // teardown loss orders after the mark and does not latch a loss. + expect(fake.markOrderlyCompletion).toHaveBeenCalledTimes(1); + fake.emitLoss("provider_exit"); + expect(fake.readDisposition().failed).toBe(false); }); }); diff --git a/packages/adapter-utils/src/acpx-engine/execute.ts b/packages/adapter-utils/src/acpx-engine/execute.ts index 925f01ae36..9210025c4d 100644 --- a/packages/adapter-utils/src/acpx-engine/execute.ts +++ b/packages/adapter-utils/src/acpx-engine/execute.ts @@ -34,7 +34,7 @@ import { type SandboxAdditionalSource, } from "@paperclipai/adapter-utils/execution-target"; import type { DuplexLossReason } from "../duplex-observability.js"; -import { DUPLEX_CHANNEL_LOST_ERROR_CODE } from "../duplex-bridge-broker.js"; +import { DUPLEX_CHANNEL_LOST_ERROR_CODE } from "../bridge-transport-contract.js"; import { DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE, applyPaperclipWorkspaceEnv, diff --git a/packages/adapter-utils/src/bridge-transport-contract.ts b/packages/adapter-utils/src/bridge-transport-contract.ts new file mode 100644 index 0000000000..5209ccd318 --- /dev/null +++ b/packages/adapter-utils/src/bridge-transport-contract.ts @@ -0,0 +1,61 @@ +/** + * Shared contract for the sandbox callback bridge transports. + * + * The retired duplex_v1 broker first defined these symbols. The host + * broker is gone, but the http2_v1 transport and the ACPX engine + * run-disposition seam still use them. This leaf module holds the + * survivors, so a caller of the run-disposition seam does not import the + * whole HTTP/2 bridge server module graph to reach one error code. + */ + +import type { DuplexLossReason } from "./duplex-observability.js"; + +/** + * The typed error code the host reports when the bridge control channel + * died before an orderly completion. Both the ACP lane and the CLI lane + * report this one code, so the run disposition is identical across the two + * lanes. + */ +export const DUPLEX_CHANNEL_LOST_ERROR_CODE = "duplex_channel_lost"; + +/** + * The terminal run disposition a bridge transport computes from its ordered + * lifecycle. A `failed` disposition means a terminal loss ordered before an + * orderly completion, so the run must not report success. The typed loss + * reason names the cause; it is `null` for a success. + */ +export interface DuplexBrokerRunDisposition { + /** True when a terminal loss ordered before an orderly completion. */ + failed: boolean; + /** The typed, closed loss reason on a failure; `null` on a success. */ + lossReason: DuplexLossReason | null; +} + +/** The nested timeout budgets. Each inner budget is smaller than its outer budget. */ +export interface DuplexBrokerBudgets { + /** The deadline for one forward call, in milliseconds. */ + forwardTimeoutMs: number; + /** The deadline for the broker to send one response frame, in milliseconds. */ + responseBudgetMs: number; + /** The deadline the in-sandbox gateway waits for the response frame, in milliseconds. */ + gatewayWaitMs: number; +} + +/** The default nested budgets: forward 30 s, response 32 s, gateway wait 35 s. */ +export const DEFAULT_DUPLEX_BROKER_BUDGETS: DuplexBrokerBudgets = { + forwardTimeoutMs: 30_000, + responseBudgetMs: 32_000, + gatewayWaitMs: 35_000, +}; + +/** + * The safe HTTP methods. RFC 7231 section 4.2.1 defines this set. A safe method + * does not change host state, so the host applies no mutation for it. A caller + * can retry a safe method after a forward failure without a double-apply risk. + */ +const SAFE_BRIDGE_METHODS = new Set(["GET", "HEAD", "OPTIONS", "TRACE"]); + +/** Report whether the method is safe, so a forward failure stays retryable. */ +export function isSafeBridgeMethod(method: string): boolean { + return SAFE_BRIDGE_METHODS.has(method.trim().toUpperCase()); +} diff --git a/packages/adapter-utils/src/duplex-bridge-broker-byte-ledger.test.ts b/packages/adapter-utils/src/duplex-bridge-broker-byte-ledger.test.ts deleted file mode 100644 index c82446343c..0000000000 --- a/packages/adapter-utils/src/duplex-bridge-broker-byte-ledger.test.ts +++ /dev/null @@ -1,384 +0,0 @@ -import { afterEach, describe, expect, it } from "vitest"; - -import { - createDuplexBridgeBroker, - type DuplexBridgeBroker, - type DuplexBrokerForwardResult, -} from "./duplex-bridge-broker.js"; -import { - DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, - DuplexAggregateByteLedger, -} from "./duplex-aggregate-byte-ledger.js"; -import { - DUPLEX_FRAME_VERSION, - DuplexFrameDecoder, - encodeDuplexFrame, - type DuplexRequestFrame, - type DuplexResponseFrame, -} from "./duplex-frame-codec.js"; -import { splitBodyIntoChunkFrames } from "./duplex-body-spool.js"; -import type { CommandManagedDuplexChannel } from "./command-managed-runtime.js"; - -/** - * Regression harness for the broker aggregate byte ledger charging. - * - * The harness feeds request frames straight into the broker through an in-memory - * channel, the same as a provider that controls the transport. A test controls - * when each forward settles, so it can assert the ledger charge while the forward - * is still in flight and after it settles. - */ - -/** One pending forward a test settles by hand. */ -interface PendingForward { - id: string; - resolve: (result: DuplexBrokerForwardResult) => void; - reject: (error: Error) => void; - settled: boolean; -} - -/** - * One reassembled response the broker wrote back. The broker writes a response as - * one envelope frame that carries `bodyByteCount`, then the `body_chunk` frames - * that carry the body. The harness reassembles the body and exposes it as `body`. - */ -type ReassembledResponse = DuplexResponseFrame & { body: string }; - -/** One request the harness feeds: the envelope frame plus its raw body text. */ -interface RequestInput { - frame: DuplexRequestFrame; - bodyText: string; -} - -/** The in-memory channel plus the levers a test uses to drive the broker. */ -interface FakeChannelHarness { - channel: CommandManagedDuplexChannel; - feed: (input: RequestInput) => void; - exit: () => void; - responses: ReassembledResponse[]; - forwards: PendingForward[]; - resolveForward: (id: string, body?: string) => void; -} - -/** Build one valid request: the envelope carries `bodyByteCount`, the body rides body_chunk frames. */ -function requestFrame(id: string, method = "POST"): RequestInput { - const bodyText = JSON.stringify({ id }); - return { - frame: { - version: DUPLEX_FRAME_VERSION, - type: "request", - id, - method, - path: `/api/issues/${id}`, - query: "", - headers: { "content-type": "application/json" }, - bodyByteCount: Buffer.byteLength(bodyText, "utf8"), - }, - bodyText, - }; -} - -function createFakeChannelHarness(): FakeChannelHarness { - const responses: ReassembledResponse[] = []; - const forwards: PendingForward[] = []; - const writtenDecoder = new DuplexFrameDecoder(); - const responseAssembly = new Map< - string, - { frame: DuplexResponseFrame; received: number; chunks: Buffer[] } - >(); - let dataListener: ((chunk: Uint8Array) => void) | null = null; - let exitListener: ((exit: { exitCode: number | null }) => void) | null = null; - - const channel: CommandManagedDuplexChannel = { - write: (data) => { - for (const result of writtenDecoder.push(data)) { - if (!result.ok) continue; - const frame = result.frame; - if (frame.type === "response") { - if (frame.bodyByteCount === 0) { - responses.push({ ...frame, body: "" }); - } else { - responseAssembly.set(frame.id, { frame, received: 0, chunks: [] }); - } - } else if (frame.type === "body_chunk") { - const assembly = responseAssembly.get(frame.id); - if (!assembly) continue; - const decoded = Buffer.from(frame.data, "base64"); - assembly.received += decoded.length; - assembly.chunks.push(decoded); - if (assembly.received >= assembly.frame.bodyByteCount) { - responseAssembly.delete(frame.id); - responses.push({ - ...assembly.frame, - body: Buffer.concat(assembly.chunks).toString("utf8"), - }); - } - } - } - }, - onData: (listener) => { - dataListener = listener; - }, - onExit: (listener) => { - exitListener = listener; - }, - stop: () => undefined, - close: () => Promise.resolve(), - }; - - return { - channel, - feed: ({ frame, bodyText }) => { - if (!dataListener) throw new Error("The broker did not bind the data listener."); - dataListener(Buffer.from(encodeDuplexFrame(frame), "utf8")); - if (bodyText.length > 0) { - for (const chunk of splitBodyIntoChunkFrames( - frame.id, - Buffer.from(bodyText, "utf8"), - DUPLEX_FRAME_VERSION, - )) { - dataListener(Buffer.from(encodeDuplexFrame(chunk), "utf8")); - } - } - }, - exit: () => { - if (!exitListener) throw new Error("The broker did not bind the exit listener."); - exitListener({ exitCode: 0 }); - }, - responses, - forwards, - resolveForward: (id, body = JSON.stringify({ ok: true })) => { - const forward = [...forwards].reverse().find((entry) => entry.id === id && !entry.settled); - if (!forward) throw new Error(`No unsettled forward for id ${id}.`); - forward.settled = true; - forward.resolve({ status: 200, headers: { "content-type": "application/json" }, body }); - }, - }; -} - -/** A forward handler that hands each call to the harness and never auto-resolves. */ -function controllableForward(harness: FakeChannelHarness) { - return (request: DuplexRequestFrame): Promise => - new Promise((resolve, reject) => { - harness.forwards.push({ id: request.id, resolve, reject, settled: false }); - }); -} - -/** - * Wait for the microtasks and one macrotask to settle, so the broker reassembles - * a request body, dispatches its forward, and runs each settled promise handler. - */ -async function flush(): Promise { - await new Promise((resolve) => setTimeout(resolve, 0)); -} - -describe("duplex bridge broker aggregate byte ledger", () => { - const brokers: DuplexBridgeBroker[] = []; - - afterEach(async () => { - while (brokers.length > 0) { - const broker = brokers.pop(); - if (broker) await broker.close(); - } - }); - - it("charges request-frame, request-payload, and seen-id tokens, then releases the request tokens on forward settlement", async () => { - const harness = createFakeChannelHarness(); - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 }); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - duplexAggregateByteLedger: ledger, - }); - brokers.push(broker); - broker.start(); - - harness.feed(requestFrame("req-1")); - // The dispatch retains three tokens at the request envelope: the raw frame, the - // normalized payload, and the no-replay set entry. The reservation is - // synchronous, so the charge holds before the body reassembles. - expect(ledger.liveTokenCount).toBe(3); - expect(ledger.bytesInUse).toBeGreaterThan(0); - const chargedInFlight = ledger.bytesInUse; - - // The broker reassembles the request body, then dispatches the forward. - await flush(); - harness.resolveForward("req-1"); - await flush(); - - // The forward settled. The finally owner released the request-frame and the - // request-payload tokens. The seen-id token stays charged for the channel - // lifetime, so exactly one token remains. - expect(ledger.liveTokenCount).toBe(1); - expect(ledger.bytesInUse).toBeGreaterThan(0); - expect(ledger.bytesInUse).toBeLessThan(chargedInFlight); - - // A close releases the seen-id token, so the ledger returns to zero. - await broker.close(); - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - }); - - it("keeps request tokens charged when the response timer answers before the forward settles, and releases them on settlement", async () => { - const harness = createFakeChannelHarness(); - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 }); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - // Squeeze the nested budgets so the response timer fires quickly while the - // forward stays unsettled. - budgets: { forwardTimeoutMs: 5, responseBudgetMs: 10, gatewayWaitMs: 20 }, - duplexAggregateByteLedger: ledger, - }); - brokers.push(broker); - broker.start(); - - harness.feed(requestFrame("req-1", "GET")); - expect(ledger.liveTokenCount).toBe(3); - - // Wait past the response budget so the backstop answers the gateway while the - // forward is still in flight. - await new Promise((resolve) => setTimeout(resolve, 40)); - expect(harness.responses.some((frame) => frame.id === "req-1")).toBe(true); - // The gateway got its answer, but the forward has not settled. The request - // tokens and the seen-id token stay charged: nothing released yet. - expect(ledger.liveTokenCount).toBe(3); - - // Settle the orphaned forward. Its finally owner now releases the two request - // tokens; the seen-id token still stays for the channel lifetime. - harness.resolveForward("req-1"); - await flush(); - expect(ledger.liveTokenCount).toBe(1); - - await broker.close(); - expect(ledger.bytesInUse).toBe(0); - }); - - it("refuses a one-byte-over dispatch with the fixed marker and retains nothing", async () => { - const harness = createFakeChannelHarness(); - // Measure the exact per-request charge with a throwaway broker, so the test - // can size a ceiling one byte below it and force the reservation to fail. - const probeHarness = createFakeChannelHarness(); - const measured = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 }); - const measuringBroker = await createDuplexBridgeBroker({ - channel: probeHarness.channel, - forwardRequest: controllableForward(probeHarness), - duplexAggregateByteLedger: measured, - }); - measuringBroker.start(); - probeHarness.feed(requestFrame("req-1")); - const perRequestBytes = measured.bytesInUse; - await measuringBroker.close(); - - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: perRequestBytes - 1 }); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - duplexAggregateByteLedger: ledger, - }); - brokers.push(broker); - broker.start(); - - harness.feed(requestFrame("req-1")); - await flush(); - - // The broker retained nothing: no token, no forward, no seen id. - expect(ledger.liveTokenCount).toBe(0); - expect(ledger.bytesInUse).toBe(0); - expect(harness.forwards.length).toBe(0); - // The refusal carries the fixed marker and is a bounded terminal response. - const refusal = harness.responses.find((frame) => frame.id === "req-1"); - expect(refusal).toBeTruthy(); - expect(refusal?.status).toBe(503); - expect(refusal?.body).toContain(DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED); - - // The refusal did not retain the id, so a resend after pressure eases is - // admitted: raise the ceiling and re-feed. - const roomyLedger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 }); - const roomyHarness = createFakeChannelHarness(); - const roomyBroker = await createDuplexBridgeBroker({ - channel: roomyHarness.channel, - forwardRequest: controllableForward(roomyHarness), - duplexAggregateByteLedger: roomyLedger, - }); - brokers.push(roomyBroker); - roomyBroker.start(); - roomyHarness.feed(requestFrame("req-1")); - await flush(); - expect(roomyHarness.forwards.length).toBe(1); - }); - - it("releases every retained token on a terminal channel loss", async () => { - const harness = createFakeChannelHarness(); - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 }); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - duplexAggregateByteLedger: ledger, - }); - brokers.push(broker); - broker.start(); - - harness.feed(requestFrame("req-1")); - harness.feed(requestFrame("req-2")); - // The reservation is synchronous, so both dispatches charge three tokens each - // at the request envelope, before the bodies reassemble. - expect(ledger.liveTokenCount).toBe(6); - - // Let both requests reassemble and start their forwards, so the loss below - // transfers live forwards to the orphan registry. - await flush(); - - // The channel exits mid-flight. The broker latches the loss, transfers the - // in-flight forwards to the orphan registry, and releases the seen-id tokens. - harness.exit(); - expect(broker.runDisposition.failed).toBe(true); - // The seen-id tokens released at loss; the two request tokens per forward stay - // charged until each aborted forward settles. - expect(ledger.liveTokenCount).toBe(4); - - // The aborted forwards settle. Each finally owner releases its two request - // tokens, so the ledger returns to zero. - harness.forwards[0]?.reject(new Error("aborted")); - harness.forwards[1]?.reject(new Error("aborted")); - await flush(); - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - }); - - it("does not double-release a token when a forward settles after the channel closed", async () => { - const harness = createFakeChannelHarness(); - let underflow = 0; - const ledger = new DuplexAggregateByteLedger({ - ceilingBytes: 1_000_000, - telemetry: { - setBytesInUse() {}, - recordReservationRejection() {}, - recordAccountingUnderflow() { - underflow += 1; - }, - }, - }); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - duplexAggregateByteLedger: ledger, - }); - brokers.push(broker); - broker.start(); - - harness.feed(requestFrame("req-1")); - // Let the request reassemble and start its forward before the close, so the - // close orphans a live forward that settles afterward. - await flush(); - await broker.close(); - // The close aborted the in-flight forward and orphaned it. The forward now - // settles after the close: its finally owner releases the request tokens once. - harness.forwards[0]?.reject(new Error("aborted")); - await flush(); - - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - // No accounting defect: every token released exactly one time. - expect(underflow).toBe(0); - }); -}); diff --git a/packages/adapter-utils/src/duplex-bridge-broker.test.ts b/packages/adapter-utils/src/duplex-bridge-broker.test.ts deleted file mode 100644 index cc7f5ce3de..0000000000 --- a/packages/adapter-utils/src/duplex-bridge-broker.test.ts +++ /dev/null @@ -1,1039 +0,0 @@ -import { afterEach, describe, expect, it } from "vitest"; - -import { - assertDuplexBrokerLimits, - createDuplexBridgeBroker, - type DuplexBridgeBroker, - type DuplexBrokerForwardResult, -} from "./duplex-bridge-broker.js"; -import { - DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES, - DUPLEX_FRAME_VERSION, - DuplexFrameDecoder, - encodeDuplexFrame, - type DuplexRequestFrame, - type DuplexResponseFrame, -} from "./duplex-frame-codec.js"; -import { splitBodyIntoChunkFrames } from "./duplex-body-spool.js"; -import { - createDuplexObservability, - DUPLEX_COUNTER_LOSS_TOTAL, - DUPLEX_SPAN_REQUEST, - type DuplexObservabilityCounterRecord, - type DuplexObservabilityEventRecord, - type DuplexObservabilitySpanRecord, -} from "./duplex-observability.js"; -import type { CommandManagedDuplexChannel } from "./command-managed-runtime.js"; - -/** - * Unit harness for the host broker limit gate. - * - * The harness drives frames straight into the broker through an in-memory - * channel. It does not spawn the gateway, so a test injects untrusted request - * frames directly, the same as a malicious provider that controls the transport. - * The channel records each frame the broker writes back, so a test asserts the - * exact response the broker returns for a refused request. - */ - -/** One pending forward the test controls. */ -interface PendingForward { - id: string; - resolve: (result: DuplexBrokerForwardResult) => void; - reject: (error: Error) => void; - settled: boolean; -} - -/** - * One reassembled response the broker wrote back. The broker writes a response as - * one envelope frame that carries `bodyByteCount`, then the `body_chunk` frames - * that carry the body. The harness reassembles the body and exposes it as `body`, - * so a test asserts the response the same way it did with the one-frame model. - */ -type ReassembledResponse = DuplexResponseFrame & { body: string }; - -/** One request the harness feeds: the envelope frame plus its raw body text. */ -interface RequestInput { - frame: DuplexRequestFrame; - bodyText: string; -} - -/** The in-memory channel plus the levers a test uses to drive the broker. */ -interface FakeChannelHarness { - channel: CommandManagedDuplexChannel; - /** Push one request (its envelope frame and its body_chunk frames) into the broker read path. */ - feed: (input: RequestInput) => void; - /** The reassembled responses the broker wrote back, in order. */ - responses: ReassembledResponse[]; - /** Every forward call the broker made, in order. */ - forwards: PendingForward[]; - /** Resolve the newest unsettled forward for one id with a 200 result. */ - resolveForward: (id: string, body?: string) => void; - /** Resolve every unsettled forward with a 200 result. */ - resolveAll: () => void; -} - -/** - * Build one valid request with distinctive, secret-looking fields. The envelope - * carries `bodyByteCount`, and the harness rides the body on `body_chunk` frames. - */ -function requestFrame( - id: string, - method = "POST", - bodyText: string = JSON.stringify({ secret: "secret-request-body" }), -): RequestInput { - return { - frame: { - version: DUPLEX_FRAME_VERSION, - type: "request", - id, - method, - path: `/api/issues/${id}`, - query: "?secret-query=leak", - headers: { authorization: "Bearer super-secret-provider-token" }, - bodyByteCount: Buffer.byteLength(bodyText, "utf8"), - }, - bodyText, - }; -} - -/** - * Build the in-memory channel harness. The channel keeps the broker read - * listener, so `feed` pushes the encoded envelope and its body_chunk frames into - * the broker. The channel decodes each written frame and reassembles a response - * body, so `responses` holds one reassembled response per delivered request. - */ -function createFakeChannelHarness(): FakeChannelHarness { - const responses: ReassembledResponse[] = []; - const forwards: PendingForward[] = []; - const writtenDecoder = new DuplexFrameDecoder(); - // The in-flight response reassembly, keyed by request id. - const responseAssembly = new Map< - string, - { frame: DuplexResponseFrame; received: number; chunks: Buffer[] } - >(); - let dataListener: ((chunk: Uint8Array) => void) | null = null; - - const channel: CommandManagedDuplexChannel = { - write: (data) => { - for (const result of writtenDecoder.push(data)) { - if (!result.ok) continue; - const frame = result.frame; - if (frame.type === "response") { - if (frame.bodyByteCount === 0) { - responses.push({ ...frame, body: "" }); - } else { - responseAssembly.set(frame.id, { frame, received: 0, chunks: [] }); - } - } else if (frame.type === "body_chunk") { - const assembly = responseAssembly.get(frame.id); - if (!assembly) continue; - const decoded = Buffer.from(frame.data, "base64"); - assembly.received += decoded.length; - assembly.chunks.push(decoded); - if (assembly.received >= assembly.frame.bodyByteCount) { - responseAssembly.delete(frame.id); - responses.push({ - ...assembly.frame, - body: Buffer.concat(assembly.chunks).toString("utf8"), - }); - } - } - } - }, - onData: (listener) => { - dataListener = listener; - }, - onExit: () => undefined, - stop: () => undefined, - close: () => Promise.resolve(), - }; - - return { - channel, - feed: ({ frame, bodyText }) => { - if (!dataListener) throw new Error("The broker did not bind the data listener."); - dataListener(Buffer.from(encodeDuplexFrame(frame), "utf8")); - if (bodyText.length > 0) { - const chunks = splitBodyIntoChunkFrames( - frame.id, - Buffer.from(bodyText, "utf8"), - DUPLEX_FRAME_VERSION, - ); - for (const chunk of chunks) dataListener(Buffer.from(encodeDuplexFrame(chunk), "utf8")); - } - }, - responses, - forwards, - resolveForward: (id, body = JSON.stringify({ ok: true })) => { - const forward = [...forwards].reverse().find((entry) => entry.id === id && !entry.settled); - if (!forward) throw new Error(`No unsettled forward for id ${id}.`); - forward.settled = true; - forward.resolve({ status: 200, headers: { "content-type": "application/json" }, body }); - }, - resolveAll: () => { - for (const forward of forwards) { - if (forward.settled) continue; - forward.settled = true; - forward.resolve({ - status: 200, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ ok: true }), - }); - } - }, - }; -} - -/** A forward handler that hands each call to the harness and never auto-resolves. */ -function controllableForward(harness: FakeChannelHarness) { - return (request: DuplexRequestFrame): Promise => - new Promise((resolve, reject) => { - harness.forwards.push({ id: request.id, resolve, reject, settled: false }); - }); -} - -/** Flush the microtasks and one macrotask, so the broker dispatches its reassembled forwards. */ -async function flush(): Promise { - await new Promise((resolve) => setTimeout(resolve, 0)); -} - -/** The telemetry sink capture. It proves the broker records nothing for a refusal. */ -interface TelemetryCapture { - spans: DuplexObservabilitySpanRecord[]; - counters: DuplexObservabilityCounterRecord[]; - events: DuplexObservabilityEventRecord[]; -} - -/** Build the real telemetry facade over a capturing recorder. */ -function createTelemetryCapture(): { telemetry: ReturnType; capture: TelemetryCapture } { - const capture: TelemetryCapture = { spans: [], counters: [], events: [] }; - const telemetry = createDuplexObservability({ - providerKey: "daytona", - recorder: { - recordSpan: (record) => capture.spans.push(record), - incrementCounter: (record) => capture.counters.push(record), - emitEvent: (record) => capture.events.push(record), - }, - }); - return { telemetry, capture }; -} - -describe("duplex bridge broker request limits", () => { - const brokers: DuplexBridgeBroker[] = []; - - afterEach(async () => { - while (brokers.length > 0) { - const broker = brokers.pop(); - if (broker) await broker.close(); - } - }); - - it("rejects a non-positive or non-integer limit at construction", () => { - expect(() => assertDuplexBrokerLimits({ maxInFlightRequests: 0, maxLifetimeRequests: 10 })).toThrow(); - expect(() => assertDuplexBrokerLimits({ maxInFlightRequests: 5, maxLifetimeRequests: 0 })).toThrow(); - expect(() => assertDuplexBrokerLimits({ maxInFlightRequests: 1.5, maxLifetimeRequests: 10 })).toThrow(); - expect(() => - assertDuplexBrokerLimits({ maxInFlightRequests: Number.POSITIVE_INFINITY, maxLifetimeRequests: 10 }), - ).toThrow(); - expect(() => assertDuplexBrokerLimits({ maxInFlightRequests: 8, maxLifetimeRequests: 16 })).not.toThrow(); - }); - - it("bounds the in-flight forwards and refuses the excess with a retryable terminal response", async () => { - const harness = createFakeChannelHarness(); - const { telemetry, capture } = createTelemetryCapture(); - const maxInFlightRequests = 4; - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - telemetry, - maxInFlightRequests, - }); - brokers.push(broker); - broker.start(); - - // Inject more than the limit of unique valid frames. Every forward hangs, so - // the broker holds the maximum number of in-flight forwards. The broker gates - // the in-flight limit at the request envelope, so it refuses the excess at - // once; it dispatches each accepted forward after it reassembles the body. - const total = 10; - const ids = Array.from({ length: total }, (_unused, index) => `flight-${index}`); - for (const id of ids) harness.feed(requestFrame(id)); - await flush(); - - // The broker forwarded only up to the limit. It refused the rest. - expect(harness.forwards).toHaveLength(maxInFlightRequests); - expect(harness.responses).toHaveLength(total - maxInFlightRequests); - for (const response of harness.responses) { - expect(response.status).toBe(503); - expect(response.outcome).toBe("unavailable"); - expect(response.headers["x-paperclip-bridge-outcome"]).toBe("unavailable"); - expect(JSON.parse(response.body)).toEqual({ - error: "Duplex broker capacity limit reached.", - outcome: "unavailable", - retryable: true, - }); - } - // The refusal leaks no route, query, body, or token. - const refusalText = harness.responses.map((response) => encodeDuplexFrame(response)).join(""); - expect(refusalText).not.toContain("secret"); - expect(refusalText).not.toContain("super-secret-provider-token"); - expect(refusalText).not.toContain("/api/issues/"); - - // The broker recorded no request span and no counter for a refusal. Only the - // delivered requests below produce a span. - expect(capture.spans).toHaveLength(0); - expect(capture.counters).toHaveLength(0); - - // Resolve the in-flight forwards. The broker delivers one response for each - // and drains the pending bookkeeping. - harness.resolveAll(); - }); - - it("frees capacity after a delivered request and forwards a resent refused id (no-replay preserved)", async () => { - const harness = createFakeChannelHarness(); - const maxInFlightRequests = 2; - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - maxInFlightRequests, - }); - brokers.push(broker); - broker.start(); - - // Saturate the in-flight limit with two hung forwards. The broker reserves the - // in-flight slot at the request envelope, then dispatches the forward after it - // reassembles the body. - harness.feed(requestFrame("keep-0")); - harness.feed(requestFrame("keep-1")); - await flush(); - expect(harness.forwards).toHaveLength(2); - - // A third unique id is refused, retryable. The broker did not forward it and - // did not retain its id. - harness.feed(requestFrame("resend-me")); - await flush(); - expect(harness.forwards).toHaveLength(2); - expect(harness.responses).toHaveLength(1); - expect(JSON.parse(harness.responses[0].body).retryable).toBe(true); - - // Free one in-flight slot. The delivered request drains the pending record. - harness.resolveForward("keep-0"); - await flush(); - - // The gateway resends the refused id. Capacity is free now, so the broker - // forwards it exactly one time. This proves the refusal did not poison the id. - harness.feed(requestFrame("resend-me")); - await flush(); - expect(harness.forwards.filter((forward) => forward.id === "resend-me")).toHaveLength(1); - - // A resend of an already-forwarded id never reaches the forward twice. - const forwardedKeep1 = harness.forwards.filter((forward) => forward.id === "keep-1").length; - harness.feed(requestFrame("keep-1")); - await flush(); - expect(harness.forwards.filter((forward) => forward.id === "keep-1")).toHaveLength(forwardedKeep1); - - harness.resolveAll(); - }); - - it("bounds the retained request ids over the channel lifetime and refuses each new id past the limit", async () => { - const harness = createFakeChannelHarness(); - const { telemetry, capture } = createTelemetryCapture(); - const maxLifetimeRequests = 3; - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - // Resolve every forward at once, so each dispatched id leaves the in-flight - // count but stays in the retained-id set. - forwardRequest: async (request: DuplexRequestFrame): Promise => { - harness.forwards.push({ id: request.id, resolve: () => undefined, reject: () => undefined, settled: true }); - return { status: 200, headers: { "content-type": "application/json" }, body: JSON.stringify({ ok: true }) }; - }, - telemetry, - maxLifetimeRequests, - }); - brokers.push(broker); - broker.start(); - - // Inject more than the lifetime limit of unique valid frames. - const total = 9; - for (let index = 0; index < total; index += 1) { - harness.feed(requestFrame(`life-${index}`)); - await flush(); - } - - // The broker forwarded only up to the lifetime limit. The retained-id set - // stays bounded, because a refused id never joins it. - expect(harness.forwards).toHaveLength(maxLifetimeRequests); - - // The broker delivered a real response for each forwarded request, then a - // bounded terminal refusal for each id past the limit. - const refusals = harness.responses.filter((response) => response.status === 503); - expect(refusals).toHaveLength(total - maxLifetimeRequests); - for (const refusal of refusals) { - expect(refusal.outcome).toBe("unavailable"); - // A lifetime refusal is terminal, so a resend does not help. - expect(JSON.parse(refusal.body).retryable).toBe(false); - } - - // A resend of a refused id stays refused, so the set never grows. - harness.feed(requestFrame("life-8")); - await flush(); - expect(harness.forwards).toHaveLength(maxLifetimeRequests); - - // The telemetry surface stayed on the fixed request span for the delivered - // requests only. No refusal produced a span, a counter, or a loss record. - expect(capture.spans).toHaveLength(maxLifetimeRequests); - for (const span of capture.spans) { - expect(span.name).toBe(DUPLEX_SPAN_REQUEST); - expect(span.dimensions.outcome).toBe("ok"); - expect(Object.keys(span.dimensions).sort()).toEqual(["outcome", "provider", "transport"]); - } - expect(capture.counters).toHaveLength(0); - expect(broker.lossRecord).toBeNull(); - expect(broker.state).toBe("open"); - }); - - it("forwards nothing and fails the channel closed under a flood of over-limit ids", async () => { - const harness = createFakeChannelHarness(); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - }); - brokers.push(broker); - broker.start(); - - // Send a sustained flood of distinct ids, each one byte over the id bound. The - // codec rejects each frame on the read path, so no over-limit id ever reaches - // the retained-id set or a forward. - const overLimitId = (index: number): string => - `${"a".repeat(DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES)}-${index}`; - for (let index = 0; index < 20; index += 1) { - harness.feed(requestFrame(overLimitId(index))); - } - - // The broker forwarded nothing. The retained-id set never grew, because the - // set only grows on a dispatched forward, and the broker dispatched none. - expect(harness.forwards).toHaveLength(0); - // The first over-limit frame is a protocol failure, so the broker fails the - // whole channel closed. It stays lost for the rest of the flood. - expect(broker.state).toBe("lost"); - expect(broker.lossRecord?.reason).toBe("protocol_failure"); - }); - - it("forwards a maximal-size id exactly once and does not forward a resend (no-replay preserved)", async () => { - const harness = createFakeChannelHarness(); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - }); - brokers.push(broker); - broker.start(); - - // An id at the maximum byte size is allowed. The broker forwards it one time. - const maxId = "a".repeat(DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES); - harness.feed(requestFrame(maxId)); - await flush(); - expect(harness.forwards.filter((forward) => forward.id === maxId)).toHaveLength(1); - - // Deliver the response, then resend the same id. The broker retained the id, - // so the resend never reaches the forward a second time. - harness.resolveForward(maxId); - harness.feed(requestFrame(maxId)); - await flush(); - expect(harness.forwards.filter((forward) => forward.id === maxId)).toHaveLength(1); - expect(broker.state).toBe("open"); - }); - - it("maps a forward that resolves with the indeterminate marker to a non-retryable response and retains the id", async () => { - const harness = createFakeChannelHarness(); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - }); - brokers.push(broker); - broker.start(); - - // The forward handler resolves with a 504 that carries the indeterminate - // marker. This is the shape the host forward returns for a response-read - // failure after the host commit. The broker must keep the marker and the - // non-retryable body, so the gateway maps it to a non-retryable status and a - // caller never repeats a possibly-committed mutation. - harness.feed(requestFrame("commit-1")); - await flush(); - const forward = harness.forwards.find((entry) => entry.id === "commit-1" && !entry.settled); - if (!forward) throw new Error("The broker did not forward the request."); - forward.settled = true; - forward.resolve({ - status: 504, - headers: { - "content-type": "application/json", - "x-paperclip-bridge-outcome": "indeterminate", - }, - body: JSON.stringify({ - error: "Bridge response body exceeded the configured size limit of 32 bytes.", - outcome: "indeterminate", - retryable: false, - }), - }); - // The broker answers inside the forward `then` microtask, so let it settle. - await Promise.resolve(); - - expect(harness.responses).toHaveLength(1); - const response = harness.responses[0]!; - expect(response.status).toBe(504); - expect(response.outcome).toBe("indeterminate"); - expect(response.headers["x-paperclip-bridge-outcome"]).toBe("indeterminate"); - expect(JSON.parse(response.body)).toEqual({ - error: "Bridge response body exceeded the configured size limit of 32 bytes.", - outcome: "indeterminate", - retryable: false, - }); - - // The broker delivered a response, so it retained the id. A resend never - // reaches the forward a second time, so a caller that ignores the - // non-retryable status still cannot repeat the mutation through the broker. - harness.feed(requestFrame("commit-1")); - expect(harness.forwards.filter((entry) => entry.id === "commit-1")).toHaveLength(1); - expect(broker.state).toBe("open"); - }); - - it("maps a forward rejection for a mutating method to a non-retryable indeterminate response and retains the id", async () => { - const harness = createFakeChannelHarness(); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - }); - brokers.push(broker); - broker.start(); - - // The forward rejects before the host delivers a response. A fetch can - // reject after the request reaches the host and the host commits, but - // before the response headers arrive. The broker did not abort the - // forward, so a POST must return a non-retryable indeterminate response. - // A retryable status would let a caller repeat a committed mutation. - harness.feed(requestFrame("mutate-1", "POST")); - await flush(); - const forward = harness.forwards.find((entry) => entry.id === "mutate-1" && !entry.settled); - if (!forward) throw new Error("The broker did not forward the request."); - forward.settled = true; - forward.reject(new Error("fetch failed before the response headers arrived")); - // The broker answers inside the forward rejection microtask, so let it settle. - await new Promise((resolve) => setTimeout(resolve, 0)); - - expect(harness.responses).toHaveLength(1); - const response = harness.responses[0]!; - expect(response.status).toBe(504); - expect(response.outcome).toBe("indeterminate"); - expect(response.headers["x-paperclip-bridge-outcome"]).toBe("indeterminate"); - expect(JSON.parse(response.body)).toEqual({ - error: "fetch failed before the response headers arrived", - outcome: "indeterminate", - retryable: false, - }); - - // The broker delivered a response, so it retained the id. A resend never - // reaches the forward a second time, so a caller that ignores the - // non-retryable status still cannot repeat the mutation through the broker. - harness.feed(requestFrame("mutate-1", "POST")); - expect(harness.forwards.filter((entry) => entry.id === "mutate-1")).toHaveLength(1); - expect(broker.state).toBe("open"); - }); - - it("substitutes a bounded terminal response for an oversized response and keeps the channel open", async () => { - const harness = createFakeChannelHarness(); - const { telemetry, capture } = createTelemetryCapture(); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - telemetry, - // A small frame bound makes a legitimately large response body exceed the - // bound, so the test exercises the encode guard without a megabyte body. - maxFrameBytes: 1000, - }); - brokers.push(broker); - broker.start(); - - // Two requests are in flight at the same time. One resolves with a normal - // body; the other resolves with a body that pushes the response frame over - // the bound. The host produced both results, so both requests reached the - // host. - harness.feed(requestFrame("small-1", "POST")); - harness.feed(requestFrame("big-1", "POST")); - await flush(); - - const smallForward = harness.forwards.find((entry) => entry.id === "small-1" && !entry.settled); - if (!smallForward) throw new Error("The broker did not forward the small request."); - smallForward.settled = true; - smallForward.resolve({ - status: 200, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ ok: true }), - }); - await new Promise((resolve) => setTimeout(resolve, 0)); - - const bigForward = harness.forwards.find((entry) => entry.id === "big-1" && !entry.settled); - if (!bigForward) throw new Error("The broker did not forward the big request."); - bigForward.settled = true; - bigForward.resolve({ - status: 200, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ data: "y".repeat(2000) }), - }); - await new Promise((resolve) => setTimeout(resolve, 0)); - - // The other in-flight request delivered its full response, unaffected by the - // oversized one. - expect(harness.responses).toHaveLength(2); - const delivered = harness.responses.find((entry) => entry.id === "small-1")!; - expect(delivered.status).toBe(200); - expect(delivered.outcome).toBe("completed"); - expect(JSON.parse(delivered.body)).toEqual({ ok: true }); - - // The oversized response became one bounded terminal response marked - // indeterminate and non-retryable, so a caller does not retry a possible - // mutation. The broker did not write the oversized frame. - const terminal = harness.responses.find((entry) => entry.id === "big-1")!; - expect(terminal.status).toBe(502); - expect(terminal.outcome).toBe("indeterminate"); - expect(terminal.headers["x-paperclip-bridge-outcome"]).toBe("indeterminate"); - expect(JSON.parse(terminal.body)).toEqual({ - error: "upstream response too large to deliver", - outcome: "indeterminate", - retryable: false, - }); - - // The oversized body never crossed the wire. The broker did not truncate it or - // pass it through. - const writtenText = harness.responses.map((entry) => encodeDuplexFrame(entry)).join(""); - expect(writtenText).not.toContain("yyyy"); - - // The broker recorded the oversized request as an error outcome, never a loss. - // The channel stays open and the run does not fail. - const errorSpans = capture.spans.filter((span) => span.dimensions.outcome === "error"); - expect(errorSpans).toHaveLength(1); - expect(errorSpans[0]!.name).toBe(DUPLEX_SPAN_REQUEST); - expect(broker.lossRecord).toBeNull(); - expect(broker.state).toBe("open"); - expect(broker.runDisposition.failed).toBe(false); - }); - - it("sends a minimal terminal response when the frame bound rejects the full replacement", async () => { - const harness = createFakeChannelHarness(); - const { telemetry, capture } = createTelemetryCapture(); - // The bound accepts a compact request envelope (128 bytes) and the minimal - // terminal response envelope (114 bytes), but rejects the full replacement - // response envelope (193 bytes). This proves the broker never drops the channel - // or strands the request when the bound is smaller than the full replacement. - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - telemetry, - maxFrameBytes: 150, - }); - brokers.push(broker); - broker.start(); - - // A compact request envelope with an empty body fits the bound, so the broker - // forwards it. - harness.feed({ - frame: { - version: DUPLEX_FRAME_VERSION, - type: "request", - id: "big-1", - method: "POST", - path: "/api/issues/big-1", - query: "", - headers: {}, - bodyByteCount: 0, - }, - bodyText: "", - }); - await flush(); - - const forward = harness.forwards.find((entry) => entry.id === "big-1" && !entry.settled); - if (!forward) throw new Error("The broker did not forward the request."); - forward.settled = true; - // The host produced a large response body that pushes the real response frame - // and the full replacement frame over the bound. - forward.resolve({ - status: 200, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ data: "y".repeat(2000) }), - }); - await new Promise((resolve) => setTimeout(resolve, 0)); - - // The broker sent one minimal terminal response, not the oversized real - // response and not the full replacement. The minimal response carries the - // indeterminate outcome, so the gateway maps it to a terminal 409. - expect(harness.responses).toHaveLength(1); - const terminal = harness.responses[0]!; - expect(terminal.id).toBe("big-1"); - expect(terminal.status).toBe(502); - expect(terminal.outcome).toBe("indeterminate"); - expect(terminal.headers).toEqual({}); - expect(terminal.body).toBe(""); - - // The oversized body never crossed the wire. - const writtenText = harness.responses.map((entry) => encodeDuplexFrame(entry)).join(""); - expect(writtenText).not.toContain("yyyy"); - - // The broker recorded the request as an error outcome, never a loss. The - // channel stays open and the run does not fail. - const errorSpans = capture.spans.filter((span) => span.dimensions.outcome === "error"); - expect(errorSpans).toHaveLength(1); - expect(errorSpans[0]!.name).toBe(DUPLEX_SPAN_REQUEST); - expect(broker.lossRecord).toBeNull(); - expect(broker.state).toBe("open"); - expect(broker.runDisposition.failed).toBe(false); - }); - - it("keeps the channel open and logs when the bound rejects even the minimal terminal response", async () => { - const harness = createFakeChannelHarness(); - const { telemetry, capture } = createTelemetryCapture(); - const logs: string[] = []; - // The bound accepts a tiny request envelope (107 bytes) but rejects even the - // minimal terminal response envelope (110 bytes). The broker cannot send any - // frame for this request, so it must keep the channel open and log a clear - // local error. - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - telemetry, - maxFrameBytes: 109, - logger: (message) => logs.push(message), - }); - brokers.push(broker); - broker.start(); - - harness.feed({ - frame: { - version: DUPLEX_FRAME_VERSION, - type: "request", - id: "x", - method: "GET", - path: "/", - query: "", - headers: {}, - bodyByteCount: 0, - }, - bodyText: "", - }); - await flush(); - - const forward = harness.forwards.find((entry) => entry.id === "x" && !entry.settled); - if (!forward) throw new Error("The broker did not forward the request."); - forward.settled = true; - forward.resolve({ - status: 200, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ data: "y".repeat(200) }), - }); - await new Promise((resolve) => setTimeout(resolve, 0)); - - // The broker sent no response frame, because no frame fits the bound. - expect(harness.responses).toHaveLength(0); - // The broker recorded the request as an error outcome, never a loss. - const errorSpans = capture.spans.filter((span) => span.dimensions.outcome === "error"); - expect(errorSpans).toHaveLength(1); - // The broker logged a clear local error and kept the channel open. - expect(logs.some((line) => line.includes("could not deliver a terminal response"))).toBe(true); - expect(broker.lossRecord).toBeNull(); - expect(broker.state).toBe("open"); - expect(broker.runDisposition.failed).toBe(false); - }); - - it("completes a request within the bound and delivers the real response", async () => { - const harness = createFakeChannelHarness(); - const { telemetry, capture } = createTelemetryCapture(); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - telemetry, - maxFrameBytes: 1000, - }); - brokers.push(broker); - broker.start(); - - harness.feed(requestFrame("ok-1", "POST")); - await flush(); - harness.resolveForward("ok-1", JSON.stringify({ ok: true })); - await new Promise((resolve) => setTimeout(resolve, 0)); - - // The broker delivered the real response promptly with the completed outcome. - expect(harness.responses).toHaveLength(1); - const delivered = harness.responses[0]!; - expect(delivered.id).toBe("ok-1"); - expect(delivered.status).toBe(200); - expect(delivered.outcome).toBe("completed"); - expect(JSON.parse(delivered.body)).toEqual({ ok: true }); - - // The delivered response records an `ok` span, never an error, and never a loss. - const okSpans = capture.spans.filter((span) => span.dimensions.outcome === "ok"); - expect(okSpans).toHaveLength(1); - const errorSpans = capture.spans.filter((span) => span.dimensions.outcome === "error"); - expect(errorSpans).toHaveLength(0); - expect(broker.lossRecord).toBeNull(); - expect(broker.state).toBe("open"); - expect(broker.runDisposition.failed).toBe(false); - }); - - it("maps a forward rejection for a safe method to a retryable response", async () => { - const harness = createFakeChannelHarness(); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - }); - brokers.push(broker); - broker.start(); - - // A safe method never changes host state, so a retry cannot double-apply a - // mutation. A forward rejection for a GET stays retryable: the broker - // returns a 502 with the completed outcome, so the gateway passes it through. - harness.feed(requestFrame("read-1", "GET")); - await flush(); - const forward = harness.forwards.find((entry) => entry.id === "read-1" && !entry.settled); - if (!forward) throw new Error("The broker did not forward the request."); - forward.settled = true; - forward.reject(new Error("fetch failed before the response headers arrived")); - await new Promise((resolve) => setTimeout(resolve, 0)); - - expect(harness.responses).toHaveLength(1); - const response = harness.responses[0]!; - expect(response.status).toBe(502); - expect(response.outcome).toBe("completed"); - expect(response.headers["x-paperclip-bridge-outcome"]).toBeUndefined(); - expect(JSON.parse(response.body)).toEqual({ - error: "fetch failed before the response headers arrived", - }); - }); - - it("maps a forward-budget timeout for a safe method to a retryable response", async () => { - const harness = createFakeChannelHarness(); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - // A tiny forward budget aborts the controller before the manual rejection. - budgets: { forwardTimeoutMs: 5 }, - }); - brokers.push(broker); - broker.start(); - - // A safe method never changes host state. A forward timeout for a GET must - // stay retryable: the broker returns a 504 with the completed outcome and no - // indeterminate marker, so the gateway does not map it to a terminal 409. - harness.feed(requestFrame("read-timeout-1", "GET")); - await flush(); - const forward = harness.forwards.find((entry) => entry.id === "read-timeout-1" && !entry.settled); - if (!forward) throw new Error("The broker did not forward the request."); - - // Wait for the forward-budget timer to abort the controller, then reject the - // forward. The rejection handler now sees the aborted signal. - await new Promise((resolve) => setTimeout(resolve, 20)); - forward.settled = true; - forward.reject(new Error("Duplex broker forward budget exceeded.")); - await new Promise((resolve) => setTimeout(resolve, 0)); - - expect(harness.responses).toHaveLength(1); - const response = harness.responses[0]!; - expect(response.status).toBe(504); - expect(response.outcome).toBe("completed"); - expect(response.headers["x-paperclip-bridge-outcome"]).toBeUndefined(); - expect(JSON.parse(response.body)).toEqual({ - error: "Duplex broker forward budget exceeded.", - retryable: true, - }); - }); - - it("maps a forward-budget timeout for a mutating method to a non-retryable indeterminate response", async () => { - const harness = createFakeChannelHarness(); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - budgets: { forwardTimeoutMs: 5 }, - }); - brokers.push(broker); - broker.start(); - - // A POST may commit before the forward budget aborts the call. The broker - // must keep the terminal contract: a 504 with the indeterminate outcome, so - // the gateway maps it to a non-retryable 409 and no caller double-applies it. - harness.feed(requestFrame("mutate-timeout-1", "POST")); - await flush(); - const forward = harness.forwards.find((entry) => entry.id === "mutate-timeout-1" && !entry.settled); - if (!forward) throw new Error("The broker did not forward the request."); - - await new Promise((resolve) => setTimeout(resolve, 20)); - forward.settled = true; - forward.reject(new Error("Duplex broker forward budget exceeded.")); - await new Promise((resolve) => setTimeout(resolve, 0)); - - expect(harness.responses).toHaveLength(1); - const response = harness.responses[0]!; - expect(response.status).toBe(504); - expect(response.outcome).toBe("indeterminate"); - expect(response.headers["x-paperclip-bridge-outcome"]).toBe("indeterminate"); - expect(JSON.parse(response.body)).toEqual({ - error: "Duplex broker forward budget exceeded.", - outcome: "indeterminate", - retryable: false, - }); - }); - - it("maps a response-budget backstop for a safe method to a retryable response", async () => { - const harness = createFakeChannelHarness(); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - // The forward budget aborts the call, but the forward promise stays - // pending. A GET receives the response headers, but the body reader stalls - // through the response budget, so the forward promise never settles. The - // response-budget backstop must answer the request. - budgets: { forwardTimeoutMs: 5, responseBudgetMs: 15, gatewayWaitMs: 40 }, - }); - brokers.push(broker); - broker.start(); - - // A safe method never changes host state, so a stalled body reader cannot - // leave a mutation half-applied. The backstop must keep the request - // retryable: a 504 with the completed outcome and no indeterminate marker, - // so the gateway does not map it to a terminal 409. - harness.feed(requestFrame("read-stall-1", "GET")); - await flush(); - const forward = harness.forwards.find((entry) => entry.id === "read-stall-1" && !entry.settled); - if (!forward) throw new Error("The broker did not forward the request."); - - // Never settle the forward. Wait past the response budget so the backstop - // fires on the stalled body reader. - await new Promise((resolve) => setTimeout(resolve, 30)); - - expect(harness.responses).toHaveLength(1); - const response = harness.responses[0]!; - expect(response.status).toBe(504); - expect(response.outcome).toBe("completed"); - expect(response.headers["x-paperclip-bridge-outcome"]).toBeUndefined(); - expect(JSON.parse(response.body)).toEqual({ - error: "Duplex broker response budget exceeded.", - retryable: true, - }); - }); - - it("maps a response-budget backstop for a mutating method to a non-retryable indeterminate response", async () => { - const harness = createFakeChannelHarness(); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - budgets: { forwardTimeoutMs: 5, responseBudgetMs: 15, gatewayWaitMs: 40 }, - }); - brokers.push(broker); - broker.start(); - - // A POST may commit before the response budget passes, and the broker cannot - // prove the host applied no mutation. The backstop must keep the terminal - // contract: a 504 with the indeterminate outcome, so the gateway maps it to a - // non-retryable 409 and no caller double-applies it. - harness.feed(requestFrame("mutate-stall-1", "POST")); - await flush(); - const forward = harness.forwards.find((entry) => entry.id === "mutate-stall-1" && !entry.settled); - if (!forward) throw new Error("The broker did not forward the request."); - - await new Promise((resolve) => setTimeout(resolve, 30)); - - expect(harness.responses).toHaveLength(1); - const response = harness.responses[0]!; - expect(response.status).toBe(504); - expect(response.outcome).toBe("indeterminate"); - expect(response.headers["x-paperclip-bridge-outcome"]).toBe("indeterminate"); - expect(JSON.parse(response.body)).toEqual({ - error: "Duplex broker response budget exceeded.", - outcome: "indeterminate", - retryable: false, - }); - }); -}); - - -/** - * A channel harness that captures the broker exit listener. `emitExit` invokes it, - * so a test drives one channel exit into the broker. - */ -function createExitChannelHarness(): { - channel: CommandManagedDuplexChannel; - emitExit: (exit: { exitCode: number | null; transportClosed?: boolean }) => void; -} { - let exitListener: - | ((exit: { exitCode: number | null; transportClosed?: boolean }) => void) - | null = null; - const channel: CommandManagedDuplexChannel = { - write: () => undefined, - onData: () => undefined, - onExit: (listener) => { - exitListener = listener; - }, - stop: () => undefined, - close: () => Promise.resolve(), - }; - return { - channel, - emitExit: (exit) => { - if (!exitListener) throw new Error("The broker did not bind the exit listener."); - exitListener(exit); - }, - }; -} - -describe("duplex bridge broker exit taxonomy", () => { - const brokers: DuplexBridgeBroker[] = []; - - afterEach(async () => { - while (brokers.length > 0) { - const broker = brokers.pop(); - if (broker) await broker.close(); - } - }); - - it("records a transport close as a distinct loss, not a process exit", async () => { - const { telemetry, capture } = createTelemetryCapture(); - const harness = createExitChannelHarness(); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: () => - Promise.resolve({ status: 200, headers: {}, body: "" }), - telemetry, - }); - brokers.push(broker); - broker.start(); - - // A reason-less transport close carries the discriminator and no exit code. - harness.emitExit({ exitCode: null, transportClosed: true }); - - expect(broker.lossRecord?.reason).toBe("transport_closed"); - expect(broker.runDisposition).toEqual({ failed: true, lossReason: "transport_closed" }); - const lossCounter = capture.counters.find( - (record) => record.metric === DUPLEX_COUNTER_LOSS_TOTAL, - ); - expect(lossCounter?.dimensions.loss_reason).toBe("transport_closed"); - }); - - it("records a numeric exit as a process exit", async () => { - const { telemetry, capture } = createTelemetryCapture(); - const harness = createExitChannelHarness(); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: () => - Promise.resolve({ status: 200, headers: {}, body: "" }), - telemetry, - }); - brokers.push(broker); - broker.start(); - - // A numeric exit code is a real process exit, so the broker keeps the existing - // channel_exit -> provider_exit mapping. - harness.emitExit({ exitCode: 0 }); - - expect(broker.lossRecord?.reason).toBe("channel_exit"); - expect(broker.runDisposition).toEqual({ failed: true, lossReason: "provider_exit" }); - const lossCounter = capture.counters.find( - (record) => record.metric === DUPLEX_COUNTER_LOSS_TOTAL, - ); - expect(lossCounter?.dimensions.loss_reason).toBe("provider_exit"); - }); -}); diff --git a/packages/adapter-utils/src/duplex-bridge-broker.ts b/packages/adapter-utils/src/duplex-bridge-broker.ts deleted file mode 100644 index 86a21ab011..0000000000 --- a/packages/adapter-utils/src/duplex-bridge-broker.ts +++ /dev/null @@ -1,1457 +0,0 @@ -/** - * Host broker for the sandbox duplex channel. - * - * The broker owns the host end of one persistent duplex channel to the sandbox - * gateway. It reads request frames from the channel, forwards each one on the - * existing Paperclip API path, and writes one response frame back. It sends a - * heartbeat frame on an interval to prove liveness. - * - * The broker does not hold the route allowlist, the token replacement, or the - * run attribution. The caller passes a forward handler, and the broker calls it - * for each request. The handler applies the real token and the signed run - * identifier, so those rules stay in one place on the existing forward path. - * - * The broker runs a set of nested timeout budgets: - * - forward budget: the deadline for one forward call. - * - response budget: the deadline for the broker to send one response frame. - * - gateway wait budget: the deadline the in-sandbox gateway waits for the - * response frame. - * Each inner budget is smaller than its outer budget, so the broker aborts and - * answers before the gateway gives up. The broker asserts this order at - * construction and fails a configuration that breaks it. - * - * The provider controls the duplex transport directly, so the broker treats each - * request frame as untrusted. The broker bounds the work one channel can force: - * - in-flight limit: the maximum number of pending forwards at one time. It - * bounds the controllers, the timers, and the concurrent authenticated - * forwards. - * - lifetime limit: the maximum number of distinct requests over the channel - * lifetime. It bounds the retained request-id memory, because the broker keeps - * one id per distinct dispatched request for the no-replay guarantee. - * The broker checks each limit before it adds the id to the seen set, allocates - * the pending record, or calls the forward handler. On a limit it answers the - * refused request with one bounded terminal response and forwards nothing. The - * refusal preserves the no-replay and no-double-dispatch rules. - * - * Loss is terminal. The broker detects loss through channel exit, a stream - * write failure, a protocol failure, a heartbeat write failure, and a close - * timeout. On loss the broker stops the heartbeat, aborts every in-flight - * forward, records the loss, and dispatches nothing more. The broker never - * reconnects and never replays a request. The broker records the loss for - * metrics only and sends nothing about the loss to the sandbox. - */ - -import type { CommandManagedDuplexChannel } from "./command-managed-runtime.js"; -import { - DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, - type DuplexAggregateByteLedger, - type ReservationToken, -} from "./duplex-aggregate-byte-ledger.js"; -import { - DUPLEX_BODY_CHUNK_RAW_BYTES, - DEFAULT_MAX_DUPLEX_FRAME_BYTES, - DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES, - DUPLEX_FRAME_VERSION, - DuplexFrameDecoder, - encodeDuplexFrame, - encodeDuplexFrameChecked, - type DuplexBodyChunkFrame, - type DuplexFrame, - type DuplexRequestFrame, - type DuplexResponseFrame, - type DuplexResponseOutcome, -} from "./duplex-frame-codec.js"; -import { - DuplexBodyError, - DuplexBodyReceiver, - splitBodyIntoChunkFrames, - type DuplexBodyReceiverConfig, - type ReassembledBody, -} from "./duplex-body-spool.js"; -import type { - DuplexLossReason, - DuplexOutcomeValue, - DuplexObservability, -} from "./duplex-observability.js"; - -/** The lifecycle states of the broker. The broker moves through them in order. */ -export type DuplexBrokerState = "opening" | "open" | "lost" | "closing" | "closed"; - -/** The reason the broker classified a loss. The broker records it for metrics only. */ -export type DuplexBrokerLossReason = - | "channel_exit" - | "transport_closed" - | "stream_failure" - | "protocol_failure" - | "heartbeat_write_failure" - | "close_timeout"; - -/** - * Map one broker loss reason to the closed, typed telemetry loss reason. The - * broker records the internal reason for its own metrics; the telemetry boundary - * carries only the closed enum value. The map is total over the internal reasons, - * so no raw text ever reaches the typed reason. - * - `channel_exit` -> `provider_exit`: the provider channel process exited. - * - `transport_closed` -> `transport_closed`: the provider transport closed with - * no exit data, so the loss is a transport close, not a process exit. - * - `stream_failure` -> `write_error`: a write to the channel failed. - * - `protocol_failure` -> `rpc_failure`: a malformed or mismatched frame. - * - `heartbeat_write_failure` -> `heartbeat_timeout`: the liveness write failed. - * - `close_timeout` -> `other`: an orderly close did not complete in the budget. - */ -const BROKER_LOSS_REASON_TO_TYPED: Readonly> = { - channel_exit: "provider_exit", - transport_closed: "transport_closed", - stream_failure: "write_error", - protocol_failure: "rpc_failure", - heartbeat_write_failure: "heartbeat_timeout", - close_timeout: "other", -}; - -/** - * Map one broker loss reason to the closed, typed telemetry loss reason. The host - * uses it to name the typed reason on a log line without the raw provider text. - */ -export function typedDuplexLossReason(reason: DuplexBrokerLossReason): DuplexLossReason { - return BROKER_LOSS_REASON_TO_TYPED[reason] ?? "other"; -} - -/** - * The typed error code the host reports when the duplex control channel died - * before an orderly completion. Both the ACP lane and the CLI lane report this - * one code, so the run disposition is identical across the two lanes. - */ -export const DUPLEX_CHANNEL_LOST_ERROR_CODE = "duplex_channel_lost"; - -/** - * The terminal run disposition the broker computes from its ordered lifecycle. A - * `failed` disposition means a terminal loss ordered before an orderly completion, - * so the run must not report success. The typed loss reason names the cause; it is - * `null` for a success. - */ -export interface DuplexBrokerRunDisposition { - /** True when a terminal loss ordered before an orderly completion. */ - failed: boolean; - /** The typed, closed loss reason on a failure; `null` on a success. */ - lossReason: DuplexLossReason | null; -} - -/** The nested timeout budgets. Each inner budget is smaller than its outer budget. */ -export interface DuplexBrokerBudgets { - /** The deadline for one forward call, in milliseconds. */ - forwardTimeoutMs: number; - /** The deadline for the broker to send one response frame, in milliseconds. */ - responseBudgetMs: number; - /** The deadline the in-sandbox gateway waits for the response frame, in milliseconds. */ - gatewayWaitMs: number; -} - -/** The default nested budgets: forward 30 s, response 32 s, gateway wait 35 s. */ -export const DEFAULT_DUPLEX_BROKER_BUDGETS: DuplexBrokerBudgets = { - forwardTimeoutMs: 30_000, - responseBudgetMs: 32_000, - gatewayWaitMs: 35_000, -}; - -/** The default interval between two heartbeat frames, in milliseconds. */ -export const DEFAULT_DUPLEX_BROKER_HEARTBEAT_INTERVAL_MS = 5_000; - -/** The default deadline for an orderly channel close, in milliseconds. */ -export const DEFAULT_DUPLEX_BROKER_CLOSE_TIMEOUT_MS = 2_000; - -/** - * The default maximum number of in-flight requests. The broker holds this many - * pending forwards at one time. It refuses a further request until an in-flight - * request completes. The provider controls the transport, so this finite limit - * bounds the controllers, the timers, and the concurrent authenticated forwards - * a provider can force on the host. - */ -export const DEFAULT_DUPLEX_BROKER_MAX_IN_FLIGHT_REQUESTS = 64; - -/** - * The default maximum number of distinct requests over the channel lifetime. The - * broker forwards this many distinct request ids, then refuses each new distinct - * id and forwards nothing more. This limit bounds the retained request-id memory, - * because the broker keeps one id per distinct dispatched request for the no-replay - * guarantee. - * - * The retained id memory has a hard ceiling. The codec bounds each id at - * `DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES` (256 bytes), so the worst-case retained id - * bytes are this count multiplied by that bound: 50,000 * 256 = 12,800,000 bytes - * (about 12.8 MB), plus the fixed per-entry overhead of the Set. This count is - * sized against that id bound to keep the ceiling small. - */ -export const DEFAULT_DUPLEX_BROKER_MAX_LIFETIME_REQUESTS = 50_000; - -/** - * The fixed, documented per-entry allocation the broker charges the aggregate byte - * ledger for one no-replay request-id set entry. The type and the cardinality of - * `seenRequestIds` are bounded: the codec caps each id at - * {@link DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES}, and the lifetime limit caps the - * entry count. The broker charges the exact raw id bytes plus this fixed entry - * overhead, so the retained set never grows uncharged. This constant models the - * fixed per-entry cost of the string key and the Set slot, not the id bytes. - */ -export const DUPLEX_SEEN_REQUEST_ID_SET_ENTRY_BYTES = 64; - -/** The result of one forward call. The broker turns it into one response frame. */ -export interface DuplexBrokerForwardResult { - status: number; - headers?: Record; - body?: string; -} - -/** - * The forward handler the broker calls for each request. The handler applies the - * real token and the run attribution, then forwards the request on the existing - * API path. The broker reassembles the request body from the `body_chunk` frames - * before it calls the handler, so the handler streams the body from - * `options.body`. The broker aborts `options.signal` when the forward budget ends - * or a loss happens, so a handler that threads the signal into its work stops - * early. - */ -export type DuplexBrokerForwardHandler = ( - request: DuplexRequestFrame, - options: { signal: AbortSignal; body: ReassembledBody }, -) => Promise; - -/** One request record. The broker captures the dispatch-start point for metrics only. */ -export interface DuplexBrokerRequestRecord { - id: string; - method: string; - path: string; - /** The point the broker started to dispatch the request, in milliseconds. */ - dispatchStartMs: number; -} - -/** One loss record. The broker reports it for metrics only. */ -export interface DuplexBrokerLossRecord { - reason: DuplexBrokerLossReason; - message: string; - /** The point the broker recorded the loss, in milliseconds. */ - atMs: number; -} - -/** The options for {@link createDuplexBridgeBroker}. */ -export interface DuplexBrokerOptions { - /** The duplex channel to the sandbox gateway. */ - channel: CommandManagedDuplexChannel; - /** The forward handler the broker calls for each request. */ - forwardRequest: DuplexBrokerForwardHandler; - /** The nested timeout budgets. The default is {@link DEFAULT_DUPLEX_BROKER_BUDGETS}. */ - budgets?: Partial; - /** The interval between two heartbeat frames, in milliseconds. */ - heartbeatIntervalMs?: number; - /** The deadline for an orderly channel close, in milliseconds. */ - closeTimeoutMs?: number; - /** The maximum size of one inbound frame, in bytes. Forwarded to the decoder. */ - maxFrameBytes?: number; - /** - * The maximum number of in-flight requests the broker holds at one time. The - * broker refuses a further request past this limit with a bounded terminal - * response and forwards nothing for it. The default is - * {@link DEFAULT_DUPLEX_BROKER_MAX_IN_FLIGHT_REQUESTS}. - */ - maxInFlightRequests?: number; - /** - * The maximum number of distinct requests the broker dispatches over the - * channel lifetime. The broker refuses each new distinct request past this - * limit with a bounded terminal response and forwards nothing more. This limit - * bounds the retained request-id memory. The default is - * {@link DEFAULT_DUPLEX_BROKER_MAX_LIFETIME_REQUESTS}. - */ - maxLifetimeRequests?: number; - /** - * The config for the receive-side request-body reassembler. It sets the spill - * threshold, the fixed raw chunk size, and the per-channel spill caps. The - * broker creates one reassembler for the channel. The config stays injectable, - * so a test lowers the spill threshold and the caps to exercise the spill path - * and the fail-closed cap behavior without a large body. - */ - bodyReceiverConfig?: DuplexBodyReceiverConfig; - /** The clock the broker reads for the metric timestamps. The default is `Date.now`. */ - now?: () => number; - /** The metrics sink for the per-request dispatch record. */ - onRequestRecord?: (record: DuplexBrokerRequestRecord) => void; - /** The metrics sink for the terminal loss record. */ - onLoss?: (record: DuplexBrokerLossRecord) => void; - /** The sink for a state change. The broker reports every transition. */ - onStateChange?: (state: DuplexBrokerState) => void; - /** The sink for a diagnostic message. The broker never writes diagnostics to the channel. */ - logger?: (message: string) => void; - /** - * The fixed observability facade. The broker records one request span per - * delivered request and one loss record per terminal loss. The facade maps each - * record to the fixed names and dimensions, so no route, query, body, token, or - * raw error rides a span or a counter. The default records nothing. - */ - telemetry?: DuplexObservability; - /** - * The process-owned aggregate byte ledger. The broker reserves the exact retained - * bytes of each dispatched request against it before it retains the frame: the - * raw request frame, the normalized request payload, and the no-replay set entry. - * A reservation that would pass the ceiling makes the broker retain nothing and - * refuse the request with the fixed marker - * {@link DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED}. When the ledger is absent the - * broker charges nothing and behaves as before, so a non-duplex or a legacy path - * stays unchanged. - */ - duplexAggregateByteLedger?: DuplexAggregateByteLedger | null; -} - -/** The broker handle the factory returns. */ -export interface DuplexBridgeBroker { - /** The current state of the broker. */ - readonly state: DuplexBrokerState; - /** The loss record, or `null` when the broker never lost the channel. */ - readonly lossRecord: DuplexBrokerLossRecord | null; - /** - * The terminal run disposition from the ordered lifecycle. It reports a failure - * when a terminal loss ordered before an orderly completion, and names the typed - * loss reason. It reports a success for a healthy channel or a normal-teardown - * loss. The host reads it at the run-disposition seam. - */ - readonly runDisposition: DuplexBrokerRunDisposition; - /** - * Mark the host-observed orderly completion of the agent turn on the ordered - * lifecycle. A loss ordered before this mark latches a failure; a loss ordered - * after it stays a success. The broker also marks it on a gateway close frame - * and on a host-initiated orderly close. Safe to call more than one time. - */ - markOrderlyCompletion(): void; - /** - * Atomically read the run disposition and mark the host-observed orderly - * completion in one synchronous step. The host calls it at the run-disposition - * seam for a success-eligible terminal. The broker marks the orderly completion - * only while no loss ordered, then returns the disposition, so no caller can - * insert an `await` between the read and the mark. A loss that already latched - * keeps the failure, because the mark no-ops after a latched loss. - */ - settleRunDisposition(): DuplexBrokerRunDisposition; - /** Start the broker. It wires the channel listeners and moves to `open`. */ - start(): void; - /** Close the channel cleanly. It moves through `closing` to `closed`. */ - close(): Promise; - /** Stop the sandbox child process. Safe to call more than one time. */ - stop(): void; -} - -/** - * Assert the nested budget order. Each inner budget must be smaller than its - * outer budget, so the broker answers before the gateway gives up. The function - * throws when the order breaks. - */ -export function assertNestedDuplexBrokerBudgets(budgets: DuplexBrokerBudgets): void { - if (!(budgets.forwardTimeoutMs < budgets.responseBudgetMs)) { - throw new Error( - `Duplex broker forward budget ${budgets.forwardTimeoutMs}ms must be smaller than the response budget ${budgets.responseBudgetMs}ms.`, - ); - } - if (!(budgets.responseBudgetMs < budgets.gatewayWaitMs)) { - throw new Error( - `Duplex broker response budget ${budgets.responseBudgetMs}ms must be smaller than the gateway wait budget ${budgets.gatewayWaitMs}ms.`, - ); - } -} - -/** The resolved request limits the broker enforces. */ -export interface DuplexBrokerLimits { - /** The maximum number of in-flight requests the broker holds at one time. */ - maxInFlightRequests: number; - /** The maximum number of distinct requests the broker dispatches over the channel lifetime. */ - maxLifetimeRequests: number; -} - -/** - * Assert the request limits. Each limit must be a finite positive integer, so the - * broker fails closed on a broken configuration instead of running with an - * unbounded or a zero limit. The function throws when a limit breaks the rule. - */ -export function assertDuplexBrokerLimits(limits: DuplexBrokerLimits): void { - const entries: ReadonlyArray = [ - ["maxInFlightRequests", limits.maxInFlightRequests], - ["maxLifetimeRequests", limits.maxLifetimeRequests], - ]; - for (const [name, value] of entries) { - if (!Number.isInteger(value) || value < 1) { - throw new Error( - `Duplex broker ${name} must be a finite positive integer; got ${String(value)}.`, - ); - } - } -} - -function errorMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error); -} - -/** - * The safe HTTP methods. RFC 7231 section 4.2.1 defines this set. A safe method - * does not change host state, so the host applies no mutation for it. A caller - * can retry a safe method after a forward failure without a double-apply risk. - */ -const SAFE_BRIDGE_METHODS = new Set(["GET", "HEAD", "OPTIONS", "TRACE"]); - -/** Report whether the method is safe, so a forward failure stays retryable. */ -export function isSafeBridgeMethod(method: string): boolean { - return SAFE_BRIDGE_METHODS.has(method.trim().toUpperCase()); -} - -/** The internal bookkeeping for one in-flight request. */ -interface PendingRequest { - controller: AbortController; - responded: boolean; - /** - * The forward-budget timer. It is `null` while the broker reassembles the - * request body, and the broker sets it when it starts the forward. So the - * forward budget bounds the forward call, not the reassembly. - */ - forwardTimer: ReturnType | null; - /** - * The response-budget backstop timer. The broker starts it at the request - * envelope, so it bounds the whole request: the body reassembly plus the - * forward. It answers a request that never completes its reassembly or forward. - */ - responseTimer: ReturnType; - /** The point the broker started to dispatch the request. It sets the span latency. */ - dispatchStartMs: number; - /** The request method. The response backstop reads it to classify the safe-method retry. */ - method: string; - /** The reassembled request body, set once reassembly completes. The broker disposes it on settle. */ - reassembled: ReassembledBody | null; - /** True once the forward promise settled. The finally owner sets it one time. */ - forwardSettled: boolean; - /** - * Release the request-frame token and the request-payload token exactly one time. - * The single forward-promise finally owner calls it after the forward settles. - * A second call is a no-op, so no token releases twice. - */ - releaseForwardTokens: () => void; -} - -/** - * One orphaned forward. The broker answered the gateway and removed the request - * from `pending`, but the forward promise or its response-body reader had not - * settled. The orphan keeps the request tokens charged and the controller live - * until the forward finally releases them, so the ledger reports nonzero ownership - * until the async work settles. - */ -interface OrphanedForward { - controller: AbortController; - /** Release the request-frame and request-payload tokens exactly one time. */ - releaseForwardTokens: () => void; -} - -/** - * Create the host duplex bridge broker. The factory asserts the budget order, - * creates the receive-side request-body reassembler, and returns a handle. It is - * asynchronous because the reassembler owns a spill directory it creates with - * `mkdtemp`. Call `start` to wire the channel and open the broker. - */ -export async function createDuplexBridgeBroker( - options: DuplexBrokerOptions, -): Promise { - const budgets: DuplexBrokerBudgets = { - ...DEFAULT_DUPLEX_BROKER_BUDGETS, - ...options.budgets, - }; - assertNestedDuplexBrokerBudgets(budgets); - - const limits: DuplexBrokerLimits = { - maxInFlightRequests: options.maxInFlightRequests ?? DEFAULT_DUPLEX_BROKER_MAX_IN_FLIGHT_REQUESTS, - maxLifetimeRequests: options.maxLifetimeRequests ?? DEFAULT_DUPLEX_BROKER_MAX_LIFETIME_REQUESTS, - }; - assertDuplexBrokerLimits(limits); - - const channel = options.channel; - const forwardRequest = options.forwardRequest; - const heartbeatIntervalMs = options.heartbeatIntervalMs ?? DEFAULT_DUPLEX_BROKER_HEARTBEAT_INTERVAL_MS; - const closeTimeoutMs = options.closeTimeoutMs ?? DEFAULT_DUPLEX_BROKER_CLOSE_TIMEOUT_MS; - const now = options.now ?? (() => Date.now()); - - // The process-owned aggregate byte ledger, or `null` when the caller injected - // none. When `null` the broker charges nothing and behaves as before. - const ledger = options.duplexAggregateByteLedger ?? null; - // The one frame size bound the broker enforces on both sides. The decoder - // rejects an inbound frame over this bound, and the encode guard refuses to - // write an outbound frame over it. Encode and decode share one value, so a - // frame the broker writes always decodes on the peer. - const maxFrameBytes = options.maxFrameBytes ?? DEFAULT_MAX_DUPLEX_FRAME_BYTES; - // The decoder charges its retained partial-frame bytes against the same host - // ledger. It receives the ledger object directly, so one gauge bounds the - // decoder buffer with every other host retention site. - const decoder = new DuplexFrameDecoder({ - maxFrameBytes, - ...(ledger ? { aggregateByteLedger: ledger } : {}), - }); - // Release one token exactly one time. A `null` token or an absent ledger is a - // no-op, so the broker never records a false accounting defect. - const releaseToken = (token: ReservationToken | null): void => { - if (ledger && token) ledger.release(token); - }; - // The byte size of the normalized request payload the broker retains across the - // forward. It counts the exact retained scalar and header bytes plus the raw body - // byte count, so the charge matches the reassembled body the broker holds and - // never a parsed object graph. The body rides `body_chunk` frames, so the - // envelope carries only `bodyByteCount`; that count is the retained body size. - const requestPayloadBytes = (frame: DuplexRequestFrame): number => { - let bytes = - Buffer.byteLength(frame.id, "utf8") + - Buffer.byteLength(frame.method, "utf8") + - Buffer.byteLength(frame.path, "utf8") + - Buffer.byteLength(frame.query, "utf8") + - frame.bodyByteCount; - for (const [key, value] of Object.entries(frame.headers)) { - bytes += Buffer.byteLength(key, "utf8") + Buffer.byteLength(value, "utf8"); - } - return bytes; - }; - - // The receive-side request-body reassembler. It reassembles each request body - // from the `body_chunk` frames, on the memory path at or below the spill - // threshold and on the spill path above it, so the broker never holds a whole - // large request body in memory. It owns a per-channel spill directory and the - // spill caps. - const receiver = await DuplexBodyReceiver.create(options.bodyReceiverConfig ?? {}); - let receiverFinalized = false; - const finalizeReceiver = (): Promise => { - // Remove the spill directory and every in-flight body once. The broker calls - // it on every terminal path: a loss, a normal teardown, and an orderly close. - if (receiverFinalized) return Promise.resolve(); - receiverFinalized = true; - return receiver.destroy().catch(() => undefined); - }; - // The fixed raw slice size for the response `body_chunk` frames. It matches the - // reassembler config, so a test that lowers the slice size splits both a request - // body and a response body the same way. - const rawChunkBytes = options.bodyReceiverConfig?.rawChunkBytes ?? DUPLEX_BODY_CHUNK_RAW_BYTES; - // The ids the broker is reassembling right now. A `body_chunk` for an id in this - // set routes to the reassembler. The broker removes an id when the body settles. - const reassembling = new Set(); - // The ids the broker refused at the envelope, with the raw bytes it still must - // drain. The sender emits the `body_chunk` frames for a refused request before - // it learns of the refusal, so the broker drains and drops those chunks instead - // of treating them as a body_chunk with no envelope. - const draining = new Map(); - - let state: DuplexBrokerState = "opening"; - let stopped = false; - let started = false; - let closePromise: Promise | null = null; - let lossRecord: DuplexBrokerLossRecord | null = null; - let heartbeatTimer: ReturnType | null = null; - - // The host-owned lifecycle sequence. The broker assigns each terminal lifecycle - // event a strictly increasing sequence number at ingress, before any - // asynchronous logging. The order of these numbers, not a wall-clock or a - // provider timestamp, decides the run disposition. - let lifecycleSeq = 0; - // The sequence number of the terminal loss, or `null` when no loss ordered. The - // broker sets it one time. A later event never clears it, so the loss latches. - let lossSeq: number | null = null; - // The typed, closed loss reason for the latched loss, or `null` on a success. - let typedLossReason: DuplexLossReason | null = null; - // The sequence number of the host-observed orderly completion, or `null` when - // none ordered. The broker sets it one time, only while the channel is healthy. - let orderlyCompletionSeq: number | null = null; - const nextLifecycleSeq = (): number => { - lifecycleSeq += 1; - return lifecycleSeq; - }; - - // Mark the host-observed orderly completion of the agent turn. The broker sets - // the sequence one time, and only while no loss has ordered. A loss that already - // latched keeps the failure, so a late completion never clears the latch. - const markOrderlyCompletion = (): void => { - if (orderlyCompletionSeq !== null || lossSeq !== null) return; - orderlyCompletionSeq = nextLifecycleSeq(); - }; - // Atomically read the run disposition and mark the host-observed orderly - // completion. The mark and the read run in one synchronous step, so no caller - // can insert an `await` between them and no teardown loss can slip in. The mark - // no-ops once a loss latched, so a real mid-run loss keeps the failure. - const settleRunDisposition = (): DuplexBrokerRunDisposition => { - markOrderlyCompletion(); - return { failed: lossSeq !== null, lossReason: typedLossReason }; - }; - // The ids the broker already dispatched. The broker forwards one id one time, - // so a repeated frame never reaches the API twice. - const seenRequestIds = new Set(); - // The aggregate-ledger tokens for the no-replay set entries. The broker holds - // one token per live set entry and releases every token one time at terminal - // teardown, so the retained set never leaves bytes charged after the channel - // ends. - const seenRequestIdTokens = new Set(); - const pending = new Map(); - // The forwards that answered the gateway but whose async work has not settled. - // The broker keeps their tokens charged and their controller live until each - // forward finally releases its tokens, so the ledger reports the real ownership. - const orphanedForwards = new Map(); - - // Release every no-replay set-entry token one time and clear the set. The - // ledger release is one way, and the cleared set stops any second release, so - // this helper is safe to call more than one time at terminal teardown. - const releaseSeenRequestIdTokens = (): void => { - for (const token of seenRequestIdTokens) releaseToken(token); - seenRequestIdTokens.clear(); - }; - - const setState = (next: DuplexBrokerState): void => { - if (state === next) return; - state = next; - options.onStateChange?.(next); - }; - - const clearHeartbeat = (): void => { - if (heartbeatTimer !== null) { - clearInterval(heartbeatTimer); - heartbeatTimer = null; - } - }; - - const clearPending = (): void => { - for (const [id, entry] of pending) { - if (entry.forwardTimer !== null) clearTimeout(entry.forwardTimer); - clearTimeout(entry.responseTimer); - entry.controller.abort(new Error("Duplex broker stopped.")); - if (entry.reassembled !== null) { - void entry.reassembled.dispose(); - entry.reassembled = null; - } - if (entry.forwardSettled) continue; - if (entry.forwardTimer !== null) { - // A live forward promise still owns its request tokens. Transfer it to the - // orphan registry, so its tokens stay charged until the forward finally - // releases them. The abort only asks the forward to stop; it never releases - // a token by itself. - orphanedForwards.set(id, { - controller: entry.controller, - releaseForwardTokens: entry.releaseForwardTokens, - }); - } else { - // The request is still reassembling, so no forward promise will settle and - // release its tokens. Release the request tokens now. The reassembler - // teardown below rejects the in-flight body and unlinks any spill file. - entry.releaseForwardTokens(); - } - } - pending.clear(); - reassembling.clear(); - draining.clear(); - // Ask every already-orphaned forward to stop as well. The forward-promise - // finally owner releases each orphan token when the forward settles. - for (const orphan of orphanedForwards.values()) { - orphan.controller.abort(new Error("Duplex broker stopped.")); - } - }; - - const recordLoss = (reason: DuplexBrokerLossReason, message: string): void => { - // Loss is terminal. Record it one time and stop every activity. - if (stopped) return; - const afterOrderlyCompletion = orderlyCompletionSeq !== null; - // A clean channel end that orders after a host-observed orderly completion is - // a normal teardown, not a loss. A process exit and a reason-less transport - // close both end the channel, so both count as the normal teardown here. Stop - // cleanly, emit no loss event, and leave the run a success. This keeps the - // closed telemetry contract: an orderly close is not a loss and emits no loss - // event. - if (afterOrderlyCompletion && (reason === "channel_exit" || reason === "transport_closed")) { - stopped = true; - clearHeartbeat(); - clearPending(); - releaseSeenRequestIdTokens(); - decoder.dispose(); - void finalizeReceiver(); - if (state !== "closing") setState("closed"); - return; - } - stopped = true; - // Classify the loss relative to the first dispatch. A loss after the broker - // dispatched a request is `post_dispatch`; a loss before any dispatch is - // `pre_dispatch`. The class rides the fixed loss counter, never the raw - // message. - const lossClass = seenRequestIds.size > 0 ? "post_dispatch" : "pre_dispatch"; - // Assign the loss its lifecycle sequence at ingress, before any logging. The - // loss latches the run as a failure only when no orderly completion ordered - // before it. A loss ordered after an orderly completion (for example a failed - // close) is a real channel loss for the telemetry and the leak metric, but it - // does not fail the run, because the run already completed. Once set, `lossSeq` - // never clears, so a later completion, exit, or activity callback cannot flip - // the latch. - const seq = nextLifecycleSeq(); - if (!afterOrderlyCompletion) { - lossSeq = seq; - typedLossReason = BROKER_LOSS_REASON_TO_TYPED[reason] ?? "other"; - } - clearHeartbeat(); - clearPending(); - releaseSeenRequestIdTokens(); - decoder.dispose(); - void finalizeReceiver(); - lossRecord = { reason, message, atMs: now() }; - setState("lost"); - // Log the internal reason only. The broker never writes the raw provider - // message to a log line, so no raw provider text rides a sink here. - options.logger?.(`Duplex broker lost the channel (${reason}).`); - options.onLoss?.(lossRecord); - options.telemetry?.recordLoss(lossClass, BROKER_LOSS_REASON_TO_TYPED[reason] ?? "other"); - }; - - const writeLine = (line: string): boolean => { - try { - // The channel carries raw bytes (see `ChannelBytesWireValue` in the plugin - // SDK's protocol.ts). Encode the frame's UTF-8 text to bytes at this one - // write seam. - channel.write(Buffer.from(line, "utf8")); - return true; - } catch (error) { - recordLoss("stream_failure", errorMessage(error)); - return false; - } - }; - - // The result of one bounded send. `sent` means every frame went out; `too_large` - // means the response envelope exceeds the bound and the broker wrote nothing; - // `lost` means a write failed and the broker recorded the channel loss. - type SendResult = "sent" | "too_large" | "lost"; - - // Send one full response as a sequence of frames: one envelope frame that - // carries `bodyByteCount`, then the `body_chunk` frames that carry the body. The - // envelope is always small. Each `body_chunk` carries one fixed raw slice, so - // each encoded chunk stays under the frame bound by construction. The result - // reports `too_large` when the envelope itself exceeds the bound and the broker - // wrote nothing, `lost` when a write failed and the broker recorded the channel - // loss, and `sent` when every frame went out. - const sendResponseFrames = ( - id: string, - status: number, - headers: Record, - bodyText: string, - outcome: DuplexResponseOutcome, - ): SendResult => { - const bodyBuffer = Buffer.from(bodyText, "utf8"); - const envelope: DuplexResponseFrame = { - version: DUPLEX_FRAME_VERSION, - type: "response", - id, - status, - headers, - bodyByteCount: bodyBuffer.length, - outcome, - }; - // Guard the envelope against the frame bound. The envelope holds no body, so a - // real bound rejects it only under an extreme small test bound. Report the - // rejection without a channel loss, so the caller decides how to answer. - const encodedEnvelope = encodeDuplexFrameChecked(envelope, maxFrameBytes); - if (!encodedEnvelope.ok) return "too_large"; - const chunkFrames = splitBodyIntoChunkFrames(id, bodyBuffer, DUPLEX_FRAME_VERSION, rawChunkBytes); - if (!writeLine(encodedEnvelope.line)) return "lost"; - for (const chunk of chunkFrames) { - const encodedChunk = encodeDuplexFrameChecked(chunk, maxFrameBytes); - if (!encodedChunk.ok) { - // A single fixed-size slice never exceeds the bound in a real configuration. - // A test bound below one chunk can reject it. The envelope already went out - // with the true `bodyByteCount`, so the broker cannot complete the body - // within the bound. Log the drop and stop; the gateway ends the request on - // its wait budget. Report `sent`, because the broker already committed the - // envelope, so no caller resends a bounded replacement over the same id. - options.logger?.("Duplex broker dropped an oversized body_chunk frame."); - return "sent"; - } - if (!writeLine(encodedChunk.line)) return "lost"; - } - return "sent"; - }; - - const sendTerminalIndeterminate = (id: string): void => { - // Answer one request the broker cannot deliver with the real result. The - // request reached the host and may have changed state, so the response is - // non-retryable and carries the `indeterminate` outcome. The broker tries the - // full replacement first. When the frame bound rejects even the full - // replacement envelope, the broker sends a minimal replacement that carries - // only the `indeterminate` outcome, so the gateway still ends the request and - // never waits for its full wait budget. When the bound rejects even the - // minimal replacement envelope, the broker logs a clear local error and keeps - // the channel open; it records no channel loss. - const fullBody = JSON.stringify({ - error: "upstream response too large to deliver", - outcome: "indeterminate", - retryable: false, - }); - if ( - sendResponseFrames( - id, - 502, - { "content-type": "application/json", "x-paperclip-bridge-outcome": "indeterminate" }, - fullBody, - "indeterminate", - ) !== "too_large" - ) { - return; - } - // The bound rejects the full replacement envelope. Send a minimal terminal - // response with empty headers and an empty body. The `indeterminate` outcome - // still rides the envelope, so the gateway maps the request to a terminal 409. - if (sendResponseFrames(id, 502, {}, "", "indeterminate") !== "too_large") return; - // The bound rejects even the minimal terminal envelope. The broker cannot - // deliver any frame for this request within the bound. Log a clear local - // error and keep the channel open for every other request. The gateway ends - // its own outstanding request on its wait budget. - options.logger?.( - `Duplex broker could not deliver a terminal response within the ${maxFrameBytes}-byte frame bound.`, - ); - }; - - const respond = ( - id: string, - result: DuplexBrokerForwardResult, - outcome: DuplexResponseOutcome, - telemetryOutcome: DuplexOutcomeValue, - ): void => { - const entry = pending.get(id); - if (!entry || entry.responded) return; - entry.responded = true; - settlePendingBookkeeping(id, entry); - // Do not write on a lost or closed channel. The gateway answers its own - // outstanding request on loss, so a late write would go to a dead channel. - if (state !== "open") return; - const bodyText = result.body ?? ""; - const bodyByteCount = Buffer.byteLength(bodyText, "utf8"); - // The response body rides `body_chunk` frames, so a large body no longer makes - // one frame too large. The gateway reassembles a response body in memory, - // though, so a body over the frame bound would force the gateway to hold an - // oversized body in memory. Treat a response body over the frame bound like the - // former oversized case: send a bounded, non-retryable indeterminate terminal - // response, and record the request outcome as an error, never a loss. - if (bodyByteCount > maxFrameBytes) { - options.telemetry?.recordRequest({ - latencyMs: now() - entry.dispatchStartMs, - outcome: "error", - }); - sendTerminalIndeterminate(id); - return; - } - // Record the request span for the delivered request. The span carries the - // latency and the outcome only; no route, query, body, or token rides it. The - // broker records the span before the writes, so the outcome is recorded even if - // a later chunk write records a channel loss. - options.telemetry?.recordRequest({ - latencyMs: now() - entry.dispatchStartMs, - outcome: telemetryOutcome, - }); - sendResponseFrames(id, result.status, result.headers ?? {}, bodyText, outcome); - }; - - const respondSaturated = (id: string, retryable: boolean): void => { - // Answer a refused request with a bounded terminal response. The broker made - // no controller, no timer, and no forward for this id, so the host API stays - // untouched. The response carries no route, no query, no body, and no token; - // it holds only the fixed error shape. The `unavailable` outcome tells the - // gateway this is not a delivered host response, so it never counts as one. - if (state !== "open") return; - sendResponseFrames( - id, - 503, - { "content-type": "application/json", "x-paperclip-bridge-outcome": "unavailable" }, - JSON.stringify({ - error: "Duplex broker capacity limit reached.", - outcome: "unavailable", - retryable, - }), - "unavailable", - ); - }; - - // Release the timers and route the reassembly and ledger resources for one - // request the broker just answered. The broker calls it exactly once per - // request, from the response path. It clears both timers and drops the pending - // record, then it settles the resources by the request phase: - // - A live forward promise (the response backstop answered while the forward - // ran) keeps its request tokens and its reassembled body until its finally - // owner releases and disposes them, so the orphan registry holds it. - // - A request still reassembling (the backstop answered before the forward - // started) has no forward promise to settle, so the broker releases its - // tokens and disposes the in-flight body now. - // - A settled forward already released and disposed through its finally owner. - const settlePendingBookkeeping = (id: string, entry: PendingRequest): void => { - if (entry.forwardTimer !== null) clearTimeout(entry.forwardTimer); - clearTimeout(entry.responseTimer); - pending.delete(id); - if (entry.forwardSettled) return; - if (entry.forwardTimer !== null) { - // A live forward still owns its request tokens and streams the reassembled - // body. Keep both charged until the forward finally owner settles them. - orphanedForwards.set(id, { - controller: entry.controller, - releaseForwardTokens: entry.releaseForwardTokens, - }); - return; - } - // The request is still reassembling, so no forward promise will settle. Release - // the request tokens and dispose the in-flight body, so no spill file, arena - // reservation, or ledger token lingers. - entry.releaseForwardTokens(); - if (reassembling.delete(id)) void receiver.disposeBody(id); - if (entry.reassembled !== null) { - void entry.reassembled.dispose(); - entry.reassembled = null; - } - }; - - // Mark a refused or duplicate request id for chunk draining. The sender emits - // the `body_chunk` frames for the request before it learns of the refusal, so - // the broker records the raw byte count it must drain and drop. A request with - // no body needs no draining, so the broker records only a non-zero count. - const markDrain = (frame: DuplexRequestFrame): void => { - if (frame.bodyByteCount > 0) draining.set(frame.id, frame.bodyByteCount); - }; - - const respondAggregateExceeded = (id: string): void => { - // Answer a request the aggregate byte ledger refused with a bounded terminal - // response. The broker reserved nothing, added no id to the seen set, and made - // no controller, timer, or forward, so the host API stays untouched. The - // response carries the fixed marker only; it holds no route, query, body, or - // token. The `unavailable` outcome tells the gateway this is not a delivered - // host response. The refusal is retryable, because the broker did not retain - // the id: the aggregate pressure can ease, and a resend can then get through. - if (state !== "open") return; - sendResponseFrames( - id, - 503, - { "content-type": "application/json", "x-paperclip-bridge-outcome": "unavailable" }, - JSON.stringify({ - error: DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, - outcome: "unavailable", - retryable: true, - }), - "unavailable", - ); - }; - - const dispatch = (frame: DuplexRequestFrame): void => { - // Dispatch only while open. After loss or close the broker forwards nothing. - if (state !== "open") return; - // Bound the id byte size before any retention or work. The codec already - // rejects an over-limit id on the read path, so this guard is defense in depth - // for a frame that reaches dispatch by another path. The broker never adds the - // id to the seen set, never allocates a controller or a timer, and never - // forwards. It answers with the bounded terminal refusal, which carries no - // route, query, body, or token, and records no telemetry. The refusal is not - // retryable, because a resend of the same over-limit id never gets past this - // bound. It drains the following chunks of the refused body. - if (Buffer.byteLength(frame.id, "utf8") > DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES) { - respondSaturated(frame.id, false); - markDrain(frame); - return; - } - // Forward one id one time. A repeated id never reaches the API twice. The - // broker already answered the first request, so it answers no second time; it - // drains the resent body chunks and drops them. - if (seenRequestIds.has(frame.id)) { - markDrain(frame); - return; - } - // Bound the retained request-id memory. The broker keeps one id per distinct - // dispatched request for the no-replay guarantee, so the set can only grow. - // Once the broker reaches the lifetime limit, it refuses each new distinct id - // and forwards nothing more. The refusal is not retryable, because a resend - // never gets past the limit. This check runs before the id joins the set, so - // the set never grows past the limit. - if (seenRequestIds.size >= limits.maxLifetimeRequests) { - respondSaturated(frame.id, false); - markDrain(frame); - return; - } - // Bound the in-flight request count. Once the broker holds the maximum number - // of pending forwards, it refuses a further request and forwards nothing for - // it. The broker does not add the id to the seen set, so the gateway can - // resend the request after an in-flight request completes. The refusal is - // retryable for that reason. This check bounds the controllers, the timers, - // the concurrent reassemblies, and the concurrent forwards a provider can - // force. - if (pending.size >= limits.maxInFlightRequests) { - respondSaturated(frame.id, true); - markDrain(frame); - return; - } - // Reserve the exact retained bytes against the aggregate ledger before the - // broker retains anything. It reserves three tokens in order: the raw request - // frame, the normalized request payload, and the no-replay set entry. A - // reservation that would pass the ceiling makes the broker retain nothing, - // release the tokens it already took, and refuse the request with the fixed - // marker. The broker adds no id to the seen set and makes no controller, timer, - // or forward, so the host API stays untouched. When the ledger is absent all - // tokens stay `null` and the broker behaves as before. - let requestFrameToken: ReservationToken | null = null; - let requestPayloadToken: ReservationToken | null = null; - let seenRequestIdToken: ReservationToken | null = null; - if (ledger) { - const rawFrameBytes = Buffer.byteLength(encodeDuplexFrame(frame), "utf8"); - requestFrameToken = ledger.reserve("request_frame", rawFrameBytes); - if (!requestFrameToken) { - respondAggregateExceeded(frame.id); - markDrain(frame); - return; - } - requestPayloadToken = ledger.reserve("request_payload", requestPayloadBytes(frame)); - if (!requestPayloadToken) { - ledger.release(requestFrameToken); - respondAggregateExceeded(frame.id); - markDrain(frame); - return; - } - const seenEntryBytes = - Buffer.byteLength(frame.id, "utf8") + DUPLEX_SEEN_REQUEST_ID_SET_ENTRY_BYTES; - seenRequestIdToken = ledger.reserve("seen_request_id", seenEntryBytes); - if (!seenRequestIdToken) { - ledger.release(requestFrameToken); - ledger.release(requestPayloadToken); - respondAggregateExceeded(frame.id); - markDrain(frame); - return; - } - } - seenRequestIds.add(frame.id); - if (seenRequestIdToken) seenRequestIdTokens.add(seenRequestIdToken); - - // Release the request-frame and the request-payload tokens exactly one time. - // The single forward-promise finally owner calls it after the forward settles. - // The seen-id token stays charged for the channel lifetime, so it is not part - // of this release; the terminal teardown releases it. - let forwardTokensReleased = false; - const releaseForwardTokens = (): void => { - if (forwardTokensReleased) return; - forwardTokensReleased = true; - releaseToken(requestFrameToken); - releaseToken(requestPayloadToken); - }; - - const record: DuplexBrokerRequestRecord = { - id: frame.id, - method: frame.method, - path: frame.path, - dispatchStartMs: now(), - }; - options.onRequestRecord?.(record); - - const controller = new AbortController(); - // Start the response-budget backstop at the request envelope, so it bounds the - // whole request: the body reassembly plus the forward. The forward-budget timer - // starts later, when the broker starts the forward, so it bounds the forward - // call alone. - const responseTimer = setTimeout(() => respondBackstop(frame.id), budgets.responseBudgetMs); - responseTimer.unref?.(); - const entry: PendingRequest = { - controller, - responded: false, - forwardTimer: null, - responseTimer, - dispatchStartMs: record.dispatchStartMs, - method: frame.method, - reassembled: null, - forwardSettled: false, - releaseForwardTokens, - }; - pending.set(frame.id, entry); - // Route the following `body_chunk` frames for this id to the reassembler. - reassembling.add(frame.id); - // Reassemble the request body from the `body_chunk` frames, then forward it. - // The reassembler chooses the memory path or the spill path from the envelope - // `bodyByteCount`, so the broker never holds a whole large request body in - // memory. A reassembly error is a terminal protocol failure. - receiver.begin(frame.id, frame.bodyByteCount).then( - (body) => onBodyReady(frame, entry, body), - (error) => onBodyError(frame.id, entry, error), - ); - }; - - const respondBackstop = (id: string): void => { - // Response-budget backstop. The forward rejection normally answers first, well - // before this deadline. This backstop answers a request whose body never - // completes its reassembly, or whose forward rejection handling itself stalls, - // so the request never strands. One stall path is a response whose headers - // arrive but whose body reader stays pending through the budget, so the forward - // promise never settles. - const entry = pending.get(id); - if (!entry || entry.responded) return; - if (isSafeBridgeMethod(entry.method)) { - // A safe method never changes host state, so a stalled body reader cannot - // leave a mutation half-applied. Keep the request retryable: return a 504 - // with the completed outcome and no indeterminate marker, so the gateway - // passes it through as a retryable status and never maps it to a terminal 409. - respond( - id, - { - status: 504, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ - error: "Duplex broker response budget exceeded.", - retryable: true, - }), - }, - "completed", - "error", - ); - return; - } - // The method may mutate host state, and the broker cannot prove the host - // applied no mutation once the response budget passes. Return a non-retryable - // 504 and mark the outcome indeterminate, so the gateway maps it to a terminal - // 409 and no caller double-applies the mutation. - respond( - id, - { - status: 504, - headers: { - "content-type": "application/json", - "x-paperclip-bridge-outcome": "indeterminate", - }, - body: JSON.stringify({ - error: "Duplex broker response budget exceeded.", - outcome: "indeterminate", - retryable: false, - }), - }, - "indeterminate", - "error", - ); - }; - - // The request body reassembled. Start the forward now, so the forward budget - // bounds the forward call alone. The broker passes the reassembled body to the - // forward handler, so the handler streams it to the host API path. - const onBodyReady = ( - frame: DuplexRequestFrame, - entry: PendingRequest, - body: ReassembledBody, - ): void => { - reassembling.delete(frame.id); - if (stopped || state !== "open" || entry.responded) { - // The channel died, or the response backstop already answered, so no forward - // runs. Dispose the body and release the request tokens, so no spill file, - // arena reservation, or ledger token lingers, and drop the request from the - // pending and orphan maps. - void body.dispose(); - pending.delete(frame.id); - orphanedForwards.delete(frame.id); - entry.releaseForwardTokens(); - return; - } - entry.reassembled = body; - const forwardTimer = setTimeout(() => { - entry.controller.abort(new Error("Duplex broker forward budget exceeded.")); - }, budgets.forwardTimeoutMs); - forwardTimer.unref?.(); - entry.forwardTimer = forwardTimer; - - // The forward promise has one cleanup owner. Both settle handlers mark the - // forward settled and answer the gateway. The `finally` then releases the - // request tokens exactly one time, disposes the reassembled body, and removes - // the request from the pending map and the orphan map. - const markForwardSettled = (): void => { - entry.forwardSettled = true; - }; - forwardRequest(frame, { signal: entry.controller.signal, body }) - .then( - (result) => { - markForwardSettled(); - // Keep the outcome classification consistent with the file path. A - // possibly-committed mutation carries the indeterminate marker header, so - // map it to the indeterminate outcome. Any other result is completed. - const outcome: DuplexResponseOutcome = - result.headers?.["x-paperclip-bridge-outcome"] === "indeterminate" - ? "indeterminate" - : "completed"; - // The host delivered a real response, so the request span outcome is `ok`. - // A host application status (200, a 4xx, a 5xx) is still a delivered - // response; only a broker-synthesized failure below is `error`. - respond(frame.id, result, outcome, "ok"); - }, - (error) => { - markForwardSettled(); - if (entry.controller.signal.aborted) { - // The forward budget aborted the call. A safe method never changes - // host state, so a forward timeout stays retryable for it. Return a - // 504 with the completed outcome and no indeterminate marker, so the - // gateway passes it through as a retryable status. - if (isSafeBridgeMethod(frame.method)) { - respond( - frame.id, - { - status: 504, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ - error: errorMessage(error), - retryable: true, - }), - }, - "completed", - "error", - ); - return; - } - // The method may mutate host state, and the forward budget aborted the - // call after the request may have committed. Return a non-retryable 504 - // and mark the outcome indeterminate, so a caller does not retry a - // committed mutation. - respond( - frame.id, - { - status: 504, - headers: { - "content-type": "application/json", - "x-paperclip-bridge-outcome": "indeterminate", - }, - body: JSON.stringify({ - error: errorMessage(error), - outcome: "indeterminate", - retryable: false, - }), - }, - "indeterminate", - "error", - ); - return; - } - // The forward rejected before the host delivered a response. This - // rejection does not prove that the host applied no mutation. A fetch - // can reject after the request bytes reach the host and the host - // commits, but before the response headers arrive. A safe method never - // changes host state, so a retry stays safe for it. For any other - // method the host may have committed, so the outcome is indeterminate. - if (isSafeBridgeMethod(frame.method)) { - // The method is safe, so a retry cannot double-apply a mutation. - // Return a 502 with the completed outcome, so the gateway passes it - // through as a retryable status. - respond( - frame.id, - { - status: 502, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ error: errorMessage(error) }), - }, - "completed", - "error", - ); - return; - } - // The method may mutate host state, so a retry with a new request id - // could apply the mutation twice. Return a non-retryable 504 and mark - // the outcome indeterminate, so the gateway maps it to a terminal 409. - respond( - frame.id, - { - status: 504, - headers: { - "content-type": "application/json", - "x-paperclip-bridge-outcome": "indeterminate", - }, - body: JSON.stringify({ - error: errorMessage(error), - outcome: "indeterminate", - retryable: false, - }), - }, - "indeterminate", - "error", - ); - }, - ) - .finally(() => { - // The single forward cleanup owner. Release the request tokens, dispose the - // reassembled body, and drop the request from the pending and orphan maps, - // each exactly one time. - pending.delete(frame.id); - orphanedForwards.delete(frame.id); - entry.releaseForwardTokens(); - if (entry.reassembled !== null) { - void entry.reassembled.dispose(); - entry.reassembled = null; - } - }); - }; - - // The request body reassembly failed. A malformed chunk, a size mismatch, an - // overrun, or a truncation is a terminal protocol failure, so the broker fails - // the channel closed. A `channel_closed` error is the broker's own teardown of an - // in-flight body, so it is not a fresh loss. - const onBodyError = (id: string, entry: PendingRequest, error: unknown): void => { - reassembling.delete(id); - // The forward never ran, so release its request tokens and drop the request - // from the pending and orphan maps. The release is idempotent, so a teardown - // that already released the tokens keeps the count correct. - pending.delete(id); - orphanedForwards.delete(id); - entry.releaseForwardTokens(); - if (stopped) return; - if (error instanceof DuplexBodyError && error.code === "channel_closed") return; - recordLoss("protocol_failure", errorMessage(error)); - }; - - // Route one `body_chunk` frame. A chunk for an id under reassembly goes to the - // reassembler, which validates it and appends it. A chunk for a refused or - // duplicate id drains against its recorded byte count and drops. A chunk with no - // matching envelope is a terminal protocol failure. - const handleBodyChunk = (frame: DuplexBodyChunkFrame): void => { - if (reassembling.has(frame.id)) { - const result = receiver.pushChunk(frame); - if (!result.ok) recordLoss("protocol_failure", result.error.message); - return; - } - const remaining = draining.get(frame.id); - if (remaining !== undefined) { - // Drain and drop a chunk of a refused body. The broker does not validate a - // drained chunk; it only tracks the byte progress, so it stops draining once - // the refused body ends. A further chunk after that is a chunk with no - // envelope. - const next = remaining - Buffer.byteLength(frame.data, "base64"); - if (next > 0) draining.set(frame.id, next); - else draining.delete(frame.id); - return; - } - recordLoss("protocol_failure", "body_chunk arrived with no matching envelope"); - }; - - const handleFrame = (frame: DuplexFrame): void => { - switch (frame.type) { - case "request": - dispatch(frame); - return; - case "body_chunk": - handleBodyChunk(frame); - return; - case "close": - // The gateway asked for an orderly close. The gateway sends this frame on - // the agent's orderly completion, so mark the completion on the ordered - // lifecycle before the close, then close the channel. - markOrderlyCompletion(); - void close(); - return; - case "ready": - case "heartbeat": - // Liveness frames. The broker reads them and dispatches nothing. - return; - case "response": - case "error": - // The host never expects these on the read path. Ignore them. - return; - default: - return; - } - }; - - const onData = (chunk: Uint8Array): void => { - if (stopped) return; - const results = decoder.push(chunk); - for (const result of results) { - if (stopped) return; - if (!result.ok) { - recordLoss("protocol_failure", result.error.message); - return; - } - handleFrame(result.frame); - } - }; - - const onExit = (exit: { exitCode: number | null; transportClosed?: boolean }): void => { - // A reason-less transport close is not a process exit. Record it as a distinct - // loss, so a transport close stays legible in the loss taxonomy. A real process - // exit stays `channel_exit` -> `provider_exit`. - if (exit.transportClosed === true) { - recordLoss("transport_closed", "The sandbox channel transport closed with no exit."); - return; - } - recordLoss("channel_exit", "The sandbox channel process exited."); - }; - - const sendHeartbeat = (): void => { - if (state !== "open") return; - try { - channel.write(Buffer.from(encodeDuplexFrame({ version: DUPLEX_FRAME_VERSION, type: "heartbeat" }), "utf8")); - } catch (error) { - recordLoss("heartbeat_write_failure", errorMessage(error)); - } - }; - - const close = (): Promise => { - if (closePromise) return closePromise; - if (stopped) return Promise.resolve(); - // A host-initiated orderly close is a host-observed orderly completion. The - // host tears the channel down on its own terms, so a channel end during the - // close is a normal teardown, not a mid-run loss. `markOrderlyCompletion` - // no-ops when a loss already latched, so a lost channel stays a failure. - markOrderlyCompletion(); - closePromise = (async () => { - setState("closing"); - clearHeartbeat(); - clearPending(); - releaseSeenRequestIdTokens(); - decoder.dispose(); - // Send an orderly close frame. Ignore a write failure here; the broker is - // already closing, so a dead channel needs no loss record. - try { - channel.write(Buffer.from(encodeDuplexFrame({ version: DUPLEX_FRAME_VERSION, type: "close" }), "utf8")); - } catch (error) { - options.logger?.(`Duplex broker could not send the close frame: ${errorMessage(error)}`); - } - let closeTimer: ReturnType | undefined; - const timeout = new Promise((_resolve, reject) => { - closeTimer = setTimeout(() => { - reject(new Error("Duplex broker channel close timed out.")); - }, closeTimeoutMs); - closeTimer.unref?.(); - }); - try { - await Promise.race([channel.close(), timeout]); - if (closeTimer !== undefined) clearTimeout(closeTimer); - stopped = true; - setState("closed"); - } catch (error) { - if (closeTimer !== undefined) clearTimeout(closeTimer); - recordLoss("close_timeout", errorMessage(error)); - } - // Remove the spill directory and every in-flight body. The broker owns the - // reassembler, so the orderly close cleans it up. - await finalizeReceiver(); - })(); - return closePromise; - }; - - const start = (): void => { - if (started) return; - started = true; - channel.onData(onData); - channel.onExit(onExit); - heartbeatTimer = setInterval(sendHeartbeat, heartbeatIntervalMs); - heartbeatTimer.unref?.(); - setState("open"); - }; - - const stop = (): void => { - try { - channel.stop(); - } catch (error) { - options.logger?.(`Duplex broker could not stop the channel: ${errorMessage(error)}`); - } - }; - - return { - get state() { - return state; - }, - get lossRecord() { - return lossRecord; - }, - get runDisposition(): DuplexBrokerRunDisposition { - // A latched loss ordered before any orderly completion is a failure. Every - // other state — a healthy channel, or a loss ordered after an orderly - // completion — is a success. - return { failed: lossSeq !== null, lossReason: typedLossReason }; - }, - markOrderlyCompletion, - settleRunDisposition, - start, - close, - stop, - }; -} diff --git a/packages/adapter-utils/src/duplex-bridge-local-e2e.test.ts b/packages/adapter-utils/src/duplex-bridge-local-e2e.test.ts deleted file mode 100644 index 91f8755c0d..0000000000 --- a/packages/adapter-utils/src/duplex-bridge-local-e2e.test.ts +++ /dev/null @@ -1,734 +0,0 @@ -import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; -import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; -import net from "node:net"; -import { mkdtemp, rm, writeFile } from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; -import { Readable } from "node:stream"; -import { afterEach, describe, expect, it } from "vitest"; - -import { - authorizeSandboxCallbackBridgeRequestWithRoutes, - getSandboxCallbackBridgeServerSource, - SANDBOX_CALLBACK_BRIDGE_DUPLEX_MODE, -} from "./sandbox-callback-bridge.js"; -import { - DuplexFrameDecoder, - DUPLEX_BODY_CHUNK_RAW_BYTES, - DUPLEX_FRAME_VERSION, - encodeDuplexFrame, - type DuplexFrame, - type DuplexReadyFrame, - type DuplexRequestFrame, -} from "./duplex-frame-codec.js"; -import { - createDuplexBridgeBroker, - type DuplexBridgeBroker, - type DuplexBrokerForwardResult, -} from "./duplex-bridge-broker.js"; -import type { ReassembledBody } from "./duplex-body-spool.js"; -import type { CommandManagedDuplexChannel } from "./command-managed-runtime.js"; - -/** - * Local real-process end-to-end harness for the composed duplex path. - * - * The harness spawns the real generated gateway with plain `node` from a - * temporary file, attaches the real host broker to the child stdin and stdout, - * and forwards each request to a local fake API server on a real HTTP call. It - * uses no provider credentials. - * - * The child stdio pipes are kernel pipes. They give partial reads, split a - * multi-byte UTF-8 sequence across two chunks, apply backpressure, and give a - * true end of file when the child dies. The harness proves the composed path - * handles each of these real conditions. - */ - -/** One recorded call the fake API server received. */ -interface FakeApiRequest { - method: string; - url: string; - auth: string | null; - runId: string | null; - body: string; -} - -/** The responder the fake API server calls for each request. */ -type FakeApiResponder = (req: IncomingMessage, res: ServerResponse, body: string) => void; - -/** The handle for one fake API server. */ -interface FakeApiServer { - origin: string; - requests: FakeApiRequest[]; - setResponder: (responder: FakeApiResponder) => void; - /** The count of sockets the server holds open right now. */ - openSocketCount: () => number; - /** True while the server still accepts connections. */ - listening: () => boolean; - close: () => Promise; -} - -/** - * Start a local fake API server. The forward handler targets it, so one real - * HTTP call proves the composed path end to end. The server tracks each open - * socket, so a test asserts teardown leaves no open socket handle. - */ -async function startFakeApiServer(): Promise { - const requests: FakeApiRequest[] = []; - const sockets = new Set(); - let responder: FakeApiResponder = (req, res) => { - const url = new URL(req.url ?? "/", "http://127.0.0.1"); - res.writeHead(200, { "content-type": "application/json" }); - res.end(JSON.stringify({ ok: true, path: url.pathname })); - }; - - const server = createServer((req, res) => { - const chunks: Buffer[] = []; - req.on("data", (chunk: Buffer) => chunks.push(chunk)); - req.on("end", () => { - const body = Buffer.concat(chunks).toString("utf8"); - requests.push({ - method: req.method ?? "GET", - url: req.url ?? "/", - auth: req.headers.authorization ?? null, - runId: - typeof req.headers["x-paperclip-run-id"] === "string" - ? req.headers["x-paperclip-run-id"] - : null, - body, - }); - responder(req, res, body); - }); - }); - server.on("connection", (socket) => { - sockets.add(socket); - socket.on("close", () => sockets.delete(socket)); - }); - - await new Promise((resolve, reject) => { - server.once("error", reject); - server.listen(0, "127.0.0.1", () => resolve()); - }); - const address = server.address(); - if (!address || typeof address === "string") { - throw new Error("The fake API server did not expose a TCP port."); - } - - return { - origin: `http://127.0.0.1:${address.port}`, - requests, - setResponder: (next) => { - responder = next; - }, - openSocketCount: () => sockets.size, - listening: () => server.listening, - close: () => - new Promise((resolve) => { - // Destroy each open socket first. A keep-alive client socket keeps the - // server open, so `server.close` alone could stall. The destroy makes the - // close callback fire and drives the open socket count to zero. - for (const socket of sockets) socket.destroy(); - server.close(() => resolve()); - }), - }; -} - -/** The channel view over the spawned child, plus the frames the child sent host-ward. */ -interface ChildDuplexChannel { - channel: CommandManagedDuplexChannel; - observedFrames: DuplexFrame[]; - stderr: () => string; -} - -/** - * Wrap the spawned child as a {@link CommandManagedDuplexChannel}. The broker - * writes to the child stdin, reads the child stdout, and learns of the child - * exit through this channel. - * - * The host end forwards each raw stdout chunk unchanged; the channel carries - * bytes, so no decode step sits between the pipe and the broker. - */ -function attachChildDuplexChannel(child: ChildProcessWithoutNullStreams): ChildDuplexChannel { - const observed = new DuplexFrameDecoder(); - const observedFrames: DuplexFrame[] = []; - let dataListener: ((chunk: Uint8Array) => void) | null = null; - let exitListener: ((exit: { exitCode: number | null }) => void) | null = null; - let pendingBytes: Buffer = Buffer.alloc(0); - let pendingExit: { exitCode: number | null } | null = null; - let stderrText = ""; - - child.stdout.on("data", (buffer: Buffer) => { - for (const result of observed.push(buffer)) { - if (result.ok) observedFrames.push(result.frame); - } - if (buffer.length === 0) return; - if (dataListener) dataListener(buffer); - else pendingBytes = pendingBytes.length === 0 ? buffer : Buffer.concat([pendingBytes, buffer]); - }); - child.stderr.on("data", (buffer: Buffer) => { - stderrText += buffer.toString("utf8"); - }); - child.on("exit", (code) => { - const exit = { exitCode: code }; - if (exitListener) exitListener(exit); - else pendingExit = exit; - }); - // Swallow a stdin EPIPE. The broker can write one more frame while the child - // exits; the write fails and the broker records the loss on its own path. - child.stdin.on("error", () => undefined); - - const channel: CommandManagedDuplexChannel = { - write: (data) => { - child.stdin.write(data); - }, - onData: (listener) => { - dataListener = listener; - if (pendingBytes.length > 0) { - const replay = pendingBytes; - pendingBytes = Buffer.alloc(0); - listener(replay); - } - }, - onExit: (listener) => { - exitListener = listener; - if (pendingExit) { - const exit = pendingExit; - pendingExit = null; - listener(exit); - } - }, - stop: () => { - child.kill("SIGKILL"); - }, - close: () => - new Promise((resolve) => { - // Close the write side. The child stdin reaches end of file, so the - // gateway sees a real EOF. - child.stdin.end(() => resolve()); - }), - }; - - return { channel, observedFrames, stderr: () => stderrText }; -} - -/** The forward-handler mode. `proxy` calls the fake API; `hang` blocks until abort. */ -type ForwardMode = "proxy" | "hang"; - -/** The options for one harness. */ -interface HarnessOptions { - hostApiToken?: string; - runId?: string; - maxBodyBytes?: number; - lossExitGraceMs?: number; -} - -/** The full harness handle for one composed duplex path. */ -interface DuplexE2EHarness { - baseUrl: string; - bridgeToken: string; - hostApiToken: string; - runId: string; - nonce: string; - broker: DuplexBridgeBroker; - api: FakeApiServer; - child: ChildProcessWithoutNullStreams; - observedFrames: DuplexFrame[]; - forwardedRequests: DuplexRequestFrame[]; - setForwardMode: (mode: ForwardMode) => void; - stderr: () => string; - killChild: () => Promise; - waitFor: (predicate: () => boolean, message: string, timeoutMs?: number) => Promise; - teardown: () => Promise; -} - -/** Reserve one free loopback port. The host assigns it to the gateway. */ -async function reserveLoopbackPort(): Promise { - return new Promise((resolve, reject) => { - const probe = net.createServer(); - probe.once("error", reject); - probe.listen(0, "127.0.0.1", () => { - const address = probe.address(); - if (!address || typeof address === "string") { - probe.close(() => reject(new Error("Could not reserve a loopback port."))); - return; - } - const reserved = address.port; - probe.close(() => resolve(reserved)); - }); - }); -} - -/** - * Build the composed duplex path: a spawned gateway child, the real broker on - * the child stdio, and a local fake API server as the forward target. - */ -async function createHarness(options: HarnessOptions = {}): Promise { - const hostApiToken = options.hostApiToken ?? "real-run-jwt"; - const runId = options.runId ?? "run-e2e"; - const maxBodyBytes = options.maxBodyBytes ?? 1_000_000; - const bridgeToken = "duplex-e2e-bridge-token"; - const nonce = "e2e112233445566778899aabbccddeeff"; - let forwardMode: ForwardMode = "proxy"; - const forwardedRequests: DuplexRequestFrame[] = []; - - const api = await startFakeApiServer(); - const assignedPort = await reserveLoopbackPort(); - - const tmpDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-duplex-e2e-")); - const entrypoint = path.join(tmpDir, "gateway.mjs"); - await writeFile(entrypoint, getSandboxCallbackBridgeServerSource(), "utf8"); - - const child = spawn(process.execPath, [entrypoint], { - stdio: ["pipe", "pipe", "pipe"], - env: { - ...process.env, - PAPERCLIP_API_BRIDGE_MODE: SANDBOX_CALLBACK_BRIDGE_DUPLEX_MODE, - PAPERCLIP_BRIDGE_HOST: "127.0.0.1", - PAPERCLIP_BRIDGE_PORT: String(assignedPort), - PAPERCLIP_BRIDGE_NONCE: nonce, - PAPERCLIP_BRIDGE_TOKEN: bridgeToken, - PAPERCLIP_BRIDGE_MAX_BODY_BYTES: String(maxBodyBytes), - ...(options.lossExitGraceMs != null - ? { PAPERCLIP_BRIDGE_LOSS_EXIT_GRACE_MS: String(options.lossExitGraceMs) } - : {}), - }, - }) as ChildProcessWithoutNullStreams; - - const { channel, observedFrames, stderr } = attachChildDuplexChannel(child); - - const forwardRequest = async ( - request: DuplexRequestFrame, - opts: { signal: AbortSignal; body: ReassembledBody }, - ): Promise => { - forwardedRequests.push(request); - // Keep the real route allowlist on the forward seam. The broker forwards - // only an allowed route; it denies any other route with a 403. - const denial = authorizeSandboxCallbackBridgeRequestWithRoutes(request); - if (denial) { - return { - status: 403, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ error: denial }), - }; - } - if (forwardMode === "hang") { - // Block until the broker aborts the forward. This lets a test hold an - // outstanding request open while it forces a loss. - await new Promise((_resolve, reject) => { - const onAbort = () => reject(new Error("The forward call was aborted.")); - if (opts.signal.aborted) { - onAbort(); - return; - } - opts.signal.addEventListener("abort", onAbort, { once: true }); - }); - } - const method = request.method.trim().toUpperCase() || "GET"; - const headers = new Headers(); - for (const [key, value] of Object.entries(request.headers)) { - if (value.trim().length === 0) continue; - headers.set(key, value); - } - // Apply the real host token and the run id, the same as the file bridge path. - headers.set("authorization", `Bearer ${hostApiToken}`); - headers.set("x-paperclip-run-id", runId); - const target = new URL(`${request.path}${request.query ?? ""}`, api.origin); - // Stream the reassembled request body to the host, the same as the real - // forward. A streamed body needs `duplex: "half"`. - const forwardInit: RequestInit & { duplex?: "half" } = { method, headers, signal: opts.signal }; - if (method !== "GET" && method !== "HEAD") { - forwardInit.body = Readable.toWeb( - opts.body.createReadStream(), - ) as unknown as ReadableStream; - forwardInit.duplex = "half"; - } - const response = await fetch(target, forwardInit); - const body = await response.text(); - const outHeaders: Record = {}; - response.headers.forEach((value, key) => { - if (key.toLowerCase() === "content-length") return; - outHeaders[key] = value; - }); - return { status: response.status, headers: outHeaders, body }; - }; - - const broker = await createDuplexBridgeBroker({ - channel, - forwardRequest, - logger: () => undefined, - }); - broker.start(); - - const waitFor = async ( - predicate: () => boolean, - message: string, - timeoutMs = 5000, - ): Promise => { - const deadline = Date.now() + timeoutMs; - for (;;) { - if (predicate()) return; - if (Date.now() > deadline) { - throw new Error(`Timed out waiting for ${message}. stderr: ${stderr()}`); - } - await new Promise((resolve) => { - const timer = setTimeout(resolve, 20); - timer.unref?.(); - }); - } - }; - - const waitForExit = (): Promise => - new Promise((resolve) => { - if (child.exitCode !== null || child.signalCode !== null) { - resolve(); - return; - } - child.once("exit", () => resolve()); - }); - - const killChild = async (): Promise => { - child.kill("SIGKILL"); - await waitForExit(); - }; - - let torndown = false; - const teardown = async (): Promise => { - if (torndown) return; - torndown = true; - child.kill("SIGKILL"); - await waitForExit(); - child.stdin.destroy(); - child.stdout.destroy(); - child.stderr.destroy(); - await api.close(); - await rm(tmpDir, { recursive: true, force: true }); - }; - - return { - baseUrl: `http://127.0.0.1:${assignedPort}`, - bridgeToken, - hostApiToken, - runId, - nonce, - broker, - api, - child, - observedFrames, - forwardedRequests, - setForwardMode: (mode) => { - forwardMode = mode; - }, - stderr, - killChild, - waitFor, - teardown, - }; -} - -/** - * Build a large body of multi-byte UTF-8 characters. "€" uses three UTF-8 bytes - * and "😀" uses four. A pipe read boundary at a 65536-byte multiple lands inside - * a "€" sequence, because 65536 is not a multiple of three. This guarantees a - * multi-byte character split across two pipe chunks. - */ -function buildMultiByteBody(targetBytes: number): string { - const marker = "😀-start-😀"; - const filler = "€".repeat(Math.ceil(targetBytes / 3)); - return `${marker}${filler}${marker}`; -} - -describe("duplex bridge local end-to-end harness", () => { - const harnesses: DuplexE2EHarness[] = []; - - afterEach(async () => { - while (harnesses.length > 0) { - const harness = harnesses.pop(); - if (harness) await harness.teardown(); - } - }); - - const readyPredicate = (harness: DuplexE2EHarness) => () => - harness.observedFrames.some((frame) => frame.type === "ready"); - - it("delivers a valid READY frame from the spawned gateway child to the attached broker", async () => { - const harness = await createHarness(); - harnesses.push(harness); - - await harness.waitFor(readyPredicate(harness), "a READY frame from the gateway child"); - - const ready = harness.observedFrames.find( - (frame) => frame.type === "ready", - ) as DuplexReadyFrame; - expect(ready.version).toBe(2); - expect(ready.nonce).toBe(harness.nonce); - // READY is liveness only. It carries no address data. - expect((ready as unknown as Record).address).toBeUndefined(); - expect((ready as unknown as Record).port).toBeUndefined(); - // The broker read the READY frame and stayed open with no loss. - expect(harness.broker.state).toBe("open"); - expect(harness.broker.lossRecord).toBeNull(); - }, 20000); - - it("carries one real HTTP request through the child and the broker to the fake API unchanged", async () => { - const harness = await createHarness(); - harnesses.push(harness); - await harness.waitFor(readyPredicate(harness), "the gateway to become ready"); - - harness.api.setResponder((req, res) => { - const url = new URL(req.url ?? "/", "http://127.0.0.1"); - res.writeHead(200, { "content-type": "application/json" }); - res.end(JSON.stringify({ ok: true, echoedPath: url.pathname })); - }); - - const response = await fetch(`${harness.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${harness.bridgeToken}` }, - }); - expect(response.status).toBe(200); - await expect(response.json()).resolves.toEqual({ ok: true, echoedPath: "/api/agents/me" }); - - // The fake API saw one request with the real host token and the run id. The - // broker replaced the bridge token, so the fake API never saw it. - expect(harness.api.requests).toHaveLength(1); - expect(harness.api.requests[0]).toMatchObject({ - method: "GET", - url: "/api/agents/me", - auth: `Bearer ${harness.hostApiToken}`, - runId: harness.runId, - }); - }, 20000); - - it("reassembles a large response that spans many pipe chunks", async () => { - const harness = await createHarness(); - harnesses.push(harness); - await harness.waitFor(readyPredicate(harness), "the gateway to become ready"); - - // The body is larger than the pipe buffer, so the response frame crosses the - // child stdin pipe in many chunks. An ASCII body isolates the chunk-span - // behavior from the multi-byte behavior the next test covers. - const largeBody = "x".repeat(256 * 1024); - expect(Buffer.byteLength(largeBody, "utf8")).toBeGreaterThan(200 * 1024); - harness.api.setResponder((_req, res) => { - res.writeHead(200, { "content-type": "text/plain; charset=utf-8" }); - res.end(largeBody); - }); - - const response = await fetch(`${harness.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${harness.bridgeToken}` }, - }); - expect(response.status).toBe(200); - const received = await response.text(); - // The whole body returns complete after it crossed the pipe in many chunks. - expect(received.length).toBe(largeBody.length); - expect(received).toBe(largeBody); - }, 20000); - - it("decodes a multi-byte UTF-8 sequence split across chunk borders", async () => { - const harness = await createHarness(); - harnesses.push(harness); - await harness.waitFor(readyPredicate(harness), "the gateway to become ready"); - - const multiByteBody = buildMultiByteBody(256 * 1024); - expect(Buffer.byteLength(multiByteBody, "utf8")).toBeGreaterThan(200 * 1024); - harness.api.setResponder((_req, res) => { - res.writeHead(200, { "content-type": "text/plain; charset=utf-8" }); - res.end(multiByteBody); - }); - - const response = await fetch(`${harness.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${harness.bridgeToken}` }, - }); - expect(response.status).toBe(200); - const received = await response.text(); - // A multi-byte character split across a pipe chunk border decodes to the - // same character, so the whole body returns unchanged. - expect(received.length).toBe(multiByteBody.length); - expect(received).toBe(multiByteBody); - }, 20000); - - it("matches each concurrent request through one child to its own response", async () => { - const harness = await createHarness(); - harnesses.push(harness); - await harness.waitFor(readyPredicate(harness), "the gateway to become ready"); - - harness.api.setResponder((req, res) => { - const url = new URL(req.url ?? "/", "http://127.0.0.1"); - res.writeHead(200, { "content-type": "application/json" }); - res.end(JSON.stringify({ path: url.pathname })); - }); - - const ids = Array.from({ length: 8 }, (_unused, index) => `issue-${index}`); - const responses = await Promise.all( - ids.map((id) => - fetch(`${harness.baseUrl}/api/issues/${id}`, { - headers: { authorization: `Bearer ${harness.bridgeToken}` }, - }).then(async (response) => ({ status: response.status, body: await response.json() })), - ), - ); - - responses.forEach((result, index) => { - expect(result.status).toBe(200); - expect(result.body).toEqual({ path: `/api/issues/${ids[index]}` }); - }); - expect(harness.api.requests).toHaveLength(8); - }, 20000); - - it("answers 409 to outstanding and 503 to new requests when the broker closes its write side", async () => { - const harness = await createHarness({ lossExitGraceMs: 3000 }); - harnesses.push(harness); - await harness.waitFor(readyPredicate(harness), "the gateway to become ready"); - - // Hold the forward open, so the broker sends no response frame. The gateway - // keeps the request outstanding until the loss. - harness.setForwardMode("hang"); - const outstanding = fetch(`${harness.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${harness.bridgeToken}` }, - }); - void outstanding.catch(() => undefined); - await harness.waitFor( - () => harness.forwardedRequests.length >= 1, - "the broker to receive the forwarded request", - ); - - // The broker closes its write side, so the child stdin reaches a real EOF. - await harness.broker.close(); - - const lossResponse = await outstanding; - expect(lossResponse.status).toBe(409); - expect(lossResponse.headers.get("x-paperclip-bridge-outcome")).toBe("indeterminate"); - await expect(lossResponse.json()).resolves.toEqual({ error: "outcome_indeterminate" }); - - const afterLoss = await fetch(`${harness.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${harness.bridgeToken}` }, - }); - expect(afterLoss.status).toBe(503); - await expect(afterLoss.json()).resolves.toEqual({ error: "bridge_unavailable" }); - }, 20000); - - it("moves the broker to lost on a child kill, dispatches nothing more, and leaks no handle after teardown", async () => { - const harness = await createHarness(); - harnesses.push(harness); - await harness.waitFor(readyPredicate(harness), "the gateway to become ready"); - - // Hold one forward open, so a request is in flight when the child dies. - harness.setForwardMode("hang"); - const outstanding = fetch(`${harness.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${harness.bridgeToken}` }, - }); - void outstanding.catch(() => undefined); - await harness.waitFor( - () => harness.forwardedRequests.length >= 1, - "the broker to receive the forwarded request", - ); - const forwardedBeforeKill = harness.forwardedRequests.length; - - // A real process kill closes the stdout pipe and ends the child. - await harness.killChild(); - await harness.waitFor( - () => harness.broker.state === "lost", - "the broker to record the channel loss", - ); - expect(harness.broker.lossRecord?.reason).toBe("channel_exit"); - - // The broker dispatches nothing more after the loss. - await new Promise((resolve) => { - const timer = setTimeout(resolve, 100); - timer.unref?.(); - }); - expect(harness.forwardedRequests.length).toBe(forwardedBeforeKill); - // The outstanding fetch fails because the child died before it answered. - await expect(outstanding).rejects.toThrow(); - - await harness.teardown(); - // Teardown left no open pipe or socket handle. - expect(harness.child.stdin.destroyed).toBe(true); - expect(harness.child.stdout.destroyed).toBe(true); - expect(harness.child.stderr.destroyed).toBe(true); - expect(harness.api.listening()).toBe(false); - expect(harness.api.openSocketCount()).toBe(0); - }, 20000); - - it("fails a request with a local 502 when the host sends a malformed response chunk", async () => { - const RAW = DUPLEX_BODY_CHUNK_RAW_BYTES; - // Each case declares one malformed body_chunk a broken or hostile host could - // send. The gateway must reject it at once with a local 502. It must not grow - // its response reassembly buffer until the response timeout. `bodyByteCount` - // sets the declared body size on the response envelope. `data` is the base64 - // payload of the one injected chunk. - const cases: Array<{ name: string; bodyByteCount: number; data: string; error: string }> = [ - { name: "an empty chunk", bodyByteCount: RAW, data: "", error: "duplex response body_chunk is empty" }, - { - name: "an undersized non-final chunk", - bodyByteCount: RAW + 8, - data: Buffer.alloc(4, 1).toString("base64"), - error: "duplex response body_chunk has the wrong size", - }, - { - name: "an oversized chunk", - bodyByteCount: RAW + 8, - data: Buffer.alloc(RAW + 4, 1).toString("base64"), - error: "duplex response body_chunk has the wrong size", - }, - { - name: "an overrun past the declared size", - bodyByteCount: 10, - data: Buffer.alloc(200, 1).toString("base64"), - error: "duplex response body overruns the declared size", - }, - { - name: "a non-canonical base64 chunk", - bodyByteCount: 10, - data: "AB==", - error: "duplex response body_chunk is not canonical base64", - }, - ]; - - const harness = await createHarness(); - harnesses.push(harness); - await harness.waitFor(readyPredicate(harness), "the gateway to become ready"); - - // Hold the broker forward open, so only the injected frames answer each - // request. The gateway reassembles the response body itself, so a raw - // malformed chunk on its stdin drives the reject path under test. - harness.setForwardMode("hang"); - - for (const testCase of cases) { - const forwardedBefore = harness.forwardedRequests.length; - const pendingFetch = fetch(`${harness.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${harness.bridgeToken}` }, - }); - void pendingFetch.catch(() => undefined); - await harness.waitFor( - () => harness.forwardedRequests.length > forwardedBefore, - `the broker to receive the forwarded request for ${testCase.name}`, - ); - const id = harness.forwardedRequests[harness.forwardedRequests.length - 1].id; - - // Inject the response envelope, then the one malformed body_chunk, straight - // to the gateway stdin. This bypasses the broker response encoder, so the - // gateway sees the exact bytes a broken or hostile host could send. - harness.child.stdin.write( - encodeDuplexFrame({ - version: DUPLEX_FRAME_VERSION, - type: "response", - id, - status: 200, - headers: {}, - bodyByteCount: testCase.bodyByteCount, - outcome: "completed", - }), - ); - harness.child.stdin.write( - encodeDuplexFrame({ - version: DUPLEX_FRAME_VERSION, - type: "body_chunk", - id, - seq: 0, - data: testCase.data, - }), - ); - - const response = await pendingFetch; - expect(response.status).toBe(502); - await expect(response.json()).resolves.toEqual({ error: testCase.error }); - } - }, 20000); -}); diff --git a/packages/adapter-utils/src/duplex-frame-codec.ts b/packages/adapter-utils/src/duplex-frame-codec.ts index 0938603139..0a658a440d 100644 --- a/packages/adapter-utils/src/duplex-frame-codec.ts +++ b/packages/adapter-utils/src/duplex-frame-codec.ts @@ -19,9 +19,9 @@ * HTTP/2 is the preferred transport. `queue_v1` is the soft-deprecated fallback. * The `http2_v1` host readiness gate imports {@link decodeDuplexLine} from this * file to read the one READY line every gateway sends, so this file's READY - * frame path stays live for both transports. `duplex-bridge-broker.ts` and - * `duplex-body-spool.ts` still import the request, response, and body-chunk - * frame types from this file, so this phase keeps every frame type here. + * frame path stays live for both transports. `duplex-body-spool.ts` still + * imports the request, response, and body-chunk frame types from this file, so + * this file keeps every frame type here. */ import { diff --git a/packages/adapter-utils/src/execution-target-sandbox.test.ts b/packages/adapter-utils/src/execution-target-sandbox.test.ts index d7aaaa2ba7..e89a07d8c7 100644 --- a/packages/adapter-utils/src/execution-target-sandbox.test.ts +++ b/packages/adapter-utils/src/execution-target-sandbox.test.ts @@ -10,15 +10,11 @@ import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { - getSandboxCallbackBridgeServerSource, - getSandboxDuplexGatewayCodecSource, -} from "./sandbox-callback-bridge.js"; +import { getSandboxDuplexGatewayCodecSource } from "./sandbox-callback-bridge.js"; import { __duplexReadinessTesting, __http2PrefaceScanTesting, - buildDuplexGatewayLaunchArgv, DEFAULT_REMOTE_SANDBOX_ADAPTER_TIMEOUT_SEC, adapterExecutionTargetDuplexObservabilityRecorder, adapterExecutionTargetEnablesSandboxDuplexBridge, @@ -62,21 +58,10 @@ import { DUPLEX_FRAME_VERSION, decodeDuplexLine, encodeDuplexFrame, - type DuplexRequestFrame, type DuplexResponseFrame, } from "./duplex-frame-codec.js"; -import { splitBodyIntoChunkFrames } from "./duplex-body-spool.js"; +import { DUPLEX_CHANNEL_LOST_ERROR_CODE } from "./bridge-transport-contract.js"; import { - assertNestedDuplexBrokerBudgets, - createDuplexBridgeBroker, - DUPLEX_CHANNEL_LOST_ERROR_CODE, - type DuplexBrokerForwardResult, - type DuplexBrokerLossRecord, - type DuplexBrokerRequestRecord, - type DuplexBrokerState, -} from "./duplex-bridge-broker.js"; -import { - createDuplexObservability, DUPLEX_AGGREGATE_BYTE_LEDGER_METRIC_NAMES, DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL, DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL, @@ -88,6 +73,7 @@ import { DUPLEX_SPAN_CHANNEL_OPEN, DUPLEX_SPAN_REQUEST, DUPLEX_TRANSPORT_EVENT, + type DuplexLossReason, type DuplexObservabilityCounterRecord, type DuplexObservabilityDimensions, type DuplexObservabilityEventRecord, @@ -97,51 +83,6 @@ import { const execFileAsync = promisify(execFile); -/** - * Emit one duplex request the way the sandbox gateway sends it under the chunked - * body protocol: the envelope frame that carries `bodyByteCount`, then the - * `body_chunk` frames that carry the body. A non-empty body rides one or more - * `body_chunk` frames that share the envelope id; an empty body sends no chunk. - */ -function emitDuplexRequest( - control: { emitData: (chunk: string) => void }, - frame: Omit, - bodyText: string, -): void { - const body = Buffer.from(bodyText, "utf8"); - control.emitData(encodeDuplexFrame({ ...frame, bodyByteCount: body.length })); - for (const chunk of splitBodyIntoChunkFrames(frame.id, body, DUPLEX_FRAME_VERSION)) { - control.emitData(encodeDuplexFrame(chunk)); - } -} - -/** - * Send one duplex response the way the host broker sends it under the chunked - * body protocol: the envelope frame that carries `bodyByteCount`, then the - * `body_chunk` frames that carry the body. The gateway reassembles the body from - * the chunk frames before it writes the HTTP response. - */ -function sendDuplexResponse( - send: (frame: Record) => void, - response: { id: string | undefined; status: number; headers: Record; outcome?: string }, - bodyText: string, -): void { - const id = response.id ?? ""; - const body = Buffer.from(bodyText, "utf8"); - send({ - version: DUPLEX_FRAME_VERSION, - type: "response", - id, - status: response.status, - headers: response.headers, - bodyByteCount: body.length, - ...(response.outcome !== undefined ? { outcome: response.outcome } : {}), - }); - for (const chunk of splitBodyIntoChunkFrames(id, body, DUPLEX_FRAME_VERSION)) { - send(chunk as unknown as Record); - } -} - type RecordedSpan = { name: string; parentName: string | null; ended: boolean }; /** @@ -5981,21 +5922,6 @@ describe("sandbox adapter execution targets", () => { }); -// One decoded stdout frame from the generated duplex gateway. The gateway writes -// newline-delimited JSON frames to stdout, so the test parses each line. -interface DecodedGatewayFrame { - version?: number; - type?: string; - id?: string; - method?: string; - path?: string; - query?: string; - headers?: Record; - body?: string; - nonce?: string; - __unparsed?: string; -} - // One decode result from the embedded codec. The shape mirrors the host codec: // a valid frame or a protocol error with a code. interface EmbeddedDecodeResult { @@ -6052,220 +5978,12 @@ interface DuplexFrameFixture { encodeVectors: DuplexEncodeVector[]; } -describe("sandbox duplex gateway", () => { - const duplexCleanupDirs: string[] = []; - const duplexChildren: Array> = []; - - afterEach(async () => { - while (duplexChildren.length > 0) { - const child = duplexChildren.pop(); - if (!child) continue; - child.kill("SIGKILL"); - } - while (duplexCleanupDirs.length > 0) { - const dir = duplexCleanupDirs.pop(); - if (!dir) continue; - await rm(dir, { recursive: true, force: true }).catch(() => undefined); - } - }); - - it("launches the gateway with `exec env`, so a POSIX shell runs the environment-assignment prefix", async () => { - // A POSIX shell accepts an environment-assignment prefix only on a plain - // command, never on `exec`. The form `exec NAME=value command` exits with - // status 127, so the gateway never starts. The launch argv must use - // `exec env NAME=value command`. This test runs the generated script in a real - // `sh` against a stub entrypoint that echoes one launch env var, so it fails on - // the old `exec NAME=value` form and passes on the `exec env` form. - const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-duplex-launch-")); - duplexCleanupDirs.push(rootDir); - const stub = path.join(rootDir, "stub-entrypoint.sh"); - // The stub stands in for the node gateway. It prints a READY frame that echoes - // the launch nonce, so the test proves the environment assignment reached the - // process the launch replaced the shell with. - await writeFile( - stub, - '#!/bin/sh\nprintf \'{"version":2,"type":"ready","nonce":"%s"}\\n\' "$PAPERCLIP_BRIDGE_NONCE"\n', - "utf8", - ); - const argv = buildDuplexGatewayLaunchArgv({ - shellCommand: "sh", - remoteEntrypoint: stub, - // Run the stub with `sh`, so the test needs no node runtime. The launch form - // is `exec env NAME=value 'sh' ''`, which exercises the exec-env fix. - nodeCommand: "sh", - env: { PAPERCLIP_BRIDGE_NONCE: "abc123def456", PAPERCLIP_BRIDGE_PORT: "40404" }, - }); - const [shell, ...shellArgs] = argv; - const { stdout } = await execFileAsync(shell, shellArgs, { encoding: "utf8" }); - const decoded = decodeDuplexLine(stdout.trim()); - expect(decoded.ok).toBe(true); - if (decoded.ok) { - expect(decoded.frame.type).toBe("ready"); - if (decoded.frame.type === "ready") { - expect(decoded.frame.nonce).toBe("abc123def456"); - } - } - }); - - interface DuplexGatewayHandle { - baseUrl: string; - frames: DecodedGatewayFrame[]; - stderr: () => string; - waitForFrame: ( - predicate: (frame: DecodedGatewayFrame) => boolean, - timeoutMs?: number, - ) => Promise; - sendFrame: (frame: Record) => void; - sendRaw: (text: string) => void; - endStdin: () => void; - exited: Promise; - stop: () => Promise; - } - - // Reserve a free loopback port. The host assigns a positive port to the duplex - // gateway; the gateway binds exactly that port. The test opens an ephemeral - // listener, reads its port, then closes it, so the number is very likely free - // when the gateway binds it a moment later. - async function reserveLoopbackPort(): Promise { - return new Promise((resolve, reject) => { - const probe = net.createServer(); - probe.once("error", reject); - probe.listen(0, "127.0.0.1", () => { - const address = probe.address(); - if (!address || typeof address === "string") { - probe.close(() => reject(new Error("Could not reserve a loopback port."))); - return; - } - const reserved = address.port; - probe.close(() => resolve(reserved)); - }); - }); - } - - // Start the generated gateway `.mjs` in duplex mode as a real child process. - // The test writes response frames to the child stdin and reads request frames - // from the child stdout, so it stands in for the host side of the channel. - // The host assigns the port and the nonce; the test builds the base URL from - // the assigned port, never from the READY frame. - async function startDuplexGateway( - env: Record, - options: { port?: number; nonce?: string } = {}, - ): Promise { - const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-duplex-gateway-")); - duplexCleanupDirs.push(rootDir); - const entrypoint = path.join(rootDir, "gateway.mjs"); - await writeFile(entrypoint, getSandboxCallbackBridgeServerSource(), "utf8"); - - const assignedPort = options.port ?? (await reserveLoopbackPort()); - const nonce = options.nonce ?? "d0c1b2a3e4f5061728394a5b6c7d8e9f"; - const child = spawn(process.execPath, [entrypoint], { - stdio: ["pipe", "pipe", "pipe"], - env: { - ...process.env, - PAPERCLIP_API_BRIDGE_MODE: "duplex_v1", - PAPERCLIP_BRIDGE_HOST: "127.0.0.1", - PAPERCLIP_BRIDGE_PORT: String(assignedPort), - PAPERCLIP_BRIDGE_NONCE: nonce, - ...env, - }, - }); - duplexChildren.push(child); - - const frames: DecodedGatewayFrame[] = []; - const waiters: Array<{ - predicate: (frame: DecodedGatewayFrame) => boolean; - resolve: (frame: DecodedGatewayFrame) => void; - }> = []; - let stdoutBuffer = ""; - let stderrText = ""; - - child.stdout?.setEncoding("utf8"); - child.stdout?.on("data", (chunk: string) => { - stdoutBuffer += chunk; - let newlineIndex = stdoutBuffer.indexOf("\n"); - while (newlineIndex !== -1) { - const line = stdoutBuffer.slice(0, newlineIndex); - stdoutBuffer = stdoutBuffer.slice(newlineIndex + 1); - if (line.length > 0) { - let frame: DecodedGatewayFrame; - try { - frame = JSON.parse(line) as DecodedGatewayFrame; - } catch { - frame = { __unparsed: line }; - } - frames.push(frame); - for (const waiter of [...waiters]) { - if (waiter.predicate(frame)) { - waiters.splice(waiters.indexOf(waiter), 1); - waiter.resolve(frame); - } - } - } - newlineIndex = stdoutBuffer.indexOf("\n"); - } - }); - child.stderr?.setEncoding("utf8"); - child.stderr?.on("data", (chunk: string) => { - stderrText += chunk; - }); - - const exited = new Promise((resolve) => { - child.on("exit", (code) => resolve(code)); - }); - - const waitForFrame = ( - predicate: (frame: DecodedGatewayFrame) => boolean, - timeoutMs = 5000, - ): Promise => { - const existing = frames.find(predicate); - if (existing) return Promise.resolve(existing); - return new Promise((resolve, reject) => { - const waiter = { - predicate, - resolve: (frame: DecodedGatewayFrame) => { - clearTimeout(timer); - resolve(frame); - }, - }; - const timer = setTimeout(() => { - const index = waiters.indexOf(waiter); - if (index !== -1) waiters.splice(index, 1); - reject(new Error(`Timed out waiting for a gateway frame. stderr: ${stderrText}`)); - }, timeoutMs); - waiters.push(waiter); - }); - }; - - const handle: DuplexGatewayHandle = { - baseUrl: "", - frames, - stderr: () => stderrText, - waitForFrame, - sendFrame: (frame) => { - child.stdin?.write(`${JSON.stringify(frame)}\n`); - }, - sendRaw: (text) => { - child.stdin?.write(text); - }, - endStdin: () => { - child.stdin?.end(); - }, - exited, - stop: async () => { - child.kill("SIGKILL"); - await exited.catch(() => null); - }, - }; - - const ready = await waitForFrame((frame) => frame.type === "ready"); - // READY carries the echoed nonce and no address data. The host builds the - // origin from the port it assigned, never from the frame. - expect(ready.nonce).toBe(nonce); - expect((ready as Record).address).toBeUndefined(); - handle.baseUrl = `http://127.0.0.1:${assignedPort}`; - return handle; - } - +// This describe block covers the zero-dependency codec every generated +// gateway embeds (`DUPLEX_GATEWAY_CODEC_SOURCE`), and the sandbox-process +// decoder byte cap. The `http2_v1` gateway embeds the same codec source to +// send its one READY line, so this coverage stays live for the active +// transport. +describe("embedded sandbox gateway codec", () => { it("embedded gateway codec passes every vector in the shared fixture", async () => { const codecFactory = new Function( `${getSandboxDuplexGatewayCodecSource()}\nreturn { encodeDuplexFrame, encodeDuplexFrameChecked, decodeDuplexLine, DuplexFrameDecoder, DUPLEX_FRAME_VERSION, DEFAULT_MAX_DUPLEX_FRAME_BYTES, DUPLEX_DECODER_SCOPE, DEFAULT_MAX_DUPLEX_DECODER_BYTES };`, @@ -6422,751 +6140,12 @@ describe("sandbox duplex gateway", () => { expect(ledger.bytesInUse).toBe(0); expect(ledger.liveTokenCount).toBe(0); }); - - it("returns the same HTTP response as the file gateway for a forwarded request", async () => { - const token = "duplex-token-forward"; - const gateway = await startDuplexGateway({ PAPERCLIP_BRIDGE_TOKEN: token }); - - const responsePromise = fetch(`${gateway.baseUrl}/api/agents/me?view=compact`, { - headers: { - authorization: `Bearer ${token}`, - accept: "application/json", - "if-none-match": '"cache-key"', - "x-bridge-debug": "drop-me", - }, - }); - const requestFrame = await gateway.waitForFrame((frame) => frame.type === "request"); - expect(requestFrame.method).toBe("GET"); - expect(requestFrame.path).toBe("/api/agents/me"); - expect(requestFrame.query).toBe("?view=compact"); - // Only allowlisted headers forward; the bearer and the debug header drop. - expect(requestFrame.headers).toEqual({ - accept: "application/json", - "if-none-match": '"cache-key"', - }); - - sendDuplexResponse( - gateway.sendFrame, - { - id: requestFrame.id, - status: 200, - headers: { "content-type": "application/json", etag: '"rev-1"', "content-length": "999" }, - outcome: "completed", - }, - JSON.stringify({ ok: true }), - ); - const response = await responsePromise; - expect(response.status).toBe(200); - expect(response.headers.get("content-type")).toContain("application/json"); - expect(response.headers.get("etag")).toBe('"rev-1"'); - await expect(response.json()).resolves.toEqual({ ok: true }); - - // An indeterminate outcome maps to a non-retryable 409, the same contract the - // file gateway applies through the outcome header. - const indeterminatePromise = fetch(`${gateway.baseUrl}/api/issues/issue-1`, { - method: "PATCH", - headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, - body: JSON.stringify({ status: "in_progress" }), - }); - const patchFrame = await gateway.waitForFrame( - (frame) => frame.type === "request" && frame.id !== requestFrame.id, - ); - sendDuplexResponse( - gateway.sendFrame, - { - id: patchFrame.id, - status: 200, - headers: { "content-type": "application/json" }, - outcome: "indeterminate", - }, - JSON.stringify({ error: "outcome_indeterminate" }), - ); - const indeterminate = await indeterminatePromise; - expect(indeterminate.status).toBe(409); - - await gateway.stop(); - }, 20000); - - it("fails a request over the body size limit locally and keeps the channel open", async () => { - const token = "duplex-token-oversize-request"; - // The chunked body protocol splits a large body into fixed-size body_chunk - // frames that each stay under the frame bound, so a body never exceeds the - // frame bound on its own. The operative local guard for a request too large to - // deliver is now the body size limit. Set it to 1,000,000 bytes here. - const gateway = await startDuplexGateway({ - PAPERCLIP_BRIDGE_TOKEN: token, - PAPERCLIP_BRIDGE_MAX_BODY_BYTES: "1000000", - }); - - // A body over the body size limit is rejected locally. The gateway must fail - // this one local request cleanly and forward no frame. - const oversizeBody = JSON.stringify({ body: "x".repeat(1_100_000) }); - const tooLarge = await fetch(`${gateway.baseUrl}/api/issues/issue-1/comments`, { - method: "POST", - headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, - body: oversizeBody, - }); - expect(tooLarge.status).toBe(502); - await expect(tooLarge.json()).resolves.toEqual({ - error: "Bridge request body exceeded the configured size limit.", - }); - - // The oversized request forwarded no frame. It never left the gateway. - expect(gateway.frames.filter((frame) => frame.type === "request")).toHaveLength(0); - // The gateway did not lose the channel: no close frame and the process runs. - expect(gateway.frames.some((frame) => frame.type === "close")).toBe(false); - - // The channel stays open, so a normal request after the oversized one still - // forwards and completes. - const okPromise = fetch(`${gateway.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${token}` }, - }); - const requestFrame = await gateway.waitForFrame((frame) => frame.type === "request"); - sendDuplexResponse( - gateway.sendFrame, - { - id: requestFrame.id, - status: 200, - headers: { "content-type": "application/json" }, - outcome: "completed", - }, - JSON.stringify({ ok: true }), - ); - const okResponse = await okPromise; - expect(okResponse.status).toBe(200); - await expect(okResponse.json()).resolves.toEqual({ ok: true }); - - await gateway.stop(); - }, 20000); - - it("enforces the bearer check, JSON-only rule, body limit, and depth limit", async () => { - const token = "duplex-token-contract"; - const gateway = await startDuplexGateway({ - PAPERCLIP_BRIDGE_TOKEN: token, - PAPERCLIP_BRIDGE_MAX_QUEUE_DEPTH: "1", - PAPERCLIP_BRIDGE_MAX_BODY_BYTES: "16", - }); - - const badAuth = await fetch(`${gateway.baseUrl}/api/agents/me`, { - headers: { authorization: "Bearer wrong-token" }, - }); - expect(badAuth.status).toBe(401); - await expect(badAuth.json()).resolves.toEqual({ error: "Invalid bridge token." }); - - const nonJson = await fetch(`${gateway.baseUrl}/api/issues/issue-1/comments`, { - method: "POST", - headers: { authorization: `Bearer ${token}`, "content-type": "text/plain" }, - body: "not json", - }); - expect(nonJson.status).toBe(415); - await expect(nonJson.json()).resolves.toEqual({ - error: "Bridge only accepts JSON request bodies.", - }); - - const oversizeBody = await fetch(`${gateway.baseUrl}/api/issues/issue-1/comments`, { - method: "POST", - headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, - body: JSON.stringify({ body: "x".repeat(64) }), - }); - expect(oversizeBody.status).toBe(502); - await expect(oversizeBody.json()).resolves.toEqual({ - error: "Bridge request body exceeded the configured size limit.", - }); - - // No request frame forwarded so far: the guards rejected before forwarding. - const framesBeforeDepth = gateway.frames.filter((frame) => frame.type === "request").length; - expect(framesBeforeDepth).toBe(0); - - // One outstanding request fills the single depth slot; the host never - // responds, so the slot stays used. - const outstanding = fetch(`${gateway.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${token}` }, - }); - void outstanding.catch(() => undefined); - await gateway.waitForFrame((frame) => frame.type === "request"); - - const queueFull = await fetch(`${gateway.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${token}` }, - }); - expect(queueFull.status).toBe(503); - await expect(queueFull.json()).resolves.toEqual({ error: "Bridge request queue is full." }); - - // Only the outstanding request forwarded a frame; the queue-full request did - // not. - const framesAfterDepth = gateway.frames.filter((frame) => frame.type === "request").length; - expect(framesAfterDepth).toBe(1); - - await gateway.stop(); - }, 20000); - - it("answers outstanding requests 409 and new requests 503 on stdin EOF, then exits", async () => { - const token = "duplex-token-eof"; - const gateway = await startDuplexGateway({ - PAPERCLIP_BRIDGE_TOKEN: token, - PAPERCLIP_BRIDGE_LOSS_EXIT_GRACE_MS: "3000", - }); - - const outstanding = fetch(`${gateway.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${token}` }, - }); - await gateway.waitForFrame((frame) => frame.type === "request"); - gateway.endStdin(); - - const lossResponse = await outstanding; - expect(lossResponse.status).toBe(409); - expect(lossResponse.headers.get("x-paperclip-bridge-outcome")).toBe("indeterminate"); - await expect(lossResponse.json()).resolves.toEqual({ error: "outcome_indeterminate" }); - - const afterLoss = await fetch(`${gateway.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${token}` }, - }); - expect(afterLoss.status).toBe(503); - await expect(afterLoss.json()).resolves.toEqual({ error: "bridge_unavailable" }); - - const exitCode = await gateway.exited; - expect(exitCode).toBe(0); - }, 20000); - - it("applies the same loss behavior on a heartbeat timeout", async () => { - const token = "duplex-token-heartbeat"; - const gateway = await startDuplexGateway({ - PAPERCLIP_BRIDGE_TOKEN: token, - PAPERCLIP_BRIDGE_HEARTBEAT_TIMEOUT_MS: "400", - PAPERCLIP_BRIDGE_LOSS_EXIT_GRACE_MS: "3000", - PAPERCLIP_BRIDGE_RESPONSE_TIMEOUT_MS: "30000", - }); - - // Never send an inbound frame: inbound silence trips the heartbeat timeout. - const outstanding = fetch(`${gateway.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${token}` }, - }); - await gateway.waitForFrame((frame) => frame.type === "request"); - - const lossResponse = await outstanding; - expect(lossResponse.status).toBe(409); - await expect(lossResponse.json()).resolves.toEqual({ error: "outcome_indeterminate" }); - - const afterLoss = await fetch(`${gateway.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${token}` }, - }); - expect(afterLoss.status).toBe(503); - await expect(afterLoss.json()).resolves.toEqual({ error: "bridge_unavailable" }); - - const exitCode = await gateway.exited; - expect(exitCode).toBe(0); - }, 20000); - - it("keeps diagnostics off stdout; stdout carries only frames", async () => { - const token = "duplex-token-stdout"; - const gateway = await startDuplexGateway({ PAPERCLIP_BRIDGE_TOKEN: token }); - - const responsePromise = fetch(`${gateway.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${token}` }, - }); - const requestFrame = await gateway.waitForFrame((frame) => frame.type === "request"); - - // Feed a malformed inbound line and an unknown response id. Both force a - // diagnostic path; none of it may reach stdout. - gateway.sendRaw("this is not a frame\n"); - sendDuplexResponse( - gateway.sendFrame, - { id: "unknown-id", status: 200, headers: {}, outcome: "completed" }, - "", - ); - sendDuplexResponse( - gateway.sendFrame, - { - id: requestFrame.id, - status: 200, - headers: { "content-type": "application/json" }, - outcome: "completed", - }, - "{}", - ); - - const response = await responsePromise; - expect(response.status).toBe(200); - - // Every stdout line parsed as a frame; none was diagnostic text. - expect(gateway.frames.some((frame) => frame.__unparsed !== undefined)).toBe(false); - expect( - gateway.frames.every( - (frame) => typeof frame.version === "number" && typeof frame.type === "string", - ), - ).toBe(true); - const allowedTypes = new Set(["ready", "heartbeat", "request"]); - expect(gateway.frames.every((frame) => allowedTypes.has(String(frame.type)))).toBe(true); - - // The malformed inbound line produced a stderr diagnostic, not a stdout one. - expect(gateway.stderr()).toContain("dropped an inbound frame"); - - await gateway.stop(); - }, 20000); - - it("defaults the wait budget to 35 s and honors the environment key override", async () => { - // The generated source carries the 35 s default. - expect(getSandboxCallbackBridgeServerSource()).toContain("35000"); - - const token = "duplex-token-budget"; - const gateway = await startDuplexGateway({ - PAPERCLIP_BRIDGE_TOKEN: token, - PAPERCLIP_BRIDGE_RESPONSE_TIMEOUT_MS: "150", - PAPERCLIP_BRIDGE_HEARTBEAT_TIMEOUT_MS: "30000", - }); - - const started = Date.now(); - const response = await fetch(`${gateway.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${token}` }, - }); - expect(response.status).toBe(502); - await expect(response.json()).resolves.toEqual({ - error: "Timed out waiting for host bridge response.", - }); - expect(Date.now() - started).toBeLessThan(4000); - - await gateway.stop(); - }, 20000); }); -/** - * A scripted fake duplex channel. The test drives the read path with - * `emitData`/`emitExit`, and reads the frames the broker wrote through `written`. - * The `writeError` and `closeBehavior` hooks let a test force a stream failure - * and a close timeout. - */ -function createFakeDuplexChannel(): { - channel: CommandManagedDuplexChannel; - emitData: (chunk: string) => void; - emitExit: (exit: { exitCode: number | null }) => void; - written: Array; - writtenTypes: string[]; - stopped: () => number; - setWriteError: (error: Error | null) => void; - setCloseBehavior: (behavior: "resolve" | "hang" | "reject") => void; -} { - let dataListener: ((chunk: Uint8Array) => void) | null = null; - let exitListener: ((exit: { exitCode: number | null }) => void) | null = null; - let writeError: Error | null = null; - let closeBehavior: "resolve" | "hang" | "reject" = "resolve"; - let stopCount = 0; - // The broker writes a response as one envelope frame that carries - // `bodyByteCount`, then the `body_chunk` frames that carry the body. Reassemble - // the body so a test reads the response the same way it did with the one-frame - // model. - const written: Array = []; - const writtenTypes: string[] = []; - const responseAssembly = new Map< - string, - { frame: DuplexResponseFrame; received: number; chunks: Buffer[] } - >(); - - const channel: CommandManagedDuplexChannel = { - write(data: Uint8Array): void { - if (writeError) throw writeError; - const decoded = decodeDuplexLine(Buffer.from(data).toString("utf8").replace(/\n$/, "")); - if (!decoded.ok) return; - const frame = decoded.frame; - writtenTypes.push(frame.type); - if (frame.type === "response") { - if (frame.bodyByteCount === 0) { - written.push({ ...frame, body: "" }); - } else { - responseAssembly.set(frame.id, { frame, received: 0, chunks: [] }); - } - } else if (frame.type === "body_chunk") { - const assembly = responseAssembly.get(frame.id); - if (!assembly) return; - const decodedChunk = Buffer.from(frame.data, "base64"); - assembly.received += decodedChunk.length; - assembly.chunks.push(decodedChunk); - if (assembly.received >= assembly.frame.bodyByteCount) { - responseAssembly.delete(frame.id); - written.push({ - ...assembly.frame, - body: Buffer.concat(assembly.chunks).toString("utf8"), - }); - } - } - }, - onData(listener: (chunk: Uint8Array) => void): void { - dataListener = listener; - }, - onExit(listener: (exit: { exitCode: number | null }) => void): void { - exitListener = listener; - }, - stop(): void { - stopCount += 1; - }, - close(): Promise { - if (closeBehavior === "resolve") return Promise.resolve(); - if (closeBehavior === "reject") return Promise.reject(new Error("close rejected")); - return new Promise(() => {}); - }, - }; - - return { - channel, - emitData: (chunk: string) => dataListener?.(new TextEncoder().encode(chunk)), - emitExit: (exit: { exitCode: number | null }) => exitListener?.(exit), - written, - writtenTypes, - stopped: () => stopCount, - setWriteError: (error: Error | null) => { - writeError = error; - }, - setCloseBehavior: (behavior: "resolve" | "hang" | "reject") => { - closeBehavior = behavior; - }, - }; -} - -/** - * Build one request as its envelope line (carrying `bodyByteCount`) plus its - * `body_chunk` lines, so the fake channel emits a whole request in one write. The - * broker reassembles the body from the chunk frames before it forwards. - */ -function requestFrameLine( - overrides: Partial> & { id: string }, - bodyText: string = JSON.stringify({ body: "hello" }), -): string { - const body = Buffer.from(bodyText, "utf8"); - const frame: DuplexRequestFrame = { - version: DUPLEX_FRAME_VERSION, - type: "request", - id: overrides.id, - method: overrides.method ?? "POST", - path: overrides.path ?? "/api/issues/PAP-1/comments", - query: overrides.query ?? "", - headers: overrides.headers ?? { authorization: "Bearer bridge-token" }, - bodyByteCount: body.length, - }; - let line = encodeDuplexFrame(frame); - for (const chunk of splitBodyIntoChunkFrames(frame.id, body, DUPLEX_FRAME_VERSION)) { - line += encodeDuplexFrame(chunk); - } - return line; -} - /** Wait for the pending microtasks and macrotasks to settle. */ function flushMacrotasks(): Promise { return new Promise((resolve) => setImmediate(resolve)); } - -describe("createDuplexBridgeBroker", () => { - it("forwards a decoded request to the handler and writes the handler response back", async () => { - const fake = createFakeDuplexChannel(); - const received: DuplexRequestFrame[] = []; - // The forward handler owns the token replacement and the run attribution. The - // broker passes the decoded request straight through, so the sandbox request - // still carries only the bridge token here, and the handler applies the real - // token and the signed run identifier on the existing forward path. - const broker = await createDuplexBridgeBroker({ - channel: fake.channel, - forwardRequest: async (request): Promise => { - received.push(request); - expect(request.headers.authorization).toBe("Bearer bridge-token"); - expect(request.headers["x-paperclip-run-id"]).toBeUndefined(); - return { - status: 201, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ ok: true }), - }; - }, - }); - - broker.start(); - fake.emitData(requestFrameLine({ id: "req-1" })); - await flushMacrotasks(); - - expect(received).toHaveLength(1); - expect(received[0].id).toBe("req-1"); - expect(fake.written).toHaveLength(1); - expect(fake.written[0]).toMatchObject({ - type: "response", - id: "req-1", - status: 201, - outcome: "completed", - }); - expect(JSON.parse(fake.written[0].body)).toEqual({ ok: true }); - - await broker.close(); - }); - - it("moves through opening, open, closing, closed in order", async () => { - const fake = createFakeDuplexChannel(); - const states: DuplexBrokerState[] = []; - const broker = await createDuplexBridgeBroker({ - channel: fake.channel, - forwardRequest: async () => ({ status: 200 }), - onStateChange: (state) => states.push(state), - }); - - expect(broker.state).toBe("opening"); - broker.start(); - expect(broker.state).toBe("open"); - await broker.close(); - expect(broker.state).toBe("closed"); - expect(states).toEqual(["open", "closing", "closed"]); - expect(fake.writtenTypes).toContain("close"); - }); - - it.each([ - { - name: "channel exit", - reason: "channel_exit" as const, - trigger: (fake: ReturnType) => - fake.emitExit({ exitCode: 1 }), - }, - { - name: "protocol failure", - reason: "protocol_failure" as const, - trigger: (fake: ReturnType) => - fake.emitData("this is not json\n"), - }, - { - name: "stream failure", - reason: "stream_failure" as const, - trigger: (fake: ReturnType) => { - fake.setWriteError(new Error("broken pipe")); - fake.emitData(requestFrameLine({ id: "req-stream" })); - }, - }, - ])("enters lost on $name", async ({ reason, trigger }) => { - const fake = createFakeDuplexChannel(); - const losses: DuplexBrokerLossRecord[] = []; - const broker = await createDuplexBridgeBroker({ - channel: fake.channel, - forwardRequest: async () => ({ status: 200 }), - onLoss: (record) => losses.push(record), - }); - broker.start(); - - trigger(fake); - await flushMacrotasks(); - - expect(broker.state).toBe("lost"); - expect(broker.lossRecord?.reason).toBe(reason); - expect(losses).toHaveLength(1); - }); - - it("enters lost on a close timeout", async () => { - const fake = createFakeDuplexChannel(); - fake.setCloseBehavior("hang"); - const broker = await createDuplexBridgeBroker({ - channel: fake.channel, - forwardRequest: async () => ({ status: 200 }), - closeTimeoutMs: 20, - }); - broker.start(); - - await broker.close(); - - expect(broker.state).toBe("lost"); - expect(broker.lossRecord?.reason).toBe("close_timeout"); - }); - - it("stops the heartbeat, marks the run bridge ended, and dispatches nothing after loss", async () => { - const fake = createFakeDuplexChannel(); - const forwarded: string[] = []; - const broker = await createDuplexBridgeBroker({ - channel: fake.channel, - forwardRequest: async (request) => { - forwarded.push(request.id); - return { status: 200 }; - }, - }); - broker.start(); - - fake.emitExit({ exitCode: 1 }); - await flushMacrotasks(); - expect(broker.state).toBe("lost"); - expect(broker.lossRecord).not.toBeNull(); - - // A request that arrives after loss reaches nothing. The broker never - // reconnects and never replays a request. - fake.emitData(requestFrameLine({ id: "after-loss" })); - await flushMacrotasks(); - expect(forwarded).toEqual([]); - }); - - it("forwards one request id one time, so a repeated frame never reaches the API twice", async () => { - const fake = createFakeDuplexChannel(); - const forwarded: string[] = []; - const broker = await createDuplexBridgeBroker({ - channel: fake.channel, - forwardRequest: async (request) => { - forwarded.push(request.id); - return { status: 200, body: "ok" }; - }, - }); - broker.start(); - - fake.emitData(requestFrameLine({ id: "dup" })); - await flushMacrotasks(); - fake.emitData(requestFrameLine({ id: "dup" })); - await flushMacrotasks(); - - expect(forwarded).toEqual(["dup"]); - expect(fake.written).toHaveLength(1); - - await broker.close(); - }); - - it("captures the dispatch-start point per request for metrics only", async () => { - const fake = createFakeDuplexChannel(); - const records: DuplexBrokerRequestRecord[] = []; - let clock = 1000; - const broker = await createDuplexBridgeBroker({ - channel: fake.channel, - forwardRequest: async () => ({ status: 200 }), - now: () => clock, - onRequestRecord: (record) => records.push(record), - }); - broker.start(); - - clock = 2500; - fake.emitData(requestFrameLine({ id: "metric-1", method: "GET", path: "/api/agents/me" })); - await flushMacrotasks(); - - expect(records).toEqual([ - { id: "metric-1", method: "GET", path: "/api/agents/me", dispatchStartMs: 2500 }, - ]); - // The record never reaches the channel. The broker writes only a response. - expect(fake.writtenTypes).not.toContain("request"); - - await broker.close(); - }); - - it("latches a failure when a loss orders before an orderly completion and names the typed reason", async () => { - const fake = createFakeDuplexChannel(); - const broker = await createDuplexBridgeBroker({ - channel: fake.channel, - forwardRequest: async () => ({ status: 200 }), - }); - broker.start(); - - // The channel dies mid-turn, before any orderly completion. - fake.emitExit({ exitCode: 1 }); - await flushMacrotasks(); - expect(broker.runDisposition).toEqual({ failed: true, lossReason: "provider_exit" }); - - // A later orderly completion cannot clear the latch. A delayed activity - // callback cannot clear the latch either, because the broker dispatches - // nothing after loss. - broker.markOrderlyCompletion(); - fake.emitData(requestFrameLine({ id: "after-loss" })); - await flushMacrotasks(); - expect(broker.runDisposition).toEqual({ failed: true, lossReason: "provider_exit" }); - }); - - it("keeps a success when a loss orders after a host-observed orderly completion, and emits no loss event", async () => { - const events: DuplexObservabilityEventRecord[] = []; - const counters: DuplexObservabilityCounterRecord[] = []; - const recorder: DuplexObservabilityRecorder = { - recordSpan() {}, - incrementCounter(record) { - counters.push(record); - }, - emitEvent(record) { - events.push(record); - }, - }; - const fake = createFakeDuplexChannel(); - const broker = await createDuplexBridgeBroker({ - channel: fake.channel, - forwardRequest: async () => ({ status: 200 }), - telemetry: createDuplexObservability({ recorder, providerKey: "daytona" }), - }); - broker.start(); - - // The agent completes its turn, then the channel ends during the teardown. - broker.markOrderlyCompletion(); - fake.emitExit({ exitCode: 0 }); - await flushMacrotasks(); - - expect(broker.runDisposition).toEqual({ failed: false, lossReason: null }); - // A normal teardown is not a loss: no loss event and no loss counter. - expect(events.some((e) => e.dimensions.loss_reason !== undefined)).toBe(false); - expect(counters.some((c) => c.metric === DUPLEX_COUNTER_LOSS_TOTAL)).toBe(false); - }); - - it("carries the typed loss_reason on the transport loss event and keeps a sentinel message off every sink", async () => { - const spans: DuplexObservabilitySpanRecord[] = []; - const counters: DuplexObservabilityCounterRecord[] = []; - const events: DuplexObservabilityEventRecord[] = []; - const recorder: DuplexObservabilityRecorder = { - recordSpan(record) { - spans.push(record); - }, - incrementCounter(record) { - counters.push(record); - }, - emitEvent(record) { - events.push(record); - }, - }; - const logLines: string[] = []; - const fake = createFakeDuplexChannel(); - const sentinel = "SENTINEL-PROVIDER-TEXT-1a2b3c"; - const broker = await createDuplexBridgeBroker({ - channel: fake.channel, - forwardRequest: async () => ({ status: 200 }), - telemetry: createDuplexObservability({ recorder, providerKey: "daytona" }), - logger: (message) => logLines.push(message), - }); - broker.start(); - - // A stream write failure carries a raw provider message. It maps to the typed - // `write_error`; the raw message must reach no sink. - fake.setWriteError(new Error(sentinel)); - fake.emitData(requestFrameLine({ id: "req-sentinel" })); - await flushMacrotasks(); - - const lossEvent = events.find((e) => e.dimensions.loss_reason !== undefined); - expect(lossEvent?.dimensions.loss_reason).toBe("write_error"); - expect(lossEvent?.dimensions).toMatchObject({ transport: "duplex", outcome: "error" }); - - // The sentinel provider text reaches no telemetry sink and no log line. - const serializedSinks = JSON.stringify({ spans, counters, events }); - expect(serializedSinks).not.toContain(sentinel); - expect(logLines.join("\n")).not.toContain(sentinel); - }); - - it("rejects a configuration where an inner budget is not smaller than its outer budget", async () => { - expect(() => - assertNestedDuplexBrokerBudgets({ - forwardTimeoutMs: 32_000, - responseBudgetMs: 32_000, - gatewayWaitMs: 35_000, - }), - ).toThrow(/forward budget/); - expect(() => - assertNestedDuplexBrokerBudgets({ - forwardTimeoutMs: 30_000, - responseBudgetMs: 35_000, - gatewayWaitMs: 35_000, - }), - ).toThrow(/response budget/); - // The async broker construction validates budgets before its first await, so - // an invalid budget set surfaces as a rejected promise. - await expect( - createDuplexBridgeBroker({ - channel: createFakeDuplexChannel().channel, - forwardRequest: async () => ({ status: 200 }), - budgets: { forwardTimeoutMs: 40_000 }, - }), - ).rejects.toThrow(/forward budget/); - // The default budget set holds the nested order. - expect(() => - assertNestedDuplexBrokerBudgets({ - forwardTimeoutMs: 30_000, - responseBudgetMs: 32_000, - gatewayWaitMs: 35_000, - }), - ).not.toThrow(); - }); -}); - describe("sandbox target spec parse: enableSandboxDuplexBridge", () => { // The minimal serialized sandbox target the host stamps and the adapter parses. // A test overrides one field per case to prove the fail-closed parse. @@ -7228,33 +6207,56 @@ describe("sandbox target spec parse: enableSandboxDuplexBridge", () => { }); }); +/** + * A minimal run-disposition latch for the seam tests below. It reproduces the + * same ordering rule the real bridge transport applies: the first ordered + * loss or orderly completion latches the terminal disposition, and a later + * call never overrides it. `emitExit` stands in for a transport-level channel + * exit — the real transport treats every channel exit as a loss candidate, + * mapped to the typed `provider_exit` reason. + */ +function createFakeBridgeTransport() { + let lossOrdered = false; + let lossReason: DuplexLossReason | null = null; + let completionOrdered = false; + const markOrderlyCompletion = (): void => { + if (completionOrdered || lossOrdered) return; + completionOrdered = true; + }; + return { + get runDisposition() { + return { failed: lossOrdered, lossReason }; + }, + settleRunDisposition() { + markOrderlyCompletion(); + return { failed: lossOrdered, lossReason }; + }, + markOrderlyCompletion, + emitExit(): void { + if (lossOrdered || completionOrdered) return; + lossOrdered = true; + lossReason = "provider_exit"; + }, + }; +} + describe("settleRunDisposition atomic read and mark", () => { it("marks the orderly completion and reports a success for a healthy channel", async () => { - const fake = createFakeDuplexChannel(); - const broker = await createDuplexBridgeBroker({ - channel: fake.channel, - forwardRequest: async () => ({ status: 200 }), - }); - broker.start(); + const broker = createFakeBridgeTransport(); // The one atomic step marks the orderly completion and reads the success. expect(broker.settleRunDisposition()).toEqual({ failed: false, lossReason: null }); // A later teardown loss orders after the mark, so it stays a normal teardown. - fake.emitExit({ exitCode: 0 }); + broker.emitExit(); await flushMacrotasks(); expect(broker.runDisposition).toEqual({ failed: false, lossReason: null }); }); it("reports the failure and does not mark for a latched loss", async () => { - const fake = createFakeDuplexChannel(); - const broker = await createDuplexBridgeBroker({ - channel: fake.channel, - forwardRequest: async () => ({ status: 200 }), - }); - broker.start(); + const broker = createFakeBridgeTransport(); // A loss ordered before any orderly completion latches the failure. - fake.emitExit({ exitCode: 1 }); + broker.emitExit(); await flushMacrotasks(); // The atomic step reads the failure and no-ops the mark, so a later // completion cannot clear the latch. @@ -7290,20 +6292,10 @@ describe("CLI-lane run-disposition seam", () => { } as AdapterSandboxExecutionTarget; } - async function startBroker(fake: ReturnType) { - const broker = await createDuplexBridgeBroker({ - channel: fake.channel, - forwardRequest: async () => ({ status: 200 }), - }); - broker.start(); - return broker; - } - - it("fails a clean CLI completion closed when the duplex channel was lost mid-turn", async () => { - const fake = createFakeDuplexChannel(); - const broker = await startBroker(fake); + it("fails a clean CLI completion closed when the bridge channel was lost mid-turn", async () => { + const broker = createFakeBridgeTransport(); // The control channel dies mid-turn, before the CLI process exits. - fake.emitExit({ exitCode: 1 }); + broker.emitExit(); await flushMacrotasks(); const runner = mockRunner({ ...CLEAN_RESULT }); @@ -7324,8 +6316,7 @@ describe("CLI-lane run-disposition seam", () => { }); it("keeps a clean CLI completion a success when the channel stays healthy, and a teardown loss stays benign", async () => { - const fake = createFakeDuplexChannel(); - const broker = await startBroker(fake); + const broker = createFakeBridgeTransport(); const runner = mockRunner({ ...CLEAN_RESULT }); const result = await runAdapterExecutionTargetProcess("run-cli-ok", sandboxTarget(runner), "agent-cli", [], { @@ -7342,17 +6333,16 @@ describe("CLI-lane run-disposition seam", () => { // The seam's atomic settle marked the orderly completion at agent // completion. A teardown loss ordered after it is a normal teardown, so the // run stays a success without any manual mark here. - fake.emitExit({ exitCode: 0 }); + broker.emitExit(); await flushMacrotasks(); expect(broker.runDisposition.failed).toBe(false); }); it("keeps a clean CLI completion a success when the gateway exits during the run-log tail finish", async () => { - const fake = createFakeDuplexChannel(); - const broker = await startBroker(fake); + const broker = createFakeBridgeTransport(); // A run-log tail whose finish emits a gateway exit. This reproduces the - // race where the duplex gateway dies after the clean process completion but + // race where the bridge gateway dies after the clean process completion but // before the host reads the disposition. The seam settles the disposition // synchronously before this finish await, so the mark orders first and the // teardown exit stays benign. @@ -7361,7 +6351,7 @@ describe("CLI-lane run-disposition seam", () => { wrapCommand: (command, args) => ({ command, args }), start: () => {}, finish: async () => { - fake.emitExit({ exitCode: 1 }); + broker.emitExit(); await flushMacrotasks(); }, abort: async () => {}, @@ -7388,10 +6378,9 @@ describe("CLI-lane run-disposition seam", () => { }); it("cannot clear the loss latch with a later completion", async () => { - const fake = createFakeDuplexChannel(); - const broker = await startBroker(fake); + const broker = createFakeBridgeTransport(); // The loss latches before the CLI process exits. - fake.emitExit({ exitCode: 1 }); + broker.emitExit(); await flushMacrotasks(); // A later orderly completion cannot clear the latch. broker.markOrderlyCompletion(); @@ -7411,10 +6400,9 @@ describe("CLI-lane run-disposition seam", () => { }); it("leaves an already-failed CLI result unchanged and never settles the disposition", async () => { - const fake = createFakeDuplexChannel(); - const broker = await startBroker(fake); + const broker = createFakeBridgeTransport(); // The control channel is lost, but the process itself also exited non-zero. - fake.emitExit({ exitCode: 1 }); + broker.emitExit(); await flushMacrotasks(); let settleCalls = 0; diff --git a/packages/adapter-utils/src/execution-target.ts b/packages/adapter-utils/src/execution-target.ts index e2238f142a..28423e1a4e 100644 --- a/packages/adapter-utils/src/execution-target.ts +++ b/packages/adapter-utils/src/execution-target.ts @@ -51,7 +51,7 @@ import { DUPLEX_CHANNEL_LOST_ERROR_CODE, isSafeBridgeMethod, type DuplexBrokerRunDisposition, -} from "./duplex-bridge-broker.js"; +} from "./bridge-transport-contract.js"; import { decodeDuplexLine, DEFAULT_MAX_DUPLEX_FRAME_BYTES } from "./duplex-frame-codec.js"; import type { ReassembledBody } from "./duplex-body-spool.js"; import { diff --git a/packages/adapter-utils/src/sandbox-callback-bridge.test.ts b/packages/adapter-utils/src/sandbox-callback-bridge.test.ts index ea69788c26..07e24ec155 100644 --- a/packages/adapter-utils/src/sandbox-callback-bridge.test.ts +++ b/packages/adapter-utils/src/sandbox-callback-bridge.test.ts @@ -2941,6 +2941,71 @@ describe("sandbox callback bridge", () => { expect(stderr).toContain("EADDRINUSE"); }, 15_000); + it("exits nonzero for the retired duplex_v1 mode instead of starting the queue gateway", async () => { + // The closed mode allowlist rejects `duplex_v1` before the queue-directory + // check, so a stale `duplex_v1` launch environment fails startup instead + // of silently falling through to the queue gateway. + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-mode-duplex-")); + cleanupDirs.push(rootDir); + const entrypoint = path.join(rootDir, "paperclip-bridge-server.mjs"); + await writeFile(entrypoint, getSandboxCallbackBridgeServerSource(), "utf8"); + const queueDir = path.join(rootDir, "queue"); + await mkdir(queueDir, { recursive: true }); + + const child = spawn(process.execPath, [entrypoint], { + env: { + ...process.env, + PAPERCLIP_API_BRIDGE_MODE: "duplex_v1", + PAPERCLIP_BRIDGE_QUEUE_DIR: queueDir, + PAPERCLIP_BRIDGE_TOKEN: "test-token", + PAPERCLIP_BRIDGE_PORT: "0", + }, + stdio: ["ignore", "ignore", "pipe"], + }); + let stderr = ""; + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + const exitCode = await new Promise((resolve) => { + child.on("close", resolve); + }); + + expect(exitCode).not.toBe(0); + expect(stderr).toContain("Unsupported PAPERCLIP_API_BRIDGE_MODE: duplex_v1"); + }, 15_000); + + it("exits nonzero for an unknown bridge mode instead of starting the queue gateway", async () => { + // The closed mode allowlist rejects every value it does not name, not + // only the retired duplex transport. + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-mode-unknown-")); + cleanupDirs.push(rootDir); + const entrypoint = path.join(rootDir, "paperclip-bridge-server.mjs"); + await writeFile(entrypoint, getSandboxCallbackBridgeServerSource(), "utf8"); + const queueDir = path.join(rootDir, "queue"); + await mkdir(queueDir, { recursive: true }); + + const child = spawn(process.execPath, [entrypoint], { + env: { + ...process.env, + PAPERCLIP_API_BRIDGE_MODE: "totally_unknown_mode", + PAPERCLIP_BRIDGE_QUEUE_DIR: queueDir, + PAPERCLIP_BRIDGE_TOKEN: "test-token", + PAPERCLIP_BRIDGE_PORT: "0", + }, + stdio: ["ignore", "ignore", "pipe"], + }); + let stderr = ""; + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + const exitCode = await new Promise((resolve) => { + child.on("close", resolve); + }); + + expect(exitCode).not.toBe(0); + expect(stderr).toContain("Unsupported PAPERCLIP_API_BRIDGE_MODE: totally_unknown_mode"); + }, 15_000); + it("test_http2_gateway_writes_no_frame_between_ready_and_the_preface", async () => { // Spawn the real generated gateway in http2_v1 mode and read its raw // stdout bytes. The only frame-codec write on this path is the READY diff --git a/packages/adapter-utils/src/sandbox-callback-bridge.ts b/packages/adapter-utils/src/sandbox-callback-bridge.ts index f66b9bdac8..cee3ce0b1d 100644 --- a/packages/adapter-utils/src/sandbox-callback-bridge.ts +++ b/packages/adapter-utils/src/sandbox-callback-bridge.ts @@ -73,35 +73,16 @@ const SANDBOX_EXEC_CHANNEL_ENV = "PAPERCLIP_SANDBOX_EXEC_CHANNEL"; const SANDBOX_EXEC_CHANNEL_BRIDGE = "bridge"; // The bridge modes the generated gateway supports. The file mode polls a -// request/response queue on disk. The retired duplex mode forwarded one -// request frame to stdout and resolved one response frame from stdin; the -// generated gateway still defines it, but no mode dispatch selects it anymore -// — the http2 mode replaced it as the active non-file transport. The http2 -// mode runs one Node HTTP/2 client session directly on stdin/stdout, after it -// sends the one READY line the host readiness gate expects. The generated -// `.mjs` selects the mode from `PAPERCLIP_API_BRIDGE_MODE`. +// request/response queue on disk. The http2 mode runs one Node HTTP/2 client +// session directly on stdin/stdout, after it sends the one READY line the +// host readiness gate expects. The generated `.mjs` selects the mode from +// `PAPERCLIP_API_BRIDGE_MODE`. The generated gateway rejects every other +// value with a fixed startup error, including the retired duplex transport. // HTTP/2 is the preferred transport. `queue_v1` is the soft-deprecated fallback. const SANDBOX_CALLBACK_BRIDGE_FILE_MODE = "queue_v1"; -export const SANDBOX_CALLBACK_BRIDGE_DUPLEX_MODE = "duplex_v1"; -/** The active non-file transport mode. It replaced {@link SANDBOX_CALLBACK_BRIDGE_DUPLEX_MODE} - * in the mode-selection path. */ +/** The active non-file transport mode. */ export const SANDBOX_CALLBACK_BRIDGE_HTTP2_MODE = "http2_v1"; -// The duplex gateway HTTP wait budget default. The gateway waits this long for a -// response frame before it answers the local caller with a 502 timeout. The -// value stays configurable through `PAPERCLIP_BRIDGE_RESPONSE_TIMEOUT_MS`, the -// same environment key the file mode reads. -const DEFAULT_DUPLEX_GATEWAY_WAIT_BUDGET_MS = 35_000; -// How often the duplex gateway writes a heartbeat frame to stdout. -const DEFAULT_DUPLEX_GATEWAY_HEARTBEAT_INTERVAL_MS = 5_000; -// The duplex gateway treats the channel as lost when no inbound frame arrives on -// stdin within this window. -const DEFAULT_DUPLEX_GATEWAY_HEARTBEAT_TIMEOUT_MS = 20_000; -// After a loss, the duplex gateway answers each new local request with a 503, -// then exits when this grace ends. The grace gives the local caller time to read -// the 409 for an outstanding request and a 503 for a new request. -const DEFAULT_DUPLEX_GATEWAY_LOSS_EXIT_GRACE_MS = 1_000; - /** Span name that wraps one Paperclip-API callback request — read the request, * write the response, and remove the request file. */ const CALLBACK_BRIDGE_RELAY_REQUEST_SPAN = "sandbox.callbackBridge.relayRequest"; @@ -1847,9 +1828,8 @@ export async function startSandboxCallbackBridgeServer(input: { // // This gateway turns each local loopback request into one HTTP/2 stream to // the host server in `http2-bridge-server.ts`. It sits beside the file-mode -// and duplex-mode gateways above; it changes neither of them, and no mode -// dispatch selects it yet — a later phase wires it into the generated -// in-sandbox entrypoint and the transport-selection path. +// gateway above; it changes neither of them. The generated in-sandbox +// entrypoint and the transport-selection path already select it. // --------------------------------------------------------------------------- /** @@ -1998,7 +1978,7 @@ function forwardOneHttp2Request( * local request the caller hands it (already checked against the bridge * token — see {@link SandboxHttp2BridgeGatewayRequest.receivedToken}) as one * HTTP/2 stream. It keeps the header allowlist on the sandbox side, exactly - * as the file-mode and duplex-mode gateways do. + * as the file-mode gateway does. */ export function createSandboxHttp2BridgeGateway( options: CreateSandboxHttp2BridgeGatewayOptions, @@ -2306,36 +2286,18 @@ const bridgeToken = process.env.PAPERCLIP_BRIDGE_TOKEN; const host = process.env.PAPERCLIP_BRIDGE_HOST || "127.0.0.1"; const port = Number(process.env.PAPERCLIP_BRIDGE_PORT || "0"); // The host assigns the loopback port and passes it through the launch -// environment. The duplex gateway binds exactly this port; it never selects a +// environment. The gateway binds exactly this port; it never selects a // different one. The host also passes one random per-open nonce here. The // gateway echoes it in the READY frame so the host correlates READY with this // channel open. The nonce is a liveness signal, not authentication. const bridgeNonce = process.env.PAPERCLIP_BRIDGE_NONCE || ""; const pollIntervalMs = Number(process.env.PAPERCLIP_BRIDGE_POLL_INTERVAL_MS || "100"); const responseTimeoutMs = Number( - process.env.PAPERCLIP_BRIDGE_RESPONSE_TIMEOUT_MS || - (bridgeMode === "${SANDBOX_CALLBACK_BRIDGE_DUPLEX_MODE}" - ? "${DEFAULT_DUPLEX_GATEWAY_WAIT_BUDGET_MS}" - : "${DEFAULT_BRIDGE_RESPONSE_TIMEOUT_MS}"), + process.env.PAPERCLIP_BRIDGE_RESPONSE_TIMEOUT_MS || "${DEFAULT_BRIDGE_RESPONSE_TIMEOUT_MS}", ); const maxQueueDepth = Number(process.env.PAPERCLIP_BRIDGE_MAX_QUEUE_DEPTH || "${DEFAULT_BRIDGE_MAX_QUEUE_DEPTH}"); const maxBodyBytes = Number(process.env.PAPERCLIP_BRIDGE_MAX_BODY_BYTES || "${DEFAULT_BRIDGE_MAX_BODY_BYTES}"); -// The host passes the separate sandbox-process raw-decoder cap here. The in-sandbox -// decoder enforces it locally under the "sandbox_process" scope; it never shares the -// host aggregate byte ledger. -const maxDuplexDecoderBytes = Number( - process.env.PAPERCLIP_BRIDGE_MAX_DUPLEX_DECODER_BYTES || "${DEFAULT_BRIDGE_MAX_DUPLEX_DECODER_BYTES}", -); -const heartbeatIntervalMs = Number( - process.env.PAPERCLIP_BRIDGE_HEARTBEAT_INTERVAL_MS || "${DEFAULT_DUPLEX_GATEWAY_HEARTBEAT_INTERVAL_MS}", -); -const heartbeatTimeoutMs = Number( - process.env.PAPERCLIP_BRIDGE_HEARTBEAT_TIMEOUT_MS || "${DEFAULT_DUPLEX_GATEWAY_HEARTBEAT_TIMEOUT_MS}", -); -const lossExitGraceMs = Number( - process.env.PAPERCLIP_BRIDGE_LOSS_EXIT_GRACE_MS || "${DEFAULT_DUPLEX_GATEWAY_LOSS_EXIT_GRACE_MS}", -); -// The header allowlist. Both the file gateway and the duplex gateway strip an +// The header allowlist. Both the file gateway and the http2 gateway strip an // inbound request to these headers before they forward it. One copy serves both // modes. The route allowlist stays on the host: both modes forward a request to // the host, and the host enforces the same route allowlist for each. @@ -2344,11 +2306,19 @@ const allowedHeaders = new Set(${JSON.stringify([...DEFAULT_SANDBOX_CALLBACK_BRI if (!bridgeToken) { throw new Error("PAPERCLIP_BRIDGE_TOKEN is required."); } +// Closed allowlist for the bridge mode. The generated gateway supports exactly +// two transports: http2 and the file-mode queue. Every other value, including +// the retired duplex transport, fails startup at once instead of falling +// through to a mode that never ran. This check runs before the queue-directory +// check below, so an unsupported mode never reaches a state where a missing +// queue directory masks the real problem. if ( - bridgeMode !== "${SANDBOX_CALLBACK_BRIDGE_DUPLEX_MODE}" && bridgeMode !== "${SANDBOX_CALLBACK_BRIDGE_HTTP2_MODE}" && - !queueDir + bridgeMode !== "${SANDBOX_CALLBACK_BRIDGE_FILE_MODE}" ) { + throw new Error("Unsupported PAPERCLIP_API_BRIDGE_MODE: " + bridgeMode); +} +if (bridgeMode !== "${SANDBOX_CALLBACK_BRIDGE_HTTP2_MODE}" && !queueDir) { throw new Error("PAPERCLIP_BRIDGE_QUEUE_DIR and PAPERCLIP_BRIDGE_TOKEN are required."); } @@ -2598,350 +2568,6 @@ async function runFileGateway() { }); } -// Split one whole body buffer into body_chunk frames. The gateway buffers the -// whole source body once, then splits it here. Each frame carries one fixed raw -// slice as base64 text, except the final frame, which carries the remaining -// bytes. A zero-length body yields no frame. Send-side true streaming is a -// separate goal; this whole-body split is acceptable for this gateway. -function splitDuplexBodyIntoChunks(id, body) { - const frames = []; - let seq = 0; - for (let offset = 0; offset < body.length; offset += DUPLEX_BODY_CHUNK_RAW_BYTES) { - const slice = body.subarray(offset, offset + DUPLEX_BODY_CHUNK_RAW_BYTES); - frames.push({ - version: DUPLEX_FRAME_VERSION, - type: "body_chunk", - id: id, - seq: seq, - data: slice.toString("base64"), - }); - seq += 1; - } - return frames; -} - -function runDuplexGateway() { - // One outstanding local request per id. Each entry holds the HTTP resolver and - // the wait-budget timer. - const pending = new Map(); - // One in-flight response reassembly per id. The host returns a response as an - // envelope frame that carries bodyByteCount, then the body_chunk frames that - // carry the body. The gateway reassembles the response body in memory here; it - // does not spill, because a production response body stays small. - const responseAssembly = new Map(); - let unavailable = false; - let lossTriggered = false; - let lastInboundAt = Date.now(); - - function diag(message) { - // Diagnostics go to stderr only. Stdout carries frames. - process.stderr.write("[paperclip-bridge] " + message + "\\n"); - } - - function writeFrame(frame) { - process.stdout.write(encodeDuplexFrame(frame)); - } - - function stopTimers() { - clearInterval(heartbeatSendTimer); - clearInterval(heartbeatWatchTimer); - } - - // Loss behavior. On stdin end, a close frame, or a heartbeat timeout, answer - // every outstanding request with a non-retryable 409 (the request may have - // reached the host), answer every new request with a retryable 503 (the - // request never left the gateway), then exit. The gateway never replays a - // request. - function triggerLoss(reason) { - if (lossTriggered) return; - lossTriggered = true; - unavailable = true; - diag("duplex channel lost: " + reason); - stopTimers(); - for (const entry of pending.values()) { - clearTimeout(entry.timer); - entry.resolve({ - status: 409, - headers: { - "content-type": "application/json", - "x-paperclip-bridge-outcome": "indeterminate", - }, - body: JSON.stringify({ error: "outcome_indeterminate" }), - }); - } - pending.clear(); - responseAssembly.clear(); - const exitTimer = setTimeout(() => process.exit(0), lossExitGraceMs); - if (typeof exitTimer.unref === "function") exitTimer.unref(); - } - - // Fail one outstanding request with a bounded local 502. The gateway calls it - // when a response reassembly breaks: a reordered seq, a base64 that is not - // canonical, or a total that overruns the declared body size. The host is the - // response peer, so this is a defensive local error, not a channel loss. - function failRequest(id, message) { - responseAssembly.delete(id); - const entry = pending.get(id); - if (!entry) return; - pending.delete(id); - clearTimeout(entry.timer); - entry.resolve({ - status: 502, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ error: message }), - }); - } - - function handleInboundFrame(frame) { - if (frame.type === "response") { - const entry = pending.get(frame.id); - if (!entry) return; - // Map an indeterminate outcome to a non-retryable 409, the same contract - // the file gateway applies through the outcome header. - const statusCode = - frame.outcome === "indeterminate" ? 409 : typeof frame.status === "number" ? frame.status : 200; - const headers = frame.headers || {}; - if (frame.bodyByteCount === 0) { - // A zero-length body accepts no body_chunk, so the response completes now. - pending.delete(frame.id); - responseAssembly.delete(frame.id); - clearTimeout(entry.timer); - entry.resolve({ status: statusCode, headers: headers, body: "" }); - return; - } - // The body rides body_chunk frames that share this id. Record the envelope - // and wait for the chunks. - responseAssembly.set(frame.id, { - status: statusCode, - headers: headers, - bodyByteCount: frame.bodyByteCount, - received: 0, - nextSeq: 0, - chunks: [], - }); - return; - } - if (frame.type === "body_chunk") { - const asm = responseAssembly.get(frame.id); - if (!asm) return; - if (frame.seq !== asm.nextSeq) { - failRequest(frame.id, "duplex response body_chunk seq is out of order"); - return; - } - const decoded = Buffer.from(frame.data, "base64"); - if (decoded.toString("base64") !== frame.data) { - failRequest(frame.id, "duplex response body_chunk is not canonical base64"); - return; - } - if (decoded.length === 0) { - failRequest(frame.id, "duplex response body_chunk is empty"); - return; - } - if (asm.received + decoded.length > asm.bodyByteCount) { - failRequest(frame.id, "duplex response body overruns the declared size"); - return; - } - const isFinal = asm.received + decoded.length === asm.bodyByteCount; - if ( - decoded.length > DUPLEX_BODY_CHUNK_RAW_BYTES || - (!isFinal && decoded.length !== DUPLEX_BODY_CHUNK_RAW_BYTES) - ) { - failRequest(frame.id, "duplex response body_chunk has the wrong size"); - return; - } - asm.nextSeq += 1; - asm.received += decoded.length; - asm.chunks.push(decoded); - if (asm.received === asm.bodyByteCount) { - const entry = pending.get(frame.id); - responseAssembly.delete(frame.id); - if (!entry) return; - pending.delete(frame.id); - clearTimeout(entry.timer); - entry.resolve({ - status: asm.status, - headers: asm.headers, - body: Buffer.concat(asm.chunks).toString("utf8"), - }); - } - return; - } - if (frame.type === "close") { - triggerLoss("host sent a close frame"); - return; - } - // A heartbeat, ready, error, or request frame proves liveness but needs no - // local action here. - } - - const decoder = new DuplexFrameDecoder({ maxAggregateBytes: maxDuplexDecoderBytes }); - process.stdin.on("data", (chunk) => { - lastInboundAt = Date.now(); - for (const result of decoder.push(chunk)) { - if (!result.ok) { - diag("dropped an inbound frame: " + result.error.code); - continue; - } - handleInboundFrame(result.frame); - } - }); - process.stdin.on("end", () => triggerLoss("stdin reached end of file")); - process.stdin.on("error", (error) => - triggerLoss("stdin error: " + (error && error.message ? error.message : String(error))), - ); - - const heartbeatSendTimer = setInterval(() => { - writeFrame({ version: DUPLEX_FRAME_VERSION, type: "heartbeat" }); - }, heartbeatIntervalMs); - const heartbeatWatchTimer = setInterval(() => { - if (Date.now() - lastInboundAt > heartbeatTimeoutMs) { - triggerLoss("no inbound frame within the heartbeat timeout"); - } - }, Math.max(200, Math.floor(heartbeatTimeoutMs / 4))); - - const server = createServer(async (req, res) => { - try { - const auth = req.headers.authorization || ""; - const receivedToken = auth.startsWith("Bearer ") ? auth.slice("Bearer ".length) : ""; - if (!tokensMatch(receivedToken)) { - writeJsonResponse(res, 401, { error: "Invalid bridge token." }); - return; - } - if (unavailable) { - writeJsonResponse(res, 503, { error: "bridge_unavailable" }); - return; - } - if (pending.size >= maxQueueDepth) { - writeJsonResponse(res, 503, { error: "Bridge request queue is full." }); - return; - } - const url = new URL(req.url || "/", "http://127.0.0.1"); - const contentType = typeof req.headers["content-type"] === "string" ? req.headers["content-type"] : ""; - if (req.method && req.method !== "GET" && req.method !== "HEAD" && !/json/i.test(contentType)) { - writeJsonResponse(res, 415, { error: "Bridge only accepts JSON request bodies." }); - return; - } - const requestId = randomUUID(); - const requestBodyBuffer = Buffer.from(await readBody(req), "utf8"); - if (unavailable) { - writeJsonResponse(res, 503, { error: "bridge_unavailable" }); - return; - } - // The request body rides body_chunk frames. The envelope carries only the - // raw byte count, so the envelope stays small and the body splits into - // fixed-size slices that each stay under the frame bound. - const requestFrame = { - version: DUPLEX_FRAME_VERSION, - type: "request", - id: requestId, - method: req.method || "GET", - path: url.pathname, - query: url.search, - headers: normalizeHeaders(req.headers), - bodyByteCount: requestBodyBuffer.length, - }; - const encodedRequest = encodeDuplexFrameChecked(requestFrame); - if (!encodedRequest.ok) { - writeJsonResponse(res, 413, { error: "request_too_large" }); - return; - } - // Pre-encode every body_chunk frame and enforce the frame size bound on - // each. A fixed raw slice never exceeds the bound, so this guard is - // defensive. On a rejection, fail this one local request with a clean 413. - // The frames never leave the gateway, so no other in-flight request is - // affected and the channel stays open. - const chunkFrames = splitDuplexBodyIntoChunks(requestId, requestBodyBuffer); - const encodedChunks = []; - let chunkTooLarge = false; - for (const chunk of chunkFrames) { - const encodedChunk = encodeDuplexFrameChecked(chunk); - if (!encodedChunk.ok) { - chunkTooLarge = true; - break; - } - encodedChunks.push(encodedChunk.line); - } - if (chunkTooLarge) { - writeJsonResponse(res, 413, { error: "request_too_large" }); - return; - } - const response = await new Promise((resolve) => { - const timer = setTimeout(() => { - responseAssembly.delete(requestId); - if (pending.delete(requestId)) { - resolve({ - status: 502, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ error: "Timed out waiting for host bridge response." }), - }); - } - }, responseTimeoutMs); - pending.set(requestId, { resolve: resolve, timer: timer }); - process.stdout.write(encodedRequest.line); - for (const line of encodedChunks) process.stdout.write(line); - }); - res.statusCode = typeof response.status === "number" ? response.status : 200; - for (const [key, value] of Object.entries(response.headers || {})) { - if (typeof value !== "string" || key.toLowerCase() === "content-length") continue; - res.setHeader(key, value); - } - res.end(typeof response.body === "string" ? response.body : ""); - } catch (error) { - writeJsonResponse(res, 502, { error: error instanceof Error ? error.message : String(error) }); - } - }); - - process.on("SIGINT", () => { - try { - server.close(); - } catch (error) { - diag("server close error: " + (error && error.message ? error.message : String(error))); - } - process.exit(0); - }); - process.on("SIGTERM", () => { - try { - server.close(); - } catch (error) { - diag("server close error: " + (error && error.message ? error.message : String(error))); - } - process.exit(0); - }); - - // Bind-or-exit. The host assigns a positive loopback port. The gateway binds - // exactly that port. On a non-positive assigned port or a bind failure it - // writes a diagnostic and exits with a nonzero code. It never selects a - // different port, so no untrusted workload can steer the endpoint. - if (!Number.isInteger(port) || port <= 0) { - diag("duplex gateway requires a positive assigned PAPERCLIP_BRIDGE_PORT; got " + String(port)); - process.exit(1); - } - server.on("error", (error) => { - diag("duplex gateway could not bind port " + String(port) + ": " + (error && error.message ? error.message : String(error))); - process.exit(1); - }); - server.listen(port, host, () => { - const address = server.address(); - if (!address || typeof address === "string") { - diag("duplex gateway did not expose a TCP address"); - process.exit(1); - return; - } - // The gateway sends READY only after the listener binds. READY is a liveness - // signal: it carries the frame version and the echoed nonce, and no address - // data. The host builds the endpoint from its own stored port. Stdout carries - // only frames. - writeFrame({ - version: DUPLEX_FRAME_VERSION, - type: "ready", - nonce: bridgeNonce, - }); - // READY is on the wire, so the host will adopt this process. From here on - // an uncaught fault must not kill the listener. - gatewayReady = true; - }); -} - // --------------------------------------------------------------------------- // http2_v1: run one Node HTTP/2 client session directly on stdin/stdout. // @@ -3160,15 +2786,14 @@ function runHttp2Gateway() { }); } +// The startup check above already rejected every value except http2 and +// queue, so this dispatch names both modes explicitly and never falls +// through to the queue gateway for an unsupported mode. if (bridgeMode === "${SANDBOX_CALLBACK_BRIDGE_HTTP2_MODE}") { runHttp2Gateway(); -} else if (bridgeMode === "${SANDBOX_CALLBACK_BRIDGE_DUPLEX_MODE}") { - // No host selection path sets this mode anymore (http2_v1 replaced it), but - // the generated gateway keeps the mode reachable: it stays defined here, - // unchanged, so nothing that still spawns the gateway directly with this - // mode name breaks. - runDuplexGateway(); -} else { +} else if (bridgeMode === "${SANDBOX_CALLBACK_BRIDGE_FILE_MODE}") { await runFileGateway(); +} else { + throw new Error("Unsupported PAPERCLIP_API_BRIDGE_MODE: " + bridgeMode); }`; }