diff --git a/packages/adapter-utils/src/acpx-engine/execute.test.ts b/packages/adapter-utils/src/acpx-engine/execute.test.ts index 217e4f582d..8e0d712b09 100644 --- a/packages/adapter-utils/src/acpx-engine/execute.test.ts +++ b/packages/adapter-utils/src/acpx-engine/execute.test.ts @@ -36,8 +36,6 @@ import { type AcpxEngineExecutorOptions, } from "./execute.js"; import { runChildProcess } from "../server-utils.js"; -import { createDuplexBridgeBroker } from "../duplex-bridge-broker.js"; -import type { CommandManagedDuplexChannel } from "../command-managed-runtime.js"; import { getActiveStepContext, runWithRuntimeParent, @@ -5734,56 +5732,55 @@ describe("ACPX engine run lifecycle corrections (F3: one teardown error policy)" }); }); -describe("ACPX engine sandbox duplex run-disposition seam (fail-closed)", () => { +describe("ACPX engine sandbox bridge run-disposition seam (fail-closed)", () => { beforeEach(() => { vi.clearAllMocks(); }); - // A minimal in-memory duplex channel. The test drives a channel exit to latch a - // real loss in a real broker, so the seam reads a real run disposition. - function createFakeDuplexChannel() { - let exitListener: ((exit: { exitCode: number | null }) => void) | null = null; - const channel: CommandManagedDuplexChannel = { - write(): void {}, - onData(): void {}, - onExit(listener: (exit: { exitCode: number | null }) => void): void { - exitListener = listener; - }, - stop(): void {}, - close(): Promise { - return Promise.resolve(); - }, - }; - return { - channel, - emitExit: (exit: { exitCode: number | null }) => exitListener?.(exit), - }; - } - - // Wrap a started real broker in a paperclip bridge handle. The handle exposes - // the same run-disposition surface the sandbox bridge exposes, so the seam runs - // against the real latch and the real orderly-completion mark. - async function bridgeOverBroker(fake: ReturnType) { - const broker = await createDuplexBridgeBroker({ - channel: fake.channel, - forwardRequest: async () => ({ status: 200 }), + /** + * A minimal run-disposition latch. It reproduces the same ordering rule the + * bridge transport applies: the first ordered loss or orderly completion + * latches the terminal disposition, and a later call never overrides it. + * The test drives the latch directly, with no channel and no live process. + */ + function createFakeBridgeHandle() { + let lossOrdered = false; + let lossReason: string | null = null; + let completionOrdered = false; + const readDisposition = () => ({ failed: lossOrdered, lossReason }); + const markOrderlyCompletion = vi.fn(() => { + if (completionOrdered || lossOrdered) return; + completionOrdered = true; + }); + const settleRunDisposition = vi.fn(() => { + markOrderlyCompletion(); + return readDisposition(); }); - broker.start(); - const markOrderlyCompletion = vi.fn(() => broker.markOrderlyCompletion()); - const settleRunDisposition = vi.fn(() => broker.settleRunDisposition()); const stop = vi.fn(async () => {}); const handle = { env: { PAPERCLIP_API_URL: "http://127.0.0.1:1", PAPERCLIP_API_KEY: "bridge-token", - PAPERCLIP_API_BRIDGE_MODE: "duplex_v1", + PAPERCLIP_API_BRIDGE_MODE: "http2_v1", }, - readRunDisposition: () => broker.runDisposition, + readRunDisposition: () => readDisposition(), settleRunDisposition, markOrderlyCompletion, stop, }; - return { broker, handle, markOrderlyCompletion, settleRunDisposition }; + return { + handle, + markOrderlyCompletion, + settleRunDisposition, + readDisposition, + // Record the first ordered loss. A loss ordered after a completion, or a + // second loss, is a no-op — the same rule the real transport applies. + emitLoss: (reason: string) => { + if (lossOrdered || completionOrdered) return; + lossOrdered = true; + lossReason = reason; + }, + }; } // A runtime whose one turn completes cleanly. The `beforeResult` hook runs at @@ -5888,14 +5885,13 @@ describe("ACPX engine sandbox duplex run-disposition seam (fail-closed)", () => } as never); } - it("fails a completed run when the duplex channel was lost before the completion", async () => { + it("fails a completed run when the bridge channel was lost before the completion", async () => { const sandbox = await setupRemoteSandbox(); - const fake = createFakeDuplexChannel(); - const { broker, handle, settleRunDisposition } = await bridgeOverBroker(fake); + const fake = createFakeBridgeHandle(); // Latch the loss before the ACP terminal resolves. - const runtime = runtimeWithControlledResult(() => fake.emitExit({ exitCode: 1 })); + const runtime = runtimeWithControlledResult(() => fake.emitLoss("provider_exit")); - const result = await runRemote(handle, runtime, sandbox); + const result = await runRemote(fake.handle, runtime, sandbox); // The lost channel overrides the nominally completed terminal to a failure. expect(result.exitCode).not.toBe(0); @@ -5905,65 +5901,62 @@ describe("ACPX engine sandbox duplex run-disposition seam (fail-closed)", () => expect(result.resultJson).toMatchObject({ status: "failed" }); // The seam read the disposition through the atomic settle step, and the // latched loss kept the failure, so no orderly completion ordered. - expect(settleRunDisposition).toHaveBeenCalledTimes(1); - expect(broker.runDisposition.failed).toBe(true); + expect(fake.settleRunDisposition).toHaveBeenCalledTimes(1); + expect(fake.readDisposition().failed).toBe(true); }); it("keeps a completed run a success when the channel stays live, and a later teardown loss is benign", async () => { const sandbox = await setupRemoteSandbox(); - const fake = createFakeDuplexChannel(); - const { broker, handle, settleRunDisposition } = await bridgeOverBroker(fake); + const fake = createFakeBridgeHandle(); // No loss before the completion. const runtime = runtimeWithControlledResult(); - const result = await runRemote(handle, runtime, sandbox); + const result = await runRemote(fake.handle, runtime, sandbox); expect(result.exitCode).toBe(0); expect(result.errorCode ?? null).toBeNull(); // The atomic settle step marked the orderly completion for the // success-eligible terminal. - expect(settleRunDisposition).toHaveBeenCalledTimes(1); + expect(fake.settleRunDisposition).toHaveBeenCalledTimes(1); // A teardown loss ordered after the orderly completion is a normal teardown, // so the run disposition stays a success. - fake.emitExit({ exitCode: 0 }); - expect(broker.runDisposition.failed).toBe(false); + fake.emitLoss("provider_exit"); + expect(fake.readDisposition().failed).toBe(false); }); it("does not let a later completion or activity clear the loss latch", async () => { const sandbox = await setupRemoteSandbox(); - const fake = createFakeDuplexChannel(); - const { broker, handle } = await bridgeOverBroker(fake); + const fake = createFakeBridgeHandle(); // Latch the loss before the ACP terminal resolves. - const runtime = runtimeWithControlledResult(() => fake.emitExit({ exitCode: 1 })); + const runtime = runtimeWithControlledResult(() => fake.emitLoss("provider_exit")); - const result = await runRemote(handle, runtime, sandbox); + const result = await runRemote(fake.handle, runtime, sandbox); expect(result.exitCode).not.toBe(0); expect(result.errorCode).toBe("duplex_channel_lost"); // A later orderly-completion mark and further channel activity cannot clear // the latched loss. - broker.markOrderlyCompletion(); - fake.emitExit({ exitCode: 0 }); - expect(broker.runDisposition.failed).toBe(true); - expect(broker.runDisposition.lossReason).toBe("provider_exit"); + fake.markOrderlyCompletion(); + fake.emitLoss("provider_exit"); + expect(fake.readDisposition().failed).toBe(true); + expect(fake.readDisposition().lossReason).toBe("provider_exit"); }); it("marks an orderly completion on a failed terminal so the teardown loss emits no false loss", async () => { const sandbox = await setupRemoteSandbox(); - const fake = createFakeDuplexChannel(); - const { broker, handle, markOrderlyCompletion } = await bridgeOverBroker(fake); + const fake = createFakeBridgeHandle(); // The turn fails, and no channel loss ordered before the finalization. const runtime = runtimeWithFailedResult(); - const result = await runRemote(handle, runtime, sandbox); + const result = await runRemote(fake.handle, runtime, sandbox); - // The failed terminal stays a failure, but not a duplex loss. + // The failed terminal stays a failure, but not a bridge-channel loss. expect(result.exitCode).not.toBe(0); expect(result.errorCode).not.toBe("duplex_channel_lost"); // The non-success-eligible terminal marked the orderly completion, so the - // teardown channel_exit orders after the mark and does not latch a loss. - expect(markOrderlyCompletion).toHaveBeenCalledTimes(1); - fake.emitExit({ exitCode: 0 }); - expect(broker.runDisposition.failed).toBe(false); + // teardown loss orders after the mark and does not latch a loss. + expect(fake.markOrderlyCompletion).toHaveBeenCalledTimes(1); + fake.emitLoss("provider_exit"); + expect(fake.readDisposition().failed).toBe(false); }); }); diff --git a/packages/adapter-utils/src/acpx-engine/execute.ts b/packages/adapter-utils/src/acpx-engine/execute.ts index 925f01ae36..9210025c4d 100644 --- a/packages/adapter-utils/src/acpx-engine/execute.ts +++ b/packages/adapter-utils/src/acpx-engine/execute.ts @@ -34,7 +34,7 @@ import { type SandboxAdditionalSource, } from "@paperclipai/adapter-utils/execution-target"; import type { DuplexLossReason } from "../duplex-observability.js"; -import { DUPLEX_CHANNEL_LOST_ERROR_CODE } from "../duplex-bridge-broker.js"; +import { DUPLEX_CHANNEL_LOST_ERROR_CODE } from "../bridge-transport-contract.js"; import { DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE, applyPaperclipWorkspaceEnv, diff --git a/packages/adapter-utils/src/bridge-transport-contract.ts b/packages/adapter-utils/src/bridge-transport-contract.ts new file mode 100644 index 0000000000..5209ccd318 --- /dev/null +++ b/packages/adapter-utils/src/bridge-transport-contract.ts @@ -0,0 +1,61 @@ +/** + * Shared contract for the sandbox callback bridge transports. + * + * The retired duplex_v1 broker first defined these symbols. The host + * broker is gone, but the http2_v1 transport and the ACPX engine + * run-disposition seam still use them. This leaf module holds the + * survivors, so a caller of the run-disposition seam does not import the + * whole HTTP/2 bridge server module graph to reach one error code. + */ + +import type { DuplexLossReason } from "./duplex-observability.js"; + +/** + * The typed error code the host reports when the bridge control channel + * died before an orderly completion. Both the ACP lane and the CLI lane + * report this one code, so the run disposition is identical across the two + * lanes. + */ +export const DUPLEX_CHANNEL_LOST_ERROR_CODE = "duplex_channel_lost"; + +/** + * The terminal run disposition a bridge transport computes from its ordered + * lifecycle. A `failed` disposition means a terminal loss ordered before an + * orderly completion, so the run must not report success. The typed loss + * reason names the cause; it is `null` for a success. + */ +export interface DuplexBrokerRunDisposition { + /** True when a terminal loss ordered before an orderly completion. */ + failed: boolean; + /** The typed, closed loss reason on a failure; `null` on a success. */ + lossReason: DuplexLossReason | null; +} + +/** The nested timeout budgets. Each inner budget is smaller than its outer budget. */ +export interface DuplexBrokerBudgets { + /** The deadline for one forward call, in milliseconds. */ + forwardTimeoutMs: number; + /** The deadline for the broker to send one response frame, in milliseconds. */ + responseBudgetMs: number; + /** The deadline the in-sandbox gateway waits for the response frame, in milliseconds. */ + gatewayWaitMs: number; +} + +/** The default nested budgets: forward 30 s, response 32 s, gateway wait 35 s. */ +export const DEFAULT_DUPLEX_BROKER_BUDGETS: DuplexBrokerBudgets = { + forwardTimeoutMs: 30_000, + responseBudgetMs: 32_000, + gatewayWaitMs: 35_000, +}; + +/** + * The safe HTTP methods. RFC 7231 section 4.2.1 defines this set. A safe method + * does not change host state, so the host applies no mutation for it. A caller + * can retry a safe method after a forward failure without a double-apply risk. + */ +const SAFE_BRIDGE_METHODS = new Set(["GET", "HEAD", "OPTIONS", "TRACE"]); + +/** Report whether the method is safe, so a forward failure stays retryable. */ +export function isSafeBridgeMethod(method: string): boolean { + return SAFE_BRIDGE_METHODS.has(method.trim().toUpperCase()); +} diff --git a/packages/adapter-utils/src/duplex-bridge-broker-byte-ledger.test.ts b/packages/adapter-utils/src/duplex-bridge-broker-byte-ledger.test.ts deleted file mode 100644 index c82446343c..0000000000 --- a/packages/adapter-utils/src/duplex-bridge-broker-byte-ledger.test.ts +++ /dev/null @@ -1,384 +0,0 @@ -import { afterEach, describe, expect, it } from "vitest"; - -import { - createDuplexBridgeBroker, - type DuplexBridgeBroker, - type DuplexBrokerForwardResult, -} from "./duplex-bridge-broker.js"; -import { - DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, - DuplexAggregateByteLedger, -} from "./duplex-aggregate-byte-ledger.js"; -import { - DUPLEX_FRAME_VERSION, - DuplexFrameDecoder, - encodeDuplexFrame, - type DuplexRequestFrame, - type DuplexResponseFrame, -} from "./duplex-frame-codec.js"; -import { splitBodyIntoChunkFrames } from "./duplex-body-spool.js"; -import type { CommandManagedDuplexChannel } from "./command-managed-runtime.js"; - -/** - * Regression harness for the broker aggregate byte ledger charging. - * - * The harness feeds request frames straight into the broker through an in-memory - * channel, the same as a provider that controls the transport. A test controls - * when each forward settles, so it can assert the ledger charge while the forward - * is still in flight and after it settles. - */ - -/** One pending forward a test settles by hand. */ -interface PendingForward { - id: string; - resolve: (result: DuplexBrokerForwardResult) => void; - reject: (error: Error) => void; - settled: boolean; -} - -/** - * One reassembled response the broker wrote back. The broker writes a response as - * one envelope frame that carries `bodyByteCount`, then the `body_chunk` frames - * that carry the body. The harness reassembles the body and exposes it as `body`. - */ -type ReassembledResponse = DuplexResponseFrame & { body: string }; - -/** One request the harness feeds: the envelope frame plus its raw body text. */ -interface RequestInput { - frame: DuplexRequestFrame; - bodyText: string; -} - -/** The in-memory channel plus the levers a test uses to drive the broker. */ -interface FakeChannelHarness { - channel: CommandManagedDuplexChannel; - feed: (input: RequestInput) => void; - exit: () => void; - responses: ReassembledResponse[]; - forwards: PendingForward[]; - resolveForward: (id: string, body?: string) => void; -} - -/** Build one valid request: the envelope carries `bodyByteCount`, the body rides body_chunk frames. */ -function requestFrame(id: string, method = "POST"): RequestInput { - const bodyText = JSON.stringify({ id }); - return { - frame: { - version: DUPLEX_FRAME_VERSION, - type: "request", - id, - method, - path: `/api/issues/${id}`, - query: "", - headers: { "content-type": "application/json" }, - bodyByteCount: Buffer.byteLength(bodyText, "utf8"), - }, - bodyText, - }; -} - -function createFakeChannelHarness(): FakeChannelHarness { - const responses: ReassembledResponse[] = []; - const forwards: PendingForward[] = []; - const writtenDecoder = new DuplexFrameDecoder(); - const responseAssembly = new Map< - string, - { frame: DuplexResponseFrame; received: number; chunks: Buffer[] } - >(); - let dataListener: ((chunk: Uint8Array) => void) | null = null; - let exitListener: ((exit: { exitCode: number | null }) => void) | null = null; - - const channel: CommandManagedDuplexChannel = { - write: (data) => { - for (const result of writtenDecoder.push(data)) { - if (!result.ok) continue; - const frame = result.frame; - if (frame.type === "response") { - if (frame.bodyByteCount === 0) { - responses.push({ ...frame, body: "" }); - } else { - responseAssembly.set(frame.id, { frame, received: 0, chunks: [] }); - } - } else if (frame.type === "body_chunk") { - const assembly = responseAssembly.get(frame.id); - if (!assembly) continue; - const decoded = Buffer.from(frame.data, "base64"); - assembly.received += decoded.length; - assembly.chunks.push(decoded); - if (assembly.received >= assembly.frame.bodyByteCount) { - responseAssembly.delete(frame.id); - responses.push({ - ...assembly.frame, - body: Buffer.concat(assembly.chunks).toString("utf8"), - }); - } - } - } - }, - onData: (listener) => { - dataListener = listener; - }, - onExit: (listener) => { - exitListener = listener; - }, - stop: () => undefined, - close: () => Promise.resolve(), - }; - - return { - channel, - feed: ({ frame, bodyText }) => { - if (!dataListener) throw new Error("The broker did not bind the data listener."); - dataListener(Buffer.from(encodeDuplexFrame(frame), "utf8")); - if (bodyText.length > 0) { - for (const chunk of splitBodyIntoChunkFrames( - frame.id, - Buffer.from(bodyText, "utf8"), - DUPLEX_FRAME_VERSION, - )) { - dataListener(Buffer.from(encodeDuplexFrame(chunk), "utf8")); - } - } - }, - exit: () => { - if (!exitListener) throw new Error("The broker did not bind the exit listener."); - exitListener({ exitCode: 0 }); - }, - responses, - forwards, - resolveForward: (id, body = JSON.stringify({ ok: true })) => { - const forward = [...forwards].reverse().find((entry) => entry.id === id && !entry.settled); - if (!forward) throw new Error(`No unsettled forward for id ${id}.`); - forward.settled = true; - forward.resolve({ status: 200, headers: { "content-type": "application/json" }, body }); - }, - }; -} - -/** A forward handler that hands each call to the harness and never auto-resolves. */ -function controllableForward(harness: FakeChannelHarness) { - return (request: DuplexRequestFrame): Promise => - new Promise((resolve, reject) => { - harness.forwards.push({ id: request.id, resolve, reject, settled: false }); - }); -} - -/** - * Wait for the microtasks and one macrotask to settle, so the broker reassembles - * a request body, dispatches its forward, and runs each settled promise handler. - */ -async function flush(): Promise { - await new Promise((resolve) => setTimeout(resolve, 0)); -} - -describe("duplex bridge broker aggregate byte ledger", () => { - const brokers: DuplexBridgeBroker[] = []; - - afterEach(async () => { - while (brokers.length > 0) { - const broker = brokers.pop(); - if (broker) await broker.close(); - } - }); - - it("charges request-frame, request-payload, and seen-id tokens, then releases the request tokens on forward settlement", async () => { - const harness = createFakeChannelHarness(); - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 }); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - duplexAggregateByteLedger: ledger, - }); - brokers.push(broker); - broker.start(); - - harness.feed(requestFrame("req-1")); - // The dispatch retains three tokens at the request envelope: the raw frame, the - // normalized payload, and the no-replay set entry. The reservation is - // synchronous, so the charge holds before the body reassembles. - expect(ledger.liveTokenCount).toBe(3); - expect(ledger.bytesInUse).toBeGreaterThan(0); - const chargedInFlight = ledger.bytesInUse; - - // The broker reassembles the request body, then dispatches the forward. - await flush(); - harness.resolveForward("req-1"); - await flush(); - - // The forward settled. The finally owner released the request-frame and the - // request-payload tokens. The seen-id token stays charged for the channel - // lifetime, so exactly one token remains. - expect(ledger.liveTokenCount).toBe(1); - expect(ledger.bytesInUse).toBeGreaterThan(0); - expect(ledger.bytesInUse).toBeLessThan(chargedInFlight); - - // A close releases the seen-id token, so the ledger returns to zero. - await broker.close(); - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - }); - - it("keeps request tokens charged when the response timer answers before the forward settles, and releases them on settlement", async () => { - const harness = createFakeChannelHarness(); - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 }); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - // Squeeze the nested budgets so the response timer fires quickly while the - // forward stays unsettled. - budgets: { forwardTimeoutMs: 5, responseBudgetMs: 10, gatewayWaitMs: 20 }, - duplexAggregateByteLedger: ledger, - }); - brokers.push(broker); - broker.start(); - - harness.feed(requestFrame("req-1", "GET")); - expect(ledger.liveTokenCount).toBe(3); - - // Wait past the response budget so the backstop answers the gateway while the - // forward is still in flight. - await new Promise((resolve) => setTimeout(resolve, 40)); - expect(harness.responses.some((frame) => frame.id === "req-1")).toBe(true); - // The gateway got its answer, but the forward has not settled. The request - // tokens and the seen-id token stay charged: nothing released yet. - expect(ledger.liveTokenCount).toBe(3); - - // Settle the orphaned forward. Its finally owner now releases the two request - // tokens; the seen-id token still stays for the channel lifetime. - harness.resolveForward("req-1"); - await flush(); - expect(ledger.liveTokenCount).toBe(1); - - await broker.close(); - expect(ledger.bytesInUse).toBe(0); - }); - - it("refuses a one-byte-over dispatch with the fixed marker and retains nothing", async () => { - const harness = createFakeChannelHarness(); - // Measure the exact per-request charge with a throwaway broker, so the test - // can size a ceiling one byte below it and force the reservation to fail. - const probeHarness = createFakeChannelHarness(); - const measured = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 }); - const measuringBroker = await createDuplexBridgeBroker({ - channel: probeHarness.channel, - forwardRequest: controllableForward(probeHarness), - duplexAggregateByteLedger: measured, - }); - measuringBroker.start(); - probeHarness.feed(requestFrame("req-1")); - const perRequestBytes = measured.bytesInUse; - await measuringBroker.close(); - - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: perRequestBytes - 1 }); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - duplexAggregateByteLedger: ledger, - }); - brokers.push(broker); - broker.start(); - - harness.feed(requestFrame("req-1")); - await flush(); - - // The broker retained nothing: no token, no forward, no seen id. - expect(ledger.liveTokenCount).toBe(0); - expect(ledger.bytesInUse).toBe(0); - expect(harness.forwards.length).toBe(0); - // The refusal carries the fixed marker and is a bounded terminal response. - const refusal = harness.responses.find((frame) => frame.id === "req-1"); - expect(refusal).toBeTruthy(); - expect(refusal?.status).toBe(503); - expect(refusal?.body).toContain(DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED); - - // The refusal did not retain the id, so a resend after pressure eases is - // admitted: raise the ceiling and re-feed. - const roomyLedger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 }); - const roomyHarness = createFakeChannelHarness(); - const roomyBroker = await createDuplexBridgeBroker({ - channel: roomyHarness.channel, - forwardRequest: controllableForward(roomyHarness), - duplexAggregateByteLedger: roomyLedger, - }); - brokers.push(roomyBroker); - roomyBroker.start(); - roomyHarness.feed(requestFrame("req-1")); - await flush(); - expect(roomyHarness.forwards.length).toBe(1); - }); - - it("releases every retained token on a terminal channel loss", async () => { - const harness = createFakeChannelHarness(); - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 }); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - duplexAggregateByteLedger: ledger, - }); - brokers.push(broker); - broker.start(); - - harness.feed(requestFrame("req-1")); - harness.feed(requestFrame("req-2")); - // The reservation is synchronous, so both dispatches charge three tokens each - // at the request envelope, before the bodies reassemble. - expect(ledger.liveTokenCount).toBe(6); - - // Let both requests reassemble and start their forwards, so the loss below - // transfers live forwards to the orphan registry. - await flush(); - - // The channel exits mid-flight. The broker latches the loss, transfers the - // in-flight forwards to the orphan registry, and releases the seen-id tokens. - harness.exit(); - expect(broker.runDisposition.failed).toBe(true); - // The seen-id tokens released at loss; the two request tokens per forward stay - // charged until each aborted forward settles. - expect(ledger.liveTokenCount).toBe(4); - - // The aborted forwards settle. Each finally owner releases its two request - // tokens, so the ledger returns to zero. - harness.forwards[0]?.reject(new Error("aborted")); - harness.forwards[1]?.reject(new Error("aborted")); - await flush(); - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - }); - - it("does not double-release a token when a forward settles after the channel closed", async () => { - const harness = createFakeChannelHarness(); - let underflow = 0; - const ledger = new DuplexAggregateByteLedger({ - ceilingBytes: 1_000_000, - telemetry: { - setBytesInUse() {}, - recordReservationRejection() {}, - recordAccountingUnderflow() { - underflow += 1; - }, - }, - }); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - duplexAggregateByteLedger: ledger, - }); - brokers.push(broker); - broker.start(); - - harness.feed(requestFrame("req-1")); - // Let the request reassemble and start its forward before the close, so the - // close orphans a live forward that settles afterward. - await flush(); - await broker.close(); - // The close aborted the in-flight forward and orphaned it. The forward now - // settles after the close: its finally owner releases the request tokens once. - harness.forwards[0]?.reject(new Error("aborted")); - await flush(); - - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - // No accounting defect: every token released exactly one time. - expect(underflow).toBe(0); - }); -}); diff --git a/packages/adapter-utils/src/duplex-bridge-broker.test.ts b/packages/adapter-utils/src/duplex-bridge-broker.test.ts deleted file mode 100644 index cc7f5ce3de..0000000000 --- a/packages/adapter-utils/src/duplex-bridge-broker.test.ts +++ /dev/null @@ -1,1039 +0,0 @@ -import { afterEach, describe, expect, it } from "vitest"; - -import { - assertDuplexBrokerLimits, - createDuplexBridgeBroker, - type DuplexBridgeBroker, - type DuplexBrokerForwardResult, -} from "./duplex-bridge-broker.js"; -import { - DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES, - DUPLEX_FRAME_VERSION, - DuplexFrameDecoder, - encodeDuplexFrame, - type DuplexRequestFrame, - type DuplexResponseFrame, -} from "./duplex-frame-codec.js"; -import { splitBodyIntoChunkFrames } from "./duplex-body-spool.js"; -import { - createDuplexObservability, - DUPLEX_COUNTER_LOSS_TOTAL, - DUPLEX_SPAN_REQUEST, - type DuplexObservabilityCounterRecord, - type DuplexObservabilityEventRecord, - type DuplexObservabilitySpanRecord, -} from "./duplex-observability.js"; -import type { CommandManagedDuplexChannel } from "./command-managed-runtime.js"; - -/** - * Unit harness for the host broker limit gate. - * - * The harness drives frames straight into the broker through an in-memory - * channel. It does not spawn the gateway, so a test injects untrusted request - * frames directly, the same as a malicious provider that controls the transport. - * The channel records each frame the broker writes back, so a test asserts the - * exact response the broker returns for a refused request. - */ - -/** One pending forward the test controls. */ -interface PendingForward { - id: string; - resolve: (result: DuplexBrokerForwardResult) => void; - reject: (error: Error) => void; - settled: boolean; -} - -/** - * One reassembled response the broker wrote back. The broker writes a response as - * one envelope frame that carries `bodyByteCount`, then the `body_chunk` frames - * that carry the body. The harness reassembles the body and exposes it as `body`, - * so a test asserts the response the same way it did with the one-frame model. - */ -type ReassembledResponse = DuplexResponseFrame & { body: string }; - -/** One request the harness feeds: the envelope frame plus its raw body text. */ -interface RequestInput { - frame: DuplexRequestFrame; - bodyText: string; -} - -/** The in-memory channel plus the levers a test uses to drive the broker. */ -interface FakeChannelHarness { - channel: CommandManagedDuplexChannel; - /** Push one request (its envelope frame and its body_chunk frames) into the broker read path. */ - feed: (input: RequestInput) => void; - /** The reassembled responses the broker wrote back, in order. */ - responses: ReassembledResponse[]; - /** Every forward call the broker made, in order. */ - forwards: PendingForward[]; - /** Resolve the newest unsettled forward for one id with a 200 result. */ - resolveForward: (id: string, body?: string) => void; - /** Resolve every unsettled forward with a 200 result. */ - resolveAll: () => void; -} - -/** - * Build one valid request with distinctive, secret-looking fields. The envelope - * carries `bodyByteCount`, and the harness rides the body on `body_chunk` frames. - */ -function requestFrame( - id: string, - method = "POST", - bodyText: string = JSON.stringify({ secret: "secret-request-body" }), -): RequestInput { - return { - frame: { - version: DUPLEX_FRAME_VERSION, - type: "request", - id, - method, - path: `/api/issues/${id}`, - query: "?secret-query=leak", - headers: { authorization: "Bearer super-secret-provider-token" }, - bodyByteCount: Buffer.byteLength(bodyText, "utf8"), - }, - bodyText, - }; -} - -/** - * Build the in-memory channel harness. The channel keeps the broker read - * listener, so `feed` pushes the encoded envelope and its body_chunk frames into - * the broker. The channel decodes each written frame and reassembles a response - * body, so `responses` holds one reassembled response per delivered request. - */ -function createFakeChannelHarness(): FakeChannelHarness { - const responses: ReassembledResponse[] = []; - const forwards: PendingForward[] = []; - const writtenDecoder = new DuplexFrameDecoder(); - // The in-flight response reassembly, keyed by request id. - const responseAssembly = new Map< - string, - { frame: DuplexResponseFrame; received: number; chunks: Buffer[] } - >(); - let dataListener: ((chunk: Uint8Array) => void) | null = null; - - const channel: CommandManagedDuplexChannel = { - write: (data) => { - for (const result of writtenDecoder.push(data)) { - if (!result.ok) continue; - const frame = result.frame; - if (frame.type === "response") { - if (frame.bodyByteCount === 0) { - responses.push({ ...frame, body: "" }); - } else { - responseAssembly.set(frame.id, { frame, received: 0, chunks: [] }); - } - } else if (frame.type === "body_chunk") { - const assembly = responseAssembly.get(frame.id); - if (!assembly) continue; - const decoded = Buffer.from(frame.data, "base64"); - assembly.received += decoded.length; - assembly.chunks.push(decoded); - if (assembly.received >= assembly.frame.bodyByteCount) { - responseAssembly.delete(frame.id); - responses.push({ - ...assembly.frame, - body: Buffer.concat(assembly.chunks).toString("utf8"), - }); - } - } - } - }, - onData: (listener) => { - dataListener = listener; - }, - onExit: () => undefined, - stop: () => undefined, - close: () => Promise.resolve(), - }; - - return { - channel, - feed: ({ frame, bodyText }) => { - if (!dataListener) throw new Error("The broker did not bind the data listener."); - dataListener(Buffer.from(encodeDuplexFrame(frame), "utf8")); - if (bodyText.length > 0) { - const chunks = splitBodyIntoChunkFrames( - frame.id, - Buffer.from(bodyText, "utf8"), - DUPLEX_FRAME_VERSION, - ); - for (const chunk of chunks) dataListener(Buffer.from(encodeDuplexFrame(chunk), "utf8")); - } - }, - responses, - forwards, - resolveForward: (id, body = JSON.stringify({ ok: true })) => { - const forward = [...forwards].reverse().find((entry) => entry.id === id && !entry.settled); - if (!forward) throw new Error(`No unsettled forward for id ${id}.`); - forward.settled = true; - forward.resolve({ status: 200, headers: { "content-type": "application/json" }, body }); - }, - resolveAll: () => { - for (const forward of forwards) { - if (forward.settled) continue; - forward.settled = true; - forward.resolve({ - status: 200, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ ok: true }), - }); - } - }, - }; -} - -/** A forward handler that hands each call to the harness and never auto-resolves. */ -function controllableForward(harness: FakeChannelHarness) { - return (request: DuplexRequestFrame): Promise => - new Promise((resolve, reject) => { - harness.forwards.push({ id: request.id, resolve, reject, settled: false }); - }); -} - -/** Flush the microtasks and one macrotask, so the broker dispatches its reassembled forwards. */ -async function flush(): Promise { - await new Promise((resolve) => setTimeout(resolve, 0)); -} - -/** The telemetry sink capture. It proves the broker records nothing for a refusal. */ -interface TelemetryCapture { - spans: DuplexObservabilitySpanRecord[]; - counters: DuplexObservabilityCounterRecord[]; - events: DuplexObservabilityEventRecord[]; -} - -/** Build the real telemetry facade over a capturing recorder. */ -function createTelemetryCapture(): { telemetry: ReturnType; capture: TelemetryCapture } { - const capture: TelemetryCapture = { spans: [], counters: [], events: [] }; - const telemetry = createDuplexObservability({ - providerKey: "daytona", - recorder: { - recordSpan: (record) => capture.spans.push(record), - incrementCounter: (record) => capture.counters.push(record), - emitEvent: (record) => capture.events.push(record), - }, - }); - return { telemetry, capture }; -} - -describe("duplex bridge broker request limits", () => { - const brokers: DuplexBridgeBroker[] = []; - - afterEach(async () => { - while (brokers.length > 0) { - const broker = brokers.pop(); - if (broker) await broker.close(); - } - }); - - it("rejects a non-positive or non-integer limit at construction", () => { - expect(() => assertDuplexBrokerLimits({ maxInFlightRequests: 0, maxLifetimeRequests: 10 })).toThrow(); - expect(() => assertDuplexBrokerLimits({ maxInFlightRequests: 5, maxLifetimeRequests: 0 })).toThrow(); - expect(() => assertDuplexBrokerLimits({ maxInFlightRequests: 1.5, maxLifetimeRequests: 10 })).toThrow(); - expect(() => - assertDuplexBrokerLimits({ maxInFlightRequests: Number.POSITIVE_INFINITY, maxLifetimeRequests: 10 }), - ).toThrow(); - expect(() => assertDuplexBrokerLimits({ maxInFlightRequests: 8, maxLifetimeRequests: 16 })).not.toThrow(); - }); - - it("bounds the in-flight forwards and refuses the excess with a retryable terminal response", async () => { - const harness = createFakeChannelHarness(); - const { telemetry, capture } = createTelemetryCapture(); - const maxInFlightRequests = 4; - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - telemetry, - maxInFlightRequests, - }); - brokers.push(broker); - broker.start(); - - // Inject more than the limit of unique valid frames. Every forward hangs, so - // the broker holds the maximum number of in-flight forwards. The broker gates - // the in-flight limit at the request envelope, so it refuses the excess at - // once; it dispatches each accepted forward after it reassembles the body. - const total = 10; - const ids = Array.from({ length: total }, (_unused, index) => `flight-${index}`); - for (const id of ids) harness.feed(requestFrame(id)); - await flush(); - - // The broker forwarded only up to the limit. It refused the rest. - expect(harness.forwards).toHaveLength(maxInFlightRequests); - expect(harness.responses).toHaveLength(total - maxInFlightRequests); - for (const response of harness.responses) { - expect(response.status).toBe(503); - expect(response.outcome).toBe("unavailable"); - expect(response.headers["x-paperclip-bridge-outcome"]).toBe("unavailable"); - expect(JSON.parse(response.body)).toEqual({ - error: "Duplex broker capacity limit reached.", - outcome: "unavailable", - retryable: true, - }); - } - // The refusal leaks no route, query, body, or token. - const refusalText = harness.responses.map((response) => encodeDuplexFrame(response)).join(""); - expect(refusalText).not.toContain("secret"); - expect(refusalText).not.toContain("super-secret-provider-token"); - expect(refusalText).not.toContain("/api/issues/"); - - // The broker recorded no request span and no counter for a refusal. Only the - // delivered requests below produce a span. - expect(capture.spans).toHaveLength(0); - expect(capture.counters).toHaveLength(0); - - // Resolve the in-flight forwards. The broker delivers one response for each - // and drains the pending bookkeeping. - harness.resolveAll(); - }); - - it("frees capacity after a delivered request and forwards a resent refused id (no-replay preserved)", async () => { - const harness = createFakeChannelHarness(); - const maxInFlightRequests = 2; - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - maxInFlightRequests, - }); - brokers.push(broker); - broker.start(); - - // Saturate the in-flight limit with two hung forwards. The broker reserves the - // in-flight slot at the request envelope, then dispatches the forward after it - // reassembles the body. - harness.feed(requestFrame("keep-0")); - harness.feed(requestFrame("keep-1")); - await flush(); - expect(harness.forwards).toHaveLength(2); - - // A third unique id is refused, retryable. The broker did not forward it and - // did not retain its id. - harness.feed(requestFrame("resend-me")); - await flush(); - expect(harness.forwards).toHaveLength(2); - expect(harness.responses).toHaveLength(1); - expect(JSON.parse(harness.responses[0].body).retryable).toBe(true); - - // Free one in-flight slot. The delivered request drains the pending record. - harness.resolveForward("keep-0"); - await flush(); - - // The gateway resends the refused id. Capacity is free now, so the broker - // forwards it exactly one time. This proves the refusal did not poison the id. - harness.feed(requestFrame("resend-me")); - await flush(); - expect(harness.forwards.filter((forward) => forward.id === "resend-me")).toHaveLength(1); - - // A resend of an already-forwarded id never reaches the forward twice. - const forwardedKeep1 = harness.forwards.filter((forward) => forward.id === "keep-1").length; - harness.feed(requestFrame("keep-1")); - await flush(); - expect(harness.forwards.filter((forward) => forward.id === "keep-1")).toHaveLength(forwardedKeep1); - - harness.resolveAll(); - }); - - it("bounds the retained request ids over the channel lifetime and refuses each new id past the limit", async () => { - const harness = createFakeChannelHarness(); - const { telemetry, capture } = createTelemetryCapture(); - const maxLifetimeRequests = 3; - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - // Resolve every forward at once, so each dispatched id leaves the in-flight - // count but stays in the retained-id set. - forwardRequest: async (request: DuplexRequestFrame): Promise => { - harness.forwards.push({ id: request.id, resolve: () => undefined, reject: () => undefined, settled: true }); - return { status: 200, headers: { "content-type": "application/json" }, body: JSON.stringify({ ok: true }) }; - }, - telemetry, - maxLifetimeRequests, - }); - brokers.push(broker); - broker.start(); - - // Inject more than the lifetime limit of unique valid frames. - const total = 9; - for (let index = 0; index < total; index += 1) { - harness.feed(requestFrame(`life-${index}`)); - await flush(); - } - - // The broker forwarded only up to the lifetime limit. The retained-id set - // stays bounded, because a refused id never joins it. - expect(harness.forwards).toHaveLength(maxLifetimeRequests); - - // The broker delivered a real response for each forwarded request, then a - // bounded terminal refusal for each id past the limit. - const refusals = harness.responses.filter((response) => response.status === 503); - expect(refusals).toHaveLength(total - maxLifetimeRequests); - for (const refusal of refusals) { - expect(refusal.outcome).toBe("unavailable"); - // A lifetime refusal is terminal, so a resend does not help. - expect(JSON.parse(refusal.body).retryable).toBe(false); - } - - // A resend of a refused id stays refused, so the set never grows. - harness.feed(requestFrame("life-8")); - await flush(); - expect(harness.forwards).toHaveLength(maxLifetimeRequests); - - // The telemetry surface stayed on the fixed request span for the delivered - // requests only. No refusal produced a span, a counter, or a loss record. - expect(capture.spans).toHaveLength(maxLifetimeRequests); - for (const span of capture.spans) { - expect(span.name).toBe(DUPLEX_SPAN_REQUEST); - expect(span.dimensions.outcome).toBe("ok"); - expect(Object.keys(span.dimensions).sort()).toEqual(["outcome", "provider", "transport"]); - } - expect(capture.counters).toHaveLength(0); - expect(broker.lossRecord).toBeNull(); - expect(broker.state).toBe("open"); - }); - - it("forwards nothing and fails the channel closed under a flood of over-limit ids", async () => { - const harness = createFakeChannelHarness(); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - }); - brokers.push(broker); - broker.start(); - - // Send a sustained flood of distinct ids, each one byte over the id bound. The - // codec rejects each frame on the read path, so no over-limit id ever reaches - // the retained-id set or a forward. - const overLimitId = (index: number): string => - `${"a".repeat(DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES)}-${index}`; - for (let index = 0; index < 20; index += 1) { - harness.feed(requestFrame(overLimitId(index))); - } - - // The broker forwarded nothing. The retained-id set never grew, because the - // set only grows on a dispatched forward, and the broker dispatched none. - expect(harness.forwards).toHaveLength(0); - // The first over-limit frame is a protocol failure, so the broker fails the - // whole channel closed. It stays lost for the rest of the flood. - expect(broker.state).toBe("lost"); - expect(broker.lossRecord?.reason).toBe("protocol_failure"); - }); - - it("forwards a maximal-size id exactly once and does not forward a resend (no-replay preserved)", async () => { - const harness = createFakeChannelHarness(); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - }); - brokers.push(broker); - broker.start(); - - // An id at the maximum byte size is allowed. The broker forwards it one time. - const maxId = "a".repeat(DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES); - harness.feed(requestFrame(maxId)); - await flush(); - expect(harness.forwards.filter((forward) => forward.id === maxId)).toHaveLength(1); - - // Deliver the response, then resend the same id. The broker retained the id, - // so the resend never reaches the forward a second time. - harness.resolveForward(maxId); - harness.feed(requestFrame(maxId)); - await flush(); - expect(harness.forwards.filter((forward) => forward.id === maxId)).toHaveLength(1); - expect(broker.state).toBe("open"); - }); - - it("maps a forward that resolves with the indeterminate marker to a non-retryable response and retains the id", async () => { - const harness = createFakeChannelHarness(); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - }); - brokers.push(broker); - broker.start(); - - // The forward handler resolves with a 504 that carries the indeterminate - // marker. This is the shape the host forward returns for a response-read - // failure after the host commit. The broker must keep the marker and the - // non-retryable body, so the gateway maps it to a non-retryable status and a - // caller never repeats a possibly-committed mutation. - harness.feed(requestFrame("commit-1")); - await flush(); - const forward = harness.forwards.find((entry) => entry.id === "commit-1" && !entry.settled); - if (!forward) throw new Error("The broker did not forward the request."); - forward.settled = true; - forward.resolve({ - status: 504, - headers: { - "content-type": "application/json", - "x-paperclip-bridge-outcome": "indeterminate", - }, - body: JSON.stringify({ - error: "Bridge response body exceeded the configured size limit of 32 bytes.", - outcome: "indeterminate", - retryable: false, - }), - }); - // The broker answers inside the forward `then` microtask, so let it settle. - await Promise.resolve(); - - expect(harness.responses).toHaveLength(1); - const response = harness.responses[0]!; - expect(response.status).toBe(504); - expect(response.outcome).toBe("indeterminate"); - expect(response.headers["x-paperclip-bridge-outcome"]).toBe("indeterminate"); - expect(JSON.parse(response.body)).toEqual({ - error: "Bridge response body exceeded the configured size limit of 32 bytes.", - outcome: "indeterminate", - retryable: false, - }); - - // The broker delivered a response, so it retained the id. A resend never - // reaches the forward a second time, so a caller that ignores the - // non-retryable status still cannot repeat the mutation through the broker. - harness.feed(requestFrame("commit-1")); - expect(harness.forwards.filter((entry) => entry.id === "commit-1")).toHaveLength(1); - expect(broker.state).toBe("open"); - }); - - it("maps a forward rejection for a mutating method to a non-retryable indeterminate response and retains the id", async () => { - const harness = createFakeChannelHarness(); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - }); - brokers.push(broker); - broker.start(); - - // The forward rejects before the host delivers a response. A fetch can - // reject after the request reaches the host and the host commits, but - // before the response headers arrive. The broker did not abort the - // forward, so a POST must return a non-retryable indeterminate response. - // A retryable status would let a caller repeat a committed mutation. - harness.feed(requestFrame("mutate-1", "POST")); - await flush(); - const forward = harness.forwards.find((entry) => entry.id === "mutate-1" && !entry.settled); - if (!forward) throw new Error("The broker did not forward the request."); - forward.settled = true; - forward.reject(new Error("fetch failed before the response headers arrived")); - // The broker answers inside the forward rejection microtask, so let it settle. - await new Promise((resolve) => setTimeout(resolve, 0)); - - expect(harness.responses).toHaveLength(1); - const response = harness.responses[0]!; - expect(response.status).toBe(504); - expect(response.outcome).toBe("indeterminate"); - expect(response.headers["x-paperclip-bridge-outcome"]).toBe("indeterminate"); - expect(JSON.parse(response.body)).toEqual({ - error: "fetch failed before the response headers arrived", - outcome: "indeterminate", - retryable: false, - }); - - // The broker delivered a response, so it retained the id. A resend never - // reaches the forward a second time, so a caller that ignores the - // non-retryable status still cannot repeat the mutation through the broker. - harness.feed(requestFrame("mutate-1", "POST")); - expect(harness.forwards.filter((entry) => entry.id === "mutate-1")).toHaveLength(1); - expect(broker.state).toBe("open"); - }); - - it("substitutes a bounded terminal response for an oversized response and keeps the channel open", async () => { - const harness = createFakeChannelHarness(); - const { telemetry, capture } = createTelemetryCapture(); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - telemetry, - // A small frame bound makes a legitimately large response body exceed the - // bound, so the test exercises the encode guard without a megabyte body. - maxFrameBytes: 1000, - }); - brokers.push(broker); - broker.start(); - - // Two requests are in flight at the same time. One resolves with a normal - // body; the other resolves with a body that pushes the response frame over - // the bound. The host produced both results, so both requests reached the - // host. - harness.feed(requestFrame("small-1", "POST")); - harness.feed(requestFrame("big-1", "POST")); - await flush(); - - const smallForward = harness.forwards.find((entry) => entry.id === "small-1" && !entry.settled); - if (!smallForward) throw new Error("The broker did not forward the small request."); - smallForward.settled = true; - smallForward.resolve({ - status: 200, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ ok: true }), - }); - await new Promise((resolve) => setTimeout(resolve, 0)); - - const bigForward = harness.forwards.find((entry) => entry.id === "big-1" && !entry.settled); - if (!bigForward) throw new Error("The broker did not forward the big request."); - bigForward.settled = true; - bigForward.resolve({ - status: 200, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ data: "y".repeat(2000) }), - }); - await new Promise((resolve) => setTimeout(resolve, 0)); - - // The other in-flight request delivered its full response, unaffected by the - // oversized one. - expect(harness.responses).toHaveLength(2); - const delivered = harness.responses.find((entry) => entry.id === "small-1")!; - expect(delivered.status).toBe(200); - expect(delivered.outcome).toBe("completed"); - expect(JSON.parse(delivered.body)).toEqual({ ok: true }); - - // The oversized response became one bounded terminal response marked - // indeterminate and non-retryable, so a caller does not retry a possible - // mutation. The broker did not write the oversized frame. - const terminal = harness.responses.find((entry) => entry.id === "big-1")!; - expect(terminal.status).toBe(502); - expect(terminal.outcome).toBe("indeterminate"); - expect(terminal.headers["x-paperclip-bridge-outcome"]).toBe("indeterminate"); - expect(JSON.parse(terminal.body)).toEqual({ - error: "upstream response too large to deliver", - outcome: "indeterminate", - retryable: false, - }); - - // The oversized body never crossed the wire. The broker did not truncate it or - // pass it through. - const writtenText = harness.responses.map((entry) => encodeDuplexFrame(entry)).join(""); - expect(writtenText).not.toContain("yyyy"); - - // The broker recorded the oversized request as an error outcome, never a loss. - // The channel stays open and the run does not fail. - const errorSpans = capture.spans.filter((span) => span.dimensions.outcome === "error"); - expect(errorSpans).toHaveLength(1); - expect(errorSpans[0]!.name).toBe(DUPLEX_SPAN_REQUEST); - expect(broker.lossRecord).toBeNull(); - expect(broker.state).toBe("open"); - expect(broker.runDisposition.failed).toBe(false); - }); - - it("sends a minimal terminal response when the frame bound rejects the full replacement", async () => { - const harness = createFakeChannelHarness(); - const { telemetry, capture } = createTelemetryCapture(); - // The bound accepts a compact request envelope (128 bytes) and the minimal - // terminal response envelope (114 bytes), but rejects the full replacement - // response envelope (193 bytes). This proves the broker never drops the channel - // or strands the request when the bound is smaller than the full replacement. - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - telemetry, - maxFrameBytes: 150, - }); - brokers.push(broker); - broker.start(); - - // A compact request envelope with an empty body fits the bound, so the broker - // forwards it. - harness.feed({ - frame: { - version: DUPLEX_FRAME_VERSION, - type: "request", - id: "big-1", - method: "POST", - path: "/api/issues/big-1", - query: "", - headers: {}, - bodyByteCount: 0, - }, - bodyText: "", - }); - await flush(); - - const forward = harness.forwards.find((entry) => entry.id === "big-1" && !entry.settled); - if (!forward) throw new Error("The broker did not forward the request."); - forward.settled = true; - // The host produced a large response body that pushes the real response frame - // and the full replacement frame over the bound. - forward.resolve({ - status: 200, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ data: "y".repeat(2000) }), - }); - await new Promise((resolve) => setTimeout(resolve, 0)); - - // The broker sent one minimal terminal response, not the oversized real - // response and not the full replacement. The minimal response carries the - // indeterminate outcome, so the gateway maps it to a terminal 409. - expect(harness.responses).toHaveLength(1); - const terminal = harness.responses[0]!; - expect(terminal.id).toBe("big-1"); - expect(terminal.status).toBe(502); - expect(terminal.outcome).toBe("indeterminate"); - expect(terminal.headers).toEqual({}); - expect(terminal.body).toBe(""); - - // The oversized body never crossed the wire. - const writtenText = harness.responses.map((entry) => encodeDuplexFrame(entry)).join(""); - expect(writtenText).not.toContain("yyyy"); - - // The broker recorded the request as an error outcome, never a loss. The - // channel stays open and the run does not fail. - const errorSpans = capture.spans.filter((span) => span.dimensions.outcome === "error"); - expect(errorSpans).toHaveLength(1); - expect(errorSpans[0]!.name).toBe(DUPLEX_SPAN_REQUEST); - expect(broker.lossRecord).toBeNull(); - expect(broker.state).toBe("open"); - expect(broker.runDisposition.failed).toBe(false); - }); - - it("keeps the channel open and logs when the bound rejects even the minimal terminal response", async () => { - const harness = createFakeChannelHarness(); - const { telemetry, capture } = createTelemetryCapture(); - const logs: string[] = []; - // The bound accepts a tiny request envelope (107 bytes) but rejects even the - // minimal terminal response envelope (110 bytes). The broker cannot send any - // frame for this request, so it must keep the channel open and log a clear - // local error. - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - telemetry, - maxFrameBytes: 109, - logger: (message) => logs.push(message), - }); - brokers.push(broker); - broker.start(); - - harness.feed({ - frame: { - version: DUPLEX_FRAME_VERSION, - type: "request", - id: "x", - method: "GET", - path: "/", - query: "", - headers: {}, - bodyByteCount: 0, - }, - bodyText: "", - }); - await flush(); - - const forward = harness.forwards.find((entry) => entry.id === "x" && !entry.settled); - if (!forward) throw new Error("The broker did not forward the request."); - forward.settled = true; - forward.resolve({ - status: 200, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ data: "y".repeat(200) }), - }); - await new Promise((resolve) => setTimeout(resolve, 0)); - - // The broker sent no response frame, because no frame fits the bound. - expect(harness.responses).toHaveLength(0); - // The broker recorded the request as an error outcome, never a loss. - const errorSpans = capture.spans.filter((span) => span.dimensions.outcome === "error"); - expect(errorSpans).toHaveLength(1); - // The broker logged a clear local error and kept the channel open. - expect(logs.some((line) => line.includes("could not deliver a terminal response"))).toBe(true); - expect(broker.lossRecord).toBeNull(); - expect(broker.state).toBe("open"); - expect(broker.runDisposition.failed).toBe(false); - }); - - it("completes a request within the bound and delivers the real response", async () => { - const harness = createFakeChannelHarness(); - const { telemetry, capture } = createTelemetryCapture(); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - telemetry, - maxFrameBytes: 1000, - }); - brokers.push(broker); - broker.start(); - - harness.feed(requestFrame("ok-1", "POST")); - await flush(); - harness.resolveForward("ok-1", JSON.stringify({ ok: true })); - await new Promise((resolve) => setTimeout(resolve, 0)); - - // The broker delivered the real response promptly with the completed outcome. - expect(harness.responses).toHaveLength(1); - const delivered = harness.responses[0]!; - expect(delivered.id).toBe("ok-1"); - expect(delivered.status).toBe(200); - expect(delivered.outcome).toBe("completed"); - expect(JSON.parse(delivered.body)).toEqual({ ok: true }); - - // The delivered response records an `ok` span, never an error, and never a loss. - const okSpans = capture.spans.filter((span) => span.dimensions.outcome === "ok"); - expect(okSpans).toHaveLength(1); - const errorSpans = capture.spans.filter((span) => span.dimensions.outcome === "error"); - expect(errorSpans).toHaveLength(0); - expect(broker.lossRecord).toBeNull(); - expect(broker.state).toBe("open"); - expect(broker.runDisposition.failed).toBe(false); - }); - - it("maps a forward rejection for a safe method to a retryable response", async () => { - const harness = createFakeChannelHarness(); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - }); - brokers.push(broker); - broker.start(); - - // A safe method never changes host state, so a retry cannot double-apply a - // mutation. A forward rejection for a GET stays retryable: the broker - // returns a 502 with the completed outcome, so the gateway passes it through. - harness.feed(requestFrame("read-1", "GET")); - await flush(); - const forward = harness.forwards.find((entry) => entry.id === "read-1" && !entry.settled); - if (!forward) throw new Error("The broker did not forward the request."); - forward.settled = true; - forward.reject(new Error("fetch failed before the response headers arrived")); - await new Promise((resolve) => setTimeout(resolve, 0)); - - expect(harness.responses).toHaveLength(1); - const response = harness.responses[0]!; - expect(response.status).toBe(502); - expect(response.outcome).toBe("completed"); - expect(response.headers["x-paperclip-bridge-outcome"]).toBeUndefined(); - expect(JSON.parse(response.body)).toEqual({ - error: "fetch failed before the response headers arrived", - }); - }); - - it("maps a forward-budget timeout for a safe method to a retryable response", async () => { - const harness = createFakeChannelHarness(); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - // A tiny forward budget aborts the controller before the manual rejection. - budgets: { forwardTimeoutMs: 5 }, - }); - brokers.push(broker); - broker.start(); - - // A safe method never changes host state. A forward timeout for a GET must - // stay retryable: the broker returns a 504 with the completed outcome and no - // indeterminate marker, so the gateway does not map it to a terminal 409. - harness.feed(requestFrame("read-timeout-1", "GET")); - await flush(); - const forward = harness.forwards.find((entry) => entry.id === "read-timeout-1" && !entry.settled); - if (!forward) throw new Error("The broker did not forward the request."); - - // Wait for the forward-budget timer to abort the controller, then reject the - // forward. The rejection handler now sees the aborted signal. - await new Promise((resolve) => setTimeout(resolve, 20)); - forward.settled = true; - forward.reject(new Error("Duplex broker forward budget exceeded.")); - await new Promise((resolve) => setTimeout(resolve, 0)); - - expect(harness.responses).toHaveLength(1); - const response = harness.responses[0]!; - expect(response.status).toBe(504); - expect(response.outcome).toBe("completed"); - expect(response.headers["x-paperclip-bridge-outcome"]).toBeUndefined(); - expect(JSON.parse(response.body)).toEqual({ - error: "Duplex broker forward budget exceeded.", - retryable: true, - }); - }); - - it("maps a forward-budget timeout for a mutating method to a non-retryable indeterminate response", async () => { - const harness = createFakeChannelHarness(); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - budgets: { forwardTimeoutMs: 5 }, - }); - brokers.push(broker); - broker.start(); - - // A POST may commit before the forward budget aborts the call. The broker - // must keep the terminal contract: a 504 with the indeterminate outcome, so - // the gateway maps it to a non-retryable 409 and no caller double-applies it. - harness.feed(requestFrame("mutate-timeout-1", "POST")); - await flush(); - const forward = harness.forwards.find((entry) => entry.id === "mutate-timeout-1" && !entry.settled); - if (!forward) throw new Error("The broker did not forward the request."); - - await new Promise((resolve) => setTimeout(resolve, 20)); - forward.settled = true; - forward.reject(new Error("Duplex broker forward budget exceeded.")); - await new Promise((resolve) => setTimeout(resolve, 0)); - - expect(harness.responses).toHaveLength(1); - const response = harness.responses[0]!; - expect(response.status).toBe(504); - expect(response.outcome).toBe("indeterminate"); - expect(response.headers["x-paperclip-bridge-outcome"]).toBe("indeterminate"); - expect(JSON.parse(response.body)).toEqual({ - error: "Duplex broker forward budget exceeded.", - outcome: "indeterminate", - retryable: false, - }); - }); - - it("maps a response-budget backstop for a safe method to a retryable response", async () => { - const harness = createFakeChannelHarness(); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - // The forward budget aborts the call, but the forward promise stays - // pending. A GET receives the response headers, but the body reader stalls - // through the response budget, so the forward promise never settles. The - // response-budget backstop must answer the request. - budgets: { forwardTimeoutMs: 5, responseBudgetMs: 15, gatewayWaitMs: 40 }, - }); - brokers.push(broker); - broker.start(); - - // A safe method never changes host state, so a stalled body reader cannot - // leave a mutation half-applied. The backstop must keep the request - // retryable: a 504 with the completed outcome and no indeterminate marker, - // so the gateway does not map it to a terminal 409. - harness.feed(requestFrame("read-stall-1", "GET")); - await flush(); - const forward = harness.forwards.find((entry) => entry.id === "read-stall-1" && !entry.settled); - if (!forward) throw new Error("The broker did not forward the request."); - - // Never settle the forward. Wait past the response budget so the backstop - // fires on the stalled body reader. - await new Promise((resolve) => setTimeout(resolve, 30)); - - expect(harness.responses).toHaveLength(1); - const response = harness.responses[0]!; - expect(response.status).toBe(504); - expect(response.outcome).toBe("completed"); - expect(response.headers["x-paperclip-bridge-outcome"]).toBeUndefined(); - expect(JSON.parse(response.body)).toEqual({ - error: "Duplex broker response budget exceeded.", - retryable: true, - }); - }); - - it("maps a response-budget backstop for a mutating method to a non-retryable indeterminate response", async () => { - const harness = createFakeChannelHarness(); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: controllableForward(harness), - budgets: { forwardTimeoutMs: 5, responseBudgetMs: 15, gatewayWaitMs: 40 }, - }); - brokers.push(broker); - broker.start(); - - // A POST may commit before the response budget passes, and the broker cannot - // prove the host applied no mutation. The backstop must keep the terminal - // contract: a 504 with the indeterminate outcome, so the gateway maps it to a - // non-retryable 409 and no caller double-applies it. - harness.feed(requestFrame("mutate-stall-1", "POST")); - await flush(); - const forward = harness.forwards.find((entry) => entry.id === "mutate-stall-1" && !entry.settled); - if (!forward) throw new Error("The broker did not forward the request."); - - await new Promise((resolve) => setTimeout(resolve, 30)); - - expect(harness.responses).toHaveLength(1); - const response = harness.responses[0]!; - expect(response.status).toBe(504); - expect(response.outcome).toBe("indeterminate"); - expect(response.headers["x-paperclip-bridge-outcome"]).toBe("indeterminate"); - expect(JSON.parse(response.body)).toEqual({ - error: "Duplex broker response budget exceeded.", - outcome: "indeterminate", - retryable: false, - }); - }); -}); - - -/** - * A channel harness that captures the broker exit listener. `emitExit` invokes it, - * so a test drives one channel exit into the broker. - */ -function createExitChannelHarness(): { - channel: CommandManagedDuplexChannel; - emitExit: (exit: { exitCode: number | null; transportClosed?: boolean }) => void; -} { - let exitListener: - | ((exit: { exitCode: number | null; transportClosed?: boolean }) => void) - | null = null; - const channel: CommandManagedDuplexChannel = { - write: () => undefined, - onData: () => undefined, - onExit: (listener) => { - exitListener = listener; - }, - stop: () => undefined, - close: () => Promise.resolve(), - }; - return { - channel, - emitExit: (exit) => { - if (!exitListener) throw new Error("The broker did not bind the exit listener."); - exitListener(exit); - }, - }; -} - -describe("duplex bridge broker exit taxonomy", () => { - const brokers: DuplexBridgeBroker[] = []; - - afterEach(async () => { - while (brokers.length > 0) { - const broker = brokers.pop(); - if (broker) await broker.close(); - } - }); - - it("records a transport close as a distinct loss, not a process exit", async () => { - const { telemetry, capture } = createTelemetryCapture(); - const harness = createExitChannelHarness(); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: () => - Promise.resolve({ status: 200, headers: {}, body: "" }), - telemetry, - }); - brokers.push(broker); - broker.start(); - - // A reason-less transport close carries the discriminator and no exit code. - harness.emitExit({ exitCode: null, transportClosed: true }); - - expect(broker.lossRecord?.reason).toBe("transport_closed"); - expect(broker.runDisposition).toEqual({ failed: true, lossReason: "transport_closed" }); - const lossCounter = capture.counters.find( - (record) => record.metric === DUPLEX_COUNTER_LOSS_TOTAL, - ); - expect(lossCounter?.dimensions.loss_reason).toBe("transport_closed"); - }); - - it("records a numeric exit as a process exit", async () => { - const { telemetry, capture } = createTelemetryCapture(); - const harness = createExitChannelHarness(); - const broker = await createDuplexBridgeBroker({ - channel: harness.channel, - forwardRequest: () => - Promise.resolve({ status: 200, headers: {}, body: "" }), - telemetry, - }); - brokers.push(broker); - broker.start(); - - // A numeric exit code is a real process exit, so the broker keeps the existing - // channel_exit -> provider_exit mapping. - harness.emitExit({ exitCode: 0 }); - - expect(broker.lossRecord?.reason).toBe("channel_exit"); - expect(broker.runDisposition).toEqual({ failed: true, lossReason: "provider_exit" }); - const lossCounter = capture.counters.find( - (record) => record.metric === DUPLEX_COUNTER_LOSS_TOTAL, - ); - expect(lossCounter?.dimensions.loss_reason).toBe("provider_exit"); - }); -}); diff --git a/packages/adapter-utils/src/duplex-bridge-broker.ts b/packages/adapter-utils/src/duplex-bridge-broker.ts deleted file mode 100644 index 86a21ab011..0000000000 --- a/packages/adapter-utils/src/duplex-bridge-broker.ts +++ /dev/null @@ -1,1457 +0,0 @@ -/** - * Host broker for the sandbox duplex channel. - * - * The broker owns the host end of one persistent duplex channel to the sandbox - * gateway. It reads request frames from the channel, forwards each one on the - * existing Paperclip API path, and writes one response frame back. It sends a - * heartbeat frame on an interval to prove liveness. - * - * The broker does not hold the route allowlist, the token replacement, or the - * run attribution. The caller passes a forward handler, and the broker calls it - * for each request. The handler applies the real token and the signed run - * identifier, so those rules stay in one place on the existing forward path. - * - * The broker runs a set of nested timeout budgets: - * - forward budget: the deadline for one forward call. - * - response budget: the deadline for the broker to send one response frame. - * - gateway wait budget: the deadline the in-sandbox gateway waits for the - * response frame. - * Each inner budget is smaller than its outer budget, so the broker aborts and - * answers before the gateway gives up. The broker asserts this order at - * construction and fails a configuration that breaks it. - * - * The provider controls the duplex transport directly, so the broker treats each - * request frame as untrusted. The broker bounds the work one channel can force: - * - in-flight limit: the maximum number of pending forwards at one time. It - * bounds the controllers, the timers, and the concurrent authenticated - * forwards. - * - lifetime limit: the maximum number of distinct requests over the channel - * lifetime. It bounds the retained request-id memory, because the broker keeps - * one id per distinct dispatched request for the no-replay guarantee. - * The broker checks each limit before it adds the id to the seen set, allocates - * the pending record, or calls the forward handler. On a limit it answers the - * refused request with one bounded terminal response and forwards nothing. The - * refusal preserves the no-replay and no-double-dispatch rules. - * - * Loss is terminal. The broker detects loss through channel exit, a stream - * write failure, a protocol failure, a heartbeat write failure, and a close - * timeout. On loss the broker stops the heartbeat, aborts every in-flight - * forward, records the loss, and dispatches nothing more. The broker never - * reconnects and never replays a request. The broker records the loss for - * metrics only and sends nothing about the loss to the sandbox. - */ - -import type { CommandManagedDuplexChannel } from "./command-managed-runtime.js"; -import { - DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, - type DuplexAggregateByteLedger, - type ReservationToken, -} from "./duplex-aggregate-byte-ledger.js"; -import { - DUPLEX_BODY_CHUNK_RAW_BYTES, - DEFAULT_MAX_DUPLEX_FRAME_BYTES, - DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES, - DUPLEX_FRAME_VERSION, - DuplexFrameDecoder, - encodeDuplexFrame, - encodeDuplexFrameChecked, - type DuplexBodyChunkFrame, - type DuplexFrame, - type DuplexRequestFrame, - type DuplexResponseFrame, - type DuplexResponseOutcome, -} from "./duplex-frame-codec.js"; -import { - DuplexBodyError, - DuplexBodyReceiver, - splitBodyIntoChunkFrames, - type DuplexBodyReceiverConfig, - type ReassembledBody, -} from "./duplex-body-spool.js"; -import type { - DuplexLossReason, - DuplexOutcomeValue, - DuplexObservability, -} from "./duplex-observability.js"; - -/** The lifecycle states of the broker. The broker moves through them in order. */ -export type DuplexBrokerState = "opening" | "open" | "lost" | "closing" | "closed"; - -/** The reason the broker classified a loss. The broker records it for metrics only. */ -export type DuplexBrokerLossReason = - | "channel_exit" - | "transport_closed" - | "stream_failure" - | "protocol_failure" - | "heartbeat_write_failure" - | "close_timeout"; - -/** - * Map one broker loss reason to the closed, typed telemetry loss reason. The - * broker records the internal reason for its own metrics; the telemetry boundary - * carries only the closed enum value. The map is total over the internal reasons, - * so no raw text ever reaches the typed reason. - * - `channel_exit` -> `provider_exit`: the provider channel process exited. - * - `transport_closed` -> `transport_closed`: the provider transport closed with - * no exit data, so the loss is a transport close, not a process exit. - * - `stream_failure` -> `write_error`: a write to the channel failed. - * - `protocol_failure` -> `rpc_failure`: a malformed or mismatched frame. - * - `heartbeat_write_failure` -> `heartbeat_timeout`: the liveness write failed. - * - `close_timeout` -> `other`: an orderly close did not complete in the budget. - */ -const BROKER_LOSS_REASON_TO_TYPED: Readonly> = { - channel_exit: "provider_exit", - transport_closed: "transport_closed", - stream_failure: "write_error", - protocol_failure: "rpc_failure", - heartbeat_write_failure: "heartbeat_timeout", - close_timeout: "other", -}; - -/** - * Map one broker loss reason to the closed, typed telemetry loss reason. The host - * uses it to name the typed reason on a log line without the raw provider text. - */ -export function typedDuplexLossReason(reason: DuplexBrokerLossReason): DuplexLossReason { - return BROKER_LOSS_REASON_TO_TYPED[reason] ?? "other"; -} - -/** - * The typed error code the host reports when the duplex control channel died - * before an orderly completion. Both the ACP lane and the CLI lane report this - * one code, so the run disposition is identical across the two lanes. - */ -export const DUPLEX_CHANNEL_LOST_ERROR_CODE = "duplex_channel_lost"; - -/** - * The terminal run disposition the broker computes from its ordered lifecycle. A - * `failed` disposition means a terminal loss ordered before an orderly completion, - * so the run must not report success. The typed loss reason names the cause; it is - * `null` for a success. - */ -export interface DuplexBrokerRunDisposition { - /** True when a terminal loss ordered before an orderly completion. */ - failed: boolean; - /** The typed, closed loss reason on a failure; `null` on a success. */ - lossReason: DuplexLossReason | null; -} - -/** The nested timeout budgets. Each inner budget is smaller than its outer budget. */ -export interface DuplexBrokerBudgets { - /** The deadline for one forward call, in milliseconds. */ - forwardTimeoutMs: number; - /** The deadline for the broker to send one response frame, in milliseconds. */ - responseBudgetMs: number; - /** The deadline the in-sandbox gateway waits for the response frame, in milliseconds. */ - gatewayWaitMs: number; -} - -/** The default nested budgets: forward 30 s, response 32 s, gateway wait 35 s. */ -export const DEFAULT_DUPLEX_BROKER_BUDGETS: DuplexBrokerBudgets = { - forwardTimeoutMs: 30_000, - responseBudgetMs: 32_000, - gatewayWaitMs: 35_000, -}; - -/** The default interval between two heartbeat frames, in milliseconds. */ -export const DEFAULT_DUPLEX_BROKER_HEARTBEAT_INTERVAL_MS = 5_000; - -/** The default deadline for an orderly channel close, in milliseconds. */ -export const DEFAULT_DUPLEX_BROKER_CLOSE_TIMEOUT_MS = 2_000; - -/** - * The default maximum number of in-flight requests. The broker holds this many - * pending forwards at one time. It refuses a further request until an in-flight - * request completes. The provider controls the transport, so this finite limit - * bounds the controllers, the timers, and the concurrent authenticated forwards - * a provider can force on the host. - */ -export const DEFAULT_DUPLEX_BROKER_MAX_IN_FLIGHT_REQUESTS = 64; - -/** - * The default maximum number of distinct requests over the channel lifetime. The - * broker forwards this many distinct request ids, then refuses each new distinct - * id and forwards nothing more. This limit bounds the retained request-id memory, - * because the broker keeps one id per distinct dispatched request for the no-replay - * guarantee. - * - * The retained id memory has a hard ceiling. The codec bounds each id at - * `DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES` (256 bytes), so the worst-case retained id - * bytes are this count multiplied by that bound: 50,000 * 256 = 12,800,000 bytes - * (about 12.8 MB), plus the fixed per-entry overhead of the Set. This count is - * sized against that id bound to keep the ceiling small. - */ -export const DEFAULT_DUPLEX_BROKER_MAX_LIFETIME_REQUESTS = 50_000; - -/** - * The fixed, documented per-entry allocation the broker charges the aggregate byte - * ledger for one no-replay request-id set entry. The type and the cardinality of - * `seenRequestIds` are bounded: the codec caps each id at - * {@link DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES}, and the lifetime limit caps the - * entry count. The broker charges the exact raw id bytes plus this fixed entry - * overhead, so the retained set never grows uncharged. This constant models the - * fixed per-entry cost of the string key and the Set slot, not the id bytes. - */ -export const DUPLEX_SEEN_REQUEST_ID_SET_ENTRY_BYTES = 64; - -/** The result of one forward call. The broker turns it into one response frame. */ -export interface DuplexBrokerForwardResult { - status: number; - headers?: Record; - body?: string; -} - -/** - * The forward handler the broker calls for each request. The handler applies the - * real token and the run attribution, then forwards the request on the existing - * API path. The broker reassembles the request body from the `body_chunk` frames - * before it calls the handler, so the handler streams the body from - * `options.body`. The broker aborts `options.signal` when the forward budget ends - * or a loss happens, so a handler that threads the signal into its work stops - * early. - */ -export type DuplexBrokerForwardHandler = ( - request: DuplexRequestFrame, - options: { signal: AbortSignal; body: ReassembledBody }, -) => Promise; - -/** One request record. The broker captures the dispatch-start point for metrics only. */ -export interface DuplexBrokerRequestRecord { - id: string; - method: string; - path: string; - /** The point the broker started to dispatch the request, in milliseconds. */ - dispatchStartMs: number; -} - -/** One loss record. The broker reports it for metrics only. */ -export interface DuplexBrokerLossRecord { - reason: DuplexBrokerLossReason; - message: string; - /** The point the broker recorded the loss, in milliseconds. */ - atMs: number; -} - -/** The options for {@link createDuplexBridgeBroker}. */ -export interface DuplexBrokerOptions { - /** The duplex channel to the sandbox gateway. */ - channel: CommandManagedDuplexChannel; - /** The forward handler the broker calls for each request. */ - forwardRequest: DuplexBrokerForwardHandler; - /** The nested timeout budgets. The default is {@link DEFAULT_DUPLEX_BROKER_BUDGETS}. */ - budgets?: Partial; - /** The interval between two heartbeat frames, in milliseconds. */ - heartbeatIntervalMs?: number; - /** The deadline for an orderly channel close, in milliseconds. */ - closeTimeoutMs?: number; - /** The maximum size of one inbound frame, in bytes. Forwarded to the decoder. */ - maxFrameBytes?: number; - /** - * The maximum number of in-flight requests the broker holds at one time. The - * broker refuses a further request past this limit with a bounded terminal - * response and forwards nothing for it. The default is - * {@link DEFAULT_DUPLEX_BROKER_MAX_IN_FLIGHT_REQUESTS}. - */ - maxInFlightRequests?: number; - /** - * The maximum number of distinct requests the broker dispatches over the - * channel lifetime. The broker refuses each new distinct request past this - * limit with a bounded terminal response and forwards nothing more. This limit - * bounds the retained request-id memory. The default is - * {@link DEFAULT_DUPLEX_BROKER_MAX_LIFETIME_REQUESTS}. - */ - maxLifetimeRequests?: number; - /** - * The config for the receive-side request-body reassembler. It sets the spill - * threshold, the fixed raw chunk size, and the per-channel spill caps. The - * broker creates one reassembler for the channel. The config stays injectable, - * so a test lowers the spill threshold and the caps to exercise the spill path - * and the fail-closed cap behavior without a large body. - */ - bodyReceiverConfig?: DuplexBodyReceiverConfig; - /** The clock the broker reads for the metric timestamps. The default is `Date.now`. */ - now?: () => number; - /** The metrics sink for the per-request dispatch record. */ - onRequestRecord?: (record: DuplexBrokerRequestRecord) => void; - /** The metrics sink for the terminal loss record. */ - onLoss?: (record: DuplexBrokerLossRecord) => void; - /** The sink for a state change. The broker reports every transition. */ - onStateChange?: (state: DuplexBrokerState) => void; - /** The sink for a diagnostic message. The broker never writes diagnostics to the channel. */ - logger?: (message: string) => void; - /** - * The fixed observability facade. The broker records one request span per - * delivered request and one loss record per terminal loss. The facade maps each - * record to the fixed names and dimensions, so no route, query, body, token, or - * raw error rides a span or a counter. The default records nothing. - */ - telemetry?: DuplexObservability; - /** - * The process-owned aggregate byte ledger. The broker reserves the exact retained - * bytes of each dispatched request against it before it retains the frame: the - * raw request frame, the normalized request payload, and the no-replay set entry. - * A reservation that would pass the ceiling makes the broker retain nothing and - * refuse the request with the fixed marker - * {@link DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED}. When the ledger is absent the - * broker charges nothing and behaves as before, so a non-duplex or a legacy path - * stays unchanged. - */ - duplexAggregateByteLedger?: DuplexAggregateByteLedger | null; -} - -/** The broker handle the factory returns. */ -export interface DuplexBridgeBroker { - /** The current state of the broker. */ - readonly state: DuplexBrokerState; - /** The loss record, or `null` when the broker never lost the channel. */ - readonly lossRecord: DuplexBrokerLossRecord | null; - /** - * The terminal run disposition from the ordered lifecycle. It reports a failure - * when a terminal loss ordered before an orderly completion, and names the typed - * loss reason. It reports a success for a healthy channel or a normal-teardown - * loss. The host reads it at the run-disposition seam. - */ - readonly runDisposition: DuplexBrokerRunDisposition; - /** - * Mark the host-observed orderly completion of the agent turn on the ordered - * lifecycle. A loss ordered before this mark latches a failure; a loss ordered - * after it stays a success. The broker also marks it on a gateway close frame - * and on a host-initiated orderly close. Safe to call more than one time. - */ - markOrderlyCompletion(): void; - /** - * Atomically read the run disposition and mark the host-observed orderly - * completion in one synchronous step. The host calls it at the run-disposition - * seam for a success-eligible terminal. The broker marks the orderly completion - * only while no loss ordered, then returns the disposition, so no caller can - * insert an `await` between the read and the mark. A loss that already latched - * keeps the failure, because the mark no-ops after a latched loss. - */ - settleRunDisposition(): DuplexBrokerRunDisposition; - /** Start the broker. It wires the channel listeners and moves to `open`. */ - start(): void; - /** Close the channel cleanly. It moves through `closing` to `closed`. */ - close(): Promise; - /** Stop the sandbox child process. Safe to call more than one time. */ - stop(): void; -} - -/** - * Assert the nested budget order. Each inner budget must be smaller than its - * outer budget, so the broker answers before the gateway gives up. The function - * throws when the order breaks. - */ -export function assertNestedDuplexBrokerBudgets(budgets: DuplexBrokerBudgets): void { - if (!(budgets.forwardTimeoutMs < budgets.responseBudgetMs)) { - throw new Error( - `Duplex broker forward budget ${budgets.forwardTimeoutMs}ms must be smaller than the response budget ${budgets.responseBudgetMs}ms.`, - ); - } - if (!(budgets.responseBudgetMs < budgets.gatewayWaitMs)) { - throw new Error( - `Duplex broker response budget ${budgets.responseBudgetMs}ms must be smaller than the gateway wait budget ${budgets.gatewayWaitMs}ms.`, - ); - } -} - -/** The resolved request limits the broker enforces. */ -export interface DuplexBrokerLimits { - /** The maximum number of in-flight requests the broker holds at one time. */ - maxInFlightRequests: number; - /** The maximum number of distinct requests the broker dispatches over the channel lifetime. */ - maxLifetimeRequests: number; -} - -/** - * Assert the request limits. Each limit must be a finite positive integer, so the - * broker fails closed on a broken configuration instead of running with an - * unbounded or a zero limit. The function throws when a limit breaks the rule. - */ -export function assertDuplexBrokerLimits(limits: DuplexBrokerLimits): void { - const entries: ReadonlyArray = [ - ["maxInFlightRequests", limits.maxInFlightRequests], - ["maxLifetimeRequests", limits.maxLifetimeRequests], - ]; - for (const [name, value] of entries) { - if (!Number.isInteger(value) || value < 1) { - throw new Error( - `Duplex broker ${name} must be a finite positive integer; got ${String(value)}.`, - ); - } - } -} - -function errorMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error); -} - -/** - * The safe HTTP methods. RFC 7231 section 4.2.1 defines this set. A safe method - * does not change host state, so the host applies no mutation for it. A caller - * can retry a safe method after a forward failure without a double-apply risk. - */ -const SAFE_BRIDGE_METHODS = new Set(["GET", "HEAD", "OPTIONS", "TRACE"]); - -/** Report whether the method is safe, so a forward failure stays retryable. */ -export function isSafeBridgeMethod(method: string): boolean { - return SAFE_BRIDGE_METHODS.has(method.trim().toUpperCase()); -} - -/** The internal bookkeeping for one in-flight request. */ -interface PendingRequest { - controller: AbortController; - responded: boolean; - /** - * The forward-budget timer. It is `null` while the broker reassembles the - * request body, and the broker sets it when it starts the forward. So the - * forward budget bounds the forward call, not the reassembly. - */ - forwardTimer: ReturnType | null; - /** - * The response-budget backstop timer. The broker starts it at the request - * envelope, so it bounds the whole request: the body reassembly plus the - * forward. It answers a request that never completes its reassembly or forward. - */ - responseTimer: ReturnType; - /** The point the broker started to dispatch the request. It sets the span latency. */ - dispatchStartMs: number; - /** The request method. The response backstop reads it to classify the safe-method retry. */ - method: string; - /** The reassembled request body, set once reassembly completes. The broker disposes it on settle. */ - reassembled: ReassembledBody | null; - /** True once the forward promise settled. The finally owner sets it one time. */ - forwardSettled: boolean; - /** - * Release the request-frame token and the request-payload token exactly one time. - * The single forward-promise finally owner calls it after the forward settles. - * A second call is a no-op, so no token releases twice. - */ - releaseForwardTokens: () => void; -} - -/** - * One orphaned forward. The broker answered the gateway and removed the request - * from `pending`, but the forward promise or its response-body reader had not - * settled. The orphan keeps the request tokens charged and the controller live - * until the forward finally releases them, so the ledger reports nonzero ownership - * until the async work settles. - */ -interface OrphanedForward { - controller: AbortController; - /** Release the request-frame and request-payload tokens exactly one time. */ - releaseForwardTokens: () => void; -} - -/** - * Create the host duplex bridge broker. The factory asserts the budget order, - * creates the receive-side request-body reassembler, and returns a handle. It is - * asynchronous because the reassembler owns a spill directory it creates with - * `mkdtemp`. Call `start` to wire the channel and open the broker. - */ -export async function createDuplexBridgeBroker( - options: DuplexBrokerOptions, -): Promise { - const budgets: DuplexBrokerBudgets = { - ...DEFAULT_DUPLEX_BROKER_BUDGETS, - ...options.budgets, - }; - assertNestedDuplexBrokerBudgets(budgets); - - const limits: DuplexBrokerLimits = { - maxInFlightRequests: options.maxInFlightRequests ?? DEFAULT_DUPLEX_BROKER_MAX_IN_FLIGHT_REQUESTS, - maxLifetimeRequests: options.maxLifetimeRequests ?? DEFAULT_DUPLEX_BROKER_MAX_LIFETIME_REQUESTS, - }; - assertDuplexBrokerLimits(limits); - - const channel = options.channel; - const forwardRequest = options.forwardRequest; - const heartbeatIntervalMs = options.heartbeatIntervalMs ?? DEFAULT_DUPLEX_BROKER_HEARTBEAT_INTERVAL_MS; - const closeTimeoutMs = options.closeTimeoutMs ?? DEFAULT_DUPLEX_BROKER_CLOSE_TIMEOUT_MS; - const now = options.now ?? (() => Date.now()); - - // The process-owned aggregate byte ledger, or `null` when the caller injected - // none. When `null` the broker charges nothing and behaves as before. - const ledger = options.duplexAggregateByteLedger ?? null; - // The one frame size bound the broker enforces on both sides. The decoder - // rejects an inbound frame over this bound, and the encode guard refuses to - // write an outbound frame over it. Encode and decode share one value, so a - // frame the broker writes always decodes on the peer. - const maxFrameBytes = options.maxFrameBytes ?? DEFAULT_MAX_DUPLEX_FRAME_BYTES; - // The decoder charges its retained partial-frame bytes against the same host - // ledger. It receives the ledger object directly, so one gauge bounds the - // decoder buffer with every other host retention site. - const decoder = new DuplexFrameDecoder({ - maxFrameBytes, - ...(ledger ? { aggregateByteLedger: ledger } : {}), - }); - // Release one token exactly one time. A `null` token or an absent ledger is a - // no-op, so the broker never records a false accounting defect. - const releaseToken = (token: ReservationToken | null): void => { - if (ledger && token) ledger.release(token); - }; - // The byte size of the normalized request payload the broker retains across the - // forward. It counts the exact retained scalar and header bytes plus the raw body - // byte count, so the charge matches the reassembled body the broker holds and - // never a parsed object graph. The body rides `body_chunk` frames, so the - // envelope carries only `bodyByteCount`; that count is the retained body size. - const requestPayloadBytes = (frame: DuplexRequestFrame): number => { - let bytes = - Buffer.byteLength(frame.id, "utf8") + - Buffer.byteLength(frame.method, "utf8") + - Buffer.byteLength(frame.path, "utf8") + - Buffer.byteLength(frame.query, "utf8") + - frame.bodyByteCount; - for (const [key, value] of Object.entries(frame.headers)) { - bytes += Buffer.byteLength(key, "utf8") + Buffer.byteLength(value, "utf8"); - } - return bytes; - }; - - // The receive-side request-body reassembler. It reassembles each request body - // from the `body_chunk` frames, on the memory path at or below the spill - // threshold and on the spill path above it, so the broker never holds a whole - // large request body in memory. It owns a per-channel spill directory and the - // spill caps. - const receiver = await DuplexBodyReceiver.create(options.bodyReceiverConfig ?? {}); - let receiverFinalized = false; - const finalizeReceiver = (): Promise => { - // Remove the spill directory and every in-flight body once. The broker calls - // it on every terminal path: a loss, a normal teardown, and an orderly close. - if (receiverFinalized) return Promise.resolve(); - receiverFinalized = true; - return receiver.destroy().catch(() => undefined); - }; - // The fixed raw slice size for the response `body_chunk` frames. It matches the - // reassembler config, so a test that lowers the slice size splits both a request - // body and a response body the same way. - const rawChunkBytes = options.bodyReceiverConfig?.rawChunkBytes ?? DUPLEX_BODY_CHUNK_RAW_BYTES; - // The ids the broker is reassembling right now. A `body_chunk` for an id in this - // set routes to the reassembler. The broker removes an id when the body settles. - const reassembling = new Set(); - // The ids the broker refused at the envelope, with the raw bytes it still must - // drain. The sender emits the `body_chunk` frames for a refused request before - // it learns of the refusal, so the broker drains and drops those chunks instead - // of treating them as a body_chunk with no envelope. - const draining = new Map(); - - let state: DuplexBrokerState = "opening"; - let stopped = false; - let started = false; - let closePromise: Promise | null = null; - let lossRecord: DuplexBrokerLossRecord | null = null; - let heartbeatTimer: ReturnType | null = null; - - // The host-owned lifecycle sequence. The broker assigns each terminal lifecycle - // event a strictly increasing sequence number at ingress, before any - // asynchronous logging. The order of these numbers, not a wall-clock or a - // provider timestamp, decides the run disposition. - let lifecycleSeq = 0; - // The sequence number of the terminal loss, or `null` when no loss ordered. The - // broker sets it one time. A later event never clears it, so the loss latches. - let lossSeq: number | null = null; - // The typed, closed loss reason for the latched loss, or `null` on a success. - let typedLossReason: DuplexLossReason | null = null; - // The sequence number of the host-observed orderly completion, or `null` when - // none ordered. The broker sets it one time, only while the channel is healthy. - let orderlyCompletionSeq: number | null = null; - const nextLifecycleSeq = (): number => { - lifecycleSeq += 1; - return lifecycleSeq; - }; - - // Mark the host-observed orderly completion of the agent turn. The broker sets - // the sequence one time, and only while no loss has ordered. A loss that already - // latched keeps the failure, so a late completion never clears the latch. - const markOrderlyCompletion = (): void => { - if (orderlyCompletionSeq !== null || lossSeq !== null) return; - orderlyCompletionSeq = nextLifecycleSeq(); - }; - // Atomically read the run disposition and mark the host-observed orderly - // completion. The mark and the read run in one synchronous step, so no caller - // can insert an `await` between them and no teardown loss can slip in. The mark - // no-ops once a loss latched, so a real mid-run loss keeps the failure. - const settleRunDisposition = (): DuplexBrokerRunDisposition => { - markOrderlyCompletion(); - return { failed: lossSeq !== null, lossReason: typedLossReason }; - }; - // The ids the broker already dispatched. The broker forwards one id one time, - // so a repeated frame never reaches the API twice. - const seenRequestIds = new Set(); - // The aggregate-ledger tokens for the no-replay set entries. The broker holds - // one token per live set entry and releases every token one time at terminal - // teardown, so the retained set never leaves bytes charged after the channel - // ends. - const seenRequestIdTokens = new Set(); - const pending = new Map(); - // The forwards that answered the gateway but whose async work has not settled. - // The broker keeps their tokens charged and their controller live until each - // forward finally releases its tokens, so the ledger reports the real ownership. - const orphanedForwards = new Map(); - - // Release every no-replay set-entry token one time and clear the set. The - // ledger release is one way, and the cleared set stops any second release, so - // this helper is safe to call more than one time at terminal teardown. - const releaseSeenRequestIdTokens = (): void => { - for (const token of seenRequestIdTokens) releaseToken(token); - seenRequestIdTokens.clear(); - }; - - const setState = (next: DuplexBrokerState): void => { - if (state === next) return; - state = next; - options.onStateChange?.(next); - }; - - const clearHeartbeat = (): void => { - if (heartbeatTimer !== null) { - clearInterval(heartbeatTimer); - heartbeatTimer = null; - } - }; - - const clearPending = (): void => { - for (const [id, entry] of pending) { - if (entry.forwardTimer !== null) clearTimeout(entry.forwardTimer); - clearTimeout(entry.responseTimer); - entry.controller.abort(new Error("Duplex broker stopped.")); - if (entry.reassembled !== null) { - void entry.reassembled.dispose(); - entry.reassembled = null; - } - if (entry.forwardSettled) continue; - if (entry.forwardTimer !== null) { - // A live forward promise still owns its request tokens. Transfer it to the - // orphan registry, so its tokens stay charged until the forward finally - // releases them. The abort only asks the forward to stop; it never releases - // a token by itself. - orphanedForwards.set(id, { - controller: entry.controller, - releaseForwardTokens: entry.releaseForwardTokens, - }); - } else { - // The request is still reassembling, so no forward promise will settle and - // release its tokens. Release the request tokens now. The reassembler - // teardown below rejects the in-flight body and unlinks any spill file. - entry.releaseForwardTokens(); - } - } - pending.clear(); - reassembling.clear(); - draining.clear(); - // Ask every already-orphaned forward to stop as well. The forward-promise - // finally owner releases each orphan token when the forward settles. - for (const orphan of orphanedForwards.values()) { - orphan.controller.abort(new Error("Duplex broker stopped.")); - } - }; - - const recordLoss = (reason: DuplexBrokerLossReason, message: string): void => { - // Loss is terminal. Record it one time and stop every activity. - if (stopped) return; - const afterOrderlyCompletion = orderlyCompletionSeq !== null; - // A clean channel end that orders after a host-observed orderly completion is - // a normal teardown, not a loss. A process exit and a reason-less transport - // close both end the channel, so both count as the normal teardown here. Stop - // cleanly, emit no loss event, and leave the run a success. This keeps the - // closed telemetry contract: an orderly close is not a loss and emits no loss - // event. - if (afterOrderlyCompletion && (reason === "channel_exit" || reason === "transport_closed")) { - stopped = true; - clearHeartbeat(); - clearPending(); - releaseSeenRequestIdTokens(); - decoder.dispose(); - void finalizeReceiver(); - if (state !== "closing") setState("closed"); - return; - } - stopped = true; - // Classify the loss relative to the first dispatch. A loss after the broker - // dispatched a request is `post_dispatch`; a loss before any dispatch is - // `pre_dispatch`. The class rides the fixed loss counter, never the raw - // message. - const lossClass = seenRequestIds.size > 0 ? "post_dispatch" : "pre_dispatch"; - // Assign the loss its lifecycle sequence at ingress, before any logging. The - // loss latches the run as a failure only when no orderly completion ordered - // before it. A loss ordered after an orderly completion (for example a failed - // close) is a real channel loss for the telemetry and the leak metric, but it - // does not fail the run, because the run already completed. Once set, `lossSeq` - // never clears, so a later completion, exit, or activity callback cannot flip - // the latch. - const seq = nextLifecycleSeq(); - if (!afterOrderlyCompletion) { - lossSeq = seq; - typedLossReason = BROKER_LOSS_REASON_TO_TYPED[reason] ?? "other"; - } - clearHeartbeat(); - clearPending(); - releaseSeenRequestIdTokens(); - decoder.dispose(); - void finalizeReceiver(); - lossRecord = { reason, message, atMs: now() }; - setState("lost"); - // Log the internal reason only. The broker never writes the raw provider - // message to a log line, so no raw provider text rides a sink here. - options.logger?.(`Duplex broker lost the channel (${reason}).`); - options.onLoss?.(lossRecord); - options.telemetry?.recordLoss(lossClass, BROKER_LOSS_REASON_TO_TYPED[reason] ?? "other"); - }; - - const writeLine = (line: string): boolean => { - try { - // The channel carries raw bytes (see `ChannelBytesWireValue` in the plugin - // SDK's protocol.ts). Encode the frame's UTF-8 text to bytes at this one - // write seam. - channel.write(Buffer.from(line, "utf8")); - return true; - } catch (error) { - recordLoss("stream_failure", errorMessage(error)); - return false; - } - }; - - // The result of one bounded send. `sent` means every frame went out; `too_large` - // means the response envelope exceeds the bound and the broker wrote nothing; - // `lost` means a write failed and the broker recorded the channel loss. - type SendResult = "sent" | "too_large" | "lost"; - - // Send one full response as a sequence of frames: one envelope frame that - // carries `bodyByteCount`, then the `body_chunk` frames that carry the body. The - // envelope is always small. Each `body_chunk` carries one fixed raw slice, so - // each encoded chunk stays under the frame bound by construction. The result - // reports `too_large` when the envelope itself exceeds the bound and the broker - // wrote nothing, `lost` when a write failed and the broker recorded the channel - // loss, and `sent` when every frame went out. - const sendResponseFrames = ( - id: string, - status: number, - headers: Record, - bodyText: string, - outcome: DuplexResponseOutcome, - ): SendResult => { - const bodyBuffer = Buffer.from(bodyText, "utf8"); - const envelope: DuplexResponseFrame = { - version: DUPLEX_FRAME_VERSION, - type: "response", - id, - status, - headers, - bodyByteCount: bodyBuffer.length, - outcome, - }; - // Guard the envelope against the frame bound. The envelope holds no body, so a - // real bound rejects it only under an extreme small test bound. Report the - // rejection without a channel loss, so the caller decides how to answer. - const encodedEnvelope = encodeDuplexFrameChecked(envelope, maxFrameBytes); - if (!encodedEnvelope.ok) return "too_large"; - const chunkFrames = splitBodyIntoChunkFrames(id, bodyBuffer, DUPLEX_FRAME_VERSION, rawChunkBytes); - if (!writeLine(encodedEnvelope.line)) return "lost"; - for (const chunk of chunkFrames) { - const encodedChunk = encodeDuplexFrameChecked(chunk, maxFrameBytes); - if (!encodedChunk.ok) { - // A single fixed-size slice never exceeds the bound in a real configuration. - // A test bound below one chunk can reject it. The envelope already went out - // with the true `bodyByteCount`, so the broker cannot complete the body - // within the bound. Log the drop and stop; the gateway ends the request on - // its wait budget. Report `sent`, because the broker already committed the - // envelope, so no caller resends a bounded replacement over the same id. - options.logger?.("Duplex broker dropped an oversized body_chunk frame."); - return "sent"; - } - if (!writeLine(encodedChunk.line)) return "lost"; - } - return "sent"; - }; - - const sendTerminalIndeterminate = (id: string): void => { - // Answer one request the broker cannot deliver with the real result. The - // request reached the host and may have changed state, so the response is - // non-retryable and carries the `indeterminate` outcome. The broker tries the - // full replacement first. When the frame bound rejects even the full - // replacement envelope, the broker sends a minimal replacement that carries - // only the `indeterminate` outcome, so the gateway still ends the request and - // never waits for its full wait budget. When the bound rejects even the - // minimal replacement envelope, the broker logs a clear local error and keeps - // the channel open; it records no channel loss. - const fullBody = JSON.stringify({ - error: "upstream response too large to deliver", - outcome: "indeterminate", - retryable: false, - }); - if ( - sendResponseFrames( - id, - 502, - { "content-type": "application/json", "x-paperclip-bridge-outcome": "indeterminate" }, - fullBody, - "indeterminate", - ) !== "too_large" - ) { - return; - } - // The bound rejects the full replacement envelope. Send a minimal terminal - // response with empty headers and an empty body. The `indeterminate` outcome - // still rides the envelope, so the gateway maps the request to a terminal 409. - if (sendResponseFrames(id, 502, {}, "", "indeterminate") !== "too_large") return; - // The bound rejects even the minimal terminal envelope. The broker cannot - // deliver any frame for this request within the bound. Log a clear local - // error and keep the channel open for every other request. The gateway ends - // its own outstanding request on its wait budget. - options.logger?.( - `Duplex broker could not deliver a terminal response within the ${maxFrameBytes}-byte frame bound.`, - ); - }; - - const respond = ( - id: string, - result: DuplexBrokerForwardResult, - outcome: DuplexResponseOutcome, - telemetryOutcome: DuplexOutcomeValue, - ): void => { - const entry = pending.get(id); - if (!entry || entry.responded) return; - entry.responded = true; - settlePendingBookkeeping(id, entry); - // Do not write on a lost or closed channel. The gateway answers its own - // outstanding request on loss, so a late write would go to a dead channel. - if (state !== "open") return; - const bodyText = result.body ?? ""; - const bodyByteCount = Buffer.byteLength(bodyText, "utf8"); - // The response body rides `body_chunk` frames, so a large body no longer makes - // one frame too large. The gateway reassembles a response body in memory, - // though, so a body over the frame bound would force the gateway to hold an - // oversized body in memory. Treat a response body over the frame bound like the - // former oversized case: send a bounded, non-retryable indeterminate terminal - // response, and record the request outcome as an error, never a loss. - if (bodyByteCount > maxFrameBytes) { - options.telemetry?.recordRequest({ - latencyMs: now() - entry.dispatchStartMs, - outcome: "error", - }); - sendTerminalIndeterminate(id); - return; - } - // Record the request span for the delivered request. The span carries the - // latency and the outcome only; no route, query, body, or token rides it. The - // broker records the span before the writes, so the outcome is recorded even if - // a later chunk write records a channel loss. - options.telemetry?.recordRequest({ - latencyMs: now() - entry.dispatchStartMs, - outcome: telemetryOutcome, - }); - sendResponseFrames(id, result.status, result.headers ?? {}, bodyText, outcome); - }; - - const respondSaturated = (id: string, retryable: boolean): void => { - // Answer a refused request with a bounded terminal response. The broker made - // no controller, no timer, and no forward for this id, so the host API stays - // untouched. The response carries no route, no query, no body, and no token; - // it holds only the fixed error shape. The `unavailable` outcome tells the - // gateway this is not a delivered host response, so it never counts as one. - if (state !== "open") return; - sendResponseFrames( - id, - 503, - { "content-type": "application/json", "x-paperclip-bridge-outcome": "unavailable" }, - JSON.stringify({ - error: "Duplex broker capacity limit reached.", - outcome: "unavailable", - retryable, - }), - "unavailable", - ); - }; - - // Release the timers and route the reassembly and ledger resources for one - // request the broker just answered. The broker calls it exactly once per - // request, from the response path. It clears both timers and drops the pending - // record, then it settles the resources by the request phase: - // - A live forward promise (the response backstop answered while the forward - // ran) keeps its request tokens and its reassembled body until its finally - // owner releases and disposes them, so the orphan registry holds it. - // - A request still reassembling (the backstop answered before the forward - // started) has no forward promise to settle, so the broker releases its - // tokens and disposes the in-flight body now. - // - A settled forward already released and disposed through its finally owner. - const settlePendingBookkeeping = (id: string, entry: PendingRequest): void => { - if (entry.forwardTimer !== null) clearTimeout(entry.forwardTimer); - clearTimeout(entry.responseTimer); - pending.delete(id); - if (entry.forwardSettled) return; - if (entry.forwardTimer !== null) { - // A live forward still owns its request tokens and streams the reassembled - // body. Keep both charged until the forward finally owner settles them. - orphanedForwards.set(id, { - controller: entry.controller, - releaseForwardTokens: entry.releaseForwardTokens, - }); - return; - } - // The request is still reassembling, so no forward promise will settle. Release - // the request tokens and dispose the in-flight body, so no spill file, arena - // reservation, or ledger token lingers. - entry.releaseForwardTokens(); - if (reassembling.delete(id)) void receiver.disposeBody(id); - if (entry.reassembled !== null) { - void entry.reassembled.dispose(); - entry.reassembled = null; - } - }; - - // Mark a refused or duplicate request id for chunk draining. The sender emits - // the `body_chunk` frames for the request before it learns of the refusal, so - // the broker records the raw byte count it must drain and drop. A request with - // no body needs no draining, so the broker records only a non-zero count. - const markDrain = (frame: DuplexRequestFrame): void => { - if (frame.bodyByteCount > 0) draining.set(frame.id, frame.bodyByteCount); - }; - - const respondAggregateExceeded = (id: string): void => { - // Answer a request the aggregate byte ledger refused with a bounded terminal - // response. The broker reserved nothing, added no id to the seen set, and made - // no controller, timer, or forward, so the host API stays untouched. The - // response carries the fixed marker only; it holds no route, query, body, or - // token. The `unavailable` outcome tells the gateway this is not a delivered - // host response. The refusal is retryable, because the broker did not retain - // the id: the aggregate pressure can ease, and a resend can then get through. - if (state !== "open") return; - sendResponseFrames( - id, - 503, - { "content-type": "application/json", "x-paperclip-bridge-outcome": "unavailable" }, - JSON.stringify({ - error: DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, - outcome: "unavailable", - retryable: true, - }), - "unavailable", - ); - }; - - const dispatch = (frame: DuplexRequestFrame): void => { - // Dispatch only while open. After loss or close the broker forwards nothing. - if (state !== "open") return; - // Bound the id byte size before any retention or work. The codec already - // rejects an over-limit id on the read path, so this guard is defense in depth - // for a frame that reaches dispatch by another path. The broker never adds the - // id to the seen set, never allocates a controller or a timer, and never - // forwards. It answers with the bounded terminal refusal, which carries no - // route, query, body, or token, and records no telemetry. The refusal is not - // retryable, because a resend of the same over-limit id never gets past this - // bound. It drains the following chunks of the refused body. - if (Buffer.byteLength(frame.id, "utf8") > DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES) { - respondSaturated(frame.id, false); - markDrain(frame); - return; - } - // Forward one id one time. A repeated id never reaches the API twice. The - // broker already answered the first request, so it answers no second time; it - // drains the resent body chunks and drops them. - if (seenRequestIds.has(frame.id)) { - markDrain(frame); - return; - } - // Bound the retained request-id memory. The broker keeps one id per distinct - // dispatched request for the no-replay guarantee, so the set can only grow. - // Once the broker reaches the lifetime limit, it refuses each new distinct id - // and forwards nothing more. The refusal is not retryable, because a resend - // never gets past the limit. This check runs before the id joins the set, so - // the set never grows past the limit. - if (seenRequestIds.size >= limits.maxLifetimeRequests) { - respondSaturated(frame.id, false); - markDrain(frame); - return; - } - // Bound the in-flight request count. Once the broker holds the maximum number - // of pending forwards, it refuses a further request and forwards nothing for - // it. The broker does not add the id to the seen set, so the gateway can - // resend the request after an in-flight request completes. The refusal is - // retryable for that reason. This check bounds the controllers, the timers, - // the concurrent reassemblies, and the concurrent forwards a provider can - // force. - if (pending.size >= limits.maxInFlightRequests) { - respondSaturated(frame.id, true); - markDrain(frame); - return; - } - // Reserve the exact retained bytes against the aggregate ledger before the - // broker retains anything. It reserves three tokens in order: the raw request - // frame, the normalized request payload, and the no-replay set entry. A - // reservation that would pass the ceiling makes the broker retain nothing, - // release the tokens it already took, and refuse the request with the fixed - // marker. The broker adds no id to the seen set and makes no controller, timer, - // or forward, so the host API stays untouched. When the ledger is absent all - // tokens stay `null` and the broker behaves as before. - let requestFrameToken: ReservationToken | null = null; - let requestPayloadToken: ReservationToken | null = null; - let seenRequestIdToken: ReservationToken | null = null; - if (ledger) { - const rawFrameBytes = Buffer.byteLength(encodeDuplexFrame(frame), "utf8"); - requestFrameToken = ledger.reserve("request_frame", rawFrameBytes); - if (!requestFrameToken) { - respondAggregateExceeded(frame.id); - markDrain(frame); - return; - } - requestPayloadToken = ledger.reserve("request_payload", requestPayloadBytes(frame)); - if (!requestPayloadToken) { - ledger.release(requestFrameToken); - respondAggregateExceeded(frame.id); - markDrain(frame); - return; - } - const seenEntryBytes = - Buffer.byteLength(frame.id, "utf8") + DUPLEX_SEEN_REQUEST_ID_SET_ENTRY_BYTES; - seenRequestIdToken = ledger.reserve("seen_request_id", seenEntryBytes); - if (!seenRequestIdToken) { - ledger.release(requestFrameToken); - ledger.release(requestPayloadToken); - respondAggregateExceeded(frame.id); - markDrain(frame); - return; - } - } - seenRequestIds.add(frame.id); - if (seenRequestIdToken) seenRequestIdTokens.add(seenRequestIdToken); - - // Release the request-frame and the request-payload tokens exactly one time. - // The single forward-promise finally owner calls it after the forward settles. - // The seen-id token stays charged for the channel lifetime, so it is not part - // of this release; the terminal teardown releases it. - let forwardTokensReleased = false; - const releaseForwardTokens = (): void => { - if (forwardTokensReleased) return; - forwardTokensReleased = true; - releaseToken(requestFrameToken); - releaseToken(requestPayloadToken); - }; - - const record: DuplexBrokerRequestRecord = { - id: frame.id, - method: frame.method, - path: frame.path, - dispatchStartMs: now(), - }; - options.onRequestRecord?.(record); - - const controller = new AbortController(); - // Start the response-budget backstop at the request envelope, so it bounds the - // whole request: the body reassembly plus the forward. The forward-budget timer - // starts later, when the broker starts the forward, so it bounds the forward - // call alone. - const responseTimer = setTimeout(() => respondBackstop(frame.id), budgets.responseBudgetMs); - responseTimer.unref?.(); - const entry: PendingRequest = { - controller, - responded: false, - forwardTimer: null, - responseTimer, - dispatchStartMs: record.dispatchStartMs, - method: frame.method, - reassembled: null, - forwardSettled: false, - releaseForwardTokens, - }; - pending.set(frame.id, entry); - // Route the following `body_chunk` frames for this id to the reassembler. - reassembling.add(frame.id); - // Reassemble the request body from the `body_chunk` frames, then forward it. - // The reassembler chooses the memory path or the spill path from the envelope - // `bodyByteCount`, so the broker never holds a whole large request body in - // memory. A reassembly error is a terminal protocol failure. - receiver.begin(frame.id, frame.bodyByteCount).then( - (body) => onBodyReady(frame, entry, body), - (error) => onBodyError(frame.id, entry, error), - ); - }; - - const respondBackstop = (id: string): void => { - // Response-budget backstop. The forward rejection normally answers first, well - // before this deadline. This backstop answers a request whose body never - // completes its reassembly, or whose forward rejection handling itself stalls, - // so the request never strands. One stall path is a response whose headers - // arrive but whose body reader stays pending through the budget, so the forward - // promise never settles. - const entry = pending.get(id); - if (!entry || entry.responded) return; - if (isSafeBridgeMethod(entry.method)) { - // A safe method never changes host state, so a stalled body reader cannot - // leave a mutation half-applied. Keep the request retryable: return a 504 - // with the completed outcome and no indeterminate marker, so the gateway - // passes it through as a retryable status and never maps it to a terminal 409. - respond( - id, - { - status: 504, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ - error: "Duplex broker response budget exceeded.", - retryable: true, - }), - }, - "completed", - "error", - ); - return; - } - // The method may mutate host state, and the broker cannot prove the host - // applied no mutation once the response budget passes. Return a non-retryable - // 504 and mark the outcome indeterminate, so the gateway maps it to a terminal - // 409 and no caller double-applies the mutation. - respond( - id, - { - status: 504, - headers: { - "content-type": "application/json", - "x-paperclip-bridge-outcome": "indeterminate", - }, - body: JSON.stringify({ - error: "Duplex broker response budget exceeded.", - outcome: "indeterminate", - retryable: false, - }), - }, - "indeterminate", - "error", - ); - }; - - // The request body reassembled. Start the forward now, so the forward budget - // bounds the forward call alone. The broker passes the reassembled body to the - // forward handler, so the handler streams it to the host API path. - const onBodyReady = ( - frame: DuplexRequestFrame, - entry: PendingRequest, - body: ReassembledBody, - ): void => { - reassembling.delete(frame.id); - if (stopped || state !== "open" || entry.responded) { - // The channel died, or the response backstop already answered, so no forward - // runs. Dispose the body and release the request tokens, so no spill file, - // arena reservation, or ledger token lingers, and drop the request from the - // pending and orphan maps. - void body.dispose(); - pending.delete(frame.id); - orphanedForwards.delete(frame.id); - entry.releaseForwardTokens(); - return; - } - entry.reassembled = body; - const forwardTimer = setTimeout(() => { - entry.controller.abort(new Error("Duplex broker forward budget exceeded.")); - }, budgets.forwardTimeoutMs); - forwardTimer.unref?.(); - entry.forwardTimer = forwardTimer; - - // The forward promise has one cleanup owner. Both settle handlers mark the - // forward settled and answer the gateway. The `finally` then releases the - // request tokens exactly one time, disposes the reassembled body, and removes - // the request from the pending map and the orphan map. - const markForwardSettled = (): void => { - entry.forwardSettled = true; - }; - forwardRequest(frame, { signal: entry.controller.signal, body }) - .then( - (result) => { - markForwardSettled(); - // Keep the outcome classification consistent with the file path. A - // possibly-committed mutation carries the indeterminate marker header, so - // map it to the indeterminate outcome. Any other result is completed. - const outcome: DuplexResponseOutcome = - result.headers?.["x-paperclip-bridge-outcome"] === "indeterminate" - ? "indeterminate" - : "completed"; - // The host delivered a real response, so the request span outcome is `ok`. - // A host application status (200, a 4xx, a 5xx) is still a delivered - // response; only a broker-synthesized failure below is `error`. - respond(frame.id, result, outcome, "ok"); - }, - (error) => { - markForwardSettled(); - if (entry.controller.signal.aborted) { - // The forward budget aborted the call. A safe method never changes - // host state, so a forward timeout stays retryable for it. Return a - // 504 with the completed outcome and no indeterminate marker, so the - // gateway passes it through as a retryable status. - if (isSafeBridgeMethod(frame.method)) { - respond( - frame.id, - { - status: 504, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ - error: errorMessage(error), - retryable: true, - }), - }, - "completed", - "error", - ); - return; - } - // The method may mutate host state, and the forward budget aborted the - // call after the request may have committed. Return a non-retryable 504 - // and mark the outcome indeterminate, so a caller does not retry a - // committed mutation. - respond( - frame.id, - { - status: 504, - headers: { - "content-type": "application/json", - "x-paperclip-bridge-outcome": "indeterminate", - }, - body: JSON.stringify({ - error: errorMessage(error), - outcome: "indeterminate", - retryable: false, - }), - }, - "indeterminate", - "error", - ); - return; - } - // The forward rejected before the host delivered a response. This - // rejection does not prove that the host applied no mutation. A fetch - // can reject after the request bytes reach the host and the host - // commits, but before the response headers arrive. A safe method never - // changes host state, so a retry stays safe for it. For any other - // method the host may have committed, so the outcome is indeterminate. - if (isSafeBridgeMethod(frame.method)) { - // The method is safe, so a retry cannot double-apply a mutation. - // Return a 502 with the completed outcome, so the gateway passes it - // through as a retryable status. - respond( - frame.id, - { - status: 502, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ error: errorMessage(error) }), - }, - "completed", - "error", - ); - return; - } - // The method may mutate host state, so a retry with a new request id - // could apply the mutation twice. Return a non-retryable 504 and mark - // the outcome indeterminate, so the gateway maps it to a terminal 409. - respond( - frame.id, - { - status: 504, - headers: { - "content-type": "application/json", - "x-paperclip-bridge-outcome": "indeterminate", - }, - body: JSON.stringify({ - error: errorMessage(error), - outcome: "indeterminate", - retryable: false, - }), - }, - "indeterminate", - "error", - ); - }, - ) - .finally(() => { - // The single forward cleanup owner. Release the request tokens, dispose the - // reassembled body, and drop the request from the pending and orphan maps, - // each exactly one time. - pending.delete(frame.id); - orphanedForwards.delete(frame.id); - entry.releaseForwardTokens(); - if (entry.reassembled !== null) { - void entry.reassembled.dispose(); - entry.reassembled = null; - } - }); - }; - - // The request body reassembly failed. A malformed chunk, a size mismatch, an - // overrun, or a truncation is a terminal protocol failure, so the broker fails - // the channel closed. A `channel_closed` error is the broker's own teardown of an - // in-flight body, so it is not a fresh loss. - const onBodyError = (id: string, entry: PendingRequest, error: unknown): void => { - reassembling.delete(id); - // The forward never ran, so release its request tokens and drop the request - // from the pending and orphan maps. The release is idempotent, so a teardown - // that already released the tokens keeps the count correct. - pending.delete(id); - orphanedForwards.delete(id); - entry.releaseForwardTokens(); - if (stopped) return; - if (error instanceof DuplexBodyError && error.code === "channel_closed") return; - recordLoss("protocol_failure", errorMessage(error)); - }; - - // Route one `body_chunk` frame. A chunk for an id under reassembly goes to the - // reassembler, which validates it and appends it. A chunk for a refused or - // duplicate id drains against its recorded byte count and drops. A chunk with no - // matching envelope is a terminal protocol failure. - const handleBodyChunk = (frame: DuplexBodyChunkFrame): void => { - if (reassembling.has(frame.id)) { - const result = receiver.pushChunk(frame); - if (!result.ok) recordLoss("protocol_failure", result.error.message); - return; - } - const remaining = draining.get(frame.id); - if (remaining !== undefined) { - // Drain and drop a chunk of a refused body. The broker does not validate a - // drained chunk; it only tracks the byte progress, so it stops draining once - // the refused body ends. A further chunk after that is a chunk with no - // envelope. - const next = remaining - Buffer.byteLength(frame.data, "base64"); - if (next > 0) draining.set(frame.id, next); - else draining.delete(frame.id); - return; - } - recordLoss("protocol_failure", "body_chunk arrived with no matching envelope"); - }; - - const handleFrame = (frame: DuplexFrame): void => { - switch (frame.type) { - case "request": - dispatch(frame); - return; - case "body_chunk": - handleBodyChunk(frame); - return; - case "close": - // The gateway asked for an orderly close. The gateway sends this frame on - // the agent's orderly completion, so mark the completion on the ordered - // lifecycle before the close, then close the channel. - markOrderlyCompletion(); - void close(); - return; - case "ready": - case "heartbeat": - // Liveness frames. The broker reads them and dispatches nothing. - return; - case "response": - case "error": - // The host never expects these on the read path. Ignore them. - return; - default: - return; - } - }; - - const onData = (chunk: Uint8Array): void => { - if (stopped) return; - const results = decoder.push(chunk); - for (const result of results) { - if (stopped) return; - if (!result.ok) { - recordLoss("protocol_failure", result.error.message); - return; - } - handleFrame(result.frame); - } - }; - - const onExit = (exit: { exitCode: number | null; transportClosed?: boolean }): void => { - // A reason-less transport close is not a process exit. Record it as a distinct - // loss, so a transport close stays legible in the loss taxonomy. A real process - // exit stays `channel_exit` -> `provider_exit`. - if (exit.transportClosed === true) { - recordLoss("transport_closed", "The sandbox channel transport closed with no exit."); - return; - } - recordLoss("channel_exit", "The sandbox channel process exited."); - }; - - const sendHeartbeat = (): void => { - if (state !== "open") return; - try { - channel.write(Buffer.from(encodeDuplexFrame({ version: DUPLEX_FRAME_VERSION, type: "heartbeat" }), "utf8")); - } catch (error) { - recordLoss("heartbeat_write_failure", errorMessage(error)); - } - }; - - const close = (): Promise => { - if (closePromise) return closePromise; - if (stopped) return Promise.resolve(); - // A host-initiated orderly close is a host-observed orderly completion. The - // host tears the channel down on its own terms, so a channel end during the - // close is a normal teardown, not a mid-run loss. `markOrderlyCompletion` - // no-ops when a loss already latched, so a lost channel stays a failure. - markOrderlyCompletion(); - closePromise = (async () => { - setState("closing"); - clearHeartbeat(); - clearPending(); - releaseSeenRequestIdTokens(); - decoder.dispose(); - // Send an orderly close frame. Ignore a write failure here; the broker is - // already closing, so a dead channel needs no loss record. - try { - channel.write(Buffer.from(encodeDuplexFrame({ version: DUPLEX_FRAME_VERSION, type: "close" }), "utf8")); - } catch (error) { - options.logger?.(`Duplex broker could not send the close frame: ${errorMessage(error)}`); - } - let closeTimer: ReturnType | undefined; - const timeout = new Promise((_resolve, reject) => { - closeTimer = setTimeout(() => { - reject(new Error("Duplex broker channel close timed out.")); - }, closeTimeoutMs); - closeTimer.unref?.(); - }); - try { - await Promise.race([channel.close(), timeout]); - if (closeTimer !== undefined) clearTimeout(closeTimer); - stopped = true; - setState("closed"); - } catch (error) { - if (closeTimer !== undefined) clearTimeout(closeTimer); - recordLoss("close_timeout", errorMessage(error)); - } - // Remove the spill directory and every in-flight body. The broker owns the - // reassembler, so the orderly close cleans it up. - await finalizeReceiver(); - })(); - return closePromise; - }; - - const start = (): void => { - if (started) return; - started = true; - channel.onData(onData); - channel.onExit(onExit); - heartbeatTimer = setInterval(sendHeartbeat, heartbeatIntervalMs); - heartbeatTimer.unref?.(); - setState("open"); - }; - - const stop = (): void => { - try { - channel.stop(); - } catch (error) { - options.logger?.(`Duplex broker could not stop the channel: ${errorMessage(error)}`); - } - }; - - return { - get state() { - return state; - }, - get lossRecord() { - return lossRecord; - }, - get runDisposition(): DuplexBrokerRunDisposition { - // A latched loss ordered before any orderly completion is a failure. Every - // other state — a healthy channel, or a loss ordered after an orderly - // completion — is a success. - return { failed: lossSeq !== null, lossReason: typedLossReason }; - }, - markOrderlyCompletion, - settleRunDisposition, - start, - close, - stop, - }; -} diff --git a/packages/adapter-utils/src/duplex-bridge-local-e2e.test.ts b/packages/adapter-utils/src/duplex-bridge-local-e2e.test.ts deleted file mode 100644 index 91f8755c0d..0000000000 --- a/packages/adapter-utils/src/duplex-bridge-local-e2e.test.ts +++ /dev/null @@ -1,734 +0,0 @@ -import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; -import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; -import net from "node:net"; -import { mkdtemp, rm, writeFile } from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; -import { Readable } from "node:stream"; -import { afterEach, describe, expect, it } from "vitest"; - -import { - authorizeSandboxCallbackBridgeRequestWithRoutes, - getSandboxCallbackBridgeServerSource, - SANDBOX_CALLBACK_BRIDGE_DUPLEX_MODE, -} from "./sandbox-callback-bridge.js"; -import { - DuplexFrameDecoder, - DUPLEX_BODY_CHUNK_RAW_BYTES, - DUPLEX_FRAME_VERSION, - encodeDuplexFrame, - type DuplexFrame, - type DuplexReadyFrame, - type DuplexRequestFrame, -} from "./duplex-frame-codec.js"; -import { - createDuplexBridgeBroker, - type DuplexBridgeBroker, - type DuplexBrokerForwardResult, -} from "./duplex-bridge-broker.js"; -import type { ReassembledBody } from "./duplex-body-spool.js"; -import type { CommandManagedDuplexChannel } from "./command-managed-runtime.js"; - -/** - * Local real-process end-to-end harness for the composed duplex path. - * - * The harness spawns the real generated gateway with plain `node` from a - * temporary file, attaches the real host broker to the child stdin and stdout, - * and forwards each request to a local fake API server on a real HTTP call. It - * uses no provider credentials. - * - * The child stdio pipes are kernel pipes. They give partial reads, split a - * multi-byte UTF-8 sequence across two chunks, apply backpressure, and give a - * true end of file when the child dies. The harness proves the composed path - * handles each of these real conditions. - */ - -/** One recorded call the fake API server received. */ -interface FakeApiRequest { - method: string; - url: string; - auth: string | null; - runId: string | null; - body: string; -} - -/** The responder the fake API server calls for each request. */ -type FakeApiResponder = (req: IncomingMessage, res: ServerResponse, body: string) => void; - -/** The handle for one fake API server. */ -interface FakeApiServer { - origin: string; - requests: FakeApiRequest[]; - setResponder: (responder: FakeApiResponder) => void; - /** The count of sockets the server holds open right now. */ - openSocketCount: () => number; - /** True while the server still accepts connections. */ - listening: () => boolean; - close: () => Promise; -} - -/** - * Start a local fake API server. The forward handler targets it, so one real - * HTTP call proves the composed path end to end. The server tracks each open - * socket, so a test asserts teardown leaves no open socket handle. - */ -async function startFakeApiServer(): Promise { - const requests: FakeApiRequest[] = []; - const sockets = new Set(); - let responder: FakeApiResponder = (req, res) => { - const url = new URL(req.url ?? "/", "http://127.0.0.1"); - res.writeHead(200, { "content-type": "application/json" }); - res.end(JSON.stringify({ ok: true, path: url.pathname })); - }; - - const server = createServer((req, res) => { - const chunks: Buffer[] = []; - req.on("data", (chunk: Buffer) => chunks.push(chunk)); - req.on("end", () => { - const body = Buffer.concat(chunks).toString("utf8"); - requests.push({ - method: req.method ?? "GET", - url: req.url ?? "/", - auth: req.headers.authorization ?? null, - runId: - typeof req.headers["x-paperclip-run-id"] === "string" - ? req.headers["x-paperclip-run-id"] - : null, - body, - }); - responder(req, res, body); - }); - }); - server.on("connection", (socket) => { - sockets.add(socket); - socket.on("close", () => sockets.delete(socket)); - }); - - await new Promise((resolve, reject) => { - server.once("error", reject); - server.listen(0, "127.0.0.1", () => resolve()); - }); - const address = server.address(); - if (!address || typeof address === "string") { - throw new Error("The fake API server did not expose a TCP port."); - } - - return { - origin: `http://127.0.0.1:${address.port}`, - requests, - setResponder: (next) => { - responder = next; - }, - openSocketCount: () => sockets.size, - listening: () => server.listening, - close: () => - new Promise((resolve) => { - // Destroy each open socket first. A keep-alive client socket keeps the - // server open, so `server.close` alone could stall. The destroy makes the - // close callback fire and drives the open socket count to zero. - for (const socket of sockets) socket.destroy(); - server.close(() => resolve()); - }), - }; -} - -/** The channel view over the spawned child, plus the frames the child sent host-ward. */ -interface ChildDuplexChannel { - channel: CommandManagedDuplexChannel; - observedFrames: DuplexFrame[]; - stderr: () => string; -} - -/** - * Wrap the spawned child as a {@link CommandManagedDuplexChannel}. The broker - * writes to the child stdin, reads the child stdout, and learns of the child - * exit through this channel. - * - * The host end forwards each raw stdout chunk unchanged; the channel carries - * bytes, so no decode step sits between the pipe and the broker. - */ -function attachChildDuplexChannel(child: ChildProcessWithoutNullStreams): ChildDuplexChannel { - const observed = new DuplexFrameDecoder(); - const observedFrames: DuplexFrame[] = []; - let dataListener: ((chunk: Uint8Array) => void) | null = null; - let exitListener: ((exit: { exitCode: number | null }) => void) | null = null; - let pendingBytes: Buffer = Buffer.alloc(0); - let pendingExit: { exitCode: number | null } | null = null; - let stderrText = ""; - - child.stdout.on("data", (buffer: Buffer) => { - for (const result of observed.push(buffer)) { - if (result.ok) observedFrames.push(result.frame); - } - if (buffer.length === 0) return; - if (dataListener) dataListener(buffer); - else pendingBytes = pendingBytes.length === 0 ? buffer : Buffer.concat([pendingBytes, buffer]); - }); - child.stderr.on("data", (buffer: Buffer) => { - stderrText += buffer.toString("utf8"); - }); - child.on("exit", (code) => { - const exit = { exitCode: code }; - if (exitListener) exitListener(exit); - else pendingExit = exit; - }); - // Swallow a stdin EPIPE. The broker can write one more frame while the child - // exits; the write fails and the broker records the loss on its own path. - child.stdin.on("error", () => undefined); - - const channel: CommandManagedDuplexChannel = { - write: (data) => { - child.stdin.write(data); - }, - onData: (listener) => { - dataListener = listener; - if (pendingBytes.length > 0) { - const replay = pendingBytes; - pendingBytes = Buffer.alloc(0); - listener(replay); - } - }, - onExit: (listener) => { - exitListener = listener; - if (pendingExit) { - const exit = pendingExit; - pendingExit = null; - listener(exit); - } - }, - stop: () => { - child.kill("SIGKILL"); - }, - close: () => - new Promise((resolve) => { - // Close the write side. The child stdin reaches end of file, so the - // gateway sees a real EOF. - child.stdin.end(() => resolve()); - }), - }; - - return { channel, observedFrames, stderr: () => stderrText }; -} - -/** The forward-handler mode. `proxy` calls the fake API; `hang` blocks until abort. */ -type ForwardMode = "proxy" | "hang"; - -/** The options for one harness. */ -interface HarnessOptions { - hostApiToken?: string; - runId?: string; - maxBodyBytes?: number; - lossExitGraceMs?: number; -} - -/** The full harness handle for one composed duplex path. */ -interface DuplexE2EHarness { - baseUrl: string; - bridgeToken: string; - hostApiToken: string; - runId: string; - nonce: string; - broker: DuplexBridgeBroker; - api: FakeApiServer; - child: ChildProcessWithoutNullStreams; - observedFrames: DuplexFrame[]; - forwardedRequests: DuplexRequestFrame[]; - setForwardMode: (mode: ForwardMode) => void; - stderr: () => string; - killChild: () => Promise; - waitFor: (predicate: () => boolean, message: string, timeoutMs?: number) => Promise; - teardown: () => Promise; -} - -/** Reserve one free loopback port. The host assigns it to the gateway. */ -async function reserveLoopbackPort(): Promise { - return new Promise((resolve, reject) => { - const probe = net.createServer(); - probe.once("error", reject); - probe.listen(0, "127.0.0.1", () => { - const address = probe.address(); - if (!address || typeof address === "string") { - probe.close(() => reject(new Error("Could not reserve a loopback port."))); - return; - } - const reserved = address.port; - probe.close(() => resolve(reserved)); - }); - }); -} - -/** - * Build the composed duplex path: a spawned gateway child, the real broker on - * the child stdio, and a local fake API server as the forward target. - */ -async function createHarness(options: HarnessOptions = {}): Promise { - const hostApiToken = options.hostApiToken ?? "real-run-jwt"; - const runId = options.runId ?? "run-e2e"; - const maxBodyBytes = options.maxBodyBytes ?? 1_000_000; - const bridgeToken = "duplex-e2e-bridge-token"; - const nonce = "e2e112233445566778899aabbccddeeff"; - let forwardMode: ForwardMode = "proxy"; - const forwardedRequests: DuplexRequestFrame[] = []; - - const api = await startFakeApiServer(); - const assignedPort = await reserveLoopbackPort(); - - const tmpDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-duplex-e2e-")); - const entrypoint = path.join(tmpDir, "gateway.mjs"); - await writeFile(entrypoint, getSandboxCallbackBridgeServerSource(), "utf8"); - - const child = spawn(process.execPath, [entrypoint], { - stdio: ["pipe", "pipe", "pipe"], - env: { - ...process.env, - PAPERCLIP_API_BRIDGE_MODE: SANDBOX_CALLBACK_BRIDGE_DUPLEX_MODE, - PAPERCLIP_BRIDGE_HOST: "127.0.0.1", - PAPERCLIP_BRIDGE_PORT: String(assignedPort), - PAPERCLIP_BRIDGE_NONCE: nonce, - PAPERCLIP_BRIDGE_TOKEN: bridgeToken, - PAPERCLIP_BRIDGE_MAX_BODY_BYTES: String(maxBodyBytes), - ...(options.lossExitGraceMs != null - ? { PAPERCLIP_BRIDGE_LOSS_EXIT_GRACE_MS: String(options.lossExitGraceMs) } - : {}), - }, - }) as ChildProcessWithoutNullStreams; - - const { channel, observedFrames, stderr } = attachChildDuplexChannel(child); - - const forwardRequest = async ( - request: DuplexRequestFrame, - opts: { signal: AbortSignal; body: ReassembledBody }, - ): Promise => { - forwardedRequests.push(request); - // Keep the real route allowlist on the forward seam. The broker forwards - // only an allowed route; it denies any other route with a 403. - const denial = authorizeSandboxCallbackBridgeRequestWithRoutes(request); - if (denial) { - return { - status: 403, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ error: denial }), - }; - } - if (forwardMode === "hang") { - // Block until the broker aborts the forward. This lets a test hold an - // outstanding request open while it forces a loss. - await new Promise((_resolve, reject) => { - const onAbort = () => reject(new Error("The forward call was aborted.")); - if (opts.signal.aborted) { - onAbort(); - return; - } - opts.signal.addEventListener("abort", onAbort, { once: true }); - }); - } - const method = request.method.trim().toUpperCase() || "GET"; - const headers = new Headers(); - for (const [key, value] of Object.entries(request.headers)) { - if (value.trim().length === 0) continue; - headers.set(key, value); - } - // Apply the real host token and the run id, the same as the file bridge path. - headers.set("authorization", `Bearer ${hostApiToken}`); - headers.set("x-paperclip-run-id", runId); - const target = new URL(`${request.path}${request.query ?? ""}`, api.origin); - // Stream the reassembled request body to the host, the same as the real - // forward. A streamed body needs `duplex: "half"`. - const forwardInit: RequestInit & { duplex?: "half" } = { method, headers, signal: opts.signal }; - if (method !== "GET" && method !== "HEAD") { - forwardInit.body = Readable.toWeb( - opts.body.createReadStream(), - ) as unknown as ReadableStream; - forwardInit.duplex = "half"; - } - const response = await fetch(target, forwardInit); - const body = await response.text(); - const outHeaders: Record = {}; - response.headers.forEach((value, key) => { - if (key.toLowerCase() === "content-length") return; - outHeaders[key] = value; - }); - return { status: response.status, headers: outHeaders, body }; - }; - - const broker = await createDuplexBridgeBroker({ - channel, - forwardRequest, - logger: () => undefined, - }); - broker.start(); - - const waitFor = async ( - predicate: () => boolean, - message: string, - timeoutMs = 5000, - ): Promise => { - const deadline = Date.now() + timeoutMs; - for (;;) { - if (predicate()) return; - if (Date.now() > deadline) { - throw new Error(`Timed out waiting for ${message}. stderr: ${stderr()}`); - } - await new Promise((resolve) => { - const timer = setTimeout(resolve, 20); - timer.unref?.(); - }); - } - }; - - const waitForExit = (): Promise => - new Promise((resolve) => { - if (child.exitCode !== null || child.signalCode !== null) { - resolve(); - return; - } - child.once("exit", () => resolve()); - }); - - const killChild = async (): Promise => { - child.kill("SIGKILL"); - await waitForExit(); - }; - - let torndown = false; - const teardown = async (): Promise => { - if (torndown) return; - torndown = true; - child.kill("SIGKILL"); - await waitForExit(); - child.stdin.destroy(); - child.stdout.destroy(); - child.stderr.destroy(); - await api.close(); - await rm(tmpDir, { recursive: true, force: true }); - }; - - return { - baseUrl: `http://127.0.0.1:${assignedPort}`, - bridgeToken, - hostApiToken, - runId, - nonce, - broker, - api, - child, - observedFrames, - forwardedRequests, - setForwardMode: (mode) => { - forwardMode = mode; - }, - stderr, - killChild, - waitFor, - teardown, - }; -} - -/** - * Build a large body of multi-byte UTF-8 characters. "€" uses three UTF-8 bytes - * and "😀" uses four. A pipe read boundary at a 65536-byte multiple lands inside - * a "€" sequence, because 65536 is not a multiple of three. This guarantees a - * multi-byte character split across two pipe chunks. - */ -function buildMultiByteBody(targetBytes: number): string { - const marker = "😀-start-😀"; - const filler = "€".repeat(Math.ceil(targetBytes / 3)); - return `${marker}${filler}${marker}`; -} - -describe("duplex bridge local end-to-end harness", () => { - const harnesses: DuplexE2EHarness[] = []; - - afterEach(async () => { - while (harnesses.length > 0) { - const harness = harnesses.pop(); - if (harness) await harness.teardown(); - } - }); - - const readyPredicate = (harness: DuplexE2EHarness) => () => - harness.observedFrames.some((frame) => frame.type === "ready"); - - it("delivers a valid READY frame from the spawned gateway child to the attached broker", async () => { - const harness = await createHarness(); - harnesses.push(harness); - - await harness.waitFor(readyPredicate(harness), "a READY frame from the gateway child"); - - const ready = harness.observedFrames.find( - (frame) => frame.type === "ready", - ) as DuplexReadyFrame; - expect(ready.version).toBe(2); - expect(ready.nonce).toBe(harness.nonce); - // READY is liveness only. It carries no address data. - expect((ready as unknown as Record).address).toBeUndefined(); - expect((ready as unknown as Record).port).toBeUndefined(); - // The broker read the READY frame and stayed open with no loss. - expect(harness.broker.state).toBe("open"); - expect(harness.broker.lossRecord).toBeNull(); - }, 20000); - - it("carries one real HTTP request through the child and the broker to the fake API unchanged", async () => { - const harness = await createHarness(); - harnesses.push(harness); - await harness.waitFor(readyPredicate(harness), "the gateway to become ready"); - - harness.api.setResponder((req, res) => { - const url = new URL(req.url ?? "/", "http://127.0.0.1"); - res.writeHead(200, { "content-type": "application/json" }); - res.end(JSON.stringify({ ok: true, echoedPath: url.pathname })); - }); - - const response = await fetch(`${harness.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${harness.bridgeToken}` }, - }); - expect(response.status).toBe(200); - await expect(response.json()).resolves.toEqual({ ok: true, echoedPath: "/api/agents/me" }); - - // The fake API saw one request with the real host token and the run id. The - // broker replaced the bridge token, so the fake API never saw it. - expect(harness.api.requests).toHaveLength(1); - expect(harness.api.requests[0]).toMatchObject({ - method: "GET", - url: "/api/agents/me", - auth: `Bearer ${harness.hostApiToken}`, - runId: harness.runId, - }); - }, 20000); - - it("reassembles a large response that spans many pipe chunks", async () => { - const harness = await createHarness(); - harnesses.push(harness); - await harness.waitFor(readyPredicate(harness), "the gateway to become ready"); - - // The body is larger than the pipe buffer, so the response frame crosses the - // child stdin pipe in many chunks. An ASCII body isolates the chunk-span - // behavior from the multi-byte behavior the next test covers. - const largeBody = "x".repeat(256 * 1024); - expect(Buffer.byteLength(largeBody, "utf8")).toBeGreaterThan(200 * 1024); - harness.api.setResponder((_req, res) => { - res.writeHead(200, { "content-type": "text/plain; charset=utf-8" }); - res.end(largeBody); - }); - - const response = await fetch(`${harness.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${harness.bridgeToken}` }, - }); - expect(response.status).toBe(200); - const received = await response.text(); - // The whole body returns complete after it crossed the pipe in many chunks. - expect(received.length).toBe(largeBody.length); - expect(received).toBe(largeBody); - }, 20000); - - it("decodes a multi-byte UTF-8 sequence split across chunk borders", async () => { - const harness = await createHarness(); - harnesses.push(harness); - await harness.waitFor(readyPredicate(harness), "the gateway to become ready"); - - const multiByteBody = buildMultiByteBody(256 * 1024); - expect(Buffer.byteLength(multiByteBody, "utf8")).toBeGreaterThan(200 * 1024); - harness.api.setResponder((_req, res) => { - res.writeHead(200, { "content-type": "text/plain; charset=utf-8" }); - res.end(multiByteBody); - }); - - const response = await fetch(`${harness.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${harness.bridgeToken}` }, - }); - expect(response.status).toBe(200); - const received = await response.text(); - // A multi-byte character split across a pipe chunk border decodes to the - // same character, so the whole body returns unchanged. - expect(received.length).toBe(multiByteBody.length); - expect(received).toBe(multiByteBody); - }, 20000); - - it("matches each concurrent request through one child to its own response", async () => { - const harness = await createHarness(); - harnesses.push(harness); - await harness.waitFor(readyPredicate(harness), "the gateway to become ready"); - - harness.api.setResponder((req, res) => { - const url = new URL(req.url ?? "/", "http://127.0.0.1"); - res.writeHead(200, { "content-type": "application/json" }); - res.end(JSON.stringify({ path: url.pathname })); - }); - - const ids = Array.from({ length: 8 }, (_unused, index) => `issue-${index}`); - const responses = await Promise.all( - ids.map((id) => - fetch(`${harness.baseUrl}/api/issues/${id}`, { - headers: { authorization: `Bearer ${harness.bridgeToken}` }, - }).then(async (response) => ({ status: response.status, body: await response.json() })), - ), - ); - - responses.forEach((result, index) => { - expect(result.status).toBe(200); - expect(result.body).toEqual({ path: `/api/issues/${ids[index]}` }); - }); - expect(harness.api.requests).toHaveLength(8); - }, 20000); - - it("answers 409 to outstanding and 503 to new requests when the broker closes its write side", async () => { - const harness = await createHarness({ lossExitGraceMs: 3000 }); - harnesses.push(harness); - await harness.waitFor(readyPredicate(harness), "the gateway to become ready"); - - // Hold the forward open, so the broker sends no response frame. The gateway - // keeps the request outstanding until the loss. - harness.setForwardMode("hang"); - const outstanding = fetch(`${harness.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${harness.bridgeToken}` }, - }); - void outstanding.catch(() => undefined); - await harness.waitFor( - () => harness.forwardedRequests.length >= 1, - "the broker to receive the forwarded request", - ); - - // The broker closes its write side, so the child stdin reaches a real EOF. - await harness.broker.close(); - - const lossResponse = await outstanding; - expect(lossResponse.status).toBe(409); - expect(lossResponse.headers.get("x-paperclip-bridge-outcome")).toBe("indeterminate"); - await expect(lossResponse.json()).resolves.toEqual({ error: "outcome_indeterminate" }); - - const afterLoss = await fetch(`${harness.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${harness.bridgeToken}` }, - }); - expect(afterLoss.status).toBe(503); - await expect(afterLoss.json()).resolves.toEqual({ error: "bridge_unavailable" }); - }, 20000); - - it("moves the broker to lost on a child kill, dispatches nothing more, and leaks no handle after teardown", async () => { - const harness = await createHarness(); - harnesses.push(harness); - await harness.waitFor(readyPredicate(harness), "the gateway to become ready"); - - // Hold one forward open, so a request is in flight when the child dies. - harness.setForwardMode("hang"); - const outstanding = fetch(`${harness.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${harness.bridgeToken}` }, - }); - void outstanding.catch(() => undefined); - await harness.waitFor( - () => harness.forwardedRequests.length >= 1, - "the broker to receive the forwarded request", - ); - const forwardedBeforeKill = harness.forwardedRequests.length; - - // A real process kill closes the stdout pipe and ends the child. - await harness.killChild(); - await harness.waitFor( - () => harness.broker.state === "lost", - "the broker to record the channel loss", - ); - expect(harness.broker.lossRecord?.reason).toBe("channel_exit"); - - // The broker dispatches nothing more after the loss. - await new Promise((resolve) => { - const timer = setTimeout(resolve, 100); - timer.unref?.(); - }); - expect(harness.forwardedRequests.length).toBe(forwardedBeforeKill); - // The outstanding fetch fails because the child died before it answered. - await expect(outstanding).rejects.toThrow(); - - await harness.teardown(); - // Teardown left no open pipe or socket handle. - expect(harness.child.stdin.destroyed).toBe(true); - expect(harness.child.stdout.destroyed).toBe(true); - expect(harness.child.stderr.destroyed).toBe(true); - expect(harness.api.listening()).toBe(false); - expect(harness.api.openSocketCount()).toBe(0); - }, 20000); - - it("fails a request with a local 502 when the host sends a malformed response chunk", async () => { - const RAW = DUPLEX_BODY_CHUNK_RAW_BYTES; - // Each case declares one malformed body_chunk a broken or hostile host could - // send. The gateway must reject it at once with a local 502. It must not grow - // its response reassembly buffer until the response timeout. `bodyByteCount` - // sets the declared body size on the response envelope. `data` is the base64 - // payload of the one injected chunk. - const cases: Array<{ name: string; bodyByteCount: number; data: string; error: string }> = [ - { name: "an empty chunk", bodyByteCount: RAW, data: "", error: "duplex response body_chunk is empty" }, - { - name: "an undersized non-final chunk", - bodyByteCount: RAW + 8, - data: Buffer.alloc(4, 1).toString("base64"), - error: "duplex response body_chunk has the wrong size", - }, - { - name: "an oversized chunk", - bodyByteCount: RAW + 8, - data: Buffer.alloc(RAW + 4, 1).toString("base64"), - error: "duplex response body_chunk has the wrong size", - }, - { - name: "an overrun past the declared size", - bodyByteCount: 10, - data: Buffer.alloc(200, 1).toString("base64"), - error: "duplex response body overruns the declared size", - }, - { - name: "a non-canonical base64 chunk", - bodyByteCount: 10, - data: "AB==", - error: "duplex response body_chunk is not canonical base64", - }, - ]; - - const harness = await createHarness(); - harnesses.push(harness); - await harness.waitFor(readyPredicate(harness), "the gateway to become ready"); - - // Hold the broker forward open, so only the injected frames answer each - // request. The gateway reassembles the response body itself, so a raw - // malformed chunk on its stdin drives the reject path under test. - harness.setForwardMode("hang"); - - for (const testCase of cases) { - const forwardedBefore = harness.forwardedRequests.length; - const pendingFetch = fetch(`${harness.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${harness.bridgeToken}` }, - }); - void pendingFetch.catch(() => undefined); - await harness.waitFor( - () => harness.forwardedRequests.length > forwardedBefore, - `the broker to receive the forwarded request for ${testCase.name}`, - ); - const id = harness.forwardedRequests[harness.forwardedRequests.length - 1].id; - - // Inject the response envelope, then the one malformed body_chunk, straight - // to the gateway stdin. This bypasses the broker response encoder, so the - // gateway sees the exact bytes a broken or hostile host could send. - harness.child.stdin.write( - encodeDuplexFrame({ - version: DUPLEX_FRAME_VERSION, - type: "response", - id, - status: 200, - headers: {}, - bodyByteCount: testCase.bodyByteCount, - outcome: "completed", - }), - ); - harness.child.stdin.write( - encodeDuplexFrame({ - version: DUPLEX_FRAME_VERSION, - type: "body_chunk", - id, - seq: 0, - data: testCase.data, - }), - ); - - const response = await pendingFetch; - expect(response.status).toBe(502); - await expect(response.json()).resolves.toEqual({ error: testCase.error }); - } - }, 20000); -}); diff --git a/packages/adapter-utils/src/duplex-frame-codec.ts b/packages/adapter-utils/src/duplex-frame-codec.ts index 0938603139..0a658a440d 100644 --- a/packages/adapter-utils/src/duplex-frame-codec.ts +++ b/packages/adapter-utils/src/duplex-frame-codec.ts @@ -19,9 +19,9 @@ * HTTP/2 is the preferred transport. `queue_v1` is the soft-deprecated fallback. * The `http2_v1` host readiness gate imports {@link decodeDuplexLine} from this * file to read the one READY line every gateway sends, so this file's READY - * frame path stays live for both transports. `duplex-bridge-broker.ts` and - * `duplex-body-spool.ts` still import the request, response, and body-chunk - * frame types from this file, so this phase keeps every frame type here. + * frame path stays live for both transports. `duplex-body-spool.ts` still + * imports the request, response, and body-chunk frame types from this file, so + * this file keeps every frame type here. */ import { diff --git a/packages/adapter-utils/src/execution-target-sandbox.test.ts b/packages/adapter-utils/src/execution-target-sandbox.test.ts index d7aaaa2ba7..e89a07d8c7 100644 --- a/packages/adapter-utils/src/execution-target-sandbox.test.ts +++ b/packages/adapter-utils/src/execution-target-sandbox.test.ts @@ -10,15 +10,11 @@ import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { - getSandboxCallbackBridgeServerSource, - getSandboxDuplexGatewayCodecSource, -} from "./sandbox-callback-bridge.js"; +import { getSandboxDuplexGatewayCodecSource } from "./sandbox-callback-bridge.js"; import { __duplexReadinessTesting, __http2PrefaceScanTesting, - buildDuplexGatewayLaunchArgv, DEFAULT_REMOTE_SANDBOX_ADAPTER_TIMEOUT_SEC, adapterExecutionTargetDuplexObservabilityRecorder, adapterExecutionTargetEnablesSandboxDuplexBridge, @@ -62,21 +58,10 @@ import { DUPLEX_FRAME_VERSION, decodeDuplexLine, encodeDuplexFrame, - type DuplexRequestFrame, type DuplexResponseFrame, } from "./duplex-frame-codec.js"; -import { splitBodyIntoChunkFrames } from "./duplex-body-spool.js"; +import { DUPLEX_CHANNEL_LOST_ERROR_CODE } from "./bridge-transport-contract.js"; import { - assertNestedDuplexBrokerBudgets, - createDuplexBridgeBroker, - DUPLEX_CHANNEL_LOST_ERROR_CODE, - type DuplexBrokerForwardResult, - type DuplexBrokerLossRecord, - type DuplexBrokerRequestRecord, - type DuplexBrokerState, -} from "./duplex-bridge-broker.js"; -import { - createDuplexObservability, DUPLEX_AGGREGATE_BYTE_LEDGER_METRIC_NAMES, DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL, DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL, @@ -88,6 +73,7 @@ import { DUPLEX_SPAN_CHANNEL_OPEN, DUPLEX_SPAN_REQUEST, DUPLEX_TRANSPORT_EVENT, + type DuplexLossReason, type DuplexObservabilityCounterRecord, type DuplexObservabilityDimensions, type DuplexObservabilityEventRecord, @@ -97,51 +83,6 @@ import { const execFileAsync = promisify(execFile); -/** - * Emit one duplex request the way the sandbox gateway sends it under the chunked - * body protocol: the envelope frame that carries `bodyByteCount`, then the - * `body_chunk` frames that carry the body. A non-empty body rides one or more - * `body_chunk` frames that share the envelope id; an empty body sends no chunk. - */ -function emitDuplexRequest( - control: { emitData: (chunk: string) => void }, - frame: Omit, - bodyText: string, -): void { - const body = Buffer.from(bodyText, "utf8"); - control.emitData(encodeDuplexFrame({ ...frame, bodyByteCount: body.length })); - for (const chunk of splitBodyIntoChunkFrames(frame.id, body, DUPLEX_FRAME_VERSION)) { - control.emitData(encodeDuplexFrame(chunk)); - } -} - -/** - * Send one duplex response the way the host broker sends it under the chunked - * body protocol: the envelope frame that carries `bodyByteCount`, then the - * `body_chunk` frames that carry the body. The gateway reassembles the body from - * the chunk frames before it writes the HTTP response. - */ -function sendDuplexResponse( - send: (frame: Record) => void, - response: { id: string | undefined; status: number; headers: Record; outcome?: string }, - bodyText: string, -): void { - const id = response.id ?? ""; - const body = Buffer.from(bodyText, "utf8"); - send({ - version: DUPLEX_FRAME_VERSION, - type: "response", - id, - status: response.status, - headers: response.headers, - bodyByteCount: body.length, - ...(response.outcome !== undefined ? { outcome: response.outcome } : {}), - }); - for (const chunk of splitBodyIntoChunkFrames(id, body, DUPLEX_FRAME_VERSION)) { - send(chunk as unknown as Record); - } -} - type RecordedSpan = { name: string; parentName: string | null; ended: boolean }; /** @@ -5981,21 +5922,6 @@ describe("sandbox adapter execution targets", () => { }); -// One decoded stdout frame from the generated duplex gateway. The gateway writes -// newline-delimited JSON frames to stdout, so the test parses each line. -interface DecodedGatewayFrame { - version?: number; - type?: string; - id?: string; - method?: string; - path?: string; - query?: string; - headers?: Record; - body?: string; - nonce?: string; - __unparsed?: string; -} - // One decode result from the embedded codec. The shape mirrors the host codec: // a valid frame or a protocol error with a code. interface EmbeddedDecodeResult { @@ -6052,220 +5978,12 @@ interface DuplexFrameFixture { encodeVectors: DuplexEncodeVector[]; } -describe("sandbox duplex gateway", () => { - const duplexCleanupDirs: string[] = []; - const duplexChildren: Array> = []; - - afterEach(async () => { - while (duplexChildren.length > 0) { - const child = duplexChildren.pop(); - if (!child) continue; - child.kill("SIGKILL"); - } - while (duplexCleanupDirs.length > 0) { - const dir = duplexCleanupDirs.pop(); - if (!dir) continue; - await rm(dir, { recursive: true, force: true }).catch(() => undefined); - } - }); - - it("launches the gateway with `exec env`, so a POSIX shell runs the environment-assignment prefix", async () => { - // A POSIX shell accepts an environment-assignment prefix only on a plain - // command, never on `exec`. The form `exec NAME=value command` exits with - // status 127, so the gateway never starts. The launch argv must use - // `exec env NAME=value command`. This test runs the generated script in a real - // `sh` against a stub entrypoint that echoes one launch env var, so it fails on - // the old `exec NAME=value` form and passes on the `exec env` form. - const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-duplex-launch-")); - duplexCleanupDirs.push(rootDir); - const stub = path.join(rootDir, "stub-entrypoint.sh"); - // The stub stands in for the node gateway. It prints a READY frame that echoes - // the launch nonce, so the test proves the environment assignment reached the - // process the launch replaced the shell with. - await writeFile( - stub, - '#!/bin/sh\nprintf \'{"version":2,"type":"ready","nonce":"%s"}\\n\' "$PAPERCLIP_BRIDGE_NONCE"\n', - "utf8", - ); - const argv = buildDuplexGatewayLaunchArgv({ - shellCommand: "sh", - remoteEntrypoint: stub, - // Run the stub with `sh`, so the test needs no node runtime. The launch form - // is `exec env NAME=value 'sh' ''`, which exercises the exec-env fix. - nodeCommand: "sh", - env: { PAPERCLIP_BRIDGE_NONCE: "abc123def456", PAPERCLIP_BRIDGE_PORT: "40404" }, - }); - const [shell, ...shellArgs] = argv; - const { stdout } = await execFileAsync(shell, shellArgs, { encoding: "utf8" }); - const decoded = decodeDuplexLine(stdout.trim()); - expect(decoded.ok).toBe(true); - if (decoded.ok) { - expect(decoded.frame.type).toBe("ready"); - if (decoded.frame.type === "ready") { - expect(decoded.frame.nonce).toBe("abc123def456"); - } - } - }); - - interface DuplexGatewayHandle { - baseUrl: string; - frames: DecodedGatewayFrame[]; - stderr: () => string; - waitForFrame: ( - predicate: (frame: DecodedGatewayFrame) => boolean, - timeoutMs?: number, - ) => Promise; - sendFrame: (frame: Record) => void; - sendRaw: (text: string) => void; - endStdin: () => void; - exited: Promise; - stop: () => Promise; - } - - // Reserve a free loopback port. The host assigns a positive port to the duplex - // gateway; the gateway binds exactly that port. The test opens an ephemeral - // listener, reads its port, then closes it, so the number is very likely free - // when the gateway binds it a moment later. - async function reserveLoopbackPort(): Promise { - return new Promise((resolve, reject) => { - const probe = net.createServer(); - probe.once("error", reject); - probe.listen(0, "127.0.0.1", () => { - const address = probe.address(); - if (!address || typeof address === "string") { - probe.close(() => reject(new Error("Could not reserve a loopback port."))); - return; - } - const reserved = address.port; - probe.close(() => resolve(reserved)); - }); - }); - } - - // Start the generated gateway `.mjs` in duplex mode as a real child process. - // The test writes response frames to the child stdin and reads request frames - // from the child stdout, so it stands in for the host side of the channel. - // The host assigns the port and the nonce; the test builds the base URL from - // the assigned port, never from the READY frame. - async function startDuplexGateway( - env: Record, - options: { port?: number; nonce?: string } = {}, - ): Promise { - const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-duplex-gateway-")); - duplexCleanupDirs.push(rootDir); - const entrypoint = path.join(rootDir, "gateway.mjs"); - await writeFile(entrypoint, getSandboxCallbackBridgeServerSource(), "utf8"); - - const assignedPort = options.port ?? (await reserveLoopbackPort()); - const nonce = options.nonce ?? "d0c1b2a3e4f5061728394a5b6c7d8e9f"; - const child = spawn(process.execPath, [entrypoint], { - stdio: ["pipe", "pipe", "pipe"], - env: { - ...process.env, - PAPERCLIP_API_BRIDGE_MODE: "duplex_v1", - PAPERCLIP_BRIDGE_HOST: "127.0.0.1", - PAPERCLIP_BRIDGE_PORT: String(assignedPort), - PAPERCLIP_BRIDGE_NONCE: nonce, - ...env, - }, - }); - duplexChildren.push(child); - - const frames: DecodedGatewayFrame[] = []; - const waiters: Array<{ - predicate: (frame: DecodedGatewayFrame) => boolean; - resolve: (frame: DecodedGatewayFrame) => void; - }> = []; - let stdoutBuffer = ""; - let stderrText = ""; - - child.stdout?.setEncoding("utf8"); - child.stdout?.on("data", (chunk: string) => { - stdoutBuffer += chunk; - let newlineIndex = stdoutBuffer.indexOf("\n"); - while (newlineIndex !== -1) { - const line = stdoutBuffer.slice(0, newlineIndex); - stdoutBuffer = stdoutBuffer.slice(newlineIndex + 1); - if (line.length > 0) { - let frame: DecodedGatewayFrame; - try { - frame = JSON.parse(line) as DecodedGatewayFrame; - } catch { - frame = { __unparsed: line }; - } - frames.push(frame); - for (const waiter of [...waiters]) { - if (waiter.predicate(frame)) { - waiters.splice(waiters.indexOf(waiter), 1); - waiter.resolve(frame); - } - } - } - newlineIndex = stdoutBuffer.indexOf("\n"); - } - }); - child.stderr?.setEncoding("utf8"); - child.stderr?.on("data", (chunk: string) => { - stderrText += chunk; - }); - - const exited = new Promise((resolve) => { - child.on("exit", (code) => resolve(code)); - }); - - const waitForFrame = ( - predicate: (frame: DecodedGatewayFrame) => boolean, - timeoutMs = 5000, - ): Promise => { - const existing = frames.find(predicate); - if (existing) return Promise.resolve(existing); - return new Promise((resolve, reject) => { - const waiter = { - predicate, - resolve: (frame: DecodedGatewayFrame) => { - clearTimeout(timer); - resolve(frame); - }, - }; - const timer = setTimeout(() => { - const index = waiters.indexOf(waiter); - if (index !== -1) waiters.splice(index, 1); - reject(new Error(`Timed out waiting for a gateway frame. stderr: ${stderrText}`)); - }, timeoutMs); - waiters.push(waiter); - }); - }; - - const handle: DuplexGatewayHandle = { - baseUrl: "", - frames, - stderr: () => stderrText, - waitForFrame, - sendFrame: (frame) => { - child.stdin?.write(`${JSON.stringify(frame)}\n`); - }, - sendRaw: (text) => { - child.stdin?.write(text); - }, - endStdin: () => { - child.stdin?.end(); - }, - exited, - stop: async () => { - child.kill("SIGKILL"); - await exited.catch(() => null); - }, - }; - - const ready = await waitForFrame((frame) => frame.type === "ready"); - // READY carries the echoed nonce and no address data. The host builds the - // origin from the port it assigned, never from the frame. - expect(ready.nonce).toBe(nonce); - expect((ready as Record).address).toBeUndefined(); - handle.baseUrl = `http://127.0.0.1:${assignedPort}`; - return handle; - } - +// This describe block covers the zero-dependency codec every generated +// gateway embeds (`DUPLEX_GATEWAY_CODEC_SOURCE`), and the sandbox-process +// decoder byte cap. The `http2_v1` gateway embeds the same codec source to +// send its one READY line, so this coverage stays live for the active +// transport. +describe("embedded sandbox gateway codec", () => { it("embedded gateway codec passes every vector in the shared fixture", async () => { const codecFactory = new Function( `${getSandboxDuplexGatewayCodecSource()}\nreturn { encodeDuplexFrame, encodeDuplexFrameChecked, decodeDuplexLine, DuplexFrameDecoder, DUPLEX_FRAME_VERSION, DEFAULT_MAX_DUPLEX_FRAME_BYTES, DUPLEX_DECODER_SCOPE, DEFAULT_MAX_DUPLEX_DECODER_BYTES };`, @@ -6422,751 +6140,12 @@ describe("sandbox duplex gateway", () => { expect(ledger.bytesInUse).toBe(0); expect(ledger.liveTokenCount).toBe(0); }); - - it("returns the same HTTP response as the file gateway for a forwarded request", async () => { - const token = "duplex-token-forward"; - const gateway = await startDuplexGateway({ PAPERCLIP_BRIDGE_TOKEN: token }); - - const responsePromise = fetch(`${gateway.baseUrl}/api/agents/me?view=compact`, { - headers: { - authorization: `Bearer ${token}`, - accept: "application/json", - "if-none-match": '"cache-key"', - "x-bridge-debug": "drop-me", - }, - }); - const requestFrame = await gateway.waitForFrame((frame) => frame.type === "request"); - expect(requestFrame.method).toBe("GET"); - expect(requestFrame.path).toBe("/api/agents/me"); - expect(requestFrame.query).toBe("?view=compact"); - // Only allowlisted headers forward; the bearer and the debug header drop. - expect(requestFrame.headers).toEqual({ - accept: "application/json", - "if-none-match": '"cache-key"', - }); - - sendDuplexResponse( - gateway.sendFrame, - { - id: requestFrame.id, - status: 200, - headers: { "content-type": "application/json", etag: '"rev-1"', "content-length": "999" }, - outcome: "completed", - }, - JSON.stringify({ ok: true }), - ); - const response = await responsePromise; - expect(response.status).toBe(200); - expect(response.headers.get("content-type")).toContain("application/json"); - expect(response.headers.get("etag")).toBe('"rev-1"'); - await expect(response.json()).resolves.toEqual({ ok: true }); - - // An indeterminate outcome maps to a non-retryable 409, the same contract the - // file gateway applies through the outcome header. - const indeterminatePromise = fetch(`${gateway.baseUrl}/api/issues/issue-1`, { - method: "PATCH", - headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, - body: JSON.stringify({ status: "in_progress" }), - }); - const patchFrame = await gateway.waitForFrame( - (frame) => frame.type === "request" && frame.id !== requestFrame.id, - ); - sendDuplexResponse( - gateway.sendFrame, - { - id: patchFrame.id, - status: 200, - headers: { "content-type": "application/json" }, - outcome: "indeterminate", - }, - JSON.stringify({ error: "outcome_indeterminate" }), - ); - const indeterminate = await indeterminatePromise; - expect(indeterminate.status).toBe(409); - - await gateway.stop(); - }, 20000); - - it("fails a request over the body size limit locally and keeps the channel open", async () => { - const token = "duplex-token-oversize-request"; - // The chunked body protocol splits a large body into fixed-size body_chunk - // frames that each stay under the frame bound, so a body never exceeds the - // frame bound on its own. The operative local guard for a request too large to - // deliver is now the body size limit. Set it to 1,000,000 bytes here. - const gateway = await startDuplexGateway({ - PAPERCLIP_BRIDGE_TOKEN: token, - PAPERCLIP_BRIDGE_MAX_BODY_BYTES: "1000000", - }); - - // A body over the body size limit is rejected locally. The gateway must fail - // this one local request cleanly and forward no frame. - const oversizeBody = JSON.stringify({ body: "x".repeat(1_100_000) }); - const tooLarge = await fetch(`${gateway.baseUrl}/api/issues/issue-1/comments`, { - method: "POST", - headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, - body: oversizeBody, - }); - expect(tooLarge.status).toBe(502); - await expect(tooLarge.json()).resolves.toEqual({ - error: "Bridge request body exceeded the configured size limit.", - }); - - // The oversized request forwarded no frame. It never left the gateway. - expect(gateway.frames.filter((frame) => frame.type === "request")).toHaveLength(0); - // The gateway did not lose the channel: no close frame and the process runs. - expect(gateway.frames.some((frame) => frame.type === "close")).toBe(false); - - // The channel stays open, so a normal request after the oversized one still - // forwards and completes. - const okPromise = fetch(`${gateway.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${token}` }, - }); - const requestFrame = await gateway.waitForFrame((frame) => frame.type === "request"); - sendDuplexResponse( - gateway.sendFrame, - { - id: requestFrame.id, - status: 200, - headers: { "content-type": "application/json" }, - outcome: "completed", - }, - JSON.stringify({ ok: true }), - ); - const okResponse = await okPromise; - expect(okResponse.status).toBe(200); - await expect(okResponse.json()).resolves.toEqual({ ok: true }); - - await gateway.stop(); - }, 20000); - - it("enforces the bearer check, JSON-only rule, body limit, and depth limit", async () => { - const token = "duplex-token-contract"; - const gateway = await startDuplexGateway({ - PAPERCLIP_BRIDGE_TOKEN: token, - PAPERCLIP_BRIDGE_MAX_QUEUE_DEPTH: "1", - PAPERCLIP_BRIDGE_MAX_BODY_BYTES: "16", - }); - - const badAuth = await fetch(`${gateway.baseUrl}/api/agents/me`, { - headers: { authorization: "Bearer wrong-token" }, - }); - expect(badAuth.status).toBe(401); - await expect(badAuth.json()).resolves.toEqual({ error: "Invalid bridge token." }); - - const nonJson = await fetch(`${gateway.baseUrl}/api/issues/issue-1/comments`, { - method: "POST", - headers: { authorization: `Bearer ${token}`, "content-type": "text/plain" }, - body: "not json", - }); - expect(nonJson.status).toBe(415); - await expect(nonJson.json()).resolves.toEqual({ - error: "Bridge only accepts JSON request bodies.", - }); - - const oversizeBody = await fetch(`${gateway.baseUrl}/api/issues/issue-1/comments`, { - method: "POST", - headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, - body: JSON.stringify({ body: "x".repeat(64) }), - }); - expect(oversizeBody.status).toBe(502); - await expect(oversizeBody.json()).resolves.toEqual({ - error: "Bridge request body exceeded the configured size limit.", - }); - - // No request frame forwarded so far: the guards rejected before forwarding. - const framesBeforeDepth = gateway.frames.filter((frame) => frame.type === "request").length; - expect(framesBeforeDepth).toBe(0); - - // One outstanding request fills the single depth slot; the host never - // responds, so the slot stays used. - const outstanding = fetch(`${gateway.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${token}` }, - }); - void outstanding.catch(() => undefined); - await gateway.waitForFrame((frame) => frame.type === "request"); - - const queueFull = await fetch(`${gateway.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${token}` }, - }); - expect(queueFull.status).toBe(503); - await expect(queueFull.json()).resolves.toEqual({ error: "Bridge request queue is full." }); - - // Only the outstanding request forwarded a frame; the queue-full request did - // not. - const framesAfterDepth = gateway.frames.filter((frame) => frame.type === "request").length; - expect(framesAfterDepth).toBe(1); - - await gateway.stop(); - }, 20000); - - it("answers outstanding requests 409 and new requests 503 on stdin EOF, then exits", async () => { - const token = "duplex-token-eof"; - const gateway = await startDuplexGateway({ - PAPERCLIP_BRIDGE_TOKEN: token, - PAPERCLIP_BRIDGE_LOSS_EXIT_GRACE_MS: "3000", - }); - - const outstanding = fetch(`${gateway.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${token}` }, - }); - await gateway.waitForFrame((frame) => frame.type === "request"); - gateway.endStdin(); - - const lossResponse = await outstanding; - expect(lossResponse.status).toBe(409); - expect(lossResponse.headers.get("x-paperclip-bridge-outcome")).toBe("indeterminate"); - await expect(lossResponse.json()).resolves.toEqual({ error: "outcome_indeterminate" }); - - const afterLoss = await fetch(`${gateway.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${token}` }, - }); - expect(afterLoss.status).toBe(503); - await expect(afterLoss.json()).resolves.toEqual({ error: "bridge_unavailable" }); - - const exitCode = await gateway.exited; - expect(exitCode).toBe(0); - }, 20000); - - it("applies the same loss behavior on a heartbeat timeout", async () => { - const token = "duplex-token-heartbeat"; - const gateway = await startDuplexGateway({ - PAPERCLIP_BRIDGE_TOKEN: token, - PAPERCLIP_BRIDGE_HEARTBEAT_TIMEOUT_MS: "400", - PAPERCLIP_BRIDGE_LOSS_EXIT_GRACE_MS: "3000", - PAPERCLIP_BRIDGE_RESPONSE_TIMEOUT_MS: "30000", - }); - - // Never send an inbound frame: inbound silence trips the heartbeat timeout. - const outstanding = fetch(`${gateway.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${token}` }, - }); - await gateway.waitForFrame((frame) => frame.type === "request"); - - const lossResponse = await outstanding; - expect(lossResponse.status).toBe(409); - await expect(lossResponse.json()).resolves.toEqual({ error: "outcome_indeterminate" }); - - const afterLoss = await fetch(`${gateway.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${token}` }, - }); - expect(afterLoss.status).toBe(503); - await expect(afterLoss.json()).resolves.toEqual({ error: "bridge_unavailable" }); - - const exitCode = await gateway.exited; - expect(exitCode).toBe(0); - }, 20000); - - it("keeps diagnostics off stdout; stdout carries only frames", async () => { - const token = "duplex-token-stdout"; - const gateway = await startDuplexGateway({ PAPERCLIP_BRIDGE_TOKEN: token }); - - const responsePromise = fetch(`${gateway.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${token}` }, - }); - const requestFrame = await gateway.waitForFrame((frame) => frame.type === "request"); - - // Feed a malformed inbound line and an unknown response id. Both force a - // diagnostic path; none of it may reach stdout. - gateway.sendRaw("this is not a frame\n"); - sendDuplexResponse( - gateway.sendFrame, - { id: "unknown-id", status: 200, headers: {}, outcome: "completed" }, - "", - ); - sendDuplexResponse( - gateway.sendFrame, - { - id: requestFrame.id, - status: 200, - headers: { "content-type": "application/json" }, - outcome: "completed", - }, - "{}", - ); - - const response = await responsePromise; - expect(response.status).toBe(200); - - // Every stdout line parsed as a frame; none was diagnostic text. - expect(gateway.frames.some((frame) => frame.__unparsed !== undefined)).toBe(false); - expect( - gateway.frames.every( - (frame) => typeof frame.version === "number" && typeof frame.type === "string", - ), - ).toBe(true); - const allowedTypes = new Set(["ready", "heartbeat", "request"]); - expect(gateway.frames.every((frame) => allowedTypes.has(String(frame.type)))).toBe(true); - - // The malformed inbound line produced a stderr diagnostic, not a stdout one. - expect(gateway.stderr()).toContain("dropped an inbound frame"); - - await gateway.stop(); - }, 20000); - - it("defaults the wait budget to 35 s and honors the environment key override", async () => { - // The generated source carries the 35 s default. - expect(getSandboxCallbackBridgeServerSource()).toContain("35000"); - - const token = "duplex-token-budget"; - const gateway = await startDuplexGateway({ - PAPERCLIP_BRIDGE_TOKEN: token, - PAPERCLIP_BRIDGE_RESPONSE_TIMEOUT_MS: "150", - PAPERCLIP_BRIDGE_HEARTBEAT_TIMEOUT_MS: "30000", - }); - - const started = Date.now(); - const response = await fetch(`${gateway.baseUrl}/api/agents/me`, { - headers: { authorization: `Bearer ${token}` }, - }); - expect(response.status).toBe(502); - await expect(response.json()).resolves.toEqual({ - error: "Timed out waiting for host bridge response.", - }); - expect(Date.now() - started).toBeLessThan(4000); - - await gateway.stop(); - }, 20000); }); -/** - * A scripted fake duplex channel. The test drives the read path with - * `emitData`/`emitExit`, and reads the frames the broker wrote through `written`. - * The `writeError` and `closeBehavior` hooks let a test force a stream failure - * and a close timeout. - */ -function createFakeDuplexChannel(): { - channel: CommandManagedDuplexChannel; - emitData: (chunk: string) => void; - emitExit: (exit: { exitCode: number | null }) => void; - written: Array; - writtenTypes: string[]; - stopped: () => number; - setWriteError: (error: Error | null) => void; - setCloseBehavior: (behavior: "resolve" | "hang" | "reject") => void; -} { - let dataListener: ((chunk: Uint8Array) => void) | null = null; - let exitListener: ((exit: { exitCode: number | null }) => void) | null = null; - let writeError: Error | null = null; - let closeBehavior: "resolve" | "hang" | "reject" = "resolve"; - let stopCount = 0; - // The broker writes a response as one envelope frame that carries - // `bodyByteCount`, then the `body_chunk` frames that carry the body. Reassemble - // the body so a test reads the response the same way it did with the one-frame - // model. - const written: Array = []; - const writtenTypes: string[] = []; - const responseAssembly = new Map< - string, - { frame: DuplexResponseFrame; received: number; chunks: Buffer[] } - >(); - - const channel: CommandManagedDuplexChannel = { - write(data: Uint8Array): void { - if (writeError) throw writeError; - const decoded = decodeDuplexLine(Buffer.from(data).toString("utf8").replace(/\n$/, "")); - if (!decoded.ok) return; - const frame = decoded.frame; - writtenTypes.push(frame.type); - if (frame.type === "response") { - if (frame.bodyByteCount === 0) { - written.push({ ...frame, body: "" }); - } else { - responseAssembly.set(frame.id, { frame, received: 0, chunks: [] }); - } - } else if (frame.type === "body_chunk") { - const assembly = responseAssembly.get(frame.id); - if (!assembly) return; - const decodedChunk = Buffer.from(frame.data, "base64"); - assembly.received += decodedChunk.length; - assembly.chunks.push(decodedChunk); - if (assembly.received >= assembly.frame.bodyByteCount) { - responseAssembly.delete(frame.id); - written.push({ - ...assembly.frame, - body: Buffer.concat(assembly.chunks).toString("utf8"), - }); - } - } - }, - onData(listener: (chunk: Uint8Array) => void): void { - dataListener = listener; - }, - onExit(listener: (exit: { exitCode: number | null }) => void): void { - exitListener = listener; - }, - stop(): void { - stopCount += 1; - }, - close(): Promise { - if (closeBehavior === "resolve") return Promise.resolve(); - if (closeBehavior === "reject") return Promise.reject(new Error("close rejected")); - return new Promise(() => {}); - }, - }; - - return { - channel, - emitData: (chunk: string) => dataListener?.(new TextEncoder().encode(chunk)), - emitExit: (exit: { exitCode: number | null }) => exitListener?.(exit), - written, - writtenTypes, - stopped: () => stopCount, - setWriteError: (error: Error | null) => { - writeError = error; - }, - setCloseBehavior: (behavior: "resolve" | "hang" | "reject") => { - closeBehavior = behavior; - }, - }; -} - -/** - * Build one request as its envelope line (carrying `bodyByteCount`) plus its - * `body_chunk` lines, so the fake channel emits a whole request in one write. The - * broker reassembles the body from the chunk frames before it forwards. - */ -function requestFrameLine( - overrides: Partial> & { id: string }, - bodyText: string = JSON.stringify({ body: "hello" }), -): string { - const body = Buffer.from(bodyText, "utf8"); - const frame: DuplexRequestFrame = { - version: DUPLEX_FRAME_VERSION, - type: "request", - id: overrides.id, - method: overrides.method ?? "POST", - path: overrides.path ?? "/api/issues/PAP-1/comments", - query: overrides.query ?? "", - headers: overrides.headers ?? { authorization: "Bearer bridge-token" }, - bodyByteCount: body.length, - }; - let line = encodeDuplexFrame(frame); - for (const chunk of splitBodyIntoChunkFrames(frame.id, body, DUPLEX_FRAME_VERSION)) { - line += encodeDuplexFrame(chunk); - } - return line; -} - /** Wait for the pending microtasks and macrotasks to settle. */ function flushMacrotasks(): Promise { return new Promise((resolve) => setImmediate(resolve)); } - -describe("createDuplexBridgeBroker", () => { - it("forwards a decoded request to the handler and writes the handler response back", async () => { - const fake = createFakeDuplexChannel(); - const received: DuplexRequestFrame[] = []; - // The forward handler owns the token replacement and the run attribution. The - // broker passes the decoded request straight through, so the sandbox request - // still carries only the bridge token here, and the handler applies the real - // token and the signed run identifier on the existing forward path. - const broker = await createDuplexBridgeBroker({ - channel: fake.channel, - forwardRequest: async (request): Promise => { - received.push(request); - expect(request.headers.authorization).toBe("Bearer bridge-token"); - expect(request.headers["x-paperclip-run-id"]).toBeUndefined(); - return { - status: 201, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ ok: true }), - }; - }, - }); - - broker.start(); - fake.emitData(requestFrameLine({ id: "req-1" })); - await flushMacrotasks(); - - expect(received).toHaveLength(1); - expect(received[0].id).toBe("req-1"); - expect(fake.written).toHaveLength(1); - expect(fake.written[0]).toMatchObject({ - type: "response", - id: "req-1", - status: 201, - outcome: "completed", - }); - expect(JSON.parse(fake.written[0].body)).toEqual({ ok: true }); - - await broker.close(); - }); - - it("moves through opening, open, closing, closed in order", async () => { - const fake = createFakeDuplexChannel(); - const states: DuplexBrokerState[] = []; - const broker = await createDuplexBridgeBroker({ - channel: fake.channel, - forwardRequest: async () => ({ status: 200 }), - onStateChange: (state) => states.push(state), - }); - - expect(broker.state).toBe("opening"); - broker.start(); - expect(broker.state).toBe("open"); - await broker.close(); - expect(broker.state).toBe("closed"); - expect(states).toEqual(["open", "closing", "closed"]); - expect(fake.writtenTypes).toContain("close"); - }); - - it.each([ - { - name: "channel exit", - reason: "channel_exit" as const, - trigger: (fake: ReturnType) => - fake.emitExit({ exitCode: 1 }), - }, - { - name: "protocol failure", - reason: "protocol_failure" as const, - trigger: (fake: ReturnType) => - fake.emitData("this is not json\n"), - }, - { - name: "stream failure", - reason: "stream_failure" as const, - trigger: (fake: ReturnType) => { - fake.setWriteError(new Error("broken pipe")); - fake.emitData(requestFrameLine({ id: "req-stream" })); - }, - }, - ])("enters lost on $name", async ({ reason, trigger }) => { - const fake = createFakeDuplexChannel(); - const losses: DuplexBrokerLossRecord[] = []; - const broker = await createDuplexBridgeBroker({ - channel: fake.channel, - forwardRequest: async () => ({ status: 200 }), - onLoss: (record) => losses.push(record), - }); - broker.start(); - - trigger(fake); - await flushMacrotasks(); - - expect(broker.state).toBe("lost"); - expect(broker.lossRecord?.reason).toBe(reason); - expect(losses).toHaveLength(1); - }); - - it("enters lost on a close timeout", async () => { - const fake = createFakeDuplexChannel(); - fake.setCloseBehavior("hang"); - const broker = await createDuplexBridgeBroker({ - channel: fake.channel, - forwardRequest: async () => ({ status: 200 }), - closeTimeoutMs: 20, - }); - broker.start(); - - await broker.close(); - - expect(broker.state).toBe("lost"); - expect(broker.lossRecord?.reason).toBe("close_timeout"); - }); - - it("stops the heartbeat, marks the run bridge ended, and dispatches nothing after loss", async () => { - const fake = createFakeDuplexChannel(); - const forwarded: string[] = []; - const broker = await createDuplexBridgeBroker({ - channel: fake.channel, - forwardRequest: async (request) => { - forwarded.push(request.id); - return { status: 200 }; - }, - }); - broker.start(); - - fake.emitExit({ exitCode: 1 }); - await flushMacrotasks(); - expect(broker.state).toBe("lost"); - expect(broker.lossRecord).not.toBeNull(); - - // A request that arrives after loss reaches nothing. The broker never - // reconnects and never replays a request. - fake.emitData(requestFrameLine({ id: "after-loss" })); - await flushMacrotasks(); - expect(forwarded).toEqual([]); - }); - - it("forwards one request id one time, so a repeated frame never reaches the API twice", async () => { - const fake = createFakeDuplexChannel(); - const forwarded: string[] = []; - const broker = await createDuplexBridgeBroker({ - channel: fake.channel, - forwardRequest: async (request) => { - forwarded.push(request.id); - return { status: 200, body: "ok" }; - }, - }); - broker.start(); - - fake.emitData(requestFrameLine({ id: "dup" })); - await flushMacrotasks(); - fake.emitData(requestFrameLine({ id: "dup" })); - await flushMacrotasks(); - - expect(forwarded).toEqual(["dup"]); - expect(fake.written).toHaveLength(1); - - await broker.close(); - }); - - it("captures the dispatch-start point per request for metrics only", async () => { - const fake = createFakeDuplexChannel(); - const records: DuplexBrokerRequestRecord[] = []; - let clock = 1000; - const broker = await createDuplexBridgeBroker({ - channel: fake.channel, - forwardRequest: async () => ({ status: 200 }), - now: () => clock, - onRequestRecord: (record) => records.push(record), - }); - broker.start(); - - clock = 2500; - fake.emitData(requestFrameLine({ id: "metric-1", method: "GET", path: "/api/agents/me" })); - await flushMacrotasks(); - - expect(records).toEqual([ - { id: "metric-1", method: "GET", path: "/api/agents/me", dispatchStartMs: 2500 }, - ]); - // The record never reaches the channel. The broker writes only a response. - expect(fake.writtenTypes).not.toContain("request"); - - await broker.close(); - }); - - it("latches a failure when a loss orders before an orderly completion and names the typed reason", async () => { - const fake = createFakeDuplexChannel(); - const broker = await createDuplexBridgeBroker({ - channel: fake.channel, - forwardRequest: async () => ({ status: 200 }), - }); - broker.start(); - - // The channel dies mid-turn, before any orderly completion. - fake.emitExit({ exitCode: 1 }); - await flushMacrotasks(); - expect(broker.runDisposition).toEqual({ failed: true, lossReason: "provider_exit" }); - - // A later orderly completion cannot clear the latch. A delayed activity - // callback cannot clear the latch either, because the broker dispatches - // nothing after loss. - broker.markOrderlyCompletion(); - fake.emitData(requestFrameLine({ id: "after-loss" })); - await flushMacrotasks(); - expect(broker.runDisposition).toEqual({ failed: true, lossReason: "provider_exit" }); - }); - - it("keeps a success when a loss orders after a host-observed orderly completion, and emits no loss event", async () => { - const events: DuplexObservabilityEventRecord[] = []; - const counters: DuplexObservabilityCounterRecord[] = []; - const recorder: DuplexObservabilityRecorder = { - recordSpan() {}, - incrementCounter(record) { - counters.push(record); - }, - emitEvent(record) { - events.push(record); - }, - }; - const fake = createFakeDuplexChannel(); - const broker = await createDuplexBridgeBroker({ - channel: fake.channel, - forwardRequest: async () => ({ status: 200 }), - telemetry: createDuplexObservability({ recorder, providerKey: "daytona" }), - }); - broker.start(); - - // The agent completes its turn, then the channel ends during the teardown. - broker.markOrderlyCompletion(); - fake.emitExit({ exitCode: 0 }); - await flushMacrotasks(); - - expect(broker.runDisposition).toEqual({ failed: false, lossReason: null }); - // A normal teardown is not a loss: no loss event and no loss counter. - expect(events.some((e) => e.dimensions.loss_reason !== undefined)).toBe(false); - expect(counters.some((c) => c.metric === DUPLEX_COUNTER_LOSS_TOTAL)).toBe(false); - }); - - it("carries the typed loss_reason on the transport loss event and keeps a sentinel message off every sink", async () => { - const spans: DuplexObservabilitySpanRecord[] = []; - const counters: DuplexObservabilityCounterRecord[] = []; - const events: DuplexObservabilityEventRecord[] = []; - const recorder: DuplexObservabilityRecorder = { - recordSpan(record) { - spans.push(record); - }, - incrementCounter(record) { - counters.push(record); - }, - emitEvent(record) { - events.push(record); - }, - }; - const logLines: string[] = []; - const fake = createFakeDuplexChannel(); - const sentinel = "SENTINEL-PROVIDER-TEXT-1a2b3c"; - const broker = await createDuplexBridgeBroker({ - channel: fake.channel, - forwardRequest: async () => ({ status: 200 }), - telemetry: createDuplexObservability({ recorder, providerKey: "daytona" }), - logger: (message) => logLines.push(message), - }); - broker.start(); - - // A stream write failure carries a raw provider message. It maps to the typed - // `write_error`; the raw message must reach no sink. - fake.setWriteError(new Error(sentinel)); - fake.emitData(requestFrameLine({ id: "req-sentinel" })); - await flushMacrotasks(); - - const lossEvent = events.find((e) => e.dimensions.loss_reason !== undefined); - expect(lossEvent?.dimensions.loss_reason).toBe("write_error"); - expect(lossEvent?.dimensions).toMatchObject({ transport: "duplex", outcome: "error" }); - - // The sentinel provider text reaches no telemetry sink and no log line. - const serializedSinks = JSON.stringify({ spans, counters, events }); - expect(serializedSinks).not.toContain(sentinel); - expect(logLines.join("\n")).not.toContain(sentinel); - }); - - it("rejects a configuration where an inner budget is not smaller than its outer budget", async () => { - expect(() => - assertNestedDuplexBrokerBudgets({ - forwardTimeoutMs: 32_000, - responseBudgetMs: 32_000, - gatewayWaitMs: 35_000, - }), - ).toThrow(/forward budget/); - expect(() => - assertNestedDuplexBrokerBudgets({ - forwardTimeoutMs: 30_000, - responseBudgetMs: 35_000, - gatewayWaitMs: 35_000, - }), - ).toThrow(/response budget/); - // The async broker construction validates budgets before its first await, so - // an invalid budget set surfaces as a rejected promise. - await expect( - createDuplexBridgeBroker({ - channel: createFakeDuplexChannel().channel, - forwardRequest: async () => ({ status: 200 }), - budgets: { forwardTimeoutMs: 40_000 }, - }), - ).rejects.toThrow(/forward budget/); - // The default budget set holds the nested order. - expect(() => - assertNestedDuplexBrokerBudgets({ - forwardTimeoutMs: 30_000, - responseBudgetMs: 32_000, - gatewayWaitMs: 35_000, - }), - ).not.toThrow(); - }); -}); - describe("sandbox target spec parse: enableSandboxDuplexBridge", () => { // The minimal serialized sandbox target the host stamps and the adapter parses. // A test overrides one field per case to prove the fail-closed parse. @@ -7228,33 +6207,56 @@ describe("sandbox target spec parse: enableSandboxDuplexBridge", () => { }); }); +/** + * A minimal run-disposition latch for the seam tests below. It reproduces the + * same ordering rule the real bridge transport applies: the first ordered + * loss or orderly completion latches the terminal disposition, and a later + * call never overrides it. `emitExit` stands in for a transport-level channel + * exit — the real transport treats every channel exit as a loss candidate, + * mapped to the typed `provider_exit` reason. + */ +function createFakeBridgeTransport() { + let lossOrdered = false; + let lossReason: DuplexLossReason | null = null; + let completionOrdered = false; + const markOrderlyCompletion = (): void => { + if (completionOrdered || lossOrdered) return; + completionOrdered = true; + }; + return { + get runDisposition() { + return { failed: lossOrdered, lossReason }; + }, + settleRunDisposition() { + markOrderlyCompletion(); + return { failed: lossOrdered, lossReason }; + }, + markOrderlyCompletion, + emitExit(): void { + if (lossOrdered || completionOrdered) return; + lossOrdered = true; + lossReason = "provider_exit"; + }, + }; +} + describe("settleRunDisposition atomic read and mark", () => { it("marks the orderly completion and reports a success for a healthy channel", async () => { - const fake = createFakeDuplexChannel(); - const broker = await createDuplexBridgeBroker({ - channel: fake.channel, - forwardRequest: async () => ({ status: 200 }), - }); - broker.start(); + const broker = createFakeBridgeTransport(); // The one atomic step marks the orderly completion and reads the success. expect(broker.settleRunDisposition()).toEqual({ failed: false, lossReason: null }); // A later teardown loss orders after the mark, so it stays a normal teardown. - fake.emitExit({ exitCode: 0 }); + broker.emitExit(); await flushMacrotasks(); expect(broker.runDisposition).toEqual({ failed: false, lossReason: null }); }); it("reports the failure and does not mark for a latched loss", async () => { - const fake = createFakeDuplexChannel(); - const broker = await createDuplexBridgeBroker({ - channel: fake.channel, - forwardRequest: async () => ({ status: 200 }), - }); - broker.start(); + const broker = createFakeBridgeTransport(); // A loss ordered before any orderly completion latches the failure. - fake.emitExit({ exitCode: 1 }); + broker.emitExit(); await flushMacrotasks(); // The atomic step reads the failure and no-ops the mark, so a later // completion cannot clear the latch. @@ -7290,20 +6292,10 @@ describe("CLI-lane run-disposition seam", () => { } as AdapterSandboxExecutionTarget; } - async function startBroker(fake: ReturnType) { - const broker = await createDuplexBridgeBroker({ - channel: fake.channel, - forwardRequest: async () => ({ status: 200 }), - }); - broker.start(); - return broker; - } - - it("fails a clean CLI completion closed when the duplex channel was lost mid-turn", async () => { - const fake = createFakeDuplexChannel(); - const broker = await startBroker(fake); + it("fails a clean CLI completion closed when the bridge channel was lost mid-turn", async () => { + const broker = createFakeBridgeTransport(); // The control channel dies mid-turn, before the CLI process exits. - fake.emitExit({ exitCode: 1 }); + broker.emitExit(); await flushMacrotasks(); const runner = mockRunner({ ...CLEAN_RESULT }); @@ -7324,8 +6316,7 @@ describe("CLI-lane run-disposition seam", () => { }); it("keeps a clean CLI completion a success when the channel stays healthy, and a teardown loss stays benign", async () => { - const fake = createFakeDuplexChannel(); - const broker = await startBroker(fake); + const broker = createFakeBridgeTransport(); const runner = mockRunner({ ...CLEAN_RESULT }); const result = await runAdapterExecutionTargetProcess("run-cli-ok", sandboxTarget(runner), "agent-cli", [], { @@ -7342,17 +6333,16 @@ describe("CLI-lane run-disposition seam", () => { // The seam's atomic settle marked the orderly completion at agent // completion. A teardown loss ordered after it is a normal teardown, so the // run stays a success without any manual mark here. - fake.emitExit({ exitCode: 0 }); + broker.emitExit(); await flushMacrotasks(); expect(broker.runDisposition.failed).toBe(false); }); it("keeps a clean CLI completion a success when the gateway exits during the run-log tail finish", async () => { - const fake = createFakeDuplexChannel(); - const broker = await startBroker(fake); + const broker = createFakeBridgeTransport(); // A run-log tail whose finish emits a gateway exit. This reproduces the - // race where the duplex gateway dies after the clean process completion but + // race where the bridge gateway dies after the clean process completion but // before the host reads the disposition. The seam settles the disposition // synchronously before this finish await, so the mark orders first and the // teardown exit stays benign. @@ -7361,7 +6351,7 @@ describe("CLI-lane run-disposition seam", () => { wrapCommand: (command, args) => ({ command, args }), start: () => {}, finish: async () => { - fake.emitExit({ exitCode: 1 }); + broker.emitExit(); await flushMacrotasks(); }, abort: async () => {}, @@ -7388,10 +6378,9 @@ describe("CLI-lane run-disposition seam", () => { }); it("cannot clear the loss latch with a later completion", async () => { - const fake = createFakeDuplexChannel(); - const broker = await startBroker(fake); + const broker = createFakeBridgeTransport(); // The loss latches before the CLI process exits. - fake.emitExit({ exitCode: 1 }); + broker.emitExit(); await flushMacrotasks(); // A later orderly completion cannot clear the latch. broker.markOrderlyCompletion(); @@ -7411,10 +6400,9 @@ describe("CLI-lane run-disposition seam", () => { }); it("leaves an already-failed CLI result unchanged and never settles the disposition", async () => { - const fake = createFakeDuplexChannel(); - const broker = await startBroker(fake); + const broker = createFakeBridgeTransport(); // The control channel is lost, but the process itself also exited non-zero. - fake.emitExit({ exitCode: 1 }); + broker.emitExit(); await flushMacrotasks(); let settleCalls = 0; diff --git a/packages/adapter-utils/src/execution-target.ts b/packages/adapter-utils/src/execution-target.ts index e2238f142a..28423e1a4e 100644 --- a/packages/adapter-utils/src/execution-target.ts +++ b/packages/adapter-utils/src/execution-target.ts @@ -51,7 +51,7 @@ import { DUPLEX_CHANNEL_LOST_ERROR_CODE, isSafeBridgeMethod, type DuplexBrokerRunDisposition, -} from "./duplex-bridge-broker.js"; +} from "./bridge-transport-contract.js"; import { decodeDuplexLine, DEFAULT_MAX_DUPLEX_FRAME_BYTES } from "./duplex-frame-codec.js"; import type { ReassembledBody } from "./duplex-body-spool.js"; import { diff --git a/packages/adapter-utils/src/sandbox-callback-bridge.test.ts b/packages/adapter-utils/src/sandbox-callback-bridge.test.ts index ea69788c26..07e24ec155 100644 --- a/packages/adapter-utils/src/sandbox-callback-bridge.test.ts +++ b/packages/adapter-utils/src/sandbox-callback-bridge.test.ts @@ -2941,6 +2941,71 @@ describe("sandbox callback bridge", () => { expect(stderr).toContain("EADDRINUSE"); }, 15_000); + it("exits nonzero for the retired duplex_v1 mode instead of starting the queue gateway", async () => { + // The closed mode allowlist rejects `duplex_v1` before the queue-directory + // check, so a stale `duplex_v1` launch environment fails startup instead + // of silently falling through to the queue gateway. + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-mode-duplex-")); + cleanupDirs.push(rootDir); + const entrypoint = path.join(rootDir, "paperclip-bridge-server.mjs"); + await writeFile(entrypoint, getSandboxCallbackBridgeServerSource(), "utf8"); + const queueDir = path.join(rootDir, "queue"); + await mkdir(queueDir, { recursive: true }); + + const child = spawn(process.execPath, [entrypoint], { + env: { + ...process.env, + PAPERCLIP_API_BRIDGE_MODE: "duplex_v1", + PAPERCLIP_BRIDGE_QUEUE_DIR: queueDir, + PAPERCLIP_BRIDGE_TOKEN: "test-token", + PAPERCLIP_BRIDGE_PORT: "0", + }, + stdio: ["ignore", "ignore", "pipe"], + }); + let stderr = ""; + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + const exitCode = await new Promise((resolve) => { + child.on("close", resolve); + }); + + expect(exitCode).not.toBe(0); + expect(stderr).toContain("Unsupported PAPERCLIP_API_BRIDGE_MODE: duplex_v1"); + }, 15_000); + + it("exits nonzero for an unknown bridge mode instead of starting the queue gateway", async () => { + // The closed mode allowlist rejects every value it does not name, not + // only the retired duplex transport. + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-mode-unknown-")); + cleanupDirs.push(rootDir); + const entrypoint = path.join(rootDir, "paperclip-bridge-server.mjs"); + await writeFile(entrypoint, getSandboxCallbackBridgeServerSource(), "utf8"); + const queueDir = path.join(rootDir, "queue"); + await mkdir(queueDir, { recursive: true }); + + const child = spawn(process.execPath, [entrypoint], { + env: { + ...process.env, + PAPERCLIP_API_BRIDGE_MODE: "totally_unknown_mode", + PAPERCLIP_BRIDGE_QUEUE_DIR: queueDir, + PAPERCLIP_BRIDGE_TOKEN: "test-token", + PAPERCLIP_BRIDGE_PORT: "0", + }, + stdio: ["ignore", "ignore", "pipe"], + }); + let stderr = ""; + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + const exitCode = await new Promise((resolve) => { + child.on("close", resolve); + }); + + expect(exitCode).not.toBe(0); + expect(stderr).toContain("Unsupported PAPERCLIP_API_BRIDGE_MODE: totally_unknown_mode"); + }, 15_000); + it("test_http2_gateway_writes_no_frame_between_ready_and_the_preface", async () => { // Spawn the real generated gateway in http2_v1 mode and read its raw // stdout bytes. The only frame-codec write on this path is the READY diff --git a/packages/adapter-utils/src/sandbox-callback-bridge.ts b/packages/adapter-utils/src/sandbox-callback-bridge.ts index f66b9bdac8..cee3ce0b1d 100644 --- a/packages/adapter-utils/src/sandbox-callback-bridge.ts +++ b/packages/adapter-utils/src/sandbox-callback-bridge.ts @@ -73,35 +73,16 @@ const SANDBOX_EXEC_CHANNEL_ENV = "PAPERCLIP_SANDBOX_EXEC_CHANNEL"; const SANDBOX_EXEC_CHANNEL_BRIDGE = "bridge"; // The bridge modes the generated gateway supports. The file mode polls a -// request/response queue on disk. The retired duplex mode forwarded one -// request frame to stdout and resolved one response frame from stdin; the -// generated gateway still defines it, but no mode dispatch selects it anymore -// — the http2 mode replaced it as the active non-file transport. The http2 -// mode runs one Node HTTP/2 client session directly on stdin/stdout, after it -// sends the one READY line the host readiness gate expects. The generated -// `.mjs` selects the mode from `PAPERCLIP_API_BRIDGE_MODE`. +// request/response queue on disk. The http2 mode runs one Node HTTP/2 client +// session directly on stdin/stdout, after it sends the one READY line the +// host readiness gate expects. The generated `.mjs` selects the mode from +// `PAPERCLIP_API_BRIDGE_MODE`. The generated gateway rejects every other +// value with a fixed startup error, including the retired duplex transport. // HTTP/2 is the preferred transport. `queue_v1` is the soft-deprecated fallback. const SANDBOX_CALLBACK_BRIDGE_FILE_MODE = "queue_v1"; -export const SANDBOX_CALLBACK_BRIDGE_DUPLEX_MODE = "duplex_v1"; -/** The active non-file transport mode. It replaced {@link SANDBOX_CALLBACK_BRIDGE_DUPLEX_MODE} - * in the mode-selection path. */ +/** The active non-file transport mode. */ export const SANDBOX_CALLBACK_BRIDGE_HTTP2_MODE = "http2_v1"; -// The duplex gateway HTTP wait budget default. The gateway waits this long for a -// response frame before it answers the local caller with a 502 timeout. The -// value stays configurable through `PAPERCLIP_BRIDGE_RESPONSE_TIMEOUT_MS`, the -// same environment key the file mode reads. -const DEFAULT_DUPLEX_GATEWAY_WAIT_BUDGET_MS = 35_000; -// How often the duplex gateway writes a heartbeat frame to stdout. -const DEFAULT_DUPLEX_GATEWAY_HEARTBEAT_INTERVAL_MS = 5_000; -// The duplex gateway treats the channel as lost when no inbound frame arrives on -// stdin within this window. -const DEFAULT_DUPLEX_GATEWAY_HEARTBEAT_TIMEOUT_MS = 20_000; -// After a loss, the duplex gateway answers each new local request with a 503, -// then exits when this grace ends. The grace gives the local caller time to read -// the 409 for an outstanding request and a 503 for a new request. -const DEFAULT_DUPLEX_GATEWAY_LOSS_EXIT_GRACE_MS = 1_000; - /** Span name that wraps one Paperclip-API callback request — read the request, * write the response, and remove the request file. */ const CALLBACK_BRIDGE_RELAY_REQUEST_SPAN = "sandbox.callbackBridge.relayRequest"; @@ -1847,9 +1828,8 @@ export async function startSandboxCallbackBridgeServer(input: { // // This gateway turns each local loopback request into one HTTP/2 stream to // the host server in `http2-bridge-server.ts`. It sits beside the file-mode -// and duplex-mode gateways above; it changes neither of them, and no mode -// dispatch selects it yet — a later phase wires it into the generated -// in-sandbox entrypoint and the transport-selection path. +// gateway above; it changes neither of them. The generated in-sandbox +// entrypoint and the transport-selection path already select it. // --------------------------------------------------------------------------- /** @@ -1998,7 +1978,7 @@ function forwardOneHttp2Request( * local request the caller hands it (already checked against the bridge * token — see {@link SandboxHttp2BridgeGatewayRequest.receivedToken}) as one * HTTP/2 stream. It keeps the header allowlist on the sandbox side, exactly - * as the file-mode and duplex-mode gateways do. + * as the file-mode gateway does. */ export function createSandboxHttp2BridgeGateway( options: CreateSandboxHttp2BridgeGatewayOptions, @@ -2306,36 +2286,18 @@ const bridgeToken = process.env.PAPERCLIP_BRIDGE_TOKEN; const host = process.env.PAPERCLIP_BRIDGE_HOST || "127.0.0.1"; const port = Number(process.env.PAPERCLIP_BRIDGE_PORT || "0"); // The host assigns the loopback port and passes it through the launch -// environment. The duplex gateway binds exactly this port; it never selects a +// environment. The gateway binds exactly this port; it never selects a // different one. The host also passes one random per-open nonce here. The // gateway echoes it in the READY frame so the host correlates READY with this // channel open. The nonce is a liveness signal, not authentication. const bridgeNonce = process.env.PAPERCLIP_BRIDGE_NONCE || ""; const pollIntervalMs = Number(process.env.PAPERCLIP_BRIDGE_POLL_INTERVAL_MS || "100"); const responseTimeoutMs = Number( - process.env.PAPERCLIP_BRIDGE_RESPONSE_TIMEOUT_MS || - (bridgeMode === "${SANDBOX_CALLBACK_BRIDGE_DUPLEX_MODE}" - ? "${DEFAULT_DUPLEX_GATEWAY_WAIT_BUDGET_MS}" - : "${DEFAULT_BRIDGE_RESPONSE_TIMEOUT_MS}"), + process.env.PAPERCLIP_BRIDGE_RESPONSE_TIMEOUT_MS || "${DEFAULT_BRIDGE_RESPONSE_TIMEOUT_MS}", ); const maxQueueDepth = Number(process.env.PAPERCLIP_BRIDGE_MAX_QUEUE_DEPTH || "${DEFAULT_BRIDGE_MAX_QUEUE_DEPTH}"); const maxBodyBytes = Number(process.env.PAPERCLIP_BRIDGE_MAX_BODY_BYTES || "${DEFAULT_BRIDGE_MAX_BODY_BYTES}"); -// The host passes the separate sandbox-process raw-decoder cap here. The in-sandbox -// decoder enforces it locally under the "sandbox_process" scope; it never shares the -// host aggregate byte ledger. -const maxDuplexDecoderBytes = Number( - process.env.PAPERCLIP_BRIDGE_MAX_DUPLEX_DECODER_BYTES || "${DEFAULT_BRIDGE_MAX_DUPLEX_DECODER_BYTES}", -); -const heartbeatIntervalMs = Number( - process.env.PAPERCLIP_BRIDGE_HEARTBEAT_INTERVAL_MS || "${DEFAULT_DUPLEX_GATEWAY_HEARTBEAT_INTERVAL_MS}", -); -const heartbeatTimeoutMs = Number( - process.env.PAPERCLIP_BRIDGE_HEARTBEAT_TIMEOUT_MS || "${DEFAULT_DUPLEX_GATEWAY_HEARTBEAT_TIMEOUT_MS}", -); -const lossExitGraceMs = Number( - process.env.PAPERCLIP_BRIDGE_LOSS_EXIT_GRACE_MS || "${DEFAULT_DUPLEX_GATEWAY_LOSS_EXIT_GRACE_MS}", -); -// The header allowlist. Both the file gateway and the duplex gateway strip an +// The header allowlist. Both the file gateway and the http2 gateway strip an // inbound request to these headers before they forward it. One copy serves both // modes. The route allowlist stays on the host: both modes forward a request to // the host, and the host enforces the same route allowlist for each. @@ -2344,11 +2306,19 @@ const allowedHeaders = new Set(${JSON.stringify([...DEFAULT_SANDBOX_CALLBACK_BRI if (!bridgeToken) { throw new Error("PAPERCLIP_BRIDGE_TOKEN is required."); } +// Closed allowlist for the bridge mode. The generated gateway supports exactly +// two transports: http2 and the file-mode queue. Every other value, including +// the retired duplex transport, fails startup at once instead of falling +// through to a mode that never ran. This check runs before the queue-directory +// check below, so an unsupported mode never reaches a state where a missing +// queue directory masks the real problem. if ( - bridgeMode !== "${SANDBOX_CALLBACK_BRIDGE_DUPLEX_MODE}" && bridgeMode !== "${SANDBOX_CALLBACK_BRIDGE_HTTP2_MODE}" && - !queueDir + bridgeMode !== "${SANDBOX_CALLBACK_BRIDGE_FILE_MODE}" ) { + throw new Error("Unsupported PAPERCLIP_API_BRIDGE_MODE: " + bridgeMode); +} +if (bridgeMode !== "${SANDBOX_CALLBACK_BRIDGE_HTTP2_MODE}" && !queueDir) { throw new Error("PAPERCLIP_BRIDGE_QUEUE_DIR and PAPERCLIP_BRIDGE_TOKEN are required."); } @@ -2598,350 +2568,6 @@ async function runFileGateway() { }); } -// Split one whole body buffer into body_chunk frames. The gateway buffers the -// whole source body once, then splits it here. Each frame carries one fixed raw -// slice as base64 text, except the final frame, which carries the remaining -// bytes. A zero-length body yields no frame. Send-side true streaming is a -// separate goal; this whole-body split is acceptable for this gateway. -function splitDuplexBodyIntoChunks(id, body) { - const frames = []; - let seq = 0; - for (let offset = 0; offset < body.length; offset += DUPLEX_BODY_CHUNK_RAW_BYTES) { - const slice = body.subarray(offset, offset + DUPLEX_BODY_CHUNK_RAW_BYTES); - frames.push({ - version: DUPLEX_FRAME_VERSION, - type: "body_chunk", - id: id, - seq: seq, - data: slice.toString("base64"), - }); - seq += 1; - } - return frames; -} - -function runDuplexGateway() { - // One outstanding local request per id. Each entry holds the HTTP resolver and - // the wait-budget timer. - const pending = new Map(); - // One in-flight response reassembly per id. The host returns a response as an - // envelope frame that carries bodyByteCount, then the body_chunk frames that - // carry the body. The gateway reassembles the response body in memory here; it - // does not spill, because a production response body stays small. - const responseAssembly = new Map(); - let unavailable = false; - let lossTriggered = false; - let lastInboundAt = Date.now(); - - function diag(message) { - // Diagnostics go to stderr only. Stdout carries frames. - process.stderr.write("[paperclip-bridge] " + message + "\\n"); - } - - function writeFrame(frame) { - process.stdout.write(encodeDuplexFrame(frame)); - } - - function stopTimers() { - clearInterval(heartbeatSendTimer); - clearInterval(heartbeatWatchTimer); - } - - // Loss behavior. On stdin end, a close frame, or a heartbeat timeout, answer - // every outstanding request with a non-retryable 409 (the request may have - // reached the host), answer every new request with a retryable 503 (the - // request never left the gateway), then exit. The gateway never replays a - // request. - function triggerLoss(reason) { - if (lossTriggered) return; - lossTriggered = true; - unavailable = true; - diag("duplex channel lost: " + reason); - stopTimers(); - for (const entry of pending.values()) { - clearTimeout(entry.timer); - entry.resolve({ - status: 409, - headers: { - "content-type": "application/json", - "x-paperclip-bridge-outcome": "indeterminate", - }, - body: JSON.stringify({ error: "outcome_indeterminate" }), - }); - } - pending.clear(); - responseAssembly.clear(); - const exitTimer = setTimeout(() => process.exit(0), lossExitGraceMs); - if (typeof exitTimer.unref === "function") exitTimer.unref(); - } - - // Fail one outstanding request with a bounded local 502. The gateway calls it - // when a response reassembly breaks: a reordered seq, a base64 that is not - // canonical, or a total that overruns the declared body size. The host is the - // response peer, so this is a defensive local error, not a channel loss. - function failRequest(id, message) { - responseAssembly.delete(id); - const entry = pending.get(id); - if (!entry) return; - pending.delete(id); - clearTimeout(entry.timer); - entry.resolve({ - status: 502, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ error: message }), - }); - } - - function handleInboundFrame(frame) { - if (frame.type === "response") { - const entry = pending.get(frame.id); - if (!entry) return; - // Map an indeterminate outcome to a non-retryable 409, the same contract - // the file gateway applies through the outcome header. - const statusCode = - frame.outcome === "indeterminate" ? 409 : typeof frame.status === "number" ? frame.status : 200; - const headers = frame.headers || {}; - if (frame.bodyByteCount === 0) { - // A zero-length body accepts no body_chunk, so the response completes now. - pending.delete(frame.id); - responseAssembly.delete(frame.id); - clearTimeout(entry.timer); - entry.resolve({ status: statusCode, headers: headers, body: "" }); - return; - } - // The body rides body_chunk frames that share this id. Record the envelope - // and wait for the chunks. - responseAssembly.set(frame.id, { - status: statusCode, - headers: headers, - bodyByteCount: frame.bodyByteCount, - received: 0, - nextSeq: 0, - chunks: [], - }); - return; - } - if (frame.type === "body_chunk") { - const asm = responseAssembly.get(frame.id); - if (!asm) return; - if (frame.seq !== asm.nextSeq) { - failRequest(frame.id, "duplex response body_chunk seq is out of order"); - return; - } - const decoded = Buffer.from(frame.data, "base64"); - if (decoded.toString("base64") !== frame.data) { - failRequest(frame.id, "duplex response body_chunk is not canonical base64"); - return; - } - if (decoded.length === 0) { - failRequest(frame.id, "duplex response body_chunk is empty"); - return; - } - if (asm.received + decoded.length > asm.bodyByteCount) { - failRequest(frame.id, "duplex response body overruns the declared size"); - return; - } - const isFinal = asm.received + decoded.length === asm.bodyByteCount; - if ( - decoded.length > DUPLEX_BODY_CHUNK_RAW_BYTES || - (!isFinal && decoded.length !== DUPLEX_BODY_CHUNK_RAW_BYTES) - ) { - failRequest(frame.id, "duplex response body_chunk has the wrong size"); - return; - } - asm.nextSeq += 1; - asm.received += decoded.length; - asm.chunks.push(decoded); - if (asm.received === asm.bodyByteCount) { - const entry = pending.get(frame.id); - responseAssembly.delete(frame.id); - if (!entry) return; - pending.delete(frame.id); - clearTimeout(entry.timer); - entry.resolve({ - status: asm.status, - headers: asm.headers, - body: Buffer.concat(asm.chunks).toString("utf8"), - }); - } - return; - } - if (frame.type === "close") { - triggerLoss("host sent a close frame"); - return; - } - // A heartbeat, ready, error, or request frame proves liveness but needs no - // local action here. - } - - const decoder = new DuplexFrameDecoder({ maxAggregateBytes: maxDuplexDecoderBytes }); - process.stdin.on("data", (chunk) => { - lastInboundAt = Date.now(); - for (const result of decoder.push(chunk)) { - if (!result.ok) { - diag("dropped an inbound frame: " + result.error.code); - continue; - } - handleInboundFrame(result.frame); - } - }); - process.stdin.on("end", () => triggerLoss("stdin reached end of file")); - process.stdin.on("error", (error) => - triggerLoss("stdin error: " + (error && error.message ? error.message : String(error))), - ); - - const heartbeatSendTimer = setInterval(() => { - writeFrame({ version: DUPLEX_FRAME_VERSION, type: "heartbeat" }); - }, heartbeatIntervalMs); - const heartbeatWatchTimer = setInterval(() => { - if (Date.now() - lastInboundAt > heartbeatTimeoutMs) { - triggerLoss("no inbound frame within the heartbeat timeout"); - } - }, Math.max(200, Math.floor(heartbeatTimeoutMs / 4))); - - const server = createServer(async (req, res) => { - try { - const auth = req.headers.authorization || ""; - const receivedToken = auth.startsWith("Bearer ") ? auth.slice("Bearer ".length) : ""; - if (!tokensMatch(receivedToken)) { - writeJsonResponse(res, 401, { error: "Invalid bridge token." }); - return; - } - if (unavailable) { - writeJsonResponse(res, 503, { error: "bridge_unavailable" }); - return; - } - if (pending.size >= maxQueueDepth) { - writeJsonResponse(res, 503, { error: "Bridge request queue is full." }); - return; - } - const url = new URL(req.url || "/", "http://127.0.0.1"); - const contentType = typeof req.headers["content-type"] === "string" ? req.headers["content-type"] : ""; - if (req.method && req.method !== "GET" && req.method !== "HEAD" && !/json/i.test(contentType)) { - writeJsonResponse(res, 415, { error: "Bridge only accepts JSON request bodies." }); - return; - } - const requestId = randomUUID(); - const requestBodyBuffer = Buffer.from(await readBody(req), "utf8"); - if (unavailable) { - writeJsonResponse(res, 503, { error: "bridge_unavailable" }); - return; - } - // The request body rides body_chunk frames. The envelope carries only the - // raw byte count, so the envelope stays small and the body splits into - // fixed-size slices that each stay under the frame bound. - const requestFrame = { - version: DUPLEX_FRAME_VERSION, - type: "request", - id: requestId, - method: req.method || "GET", - path: url.pathname, - query: url.search, - headers: normalizeHeaders(req.headers), - bodyByteCount: requestBodyBuffer.length, - }; - const encodedRequest = encodeDuplexFrameChecked(requestFrame); - if (!encodedRequest.ok) { - writeJsonResponse(res, 413, { error: "request_too_large" }); - return; - } - // Pre-encode every body_chunk frame and enforce the frame size bound on - // each. A fixed raw slice never exceeds the bound, so this guard is - // defensive. On a rejection, fail this one local request with a clean 413. - // The frames never leave the gateway, so no other in-flight request is - // affected and the channel stays open. - const chunkFrames = splitDuplexBodyIntoChunks(requestId, requestBodyBuffer); - const encodedChunks = []; - let chunkTooLarge = false; - for (const chunk of chunkFrames) { - const encodedChunk = encodeDuplexFrameChecked(chunk); - if (!encodedChunk.ok) { - chunkTooLarge = true; - break; - } - encodedChunks.push(encodedChunk.line); - } - if (chunkTooLarge) { - writeJsonResponse(res, 413, { error: "request_too_large" }); - return; - } - const response = await new Promise((resolve) => { - const timer = setTimeout(() => { - responseAssembly.delete(requestId); - if (pending.delete(requestId)) { - resolve({ - status: 502, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ error: "Timed out waiting for host bridge response." }), - }); - } - }, responseTimeoutMs); - pending.set(requestId, { resolve: resolve, timer: timer }); - process.stdout.write(encodedRequest.line); - for (const line of encodedChunks) process.stdout.write(line); - }); - res.statusCode = typeof response.status === "number" ? response.status : 200; - for (const [key, value] of Object.entries(response.headers || {})) { - if (typeof value !== "string" || key.toLowerCase() === "content-length") continue; - res.setHeader(key, value); - } - res.end(typeof response.body === "string" ? response.body : ""); - } catch (error) { - writeJsonResponse(res, 502, { error: error instanceof Error ? error.message : String(error) }); - } - }); - - process.on("SIGINT", () => { - try { - server.close(); - } catch (error) { - diag("server close error: " + (error && error.message ? error.message : String(error))); - } - process.exit(0); - }); - process.on("SIGTERM", () => { - try { - server.close(); - } catch (error) { - diag("server close error: " + (error && error.message ? error.message : String(error))); - } - process.exit(0); - }); - - // Bind-or-exit. The host assigns a positive loopback port. The gateway binds - // exactly that port. On a non-positive assigned port or a bind failure it - // writes a diagnostic and exits with a nonzero code. It never selects a - // different port, so no untrusted workload can steer the endpoint. - if (!Number.isInteger(port) || port <= 0) { - diag("duplex gateway requires a positive assigned PAPERCLIP_BRIDGE_PORT; got " + String(port)); - process.exit(1); - } - server.on("error", (error) => { - diag("duplex gateway could not bind port " + String(port) + ": " + (error && error.message ? error.message : String(error))); - process.exit(1); - }); - server.listen(port, host, () => { - const address = server.address(); - if (!address || typeof address === "string") { - diag("duplex gateway did not expose a TCP address"); - process.exit(1); - return; - } - // The gateway sends READY only after the listener binds. READY is a liveness - // signal: it carries the frame version and the echoed nonce, and no address - // data. The host builds the endpoint from its own stored port. Stdout carries - // only frames. - writeFrame({ - version: DUPLEX_FRAME_VERSION, - type: "ready", - nonce: bridgeNonce, - }); - // READY is on the wire, so the host will adopt this process. From here on - // an uncaught fault must not kill the listener. - gatewayReady = true; - }); -} - // --------------------------------------------------------------------------- // http2_v1: run one Node HTTP/2 client session directly on stdin/stdout. // @@ -3160,15 +2786,14 @@ function runHttp2Gateway() { }); } +// The startup check above already rejected every value except http2 and +// queue, so this dispatch names both modes explicitly and never falls +// through to the queue gateway for an unsupported mode. if (bridgeMode === "${SANDBOX_CALLBACK_BRIDGE_HTTP2_MODE}") { runHttp2Gateway(); -} else if (bridgeMode === "${SANDBOX_CALLBACK_BRIDGE_DUPLEX_MODE}") { - // No host selection path sets this mode anymore (http2_v1 replaced it), but - // the generated gateway keeps the mode reachable: it stays defined here, - // unchanged, so nothing that still spawns the gateway directly with this - // mode name breaks. - runDuplexGateway(); -} else { +} else if (bridgeMode === "${SANDBOX_CALLBACK_BRIDGE_FILE_MODE}") { await runFileGateway(); +} else { + throw new Error("Unsupported PAPERCLIP_API_BRIDGE_MODE: " + bridgeMode); }`; }