From 16b59c9315eb2fa2d61facc63ad3d678f32cea1e Mon Sep 17 00:00:00 2001 From: Nicky Leach Date: Sun, 23 Aug 2026 08:52:31 -0700 Subject: [PATCH] feat(adapter-utils): stream duplex bridge bodies as sequenced chunks with receive-side spill (#12006) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Agent adapters use a duplex bridge to send requests and responses across an isolated boundary > - The bridge held each request body and response body in memory on both ends > - Large bodies can exhaust memory and reduce the safe size of adapter traffic > - This pull request sends receive-side bodies as sequenced chunks and spills large bodies to disk > - The benefit is bounded memory use with strict size, order, and cleanup checks ## Linked Issues or Issue Description **What existing behavior does this improve?** The adapter-utils duplex bridge transports request and response bodies across the sandbox boundary. **Current behavior** The bridge stores each complete body in memory on both ends of the duplex channel. **Proposed behavior** The bridge sends body chunks with sequence checks. The receive side keeps bodies up to 1 MiB in memory and spills larger bodies to a temporary file. **Reason and benefit** This change reduces memory pressure and keeps malformed or oversized input on a terminal error path. **Breaking changes** The duplex frame version changes to version 2. The request and response envelopes now carry bodyByteCount, and body_chunk frames carry the body data. ## What Changed - Add version 2 body_chunk frames with 256 KiB raw slices encoded as canonical base64 text. - Add receive-side memory and spill reassembly with per-channel disk and file limits. - Reject malformed, reordered, oversized, truncated, and non-canonical body chunks. - Stream reassembled request bodies to the host forward handler with a web stream and half-duplex request. - Remove spill files on success, failure, channel death, and startup cleanup. ## Verification - Run the adapter-utils type-check. - Run the adapter-utils duplex test suite. - Run all pull request checks. - Run the Greptile review and confirm a 5/5 score with no open findings. ## Risks The wire format changes from version 1 to version 2. Older bridge peers cannot use this protocol. The receive path adds temporary file operations and cleanup paths. The implementation fails closed when a body violates size or sequence rules. ## Model Used OpenAI GPT-5 Codex, tool-enabled coding agent. The exact context-window size and reasoning mode are not exposed by the runtime. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with Fixes: # / Closes # / Refs # OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub #NNN / github.com/paperclipai/paperclip URLs) - [x] My branch name describes the change (e.g. docs/..., fix/...) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- .../src/acpx-engine/execute.test.ts | 12 +- .../src/duplex-body-spool.test.ts | 306 +++++++++ .../adapter-utils/src/duplex-body-spool.ts | 596 ++++++++++++++++ .../duplex-bridge-broker-byte-ledger.test.ts | 121 +++- .../src/duplex-bridge-broker.test.ts | 241 +++++-- .../adapter-utils/src/duplex-bridge-broker.ts | 643 ++++++++++++------ .../src/duplex-bridge-local-e2e.test.ts | 113 ++- .../src/duplex-frame-codec.test.ts | 31 +- .../adapter-utils/src/duplex-frame-codec.ts | 82 ++- .../src/duplex-frame-vectors.json | 359 +++++++--- .../src/execution-target-sandbox.test.ts | 388 +++++++---- .../adapter-utils/src/execution-target.ts | 49 +- .../src/sandbox-callback-bridge.ts | 172 ++++- 13 files changed, 2493 insertions(+), 620 deletions(-) create mode 100644 packages/adapter-utils/src/duplex-body-spool.test.ts create mode 100644 packages/adapter-utils/src/duplex-body-spool.ts diff --git a/packages/adapter-utils/src/acpx-engine/execute.test.ts b/packages/adapter-utils/src/acpx-engine/execute.test.ts index 6f5ae7dff6..217e4f582d 100644 --- a/packages/adapter-utils/src/acpx-engine/execute.test.ts +++ b/packages/adapter-utils/src/acpx-engine/execute.test.ts @@ -5763,8 +5763,8 @@ describe("ACPX engine sandbox duplex run-disposition seam (fail-closed)", () => // Wrap a started real broker in a paperclip bridge handle. The handle exposes // the same run-disposition surface the sandbox bridge exposes, so the seam runs // against the real latch and the real orderly-completion mark. - function bridgeOverBroker(fake: ReturnType) { - const broker = createDuplexBridgeBroker({ + async function bridgeOverBroker(fake: ReturnType) { + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), }); @@ -5891,7 +5891,7 @@ describe("ACPX engine sandbox duplex run-disposition seam (fail-closed)", () => it("fails a completed run when the duplex channel was lost before the completion", async () => { const sandbox = await setupRemoteSandbox(); const fake = createFakeDuplexChannel(); - const { broker, handle, settleRunDisposition } = bridgeOverBroker(fake); + const { broker, handle, settleRunDisposition } = await bridgeOverBroker(fake); // Latch the loss before the ACP terminal resolves. const runtime = runtimeWithControlledResult(() => fake.emitExit({ exitCode: 1 })); @@ -5912,7 +5912,7 @@ describe("ACPX engine sandbox duplex run-disposition seam (fail-closed)", () => it("keeps a completed run a success when the channel stays live, and a later teardown loss is benign", async () => { const sandbox = await setupRemoteSandbox(); const fake = createFakeDuplexChannel(); - const { broker, handle, settleRunDisposition } = bridgeOverBroker(fake); + const { broker, handle, settleRunDisposition } = await bridgeOverBroker(fake); // No loss before the completion. const runtime = runtimeWithControlledResult(); @@ -5932,7 +5932,7 @@ describe("ACPX engine sandbox duplex run-disposition seam (fail-closed)", () => it("does not let a later completion or activity clear the loss latch", async () => { const sandbox = await setupRemoteSandbox(); const fake = createFakeDuplexChannel(); - const { broker, handle } = bridgeOverBroker(fake); + const { broker, handle } = await bridgeOverBroker(fake); // Latch the loss before the ACP terminal resolves. const runtime = runtimeWithControlledResult(() => fake.emitExit({ exitCode: 1 })); @@ -5951,7 +5951,7 @@ describe("ACPX engine sandbox duplex run-disposition seam (fail-closed)", () => it("marks an orderly completion on a failed terminal so the teardown loss emits no false loss", async () => { const sandbox = await setupRemoteSandbox(); const fake = createFakeDuplexChannel(); - const { broker, handle, markOrderlyCompletion } = bridgeOverBroker(fake); + const { broker, handle, markOrderlyCompletion } = await bridgeOverBroker(fake); // The turn fails, and no channel loss ordered before the finalization. const runtime = runtimeWithFailedResult(); diff --git a/packages/adapter-utils/src/duplex-body-spool.test.ts b/packages/adapter-utils/src/duplex-body-spool.test.ts new file mode 100644 index 0000000000..37307f2e67 --- /dev/null +++ b/packages/adapter-utils/src/duplex-body-spool.test.ts @@ -0,0 +1,306 @@ +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { DUPLEX_FRAME_VERSION, type DuplexBodyChunkFrame } from "./duplex-frame-codec.js"; +import { + DuplexBodyReceiver, + type DuplexBodyReceiverConfig, + splitBodyIntoChunkFrames, + sweepStaleSpillDirectories, +} from "./duplex-body-spool.js"; + +// Read the whole reassembled body back, whatever path it took. A memory body +// returns its buffer; a spilled body streams from disk. The test uses this only +// to assert the round-trip content, never on a hot path. +async function readBodyBytes(body: { + spilled: boolean; + createReadStream: () => NodeJS.ReadableStream; +}): Promise { + const chunks: Buffer[] = []; + for await (const chunk of body.createReadStream()) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + return Buffer.concat(chunks); +} + +let spillRoot: string; + +beforeEach(async () => { + spillRoot = await fs.mkdtemp(path.join(os.tmpdir(), "duplex-spool-test-")); +}); + +afterEach(async () => { + await fs.rm(spillRoot, { recursive: true, force: true }).catch(() => undefined); +}); + +function makeReceiver(overrides: DuplexBodyReceiverConfig = {}): Promise { + return DuplexBodyReceiver.create({ spillRootDir: spillRoot, ...overrides }); +} + +const RAW = 8; + +// Feed a whole body through a receiver as an envelope plus chunk frames. Return +// the settled body handle. The caller pushes the chunks the split produces. +async function transfer( + receiver: DuplexBodyReceiver, + id: string, + body: Buffer, +): ReturnType { + const promise = receiver.begin(id, body.length); + const frames = splitBodyIntoChunkFrames(id, body, DUPLEX_FRAME_VERSION, RAW); + for (const frame of frames) { + const result = receiver.pushChunk(frame); + expect(result.ok).toBe(true); + } + return promise; +} + +describe("duplex body reassembler — round trip", () => { + it("reassembles a small body on the memory path", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + const source = Buffer.from("hello duplex body"); + const body = await transfer(receiver, "req-mem", source); + expect(body.spilled).toBe(false); + expect(body.byteCount).toBe(source.length); + expect(body.readMemory().equals(source)).toBe(true); + expect((await readBodyBytes(body)).equals(source)).toBe(true); + await body.dispose(); + }); + + it("reassembles a large body on the spill path without holding it in memory", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 16, rawChunkBytes: RAW }); + const source = Buffer.from("abcdefghijklmnopqrstuvwxyz0123456789ABCD"); // 40 bytes + const body = await transfer(receiver, "req-spill", source); + expect(body.spilled).toBe(true); + expect(body.byteCount).toBe(source.length); + // A spilled body never loads the whole body into memory on the receive side. + expect(() => body.readMemory()).toThrow(); + expect(receiver.spillUsedBytes).toBe(source.length); + expect(receiver.spillOpenFiles).toBe(1); + expect((await readBodyBytes(body)).equals(source)).toBe(true); + await body.dispose(); + // The dispose path unlinks the file and releases the reservation. + expect(receiver.spillUsedBytes).toBe(0); + expect(receiver.spillOpenFiles).toBe(0); + const entries = await fs.readdir(receiver.spillDir); + expect(entries).toHaveLength(0); + await receiver.destroy(); + }); + + it("completes a zero-length body at once and accepts no chunk", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 16, rawChunkBytes: RAW }); + const body = await receiver.begin("req-zero", 0); + expect(body.byteCount).toBe(0); + expect(body.readMemory().length).toBe(0); + const stray = receiver.pushChunk({ + version: DUPLEX_FRAME_VERSION, + type: "body_chunk", + id: "req-zero", + seq: 0, + data: Buffer.from("x").toString("base64"), + }); + expect(stray.ok).toBe(false); + if (!stray.ok) expect(stray.error.code).toBe("zero_body_has_chunk"); + await body.dispose(); + }); +}); + +describe("duplex body reassembler — rejection matrix", () => { + function chunk(id: string, seq: number, data: string): DuplexBodyChunkFrame { + return { version: DUPLEX_FRAME_VERSION, type: "body_chunk", id, seq, data }; + } + + it("rejects a duplicate seq", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + void receiver.begin("d", 16); + expect(receiver.pushChunk(chunk("d", 0, Buffer.alloc(RAW, 1).toString("base64"))).ok).toBe(true); + const dup = receiver.pushChunk(chunk("d", 0, Buffer.alloc(RAW, 2).toString("base64"))); + expect(dup.ok).toBe(false); + if (!dup.ok) expect(dup.error.code).toBe("seq_out_of_order"); + }); + + it("rejects a gap in the seq order", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + void receiver.begin("g", 24); + expect(receiver.pushChunk(chunk("g", 0, Buffer.alloc(RAW, 1).toString("base64"))).ok).toBe(true); + const gap = receiver.pushChunk(chunk("g", 2, Buffer.alloc(RAW, 2).toString("base64"))); + expect(gap.ok).toBe(false); + if (!gap.ok) expect(gap.error.code).toBe("seq_out_of_order"); + }); + + it("rejects a reordered seq", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + void receiver.begin("r", 24); + const reorder = receiver.pushChunk(chunk("r", 1, Buffer.alloc(RAW, 1).toString("base64"))); + expect(reorder.ok).toBe(false); + if (!reorder.ok) expect(reorder.error.code).toBe("seq_out_of_order"); + }); + + it("rejects non-canonical base64", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + const begun = receiver.begin("b", 8); + begun.catch(() => undefined); + // "AB==" carries non-zero bits in the final sextet, so it decodes to one + // byte and re-encodes to "AA==". The round-trip mismatch marks it not + // canonical. + const bad = receiver.pushChunk(chunk("b", 0, "AB==")); + expect(bad.ok).toBe(false); + if (!bad.ok) expect(bad.error.code).toBe("chunk_not_canonical_base64"); + }); + + it("rejects a short non-final chunk (truncation)", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + void receiver.begin("t", 24); + // A non-final chunk must be exactly RAW bytes. A 4-byte chunk that does not + // complete the 24-byte body is a truncation. + const truncation = receiver.pushChunk(chunk("t", 0, Buffer.alloc(4, 1).toString("base64"))); + expect(truncation.ok).toBe(false); + if (!truncation.ok) expect(truncation.error.code).toBe("chunk_wrong_size"); + }); + + it("rejects an empty non-final chunk", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + void receiver.begin("e", 16); + const empty = receiver.pushChunk(chunk("e", 0, "")); + expect(empty.ok).toBe(false); + if (!empty.ok) expect(empty.error.code).toBe("empty_chunk"); + }); + + it("rejects an overrun past the declared body size", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + // A 12-byte body takes one full RAW chunk, then a final chunk of 4 bytes. A + // second full RAW chunk overruns the declared size. + void receiver.begin("o", 12); + expect(receiver.pushChunk(chunk("o", 0, Buffer.alloc(RAW, 1).toString("base64"))).ok).toBe(true); + const overrun = receiver.pushChunk(chunk("o", 1, Buffer.alloc(RAW, 2).toString("base64"))); + expect(overrun.ok).toBe(false); + if (!overrun.ok) expect(overrun.error.code).toBe("body_overrun"); + }); + + it("rejects an over-size final chunk", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + void receiver.begin("os", 40); + const over = receiver.pushChunk(chunk("os", 0, Buffer.alloc(RAW + 4, 1).toString("base64"))); + expect(over.ok).toBe(false); + if (!over.ok) expect(over.error.code).toBe("chunk_wrong_size"); + }); + + it("rejects a chunk that arrives with no matching envelope", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + const orphan = receiver.pushChunk(chunk("missing", 0, Buffer.alloc(RAW, 1).toString("base64"))); + expect(orphan.ok).toBe(false); + if (!orphan.ok) expect(orphan.error.code).toBe("chunk_without_envelope"); + }); + + it("rejects a chunk that arrives after completion", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + const promise = receiver.begin("c", RAW); + expect(receiver.pushChunk(chunk("c", 0, Buffer.alloc(RAW, 1).toString("base64"))).ok).toBe(true); + await promise; + const late = receiver.pushChunk(chunk("c", 1, Buffer.alloc(RAW, 2).toString("base64"))); + expect(late.ok).toBe(false); + if (!late.ok) expect(late.error.code).toBe("chunk_after_completion"); + }); +}); + +describe("duplex body reassembler — spill caps fail closed", () => { + it("fails closed when a write exceeds the per-channel byte cap and unlinks the partial file", async () => { + // A 5-byte cap admits the first RAW=8 chunk write? No: reserve 8 > cap 5, so + // the first write fails closed. Use a cap that admits one chunk but not two. + const receiver = await makeReceiver({ + spillThresholdBytes: 4, + rawChunkBytes: RAW, + channelSpillByteCap: RAW + 2, + }); + const promise = receiver.begin("cap", 24); + expect(receiver.pushChunk({ version: DUPLEX_FRAME_VERSION, type: "body_chunk", id: "cap", seq: 0, data: Buffer.alloc(RAW, 1).toString("base64") }).ok).toBe(true); + // The second chunk write reserves past the byte cap. + receiver.pushChunk({ version: DUPLEX_FRAME_VERSION, type: "body_chunk", id: "cap", seq: 1, data: Buffer.alloc(RAW, 2).toString("base64") }); + await expect(promise).rejects.toMatchObject({ code: "spill_cap_exceeded" }); + // The partial file is gone and the reservation released. + expect(receiver.spillUsedBytes).toBe(0); + expect(receiver.spillOpenFiles).toBe(0); + const entries = await fs.readdir(receiver.spillDir); + expect(entries).toHaveLength(0); + await receiver.destroy(); + }); + + it("fails closed when the open-file cap is reached", async () => { + const receiver = await makeReceiver({ + spillThresholdBytes: 4, + rawChunkBytes: RAW, + channelSpillFileCap: 1, + }); + // The first spill body claims the one file slot and stays open. The channel + // dispose rejects this in-flight body, so attach the catch at once. + const first = receiver.begin("f1", 24); + first.catch(() => undefined); + expect(receiver.pushChunk({ version: DUPLEX_FRAME_VERSION, type: "body_chunk", id: "f1", seq: 0, data: Buffer.alloc(RAW, 1).toString("base64") }).ok).toBe(true); + // The second spill body cannot claim a file slot. + await expect(receiver.begin("f2", 24)).rejects.toMatchObject({ code: "spill_cap_exceeded" }); + expect(receiver.spillOpenFiles).toBe(1); + // The dispose of the in-flight first body rejects its begin promise with a + // terminal channel-closed error and frees the file slot. + await receiver.disposeBody("f1"); + await expect(first).rejects.toMatchObject({ code: "channel_closed" }); + expect(receiver.spillOpenFiles).toBe(0); + await receiver.destroy(); + }); +}); + +describe("duplex body reassembler — lifecycle cleanup", () => { + it("releases the reservation and unlinks the file on channel death", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 4, rawChunkBytes: RAW }); + // The channel death rejects this in-flight body, so attach the catch at once. + const dying = receiver.begin("death", 24); + dying.catch(() => undefined); + expect(receiver.pushChunk({ version: DUPLEX_FRAME_VERSION, type: "body_chunk", id: "death", seq: 0, data: Buffer.alloc(RAW, 1).toString("base64") }).ok).toBe(true); + // Wait for the write to land, so the file exists before the channel dies. + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(receiver.spillOpenFiles).toBe(1); + await receiver.disposeAll(); + await expect(dying).rejects.toMatchObject({ code: "channel_closed" }); + expect(receiver.spillUsedBytes).toBe(0); + expect(receiver.spillOpenFiles).toBe(0); + const entries = await fs.readdir(receiver.spillDir); + expect(entries).toHaveLength(0); + await receiver.destroy(); + }); +}); + +describe("duplex body reassembler — spill file safety", () => { + it("creates the spill directory with mode 0700 and each spill file with mode 0600", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 4, rawChunkBytes: RAW }); + const dirStat = await fs.stat(receiver.spillDir); + expect(dirStat.mode & 0o777).toBe(0o700); + // The final destroy rejects this in-flight body, so attach the catch at once. + const pending = receiver.begin("mode", 8); + pending.catch(() => undefined); + receiver.pushChunk({ version: DUPLEX_FRAME_VERSION, type: "body_chunk", id: "mode", seq: 0, data: Buffer.alloc(RAW, 1).toString("base64") }); + await new Promise((resolve) => setTimeout(resolve, 20)); + const entries = await fs.readdir(receiver.spillDir); + expect(entries).toHaveLength(1); + // The file name is random, so no frame id rides the path. + expect(entries[0]).not.toContain("mode"); + const fileStat = await fs.stat(path.join(receiver.spillDir, entries[0])); + expect(fileStat.mode & 0o777).toBe(0o600); + await receiver.destroy(); + }); +}); + +describe("duplex body reassembler — startup sweep", () => { + it("removes a stale spill directory from a dead process but keeps a live one", async () => { + const deadPid = 999999; // A pid that is almost never alive. + const staleDir = path.join(spillRoot, `paperclip-duplex-spill-${deadPid}-stale`); + await fs.mkdir(staleDir); + await fs.writeFile(path.join(staleDir, "leftover.body"), "x"); + const liveDir = path.join(spillRoot, `paperclip-duplex-spill-${process.pid}-live`); + await fs.mkdir(liveDir); + await sweepStaleSpillDirectories(spillRoot); + await expect(fs.stat(staleDir)).rejects.toThrow(); + // The sweep keeps a directory that belongs to a live process. + await expect(fs.stat(liveDir)).resolves.toBeDefined(); + }); +}); diff --git a/packages/adapter-utils/src/duplex-body-spool.ts b/packages/adapter-utils/src/duplex-body-spool.ts new file mode 100644 index 0000000000..576caca5af --- /dev/null +++ b/packages/adapter-utils/src/duplex-body-spool.ts @@ -0,0 +1,596 @@ +/** + * Receive-side body reassembly and spill for the sandbox duplex channel. + * + * The duplex codec splits one request body or one response body into an envelope + * frame plus a sequence of `body_chunk` frames. This module reassembles those + * chunks back into one body. The receiver reads `bodyByteCount` from the envelope + * first, then chooses one of two paths before it reads the first chunk: + * - memory path: a body at or below the spill threshold stays in a memory + * buffer. + * - spill path: a body above the spill threshold streams to a temporary file, + * so the receiver never holds a whole large body in memory. + * + * The spill path bounds the disk a single channel can consume. A per-channel + * arena caps the bytes on disk and the number of open spill files. The receiver + * reserves a file slot and reserves bytes before it opens a file and before each + * write. A reserve that exceeds a cap fails closed: it rejects the body as a + * resource error and unlinks any partial file. + * + * The reassembler treats every frame as untrusted. It rejects a bad chunk as a + * loud, terminal protocol error: a non-canonical base64 payload, a wrong chunk + * size, an out-of-order `seq`, a total that does not equal `bodyByteCount`, or a + * chunk that arrives for a completed or unknown body. + */ + +import { randomBytes } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { promises as fs } from "node:fs"; +import type { FileHandle } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { Readable } from "node:stream"; + +import { + DUPLEX_BODY_CHUNK_RAW_BYTES, + type DuplexBodyChunkFrame, +} from "./duplex-frame-codec.js"; + +/** + * The default spill threshold, in bytes. A body at or below this size stays in + * memory. A body above it spills to disk. The value is 1 MiB. It stays + * injectable so a test can lower it and exercise the spill path. + */ +export const DEFAULT_DUPLEX_BODY_SPILL_THRESHOLD_BYTES = 1024 * 1024; + +/** + * The default per-channel spill byte cap. The receiver keeps at most this many + * bytes on disk at one time for one channel. The value is 64 MiB. It is about + * 0.64% of the default 10 GiB sandbox disk and stays far below the security + * ceiling of 25% of the disk quota. It stays injectable so a test can lower it. + */ +export const DEFAULT_DUPLEX_CHANNEL_SPILL_BYTE_CAP = 64 * 1024 * 1024; + +/** + * The default per-channel open-file cap. The receiver keeps at most this many + * spill files open at one time for one channel. The value derives from the byte + * cap and the spill threshold: 64 MiB divided by the 1 MiB spill floor gives 64. + * It stays injectable so a test can lower it. + */ +export const DEFAULT_DUPLEX_CHANNEL_SPILL_FILE_CAP = 64; + +/** The prefix of every spill directory name. The startup sweep matches it. */ +const SPILL_DIR_PREFIX = "paperclip-duplex-spill-"; + +/** The config a caller passes to build a {@link DuplexBodyReceiver}. */ +export interface DuplexBodyReceiverConfig { + /** The spill threshold in bytes. Defaults to {@link DEFAULT_DUPLEX_BODY_SPILL_THRESHOLD_BYTES}. */ + spillThresholdBytes?: number; + /** The fixed raw chunk size in bytes. Defaults to {@link DUPLEX_BODY_CHUNK_RAW_BYTES}. */ + rawChunkBytes?: number; + /** The per-channel spill byte cap. Defaults to {@link DEFAULT_DUPLEX_CHANNEL_SPILL_BYTE_CAP}. */ + channelSpillByteCap?: number; + /** The per-channel open-file cap. Defaults to {@link DEFAULT_DUPLEX_CHANNEL_SPILL_FILE_CAP}. */ + channelSpillFileCap?: number; + /** The root directory that holds the spill directory. Defaults to the OS temp dir. */ + spillRootDir?: string; + /** A diagnostic sink for a non-fatal cleanup note. The receiver names no body id here. */ + logger?: (message: string) => void; +} + +/** The reason the reassembler rejected a body. Each reason is a terminal protocol error. */ +export type DuplexBodyErrorCode = + | "chunk_without_envelope" + | "chunk_after_completion" + | "seq_out_of_order" + | "chunk_not_canonical_base64" + | "chunk_wrong_size" + | "empty_chunk" + | "body_overrun" + | "zero_body_has_chunk" + | "spill_cap_exceeded" + | "spill_write_failed" + | "path_escape" + | "channel_closed"; + +/** A reassembly error. The receiver returns it; it never leaks a raw path or a raw provider string. */ +export class DuplexBodyError extends Error { + readonly code: DuplexBodyErrorCode; + constructor(code: DuplexBodyErrorCode, message: string) { + super(message); + this.name = "DuplexBodyError"; + this.code = code; + } +} + +/** + * One reassembled body. The memory path holds the whole body in a buffer. The + * spill path holds a temporary file. A caller reads the memory buffer, or opens + * a fresh read stream, then disposes the body once. + */ +export interface ReassembledBody { + /** The raw byte length of the body. */ + readonly byteCount: number; + /** True when the body spilled to disk. */ + readonly spilled: boolean; + /** + * The whole body as a buffer. The memory path returns it. The spill path + * throws, because the spill path must never load a whole large body into + * memory on the receive side. A caller streams a spilled body instead. + */ + readMemory(): Buffer; + /** A fresh read stream over the body. The memory path streams one buffer; the spill path streams the file. */ + createReadStream(): Readable; + /** Release the body. The spill path unlinks the file and releases the reservation exactly once. */ + dispose(): Promise; +} + +/** + * The per-channel arena that bounds the spill disk use. It tracks the bytes on + * disk and the open files, and it owns the spill directory. It hands out + * reservations and takes them back. The reservation math is synchronous, so a + * concurrent reserve never races past a cap. + */ +class SpillArena { + usedBytes = 0; + openFiles = 0; + constructor( + readonly dir: string, + private readonly byteCap: number, + private readonly fileCap: number, + ) {} + + /** Reserve one open-file slot. Return false when the file cap is already at the limit. */ + reserveFile(): boolean { + if (this.openFiles >= this.fileCap) return false; + this.openFiles += 1; + return true; + } + + /** Release one open-file slot. */ + releaseFile(): void { + if (this.openFiles > 0) this.openFiles -= 1; + } + + /** Reserve `n` bytes of disk. Return false when the byte cap would be exceeded. */ + reserveBytes(n: number): boolean { + if (this.usedBytes + n > this.byteCap) return false; + this.usedBytes += n; + return true; + } + + /** Release `n` bytes of disk. */ + releaseBytes(n: number): void { + this.usedBytes -= n; + if (this.usedBytes < 0) this.usedBytes = 0; + } +} + +/** The internal state of one in-flight body. */ +interface ActiveBody { + bodyByteCount: number; + nextSeq: number; + receivedBytes: number; + spilled: boolean; + /** Memory-path accumulation. Empty for the spill path. */ + memoryChunks: Buffer[]; + /** The spill file path. Null for the memory path. */ + filePath: string | null; + /** The open spill file handle. Null for the memory path or before the first write. */ + handle: FileHandle | null; + /** The bytes this body reserved in the arena. The dispose path releases exactly this many. */ + reservedBytes: number; + /** True when the body holds an arena file slot that the dispose path must release. */ + holdsFileSlot: boolean; + /** True once the dispose path ran, so it runs its release once. */ + disposed: boolean; + /** Resolve the completion promise the caller awaits. */ + resolve: (body: ReassembledBody) => void; + /** Reject the completion promise on a terminal error. */ + reject: (error: DuplexBodyError) => void; + /** Serialize the spill writes, so the chunks land on disk in `seq` order. */ + writeChain: Promise; + /** True once the body settled (resolved or rejected), so a late chunk never double-settles. */ + settled: boolean; +} + +/** The synchronous result of one `pushChunk` call. */ +export type DuplexBodyPushResult = + | { ok: true } + | { ok: false; error: DuplexBodyError }; + +/** + * A per-channel receive-side body reassembler. One instance serves one channel. + * The caller feeds it one envelope per body, then the `body_chunk` frames for + * that body. The reassembler reassembles the body on the memory path or the + * spill path and settles one completion promise per body. + */ +export class DuplexBodyReceiver { + private readonly spillThresholdBytes: number; + private readonly rawChunkBytes: number; + private readonly arena: SpillArena; + private readonly logger?: (message: string) => void; + private readonly active = new Map(); + + private constructor(arena: SpillArena, config: DuplexBodyReceiverConfig) { + this.arena = arena; + this.spillThresholdBytes = + config.spillThresholdBytes ?? DEFAULT_DUPLEX_BODY_SPILL_THRESHOLD_BYTES; + this.rawChunkBytes = config.rawChunkBytes ?? DUPLEX_BODY_CHUNK_RAW_BYTES; + this.logger = config.logger; + } + + /** + * Build a receiver. The receiver owns the spill directory. It creates the + * directory with `mkdtemp` and mode 0700, then sweeps stale spill directories + * from a prior process. The sweep runs and awaits before the receiver serves a + * body, so no caller races a half-swept directory. + */ + static async create(config: DuplexBodyReceiverConfig = {}): Promise { + const root = config.spillRootDir ?? os.tmpdir(); + await sweepStaleSpillDirectories(root, config.logger); + const dir = await fs.mkdtemp(path.join(root, `${SPILL_DIR_PREFIX}${process.pid}-`)); + // `mkdtemp` creates the directory with mode 0700 on a POSIX host by default, + // but a permissive umask can widen it. Set the mode so the owner alone reads, + // writes, and lists the spill directory. + await fs.chmod(dir, 0o700); + const arena = new SpillArena( + dir, + config.channelSpillByteCap ?? DEFAULT_DUPLEX_CHANNEL_SPILL_BYTE_CAP, + config.channelSpillFileCap ?? DEFAULT_DUPLEX_CHANNEL_SPILL_FILE_CAP, + ); + return new DuplexBodyReceiver(arena, config); + } + + /** The spill directory this receiver owns. A test reads it to assert containment and cleanup. */ + get spillDir(): string { + return this.arena.dir; + } + + /** The bytes on disk across all in-flight bodies for this channel. A test asserts the reservation. */ + get spillUsedBytes(): number { + return this.arena.usedBytes; + } + + /** The open spill files for this channel. A test asserts the file-slot reservation. */ + get spillOpenFiles(): number { + return this.arena.openFiles; + } + + /** + * Begin one body. The caller passes the envelope `id` and `bodyByteCount`. The + * receiver chooses the memory path or the spill path from `bodyByteCount` + * before the first chunk arrives. A `bodyByteCount` of 0 completes at once with + * an empty body and accepts no chunk. The returned promise resolves with the + * reassembled body, or rejects with a {@link DuplexBodyError} on a terminal + * protocol or resource error. + */ + begin(id: string, bodyByteCount: number): Promise { + return new Promise((resolve, reject) => { + // A repeated envelope id for an in-flight body is a protocol violation. The + // caller must not begin the same id twice. Reject the new begin and leave + // the in-flight body untouched. + if (this.active.has(id)) { + reject(new DuplexBodyError("chunk_after_completion", "duplicate body envelope id")); + return; + } + const body: ActiveBody = { + bodyByteCount, + nextSeq: 0, + receivedBytes: 0, + spilled: bodyByteCount > this.spillThresholdBytes, + memoryChunks: [], + filePath: null, + handle: null, + reservedBytes: 0, + holdsFileSlot: false, + disposed: false, + resolve, + reject, + writeChain: Promise.resolve(), + settled: false, + }; + this.active.set(id, body); + if (bodyByteCount === 0) { + // The zero-length body completes at once. It accepts no chunk, so the + // receiver keeps the id in a completed state to reject a stray chunk. + this.complete(id, body); + return; + } + if (body.spilled) { + // Reserve one file slot before the receiver opens the spill file. A + // reserve past the file cap fails closed at once. + if (!this.arena.reserveFile()) { + this.settleReject(id, body, new DuplexBodyError("spill_cap_exceeded", "spill file cap reached")); + return; + } + body.holdsFileSlot = true; + // Open the spill file with a random name and mode 0600. The name never + // derives from the frame id, so no id rides a path. Verify the resolved + // path stays inside the spill directory before the receiver writes. + this.openSpillFile(id, body); + } + }); + } + + /** + * Push one `body_chunk` frame. The receiver validates the chunk against the + * in-flight body, then appends it on the memory path or the spill path. The + * return value reports a synchronous validation outcome. A spill write failure + * or a completion rejects the begin promise instead. + */ + pushChunk(frame: DuplexBodyChunkFrame): DuplexBodyPushResult { + const body = this.active.get(frame.id); + if (!body) { + return fail("chunk_without_envelope", "body_chunk arrived with no matching envelope"); + } + if (body.bodyByteCount === 0) { + return fail("zero_body_has_chunk", "a zero-length body accepts no body_chunk"); + } + if (body.settled) { + return fail("chunk_after_completion", "body_chunk arrived after the body completed"); + } + if (frame.seq !== body.nextSeq) { + return fail("seq_out_of_order", "body_chunk seq is a duplicate, a gap, or a reorder"); + } + // Decode the base64 payload and confirm it is canonical. A non-canonical + // payload re-encodes to a different string, so the round-trip catches it. + const decoded = Buffer.from(frame.data, "base64"); + if (decoded.toString("base64") !== frame.data) { + return fail("chunk_not_canonical_base64", "body_chunk data is not canonical base64"); + } + if (decoded.length === 0) { + return fail("empty_chunk", "body_chunk is empty"); + } + const nextReceived = body.receivedBytes + decoded.length; + if (nextReceived > body.bodyByteCount) { + return fail("body_overrun", "body_chunk overruns the declared body size"); + } + const isFinal = nextReceived === body.bodyByteCount; + if (!isFinal && decoded.length !== this.rawChunkBytes) { + // A non-final chunk must be exactly the fixed raw slice size. A short + // non-final chunk is a truncation; an over-size chunk is a framing fault. + return fail("chunk_wrong_size", "a non-final body_chunk is not the fixed slice size"); + } + if (decoded.length > this.rawChunkBytes) { + // The final chunk must also stay within the fixed raw slice size. + return fail("chunk_wrong_size", "a body_chunk exceeds the fixed slice size"); + } + body.nextSeq += 1; + body.receivedBytes = nextReceived; + if (body.spilled) { + this.appendSpill(frame.id, body, decoded, isFinal); + } else { + body.memoryChunks.push(decoded); + if (isFinal) this.complete(frame.id, body); + } + return { ok: true }; + } + + /** + * Dispose one body by id, whatever its state. The dispose path unlinks the + * spill file and releases the arena reservation exactly once. The caller runs + * it on every settle path and on channel death, so a spill file never lingers. + */ + async disposeBody(id: string): Promise { + const body = this.active.get(id); + if (!body) return; + this.active.delete(id); + await this.release(body); + } + + /** + * Dispose every in-flight body. The caller runs it on channel death, so a + * channel that dies mid-body leaves no spill file behind. + */ + async disposeAll(): Promise { + const ids = [...this.active.keys()]; + await Promise.all(ids.map((id) => this.disposeBody(id))); + } + + /** + * Remove the spill directory this receiver owns. The caller runs it after it + * disposes every body, on channel shutdown. + */ + async destroy(): Promise { + await this.disposeAll(); + await fs.rm(this.arena.dir, { recursive: true, force: true }).catch(() => undefined); + } + + private openSpillFile(id: string, body: ActiveBody): void { + const name = `${randomBytes(24).toString("hex")}.body`; + const filePath = path.join(this.arena.dir, name); + // Path containment. The resolved file path must stay inside the spill + // directory. A generated random name never escapes, but the check is defense + // in depth against a future name scheme. + const resolvedDir = path.resolve(this.arena.dir); + const resolvedFile = path.resolve(filePath); + if (resolvedFile !== path.join(resolvedDir, name) || !resolvedFile.startsWith(resolvedDir + path.sep)) { + this.settleReject(id, body, new DuplexBodyError("path_escape", "spill path escapes the spill directory")); + return; + } + body.filePath = filePath; + // Open the file once, exclusive, mode 0600. The write chain appends each + // chunk in order to this one handle. + body.writeChain = body.writeChain.then(async () => { + body.handle = await fs.open(filePath, "wx", 0o600); + }); + body.writeChain.catch((error) => { + this.settleReject(id, body, new DuplexBodyError("spill_write_failed", errorText(error))); + }); + } + + private appendSpill(id: string, body: ActiveBody, decoded: Buffer, isFinal: boolean): void { + // Reserve the bytes before the write. A reserve past the byte cap fails + // closed: reject the body and unlink the partial file. + if (!this.arena.reserveBytes(decoded.length)) { + this.settleReject(id, body, new DuplexBodyError("spill_cap_exceeded", "spill byte cap reached")); + return; + } + body.reservedBytes += decoded.length; + body.writeChain = body.writeChain.then(async () => { + if (body.settled) return; + if (!body.handle) throw new Error("spill file handle is not open"); + await body.handle.write(decoded); + if (isFinal) { + await body.handle.close(); + body.handle = null; + this.complete(id, body); + } + }); + body.writeChain.catch((error) => { + if (error instanceof DuplexBodyError) return; + this.settleReject(id, body, new DuplexBodyError("spill_write_failed", errorText(error))); + }); + } + + private complete(id: string, body: ActiveBody): void { + if (body.settled) return; + body.settled = true; + const receiver = this; + const handle: ReassembledBody = { + byteCount: body.bodyByteCount, + spilled: body.spilled, + readMemory(): Buffer { + if (body.spilled) { + throw new DuplexBodyError("spill_write_failed", "a spilled body has no in-memory buffer"); + } + return Buffer.concat(body.memoryChunks); + }, + createReadStream(): Readable { + if (body.spilled && body.filePath) return createReadStream(body.filePath); + return Readable.from(Buffer.concat(body.memoryChunks)); + }, + async dispose(): Promise { + await receiver.disposeBody(id); + }, + }; + body.resolve(handle); + } + + private settleReject(id: string, body: ActiveBody, error: DuplexBodyError): void { + if (body.settled) return; + body.settled = true; + // Release the reservation and unlink the partial file before the reject, so + // a rejected body never keeps a spill file or a reservation. The reject fires + // after the release, so a caller that awaits the reject sees a clean arena. + void this.disposeBody(id).finally(() => body.reject(error)); + } + + private async release(body: ActiveBody): Promise { + if (body.disposed) return; + body.disposed = true; + // Wait for the pending writes to settle, so the unlink never races an open + // handle. Swallow a write error here; the reject path already reported it. + await body.writeChain.catch(() => undefined); + if (body.handle) { + await body.handle.close().catch(() => undefined); + body.handle = null; + } + if (body.filePath) { + await fs.rm(body.filePath, { force: true }).catch((error) => { + this.logger?.(`Duplex spill unlink failed: ${errorText(error)}`); + }); + } + if (body.reservedBytes > 0) { + this.arena.releaseBytes(body.reservedBytes); + body.reservedBytes = 0; + } + if (body.holdsFileSlot) { + this.arena.releaseFile(); + body.holdsFileSlot = false; + } + // A body that the channel disposes before it settles must not leave its + // begin promise pending. Reject it with a terminal channel-closed error, so + // a caller that awaits the body sees the channel death. A body that already + // settled keeps its outcome: `complete` resolves it, and `settleReject` sets + // `settled` before it calls dispose, so this guard never double-settles. + if (!body.settled) { + body.settled = true; + body.reject( + new DuplexBodyError("channel_closed", "the channel closed before the body completed"), + ); + } + } +} + +function fail(code: DuplexBodyErrorCode, message: string): DuplexBodyPushResult { + return { ok: false, error: new DuplexBodyError(code, message) }; +} + +function errorText(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +/** + * Split a whole body buffer into `body_chunk` frames. The send side reads the + * source body once, then calls this to build the frames. Each frame carries one + * raw slice of `rawChunkBytes` as base64 text, except the final frame, which + * carries the remaining bytes. A zero-length body yields no frame. + */ +export function splitBodyIntoChunkFrames( + id: string, + body: Buffer, + version: number, + rawChunkBytes: number = DUPLEX_BODY_CHUNK_RAW_BYTES, +): DuplexBodyChunkFrame[] { + const frames: DuplexBodyChunkFrame[] = []; + let seq = 0; + for (let offset = 0; offset < body.length; offset += rawChunkBytes) { + const slice = body.subarray(offset, offset + rawChunkBytes); + frames.push({ + version, + type: "body_chunk", + id, + seq, + data: slice.toString("base64"), + }); + seq += 1; + } + return frames; +} + +/** + * Sweep stale spill directories from a prior process. The receiver runs it at + * startup, before it serves a body. It removes a spill directory whose embedded + * pid is not this process and is not a live process, so a crash that left a + * directory behind does not leak disk. It never removes a directory of a live + * process, so a concurrent channel keeps its own spill directory. + */ +export async function sweepStaleSpillDirectories( + root: string, + logger?: (message: string) => void, +): Promise { + const entries = await fs.readdir(root, { withFileTypes: true }).catch(() => []); + await Promise.all( + entries.map(async (entry) => { + if (!entry.isDirectory() || !entry.name.startsWith(SPILL_DIR_PREFIX)) return; + const pid = parsePidFromSpillDir(entry.name); + if (pid !== null && pid !== process.pid && isProcessAlive(pid)) return; + if (pid === process.pid) return; + await fs.rm(path.join(root, entry.name), { recursive: true, force: true }).catch((error) => { + logger?.(`Duplex stale spill sweep failed: ${errorText(error)}`); + }); + }), + ); +} + +/** Read the pid segment from a spill directory name, or null when it is absent. */ +function parsePidFromSpillDir(name: string): number | null { + const rest = name.slice(SPILL_DIR_PREFIX.length); + const dash = rest.indexOf("-"); + const pidText = dash === -1 ? rest : rest.slice(0, dash); + const pid = Number(pidText); + return Number.isInteger(pid) && pid > 0 ? pid : null; +} + +/** Return true when a process with the pid is alive. A signal 0 probes without a real signal. */ +function isProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + // EPERM means the process exists but this process cannot signal it. ESRCH + // means no such process. Treat only ESRCH as dead. + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +} diff --git a/packages/adapter-utils/src/duplex-bridge-broker-byte-ledger.test.ts b/packages/adapter-utils/src/duplex-bridge-broker-byte-ledger.test.ts index 78780c9ff5..ff901dd1d9 100644 --- a/packages/adapter-utils/src/duplex-bridge-broker-byte-ledger.test.ts +++ b/packages/adapter-utils/src/duplex-bridge-broker-byte-ledger.test.ts @@ -16,6 +16,7 @@ import { type DuplexRequestFrame, type DuplexResponseFrame, } from "./duplex-frame-codec.js"; +import { splitBodyIntoChunkFrames } from "./duplex-body-spool.js"; import type { CommandManagedDuplexChannel } from "./command-managed-runtime.js"; /** @@ -35,41 +36,83 @@ interface PendingForward { settled: boolean; } +/** + * One reassembled response the broker wrote back. The broker writes a response as + * one envelope frame that carries `bodyByteCount`, then the `body_chunk` frames + * that carry the body. The harness reassembles the body and exposes it as `body`. + */ +type ReassembledResponse = DuplexResponseFrame & { body: string }; + +/** One request the harness feeds: the envelope frame plus its raw body text. */ +interface RequestInput { + frame: DuplexRequestFrame; + bodyText: string; +} + /** The in-memory channel plus the levers a test uses to drive the broker. */ interface FakeChannelHarness { channel: CommandManagedDuplexChannel; - feed: (frame: DuplexRequestFrame) => void; + feed: (input: RequestInput) => void; exit: () => void; - responses: DuplexResponseFrame[]; + responses: ReassembledResponse[]; forwards: PendingForward[]; resolveForward: (id: string, body?: string) => void; } -/** Build one valid request frame. */ -function requestFrame(id: string, method = "POST"): DuplexRequestFrame { +/** Build one valid request: the envelope carries `bodyByteCount`, the body rides body_chunk frames. */ +function requestFrame(id: string, method = "POST"): RequestInput { + const bodyText = JSON.stringify({ id }); return { - version: DUPLEX_FRAME_VERSION, - type: "request", - id, - method, - path: `/api/issues/${id}`, - query: "", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ id }), + frame: { + version: DUPLEX_FRAME_VERSION, + type: "request", + id, + method, + path: `/api/issues/${id}`, + query: "", + headers: { "content-type": "application/json" }, + bodyByteCount: Buffer.byteLength(bodyText, "utf8"), + }, + bodyText, }; } function createFakeChannelHarness(): FakeChannelHarness { - const responses: DuplexResponseFrame[] = []; + const responses: ReassembledResponse[] = []; const forwards: PendingForward[] = []; const writtenDecoder = new DuplexFrameDecoder(); + const responseAssembly = new Map< + string, + { frame: DuplexResponseFrame; received: number; chunks: Buffer[] } + >(); let dataListener: ((chunk: string) => void) | null = null; let exitListener: ((exit: { exitCode: number | null }) => void) | null = null; const channel: CommandManagedDuplexChannel = { write: (data) => { for (const result of writtenDecoder.push(data)) { - if (result.ok && result.frame.type === "response") responses.push(result.frame); + if (!result.ok) continue; + const frame = result.frame; + if (frame.type === "response") { + if (frame.bodyByteCount === 0) { + responses.push({ ...frame, body: "" }); + } else { + responseAssembly.set(frame.id, { frame, received: 0, chunks: [] }); + } + } else if (frame.type === "body_chunk") { + const assembly = responseAssembly.get(frame.id); + if (!assembly) continue; + const decoded = Buffer.from(frame.data, "base64"); + assembly.received += decoded.length; + assembly.chunks.push(decoded); + if (assembly.received >= assembly.frame.bodyByteCount) { + responseAssembly.delete(frame.id); + responses.push({ + ...assembly.frame, + body: Buffer.concat(assembly.chunks).toString("utf8"), + }); + } + } } }, onData: (listener) => { @@ -84,9 +127,18 @@ function createFakeChannelHarness(): FakeChannelHarness { return { channel, - feed: (frame) => { + feed: ({ frame, bodyText }) => { if (!dataListener) throw new Error("The broker did not bind the data listener."); dataListener(encodeDuplexFrame(frame)); + if (bodyText.length > 0) { + for (const chunk of splitBodyIntoChunkFrames( + frame.id, + Buffer.from(bodyText, "utf8"), + DUPLEX_FRAME_VERSION, + )) { + dataListener(encodeDuplexFrame(chunk)); + } + } }, exit: () => { if (!exitListener) throw new Error("The broker did not bind the exit listener."); @@ -111,10 +163,12 @@ function controllableForward(harness: FakeChannelHarness) { }); } -/** Wait for the microtask queue to drain, so a settled promise runs its handlers. */ +/** + * Wait for the microtasks and one macrotask to settle, so the broker reassembles + * a request body, dispatches its forward, and runs each settled promise handler. + */ async function flush(): Promise { - await Promise.resolve(); - await Promise.resolve(); + await new Promise((resolve) => setTimeout(resolve, 0)); } describe("duplex bridge broker aggregate byte ledger", () => { @@ -130,7 +184,7 @@ describe("duplex bridge broker aggregate byte ledger", () => { it("charges request-frame, request-payload, and seen-id tokens, then releases the request tokens on forward settlement", async () => { const harness = createFakeChannelHarness(); const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 }); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), duplexAggregateByteLedger: ledger, @@ -139,12 +193,15 @@ describe("duplex bridge broker aggregate byte ledger", () => { broker.start(); harness.feed(requestFrame("req-1")); - // The dispatch retains three tokens: the raw frame, the normalized payload, - // and the no-replay set entry. + // The dispatch retains three tokens at the request envelope: the raw frame, the + // normalized payload, and the no-replay set entry. The reservation is + // synchronous, so the charge holds before the body reassembles. expect(ledger.liveTokenCount).toBe(3); expect(ledger.bytesInUse).toBeGreaterThan(0); const chargedInFlight = ledger.bytesInUse; + // The broker reassembles the request body, then dispatches the forward. + await flush(); harness.resolveForward("req-1"); await flush(); @@ -164,7 +221,7 @@ describe("duplex bridge broker aggregate byte ledger", () => { it("keeps request tokens charged when the response timer answers before the forward settles, and releases them on settlement", async () => { const harness = createFakeChannelHarness(); const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 }); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), // Squeeze the nested budgets so the response timer fires quickly while the @@ -202,7 +259,7 @@ describe("duplex bridge broker aggregate byte ledger", () => { // can size a ceiling one byte below it and force the reservation to fail. const probeHarness = createFakeChannelHarness(); const measured = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 }); - const measuringBroker = createDuplexBridgeBroker({ + const measuringBroker = await createDuplexBridgeBroker({ channel: probeHarness.channel, forwardRequest: controllableForward(probeHarness), duplexAggregateByteLedger: measured, @@ -213,7 +270,7 @@ describe("duplex bridge broker aggregate byte ledger", () => { await measuringBroker.close(); const ledger = new DuplexAggregateByteLedger({ ceilingBytes: perRequestBytes - 1 }); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), duplexAggregateByteLedger: ledger, @@ -238,7 +295,7 @@ describe("duplex bridge broker aggregate byte ledger", () => { // admitted: raise the ceiling and re-feed. const roomyLedger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 }); const roomyHarness = createFakeChannelHarness(); - const roomyBroker = createDuplexBridgeBroker({ + const roomyBroker = await createDuplexBridgeBroker({ channel: roomyHarness.channel, forwardRequest: controllableForward(roomyHarness), duplexAggregateByteLedger: roomyLedger, @@ -246,13 +303,14 @@ describe("duplex bridge broker aggregate byte ledger", () => { brokers.push(roomyBroker); roomyBroker.start(); roomyHarness.feed(requestFrame("req-1")); + await flush(); expect(roomyHarness.forwards.length).toBe(1); }); it("releases every retained token on a terminal channel loss", async () => { const harness = createFakeChannelHarness(); const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 }); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), duplexAggregateByteLedger: ledger, @@ -262,8 +320,14 @@ describe("duplex bridge broker aggregate byte ledger", () => { harness.feed(requestFrame("req-1")); harness.feed(requestFrame("req-2")); + // The reservation is synchronous, so both dispatches charge three tokens each + // at the request envelope, before the bodies reassemble. expect(ledger.liveTokenCount).toBe(6); + // Let both requests reassemble and start their forwards, so the loss below + // transfers live forwards to the orphan registry. + await flush(); + // The channel exits mid-flight. The broker latches the loss, transfers the // in-flight forwards to the orphan registry, and releases the seen-id tokens. harness.exit(); @@ -294,7 +358,7 @@ describe("duplex bridge broker aggregate byte ledger", () => { }, }, }); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), duplexAggregateByteLedger: ledger, @@ -303,6 +367,9 @@ describe("duplex bridge broker aggregate byte ledger", () => { broker.start(); harness.feed(requestFrame("req-1")); + // Let the request reassemble and start its forward before the close, so the + // close orphans a live forward that settles afterward. + await flush(); await broker.close(); // The close aborted the in-flight forward and orphaned it. The forward now // settles after the close: its finally owner releases the request tokens once. diff --git a/packages/adapter-utils/src/duplex-bridge-broker.test.ts b/packages/adapter-utils/src/duplex-bridge-broker.test.ts index 6c87a1ae87..346c0cbf39 100644 --- a/packages/adapter-utils/src/duplex-bridge-broker.test.ts +++ b/packages/adapter-utils/src/duplex-bridge-broker.test.ts @@ -14,6 +14,7 @@ import { type DuplexRequestFrame, type DuplexResponseFrame, } from "./duplex-frame-codec.js"; +import { splitBodyIntoChunkFrames } from "./duplex-body-spool.js"; import { createDuplexTelemetry, DUPLEX_COUNTER_LOSS_TOTAL, @@ -42,13 +43,27 @@ interface PendingForward { settled: boolean; } +/** + * One reassembled response the broker wrote back. The broker writes a response as + * one envelope frame that carries `bodyByteCount`, then the `body_chunk` frames + * that carry the body. The harness reassembles the body and exposes it as `body`, + * so a test asserts the response the same way it did with the one-frame model. + */ +type ReassembledResponse = DuplexResponseFrame & { body: string }; + +/** One request the harness feeds: the envelope frame plus its raw body text. */ +interface RequestInput { + frame: DuplexRequestFrame; + bodyText: string; +} + /** The in-memory channel plus the levers a test uses to drive the broker. */ interface FakeChannelHarness { channel: CommandManagedDuplexChannel; - /** Push one request frame into the broker read path. */ - feed: (frame: DuplexRequestFrame) => void; - /** The response frames the broker wrote back, in order. */ - responses: DuplexResponseFrame[]; + /** Push one request (its envelope frame and its body_chunk frames) into the broker read path. */ + feed: (input: RequestInput) => void; + /** The reassembled responses the broker wrote back, in order. */ + responses: ReassembledResponse[]; /** Every forward call the broker made, in order. */ forwards: PendingForward[]; /** Resolve the newest unsettled forward for one id with a 200 result. */ @@ -57,36 +72,71 @@ interface FakeChannelHarness { resolveAll: () => void; } -/** Build one valid request frame with distinctive, secret-looking fields. */ -function requestFrame(id: string, method = "POST"): DuplexRequestFrame { +/** + * Build one valid request with distinctive, secret-looking fields. The envelope + * carries `bodyByteCount`, and the harness rides the body on `body_chunk` frames. + */ +function requestFrame( + id: string, + method = "POST", + bodyText: string = JSON.stringify({ secret: "secret-request-body" }), +): RequestInput { return { - version: DUPLEX_FRAME_VERSION, - type: "request", - id, - method, - path: `/api/issues/${id}`, - query: "?secret-query=leak", - headers: { authorization: "Bearer super-secret-provider-token" }, - body: JSON.stringify({ secret: "secret-request-body" }), + frame: { + version: DUPLEX_FRAME_VERSION, + type: "request", + id, + method, + path: `/api/issues/${id}`, + query: "?secret-query=leak", + headers: { authorization: "Bearer super-secret-provider-token" }, + bodyByteCount: Buffer.byteLength(bodyText, "utf8"), + }, + bodyText, }; } /** * Build the in-memory channel harness. The channel keeps the broker read - * listener, so `feed` pushes an encoded frame into the broker. The channel - * decodes each written frame, so `responses` holds the response frames only. + * listener, so `feed` pushes the encoded envelope and its body_chunk frames into + * the broker. The channel decodes each written frame and reassembles a response + * body, so `responses` holds one reassembled response per delivered request. */ function createFakeChannelHarness(): FakeChannelHarness { - const responses: DuplexResponseFrame[] = []; + const responses: ReassembledResponse[] = []; const forwards: PendingForward[] = []; const writtenDecoder = new DuplexFrameDecoder(); + // The in-flight response reassembly, keyed by request id. + const responseAssembly = new Map< + string, + { frame: DuplexResponseFrame; received: number; chunks: Buffer[] } + >(); let dataListener: ((chunk: string) => void) | null = null; const channel: CommandManagedDuplexChannel = { write: (data) => { for (const result of writtenDecoder.push(data)) { - if (result.ok && result.frame.type === "response") { - responses.push(result.frame); + if (!result.ok) continue; + const frame = result.frame; + if (frame.type === "response") { + if (frame.bodyByteCount === 0) { + responses.push({ ...frame, body: "" }); + } else { + responseAssembly.set(frame.id, { frame, received: 0, chunks: [] }); + } + } else if (frame.type === "body_chunk") { + const assembly = responseAssembly.get(frame.id); + if (!assembly) continue; + const decoded = Buffer.from(frame.data, "base64"); + assembly.received += decoded.length; + assembly.chunks.push(decoded); + if (assembly.received >= assembly.frame.bodyByteCount) { + responseAssembly.delete(frame.id); + responses.push({ + ...assembly.frame, + body: Buffer.concat(assembly.chunks).toString("utf8"), + }); + } } } }, @@ -100,9 +150,17 @@ function createFakeChannelHarness(): FakeChannelHarness { return { channel, - feed: (frame) => { + feed: ({ frame, bodyText }) => { if (!dataListener) throw new Error("The broker did not bind the data listener."); dataListener(encodeDuplexFrame(frame)); + if (bodyText.length > 0) { + const chunks = splitBodyIntoChunkFrames( + frame.id, + Buffer.from(bodyText, "utf8"), + DUPLEX_FRAME_VERSION, + ); + for (const chunk of chunks) dataListener(encodeDuplexFrame(chunk)); + } }, responses, forwards, @@ -134,6 +192,11 @@ function controllableForward(harness: FakeChannelHarness) { }); } +/** Flush the microtasks and one macrotask, so the broker dispatches its reassembled forwards. */ +async function flush(): Promise { + await new Promise((resolve) => setTimeout(resolve, 0)); +} + /** The telemetry sink capture. It proves the broker records nothing for a refusal. */ interface TelemetryCapture { spans: DuplexTelemetrySpanRecord[]; @@ -175,11 +238,11 @@ describe("duplex bridge broker request limits", () => { expect(() => assertDuplexBrokerLimits({ maxInFlightRequests: 8, maxLifetimeRequests: 16 })).not.toThrow(); }); - it("bounds the in-flight forwards and refuses the excess with a retryable terminal response", () => { + it("bounds the in-flight forwards and refuses the excess with a retryable terminal response", async () => { const harness = createFakeChannelHarness(); const { telemetry, capture } = createTelemetryCapture(); const maxInFlightRequests = 4; - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), telemetry, @@ -189,10 +252,13 @@ describe("duplex bridge broker request limits", () => { broker.start(); // Inject more than the limit of unique valid frames. Every forward hangs, so - // the broker holds the maximum number of in-flight forwards. + // the broker holds the maximum number of in-flight forwards. The broker gates + // the in-flight limit at the request envelope, so it refuses the excess at + // once; it dispatches each accepted forward after it reassembles the body. const total = 10; const ids = Array.from({ length: total }, (_unused, index) => `flight-${index}`); for (const id of ids) harness.feed(requestFrame(id)); + await flush(); // The broker forwarded only up to the limit. It refused the rest. expect(harness.forwards).toHaveLength(maxInFlightRequests); @@ -226,7 +292,7 @@ describe("duplex bridge broker request limits", () => { it("frees capacity after a delivered request and forwards a resent refused id (no-replay preserved)", async () => { const harness = createFakeChannelHarness(); const maxInFlightRequests = 2; - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), maxInFlightRequests, @@ -234,30 +300,36 @@ describe("duplex bridge broker request limits", () => { brokers.push(broker); broker.start(); - // Saturate the in-flight limit with two hung forwards. + // Saturate the in-flight limit with two hung forwards. The broker reserves the + // in-flight slot at the request envelope, then dispatches the forward after it + // reassembles the body. harness.feed(requestFrame("keep-0")); harness.feed(requestFrame("keep-1")); + await flush(); expect(harness.forwards).toHaveLength(2); // A third unique id is refused, retryable. The broker did not forward it and // did not retain its id. harness.feed(requestFrame("resend-me")); + await flush(); expect(harness.forwards).toHaveLength(2); expect(harness.responses).toHaveLength(1); expect(JSON.parse(harness.responses[0].body).retryable).toBe(true); // Free one in-flight slot. The delivered request drains the pending record. harness.resolveForward("keep-0"); - await Promise.resolve(); + await flush(); // The gateway resends the refused id. Capacity is free now, so the broker // forwards it exactly one time. This proves the refusal did not poison the id. harness.feed(requestFrame("resend-me")); + await flush(); expect(harness.forwards.filter((forward) => forward.id === "resend-me")).toHaveLength(1); // A resend of an already-forwarded id never reaches the forward twice. const forwardedKeep1 = harness.forwards.filter((forward) => forward.id === "keep-1").length; harness.feed(requestFrame("keep-1")); + await flush(); expect(harness.forwards.filter((forward) => forward.id === "keep-1")).toHaveLength(forwardedKeep1); harness.resolveAll(); @@ -267,7 +339,7 @@ describe("duplex bridge broker request limits", () => { const harness = createFakeChannelHarness(); const { telemetry, capture } = createTelemetryCapture(); const maxLifetimeRequests = 3; - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, // Resolve every forward at once, so each dispatched id leaves the in-flight // count but stays in the retained-id set. @@ -285,7 +357,7 @@ describe("duplex bridge broker request limits", () => { const total = 9; for (let index = 0; index < total; index += 1) { harness.feed(requestFrame(`life-${index}`)); - await Promise.resolve(); + await flush(); } // The broker forwarded only up to the lifetime limit. The retained-id set @@ -304,7 +376,7 @@ describe("duplex bridge broker request limits", () => { // A resend of a refused id stays refused, so the set never grows. harness.feed(requestFrame("life-8")); - await Promise.resolve(); + await flush(); expect(harness.forwards).toHaveLength(maxLifetimeRequests); // The telemetry surface stayed on the fixed request span for the delivered @@ -320,9 +392,9 @@ describe("duplex bridge broker request limits", () => { expect(broker.state).toBe("open"); }); - it("forwards nothing and fails the channel closed under a flood of over-limit ids", () => { + it("forwards nothing and fails the channel closed under a flood of over-limit ids", async () => { const harness = createFakeChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), }); @@ -347,9 +419,9 @@ describe("duplex bridge broker request limits", () => { expect(broker.lossRecord?.reason).toBe("protocol_failure"); }); - it("forwards a maximal-size id exactly once and does not forward a resend (no-replay preserved)", () => { + it("forwards a maximal-size id exactly once and does not forward a resend (no-replay preserved)", async () => { const harness = createFakeChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), }); @@ -359,19 +431,21 @@ describe("duplex bridge broker request limits", () => { // An id at the maximum byte size is allowed. The broker forwards it one time. const maxId = "a".repeat(DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES); harness.feed(requestFrame(maxId)); + await flush(); expect(harness.forwards.filter((forward) => forward.id === maxId)).toHaveLength(1); // Deliver the response, then resend the same id. The broker retained the id, // so the resend never reaches the forward a second time. harness.resolveForward(maxId); harness.feed(requestFrame(maxId)); + await flush(); expect(harness.forwards.filter((forward) => forward.id === maxId)).toHaveLength(1); expect(broker.state).toBe("open"); }); it("maps a forward that resolves with the indeterminate marker to a non-retryable response and retains the id", async () => { const harness = createFakeChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), }); @@ -384,6 +458,7 @@ describe("duplex bridge broker request limits", () => { // non-retryable body, so the gateway maps it to a non-retryable status and a // caller never repeats a possibly-committed mutation. harness.feed(requestFrame("commit-1")); + await flush(); const forward = harness.forwards.find((entry) => entry.id === "commit-1" && !entry.settled); if (!forward) throw new Error("The broker did not forward the request."); forward.settled = true; @@ -423,7 +498,7 @@ describe("duplex bridge broker request limits", () => { it("maps a forward rejection for a mutating method to a non-retryable indeterminate response and retains the id", async () => { const harness = createFakeChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), }); @@ -436,6 +511,7 @@ describe("duplex bridge broker request limits", () => { // forward, so a POST must return a non-retryable indeterminate response. // A retryable status would let a caller repeat a committed mutation. harness.feed(requestFrame("mutate-1", "POST")); + await flush(); const forward = harness.forwards.find((entry) => entry.id === "mutate-1" && !entry.settled); if (!forward) throw new Error("The broker did not forward the request."); forward.settled = true; @@ -465,7 +541,7 @@ describe("duplex bridge broker request limits", () => { it("substitutes a bounded terminal response for an oversized response and keeps the channel open", async () => { const harness = createFakeChannelHarness(); const { telemetry, capture } = createTelemetryCapture(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), telemetry, @@ -482,6 +558,7 @@ describe("duplex bridge broker request limits", () => { // host. harness.feed(requestFrame("small-1", "POST")); harness.feed(requestFrame("big-1", "POST")); + await flush(); const smallForward = harness.forwards.find((entry) => entry.id === "small-1" && !entry.settled); if (!smallForward) throw new Error("The broker did not forward the small request."); @@ -542,30 +619,35 @@ describe("duplex bridge broker request limits", () => { it("sends a minimal terminal response when the frame bound rejects the full replacement", async () => { const harness = createFakeChannelHarness(); const { telemetry, capture } = createTelemetryCapture(); - // The bound accepts a compact request (130 bytes) and the minimal terminal - // response (106 bytes), but rejects the full replacement response (288 bytes). - // This proves the broker never drops the channel or strands the request when - // the configured bound is smaller than the full replacement response. - const broker = createDuplexBridgeBroker({ + // The bound accepts a compact request envelope (128 bytes) and the minimal + // terminal response envelope (114 bytes), but rejects the full replacement + // response envelope (193 bytes). This proves the broker never drops the channel + // or strands the request when the bound is smaller than the full replacement. + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), telemetry, - maxFrameBytes: 200, + maxFrameBytes: 150, }); brokers.push(broker); broker.start(); - // A compact request frame fits the bound, so the broker forwards it. + // A compact request envelope with an empty body fits the bound, so the broker + // forwards it. harness.feed({ - version: DUPLEX_FRAME_VERSION, - type: "request", - id: "big-1", - method: "POST", - path: "/api/issues/big-1", - query: "", - headers: {}, - body: JSON.stringify({ go: 1 }), + frame: { + version: DUPLEX_FRAME_VERSION, + type: "request", + id: "big-1", + method: "POST", + path: "/api/issues/big-1", + query: "", + headers: {}, + bodyByteCount: 0, + }, + bodyText: "", }); + await flush(); const forward = harness.forwards.find((entry) => entry.id === "big-1" && !entry.settled); if (!forward) throw new Error("The broker did not forward the request."); @@ -608,29 +690,34 @@ describe("duplex bridge broker request limits", () => { const harness = createFakeChannelHarness(); const { telemetry, capture } = createTelemetryCapture(); const logs: string[] = []; - // The bound accepts a tiny request (99 bytes) but rejects even the minimal - // terminal response (102 bytes). The broker cannot send any frame for this - // request, so it must keep the channel open and log a clear local error. - const broker = createDuplexBridgeBroker({ + // The bound accepts a tiny request envelope (107 bytes) but rejects even the + // minimal terminal response envelope (110 bytes). The broker cannot send any + // frame for this request, so it must keep the channel open and log a clear + // local error. + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), telemetry, - maxFrameBytes: 100, + maxFrameBytes: 109, logger: (message) => logs.push(message), }); brokers.push(broker); broker.start(); harness.feed({ - version: DUPLEX_FRAME_VERSION, - type: "request", - id: "x", - method: "GET", - path: "/", - query: "", - headers: {}, - body: "", + frame: { + version: DUPLEX_FRAME_VERSION, + type: "request", + id: "x", + method: "GET", + path: "/", + query: "", + headers: {}, + bodyByteCount: 0, + }, + bodyText: "", }); + await flush(); const forward = harness.forwards.find((entry) => entry.id === "x" && !entry.settled); if (!forward) throw new Error("The broker did not forward the request."); @@ -657,7 +744,7 @@ describe("duplex bridge broker request limits", () => { it("completes a request within the bound and delivers the real response", async () => { const harness = createFakeChannelHarness(); const { telemetry, capture } = createTelemetryCapture(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), telemetry, @@ -667,6 +754,7 @@ describe("duplex bridge broker request limits", () => { broker.start(); harness.feed(requestFrame("ok-1", "POST")); + await flush(); harness.resolveForward("ok-1", JSON.stringify({ ok: true })); await new Promise((resolve) => setTimeout(resolve, 0)); @@ -690,7 +778,7 @@ describe("duplex bridge broker request limits", () => { it("maps a forward rejection for a safe method to a retryable response", async () => { const harness = createFakeChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), }); @@ -701,6 +789,7 @@ describe("duplex bridge broker request limits", () => { // mutation. A forward rejection for a GET stays retryable: the broker // returns a 502 with the completed outcome, so the gateway passes it through. harness.feed(requestFrame("read-1", "GET")); + await flush(); const forward = harness.forwards.find((entry) => entry.id === "read-1" && !entry.settled); if (!forward) throw new Error("The broker did not forward the request."); forward.settled = true; @@ -719,7 +808,7 @@ describe("duplex bridge broker request limits", () => { it("maps a forward-budget timeout for a safe method to a retryable response", async () => { const harness = createFakeChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), // A tiny forward budget aborts the controller before the manual rejection. @@ -732,6 +821,7 @@ describe("duplex bridge broker request limits", () => { // stay retryable: the broker returns a 504 with the completed outcome and no // indeterminate marker, so the gateway does not map it to a terminal 409. harness.feed(requestFrame("read-timeout-1", "GET")); + await flush(); const forward = harness.forwards.find((entry) => entry.id === "read-timeout-1" && !entry.settled); if (!forward) throw new Error("The broker did not forward the request."); @@ -755,7 +845,7 @@ describe("duplex bridge broker request limits", () => { it("maps a forward-budget timeout for a mutating method to a non-retryable indeterminate response", async () => { const harness = createFakeChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), budgets: { forwardTimeoutMs: 5 }, @@ -767,6 +857,7 @@ describe("duplex bridge broker request limits", () => { // must keep the terminal contract: a 504 with the indeterminate outcome, so // the gateway maps it to a non-retryable 409 and no caller double-applies it. harness.feed(requestFrame("mutate-timeout-1", "POST")); + await flush(); const forward = harness.forwards.find((entry) => entry.id === "mutate-timeout-1" && !entry.settled); if (!forward) throw new Error("The broker did not forward the request."); @@ -789,7 +880,7 @@ describe("duplex bridge broker request limits", () => { it("maps a response-budget backstop for a safe method to a retryable response", async () => { const harness = createFakeChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), // The forward budget aborts the call, but the forward promise stays @@ -806,6 +897,7 @@ describe("duplex bridge broker request limits", () => { // retryable: a 504 with the completed outcome and no indeterminate marker, // so the gateway does not map it to a terminal 409. harness.feed(requestFrame("read-stall-1", "GET")); + await flush(); const forward = harness.forwards.find((entry) => entry.id === "read-stall-1" && !entry.settled); if (!forward) throw new Error("The broker did not forward the request."); @@ -826,7 +918,7 @@ describe("duplex bridge broker request limits", () => { it("maps a response-budget backstop for a mutating method to a non-retryable indeterminate response", async () => { const harness = createFakeChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), budgets: { forwardTimeoutMs: 5, responseBudgetMs: 15, gatewayWaitMs: 40 }, @@ -839,6 +931,7 @@ describe("duplex bridge broker request limits", () => { // contract: a 504 with the indeterminate outcome, so the gateway maps it to a // non-retryable 409 and no caller double-applies it. harness.feed(requestFrame("mutate-stall-1", "POST")); + await flush(); const forward = harness.forwards.find((entry) => entry.id === "mutate-stall-1" && !entry.settled); if (!forward) throw new Error("The broker did not forward the request."); @@ -897,10 +990,10 @@ describe("duplex bridge broker exit taxonomy", () => { } }); - it("records a transport close as a distinct loss, not a process exit", () => { + it("records a transport close as a distinct loss, not a process exit", async () => { const { telemetry, capture } = createTelemetryCapture(); const harness = createExitChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: () => Promise.resolve({ status: 200, headers: {}, body: "" }), @@ -920,10 +1013,10 @@ describe("duplex bridge broker exit taxonomy", () => { expect(lossCounter?.dimensions.loss_reason).toBe("transport_closed"); }); - it("records a numeric exit as a process exit", () => { + it("records a numeric exit as a process exit", async () => { const { telemetry, capture } = createTelemetryCapture(); const harness = createExitChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: () => Promise.resolve({ status: 200, headers: {}, body: "" }), diff --git a/packages/adapter-utils/src/duplex-bridge-broker.ts b/packages/adapter-utils/src/duplex-bridge-broker.ts index b7a04021a1..1ae50d92bb 100644 --- a/packages/adapter-utils/src/duplex-bridge-broker.ts +++ b/packages/adapter-utils/src/duplex-bridge-broker.ts @@ -48,17 +48,26 @@ import { type ReservationToken, } from "./duplex-aggregate-byte-ledger.js"; import { + DUPLEX_BODY_CHUNK_RAW_BYTES, DEFAULT_MAX_DUPLEX_FRAME_BYTES, DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES, DUPLEX_FRAME_VERSION, DuplexFrameDecoder, encodeDuplexFrame, encodeDuplexFrameChecked, + type DuplexBodyChunkFrame, type DuplexFrame, type DuplexRequestFrame, type DuplexResponseFrame, type DuplexResponseOutcome, } from "./duplex-frame-codec.js"; +import { + DuplexBodyError, + DuplexBodyReceiver, + splitBodyIntoChunkFrames, + type DuplexBodyReceiverConfig, + type ReassembledBody, +} from "./duplex-body-spool.js"; import type { DuplexLossReason, DuplexOutcomeValue, @@ -195,12 +204,15 @@ export interface DuplexBrokerForwardResult { /** * The forward handler the broker calls for each request. The handler applies the * real token and the run attribution, then forwards the request on the existing - * API path. The broker aborts `options.signal` when the forward budget ends or a - * loss happens, so a handler that threads the signal into its work stops early. + * API path. The broker reassembles the request body from the `body_chunk` frames + * before it calls the handler, so the handler streams the body from + * `options.body`. The broker aborts `options.signal` when the forward budget ends + * or a loss happens, so a handler that threads the signal into its work stops + * early. */ export type DuplexBrokerForwardHandler = ( request: DuplexRequestFrame, - options: { signal: AbortSignal }, + options: { signal: AbortSignal; body: ReassembledBody }, ) => Promise; /** One request record. The broker captures the dispatch-start point for metrics only. */ @@ -249,6 +261,14 @@ export interface DuplexBrokerOptions { * {@link DEFAULT_DUPLEX_BROKER_MAX_LIFETIME_REQUESTS}. */ maxLifetimeRequests?: number; + /** + * The config for the receive-side request-body reassembler. It sets the spill + * threshold, the fixed raw chunk size, and the per-channel spill caps. The + * broker creates one reassembler for the channel. The config stays injectable, + * so a test lowers the spill threshold and the caps to exercise the spill path + * and the fail-closed cap behavior without a large body. + */ + bodyReceiverConfig?: DuplexBodyReceiverConfig; /** The clock the broker reads for the metric timestamps. The default is `Date.now`. */ now?: () => number; /** The metrics sink for the per-request dispatch record. */ @@ -381,10 +401,24 @@ export function isSafeBridgeMethod(method: string): boolean { interface PendingRequest { controller: AbortController; responded: boolean; - forwardTimer: ReturnType; + /** + * The forward-budget timer. It is `null` while the broker reassembles the + * request body, and the broker sets it when it starts the forward. So the + * forward budget bounds the forward call, not the reassembly. + */ + forwardTimer: ReturnType | null; + /** + * The response-budget backstop timer. The broker starts it at the request + * envelope, so it bounds the whole request: the body reassembly plus the + * forward. It answers a request that never completes its reassembly or forward. + */ responseTimer: ReturnType; /** The point the broker started to dispatch the request. It sets the span latency. */ dispatchStartMs: number; + /** The request method. The response backstop reads it to classify the safe-method retry. */ + method: string; + /** The reassembled request body, set once reassembly completes. The broker disposes it on settle. */ + reassembled: ReassembledBody | null; /** True once the forward promise settled. The finally owner sets it one time. */ forwardSettled: boolean; /** @@ -409,10 +443,14 @@ interface OrphanedForward { } /** - * Create the host duplex bridge broker. The factory asserts the budget order and - * returns a handle. Call `start` to wire the channel and open the broker. + * Create the host duplex bridge broker. The factory asserts the budget order, + * creates the receive-side request-body reassembler, and returns a handle. It is + * asynchronous because the reassembler owns a spill directory it creates with + * `mkdtemp`. Call `start` to wire the channel and open the broker. */ -export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBridgeBroker { +export async function createDuplexBridgeBroker( + options: DuplexBrokerOptions, +): Promise { const budgets: DuplexBrokerBudgets = { ...DEFAULT_DUPLEX_BROKER_BUDGETS, ...options.budgets, @@ -452,21 +490,50 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr if (ledger && token) ledger.release(token); }; // The byte size of the normalized request payload the broker retains across the - // forward. It counts the exact retained scalar and header bytes, so the charge - // matches the retained bytes and never a parsed object graph. + // forward. It counts the exact retained scalar and header bytes plus the raw body + // byte count, so the charge matches the reassembled body the broker holds and + // never a parsed object graph. The body rides `body_chunk` frames, so the + // envelope carries only `bodyByteCount`; that count is the retained body size. const requestPayloadBytes = (frame: DuplexRequestFrame): number => { let bytes = Buffer.byteLength(frame.id, "utf8") + Buffer.byteLength(frame.method, "utf8") + Buffer.byteLength(frame.path, "utf8") + Buffer.byteLength(frame.query, "utf8") + - Buffer.byteLength(frame.body, "utf8"); + frame.bodyByteCount; for (const [key, value] of Object.entries(frame.headers)) { bytes += Buffer.byteLength(key, "utf8") + Buffer.byteLength(value, "utf8"); } return bytes; }; + // The receive-side request-body reassembler. It reassembles each request body + // from the `body_chunk` frames, on the memory path at or below the spill + // threshold and on the spill path above it, so the broker never holds a whole + // large request body in memory. It owns a per-channel spill directory and the + // spill caps. + const receiver = await DuplexBodyReceiver.create(options.bodyReceiverConfig ?? {}); + let receiverFinalized = false; + const finalizeReceiver = (): Promise => { + // Remove the spill directory and every in-flight body once. The broker calls + // it on every terminal path: a loss, a normal teardown, and an orderly close. + if (receiverFinalized) return Promise.resolve(); + receiverFinalized = true; + return receiver.destroy().catch(() => undefined); + }; + // The fixed raw slice size for the response `body_chunk` frames. It matches the + // reassembler config, so a test that lowers the slice size splits both a request + // body and a response body the same way. + const rawChunkBytes = options.bodyReceiverConfig?.rawChunkBytes ?? DUPLEX_BODY_CHUNK_RAW_BYTES; + // The ids the broker is reassembling right now. A `body_chunk` for an id in this + // set routes to the reassembler. The broker removes an id when the body settles. + const reassembling = new Set(); + // The ids the broker refused at the envelope, with the raw bytes it still must + // drain. The sender emits the `body_chunk` frames for a refused request before + // it learns of the refusal, so the broker drains and drops those chunks instead + // of treating them as a body_chunk with no envelope. + const draining = new Map(); + let state: DuplexBrokerState = "opening"; let stopped = false; let started = false; @@ -544,21 +611,33 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr const clearPending = (): void => { for (const [id, entry] of pending) { - clearTimeout(entry.forwardTimer); + if (entry.forwardTimer !== null) clearTimeout(entry.forwardTimer); clearTimeout(entry.responseTimer); entry.controller.abort(new Error("Duplex broker stopped.")); - // A forward that has not settled still owns its request tokens. Transfer it - // to the orphan registry, so its tokens stay charged until the forward - // finally releases them. The abort only asks the forward to stop; it never - // releases a token by itself. - if (!entry.forwardSettled) { + if (entry.reassembled !== null) { + void entry.reassembled.dispose(); + entry.reassembled = null; + } + if (entry.forwardSettled) continue; + if (entry.forwardTimer !== null) { + // A live forward promise still owns its request tokens. Transfer it to the + // orphan registry, so its tokens stay charged until the forward finally + // releases them. The abort only asks the forward to stop; it never releases + // a token by itself. orphanedForwards.set(id, { controller: entry.controller, releaseForwardTokens: entry.releaseForwardTokens, }); + } else { + // The request is still reassembling, so no forward promise will settle and + // release its tokens. Release the request tokens now. The reassembler + // teardown below rejects the in-flight body and unlinks any spill file. + entry.releaseForwardTokens(); } } pending.clear(); + reassembling.clear(); + draining.clear(); // Ask every already-orphaned forward to stop as well. The forward-promise // finally owner releases each orphan token when the forward settles. for (const orphan of orphanedForwards.values()) { @@ -582,6 +661,7 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr clearPending(); releaseSeenRequestIdTokens(); decoder.dispose(); + void finalizeReceiver(); if (state !== "closing") setState("closed"); return; } @@ -607,6 +687,7 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr clearPending(); releaseSeenRequestIdTokens(); decoder.dispose(); + void finalizeReceiver(); lossRecord = { reason, message, atMs: now() }; setState("lost"); // Log the internal reason only. The broker never writes the raw provider @@ -626,31 +707,57 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr } }; - // The result of one bounded send. `sent` means the frame went out; `too_large` - // means the encoded frame exceeds the bound and the broker wrote nothing; - // `lost` means the write failed and the broker recorded the channel loss. + // The result of one bounded send. `sent` means every frame went out; `too_large` + // means the response envelope exceeds the bound and the broker wrote nothing; + // `lost` means a write failed and the broker recorded the channel loss. type SendResult = "sent" | "too_large" | "lost"; - const trySendFrame = (frame: DuplexFrame): SendResult => { - // Enforce the frame size bound on every broker write. Report a size rejection - // as `too_large` without a channel loss; a broker-built frame over the bound - // is a defect, not a transport failure. A caller decides how to answer the - // request, so no size rejection ever drops the channel. - const encoded = encodeDuplexFrameChecked(frame, maxFrameBytes); - if (!encoded.ok) return "too_large"; - return writeLine(encoded.line) ? "sent" : "lost"; - }; - - const writeFrame = (frame: DuplexFrame): boolean => { - // A control frame and the bounded terminal responses are always small, so the - // size guard is a no-op for them. Drop an oversized frame here without a - // channel loss and report the drop to the caller. - const result = trySendFrame(frame); - if (result === "too_large") { - options.logger?.(`Duplex broker dropped an oversized ${frame.type} frame.`); - return false; + // Send one full response as a sequence of frames: one envelope frame that + // carries `bodyByteCount`, then the `body_chunk` frames that carry the body. The + // envelope is always small. Each `body_chunk` carries one fixed raw slice, so + // each encoded chunk stays under the frame bound by construction. The result + // reports `too_large` when the envelope itself exceeds the bound and the broker + // wrote nothing, `lost` when a write failed and the broker recorded the channel + // loss, and `sent` when every frame went out. + const sendResponseFrames = ( + id: string, + status: number, + headers: Record, + bodyText: string, + outcome: DuplexResponseOutcome, + ): SendResult => { + const bodyBuffer = Buffer.from(bodyText, "utf8"); + const envelope: DuplexResponseFrame = { + version: DUPLEX_FRAME_VERSION, + type: "response", + id, + status, + headers, + bodyByteCount: bodyBuffer.length, + outcome, + }; + // Guard the envelope against the frame bound. The envelope holds no body, so a + // real bound rejects it only under an extreme small test bound. Report the + // rejection without a channel loss, so the caller decides how to answer. + const encodedEnvelope = encodeDuplexFrameChecked(envelope, maxFrameBytes); + if (!encodedEnvelope.ok) return "too_large"; + const chunkFrames = splitBodyIntoChunkFrames(id, bodyBuffer, DUPLEX_FRAME_VERSION, rawChunkBytes); + if (!writeLine(encodedEnvelope.line)) return "lost"; + for (const chunk of chunkFrames) { + const encodedChunk = encodeDuplexFrameChecked(chunk, maxFrameBytes); + if (!encodedChunk.ok) { + // A single fixed-size slice never exceeds the bound in a real configuration. + // A test bound below one chunk can reject it. The envelope already went out + // with the true `bodyByteCount`, so the broker cannot complete the body + // within the bound. Log the drop and stop; the gateway ends the request on + // its wait budget. Report `sent`, because the broker already committed the + // envelope, so no caller resends a bounded replacement over the same id. + options.logger?.("Duplex broker dropped an oversized body_chunk frame."); + return "sent"; + } + if (!writeLine(encodedChunk.line)) return "lost"; } - return result === "sent"; + return "sent"; }; const sendTerminalIndeterminate = (id: string): void => { @@ -658,42 +765,32 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr // request reached the host and may have changed state, so the response is // non-retryable and carries the `indeterminate` outcome. The broker tries the // full replacement first. When the frame bound rejects even the full - // replacement, the broker sends a minimal replacement that carries only the - // `indeterminate` outcome, so the gateway still ends the request and never - // waits for its full wait budget. When the bound rejects even the minimal - // replacement, the broker logs a clear local error and keeps the channel - // open; it records no channel loss. - const fullReplacement: DuplexResponseFrame = { - version: DUPLEX_FRAME_VERSION, - type: "response", - id, - status: 502, - headers: { - "content-type": "application/json", - "x-paperclip-bridge-outcome": "indeterminate", - }, - body: JSON.stringify({ - error: "upstream response too large to deliver", - outcome: "indeterminate", - retryable: false, - }), + // replacement envelope, the broker sends a minimal replacement that carries + // only the `indeterminate` outcome, so the gateway still ends the request and + // never waits for its full wait budget. When the bound rejects even the + // minimal replacement envelope, the broker logs a clear local error and keeps + // the channel open; it records no channel loss. + const fullBody = JSON.stringify({ + error: "upstream response too large to deliver", outcome: "indeterminate", - }; - if (trySendFrame(fullReplacement) !== "too_large") return; - // The bound rejects the full replacement. Send a minimal terminal response - // with empty headers and an empty body. The `indeterminate` outcome still - // rides the frame, so the gateway maps the request to a terminal 409. - const minimalReplacement: DuplexResponseFrame = { - version: DUPLEX_FRAME_VERSION, - type: "response", - id, - status: 502, - headers: {}, - body: "", - outcome: "indeterminate", - }; - if (trySendFrame(minimalReplacement) !== "too_large") return; - // The bound rejects even the minimal terminal response. The broker cannot + retryable: false, + }); + if ( + sendResponseFrames( + id, + 502, + { "content-type": "application/json", "x-paperclip-bridge-outcome": "indeterminate" }, + fullBody, + "indeterminate", + ) !== "too_large" + ) { + return; + } + // The bound rejects the full replacement envelope. Send a minimal terminal + // response with empty headers and an empty body. The `indeterminate` outcome + // still rides the envelope, so the gateway maps the request to a terminal 409. + if (sendResponseFrames(id, 502, {}, "", "indeterminate") !== "too_large") return; + // The bound rejects even the minimal terminal envelope. The broker cannot // deliver any frame for this request within the bound. Log a clear local // error and keep the channel open for every other request. The gateway ends // its own outstanding request on its wait budget. @@ -711,40 +808,19 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr const entry = pending.get(id); if (!entry || entry.responded) return; entry.responded = true; - clearTimeout(entry.forwardTimer); - clearTimeout(entry.responseTimer); - pending.delete(id); - // The response timer, an abort, or a broker loss can answer the gateway before - // the forward promise settles. When the forward has not settled, transfer it to - // the orphan registry so its request tokens stay charged until the forward - // finally releases them. A settled forward already released or will release - // through its finally owner, so it needs no orphan record. - if (!entry.forwardSettled) { - orphanedForwards.set(id, { - controller: entry.controller, - releaseForwardTokens: entry.releaseForwardTokens, - }); - } + settlePendingBookkeeping(id, entry); // Do not write on a lost or closed channel. The gateway answers its own // outstanding request on loss, so a late write would go to a dead channel. if (state !== "open") return; - const frame: DuplexResponseFrame = { - version: DUPLEX_FRAME_VERSION, - type: "response", - id, - status: result.status, - headers: result.headers ?? {}, - body: result.body ?? "", - outcome, - }; - const encoded = encodeDuplexFrameChecked(frame, maxFrameBytes); - if (!encoded.ok) { - // The host produced a result, but the response frame exceeds the size - // bound. Do not write the oversized frame and do not record a channel loss. - // The request reached the host and may have changed state, so the caller - // must not retry a possible mutation. Send a bounded, non-retryable terminal - // response marked `indeterminate` in its place. Record the request outcome - // as an error, never a loss. The channel stays open for every other request. + const bodyText = result.body ?? ""; + const bodyByteCount = Buffer.byteLength(bodyText, "utf8"); + // The response body rides `body_chunk` frames, so a large body no longer makes + // one frame too large. The gateway reassembles a response body in memory, + // though, so a body over the frame bound would force the gateway to hold an + // oversized body in memory. Treat a response body over the frame bound like the + // former oversized case: send a bounded, non-retryable indeterminate terminal + // response, and record the request outcome as an error, never a loss. + if (bodyByteCount > maxFrameBytes) { options.telemetry?.recordRequest({ latencyMs: now() - entry.dispatchStartMs, outcome: "error", @@ -753,12 +829,14 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr return; } // Record the request span for the delivered request. The span carries the - // latency and the outcome only; no route, query, body, or token rides it. + // latency and the outcome only; no route, query, body, or token rides it. The + // broker records the span before the writes, so the outcome is recorded even if + // a later chunk write records a channel loss. options.telemetry?.recordRequest({ latencyMs: now() - entry.dispatchStartMs, outcome: telemetryOutcome, }); - writeLine(encoded.line); + sendResponseFrames(id, result.status, result.headers ?? {}, bodyText, outcome); }; const respondSaturated = (id: string, retryable: boolean): void => { @@ -768,23 +846,61 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr // it holds only the fixed error shape. The `unavailable` outcome tells the // gateway this is not a delivered host response, so it never counts as one. if (state !== "open") return; - const frame: DuplexResponseFrame = { - version: DUPLEX_FRAME_VERSION, - type: "response", + sendResponseFrames( id, - status: 503, - headers: { - "content-type": "application/json", - "x-paperclip-bridge-outcome": "unavailable", - }, - body: JSON.stringify({ + 503, + { "content-type": "application/json", "x-paperclip-bridge-outcome": "unavailable" }, + JSON.stringify({ error: "Duplex broker capacity limit reached.", outcome: "unavailable", retryable, }), - outcome: "unavailable", - }; - writeFrame(frame); + "unavailable", + ); + }; + + // Release the timers and route the reassembly and ledger resources for one + // request the broker just answered. The broker calls it exactly once per + // request, from the response path. It clears both timers and drops the pending + // record, then it settles the resources by the request phase: + // - A live forward promise (the response backstop answered while the forward + // ran) keeps its request tokens and its reassembled body until its finally + // owner releases and disposes them, so the orphan registry holds it. + // - A request still reassembling (the backstop answered before the forward + // started) has no forward promise to settle, so the broker releases its + // tokens and disposes the in-flight body now. + // - A settled forward already released and disposed through its finally owner. + const settlePendingBookkeeping = (id: string, entry: PendingRequest): void => { + if (entry.forwardTimer !== null) clearTimeout(entry.forwardTimer); + clearTimeout(entry.responseTimer); + pending.delete(id); + if (entry.forwardSettled) return; + if (entry.forwardTimer !== null) { + // A live forward still owns its request tokens and streams the reassembled + // body. Keep both charged until the forward finally owner settles them. + orphanedForwards.set(id, { + controller: entry.controller, + releaseForwardTokens: entry.releaseForwardTokens, + }); + return; + } + // The request is still reassembling, so no forward promise will settle. Release + // the request tokens and dispose the in-flight body, so no spill file, arena + // reservation, or ledger token lingers. + entry.releaseForwardTokens(); + if (reassembling.delete(id)) void receiver.disposeBody(id); + if (entry.reassembled !== null) { + void entry.reassembled.dispose(); + entry.reassembled = null; + } + }; + + // Mark a refused or duplicate request id for chunk draining. The sender emits + // the `body_chunk` frames for the request before it learns of the refusal, so + // the broker records the raw byte count it must drain and drop. A request with + // no body needs no draining, so the broker records only a non-zero count. + const markDrain = (frame: DuplexRequestFrame): void => { + if (frame.bodyByteCount > 0) draining.set(frame.id, frame.bodyByteCount); }; const respondAggregateExceeded = (id: string): void => { @@ -796,23 +912,17 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr // host response. The refusal is retryable, because the broker did not retain // the id: the aggregate pressure can ease, and a resend can then get through. if (state !== "open") return; - const frame: DuplexResponseFrame = { - version: DUPLEX_FRAME_VERSION, - type: "response", + sendResponseFrames( id, - status: 503, - headers: { - "content-type": "application/json", - "x-paperclip-bridge-outcome": "unavailable", - }, - body: JSON.stringify({ + 503, + { "content-type": "application/json", "x-paperclip-bridge-outcome": "unavailable" }, + JSON.stringify({ error: DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, outcome: "unavailable", retryable: true, }), - outcome: "unavailable", - }; - writeFrame(frame); + "unavailable", + ); }; const dispatch = (frame: DuplexRequestFrame): void => { @@ -825,13 +935,19 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr // forwards. It answers with the bounded terminal refusal, which carries no // route, query, body, or token, and records no telemetry. The refusal is not // retryable, because a resend of the same over-limit id never gets past this - // bound. + // bound. It drains the following chunks of the refused body. if (Buffer.byteLength(frame.id, "utf8") > DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES) { respondSaturated(frame.id, false); + markDrain(frame); + return; + } + // Forward one id one time. A repeated id never reaches the API twice. The + // broker already answered the first request, so it answers no second time; it + // drains the resent body chunks and drops them. + if (seenRequestIds.has(frame.id)) { + markDrain(frame); return; } - // Forward one id one time. A repeated id never reaches the API twice. - if (seenRequestIds.has(frame.id)) return; // Bound the retained request-id memory. The broker keeps one id per distinct // dispatched request for the no-replay guarantee, so the set can only grow. // Once the broker reaches the lifetime limit, it refuses each new distinct id @@ -840,6 +956,7 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr // the set never grows past the limit. if (seenRequestIds.size >= limits.maxLifetimeRequests) { respondSaturated(frame.id, false); + markDrain(frame); return; } // Bound the in-flight request count. Once the broker holds the maximum number @@ -847,9 +964,11 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr // it. The broker does not add the id to the seen set, so the gateway can // resend the request after an in-flight request completes. The refusal is // retryable for that reason. This check bounds the controllers, the timers, - // and the concurrent forwards a provider can force. + // the concurrent reassemblies, and the concurrent forwards a provider can + // force. if (pending.size >= limits.maxInFlightRequests) { respondSaturated(frame.id, true); + markDrain(frame); return; } // Reserve the exact retained bytes against the aggregate ledger before the @@ -868,12 +987,14 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr requestFrameToken = ledger.reserve("request_frame", rawFrameBytes); if (!requestFrameToken) { respondAggregateExceeded(frame.id); + markDrain(frame); return; } requestPayloadToken = ledger.reserve("request_payload", requestPayloadBytes(frame)); if (!requestPayloadToken) { ledger.release(requestFrameToken); respondAggregateExceeded(frame.id); + markDrain(frame); return; } const seenEntryBytes = @@ -883,6 +1004,7 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr ledger.release(requestFrameToken); ledger.release(requestPayloadToken); respondAggregateExceeded(frame.id); + markDrain(frame); return; } } @@ -910,83 +1032,123 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr options.onRequestRecord?.(record); const controller = new AbortController(); - const forwardTimer = setTimeout(() => { - controller.abort(new Error("Duplex broker forward budget exceeded.")); - }, budgets.forwardTimeoutMs); - const responseTimer = setTimeout(() => { - // Response-budget backstop. The forward rejection normally answers first, - // well before this deadline. This backstop answers a request whose forward - // rejection handling itself stalls, so the request never strands. One stall - // path is a response whose headers arrive but whose body reader stays - // pending through the budget, so the forward promise never settles. - if (isSafeBridgeMethod(frame.method)) { - // A safe method never changes host state, so a stalled body reader - // cannot leave a mutation half-applied. Keep the request retryable: - // return a 504 with the completed outcome and no indeterminate marker, - // so the gateway passes it through as a retryable status and never maps - // it to a terminal 409. - respond( - frame.id, - { - status: 504, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ - error: "Duplex broker response budget exceeded.", - retryable: true, - }), - }, - "completed", - "error", - ); - return; - } - // The method may mutate host state, and the broker cannot prove the host - // applied no mutation once the response budget passes. Return a - // non-retryable 504 and mark the outcome indeterminate, so the gateway - // maps it to a terminal 409 and no caller double-applies the mutation. - respond( - frame.id, - { - status: 504, - headers: { - "content-type": "application/json", - "x-paperclip-bridge-outcome": "indeterminate", - }, - body: JSON.stringify({ - error: "Duplex broker response budget exceeded.", - outcome: "indeterminate", - retryable: false, - }), - }, - "indeterminate", - "error", - ); - }, budgets.responseBudgetMs); - forwardTimer.unref?.(); + // Start the response-budget backstop at the request envelope, so it bounds the + // whole request: the body reassembly plus the forward. The forward-budget timer + // starts later, when the broker starts the forward, so it bounds the forward + // call alone. + const responseTimer = setTimeout(() => respondBackstop(frame.id), budgets.responseBudgetMs); responseTimer.unref?.(); - pending.set(frame.id, { + const entry: PendingRequest = { controller, responded: false, - forwardTimer, + forwardTimer: null, responseTimer, dispatchStartMs: record.dispatchStartMs, + method: frame.method, + reassembled: null, forwardSettled: false, releaseForwardTokens, - }); - - // The forward promise has one cleanup owner. The two settle handlers mark the - // forward settled and answer the gateway. The `finally` then releases the - // request tokens exactly one time and removes the request from both the pending - // map and the orphan map. The forward resolves only after the forward and its - // response-body reader both settle, so this owner waits for both. - const markForwardSettled = (): void => { - // Mark the pending entry settled before `respond` runs, so `respond` never - // moves a just-settled forward to the orphan registry. An already-orphaned - // forward needs no flag; its finally owner releases and drains it. - const entry = pending.get(frame.id); - if (entry) entry.forwardSettled = true; }; - forwardRequest(frame, { signal: controller.signal }) + pending.set(frame.id, entry); + // Route the following `body_chunk` frames for this id to the reassembler. + reassembling.add(frame.id); + // Reassemble the request body from the `body_chunk` frames, then forward it. + // The reassembler chooses the memory path or the spill path from the envelope + // `bodyByteCount`, so the broker never holds a whole large request body in + // memory. A reassembly error is a terminal protocol failure. + receiver.begin(frame.id, frame.bodyByteCount).then( + (body) => onBodyReady(frame, entry, body), + (error) => onBodyError(frame.id, entry, error), + ); + }; + + const respondBackstop = (id: string): void => { + // Response-budget backstop. The forward rejection normally answers first, well + // before this deadline. This backstop answers a request whose body never + // completes its reassembly, or whose forward rejection handling itself stalls, + // so the request never strands. One stall path is a response whose headers + // arrive but whose body reader stays pending through the budget, so the forward + // promise never settles. + const entry = pending.get(id); + if (!entry || entry.responded) return; + if (isSafeBridgeMethod(entry.method)) { + // A safe method never changes host state, so a stalled body reader cannot + // leave a mutation half-applied. Keep the request retryable: return a 504 + // with the completed outcome and no indeterminate marker, so the gateway + // passes it through as a retryable status and never maps it to a terminal 409. + respond( + id, + { + status: 504, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + error: "Duplex broker response budget exceeded.", + retryable: true, + }), + }, + "completed", + "error", + ); + return; + } + // The method may mutate host state, and the broker cannot prove the host + // applied no mutation once the response budget passes. Return a non-retryable + // 504 and mark the outcome indeterminate, so the gateway maps it to a terminal + // 409 and no caller double-applies the mutation. + respond( + id, + { + status: 504, + headers: { + "content-type": "application/json", + "x-paperclip-bridge-outcome": "indeterminate", + }, + body: JSON.stringify({ + error: "Duplex broker response budget exceeded.", + outcome: "indeterminate", + retryable: false, + }), + }, + "indeterminate", + "error", + ); + }; + + // The request body reassembled. Start the forward now, so the forward budget + // bounds the forward call alone. The broker passes the reassembled body to the + // forward handler, so the handler streams it to the host API path. + const onBodyReady = ( + frame: DuplexRequestFrame, + entry: PendingRequest, + body: ReassembledBody, + ): void => { + reassembling.delete(frame.id); + if (stopped || state !== "open" || entry.responded) { + // The channel died, or the response backstop already answered, so no forward + // runs. Dispose the body and release the request tokens, so no spill file, + // arena reservation, or ledger token lingers, and drop the request from the + // pending and orphan maps. + void body.dispose(); + pending.delete(frame.id); + orphanedForwards.delete(frame.id); + entry.releaseForwardTokens(); + return; + } + entry.reassembled = body; + const forwardTimer = setTimeout(() => { + entry.controller.abort(new Error("Duplex broker forward budget exceeded.")); + }, budgets.forwardTimeoutMs); + forwardTimer.unref?.(); + entry.forwardTimer = forwardTimer; + + // The forward promise has one cleanup owner. Both settle handlers mark the + // forward settled and answer the gateway. The `finally` then releases the + // request tokens exactly one time, disposes the reassembled body, and removes + // the request from the pending map and the orphan map. + const markForwardSettled = (): void => { + entry.forwardSettled = true; + }; + forwardRequest(frame, { signal: entry.controller.signal, body }) .then( (result) => { markForwardSettled(); @@ -1004,20 +1166,7 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr }, (error) => { markForwardSettled(); - return handleForwardRejection(error); - }, - ) - .finally(() => { - pending.delete(frame.id); - orphanedForwards.delete(frame.id); - releaseForwardTokens(); - }); - - // The forward rejection handler. It classifies the rejection by method safety - // and answers the gateway. The `markForwardSettled` call above runs first, so - // the response never re-orphans a settled forward. - function handleForwardRejection(error: unknown): void { - if (controller.signal.aborted) { + if (entry.controller.signal.aborted) { // The forward budget aborted the call. A safe method never changes // host state, so a forward timeout stays retryable for it. Return a // 504 with the completed outcome and no indeterminate marker, so the @@ -1103,7 +1252,61 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr "indeterminate", "error", ); + }, + ) + .finally(() => { + // The single forward cleanup owner. Release the request tokens, dispose the + // reassembled body, and drop the request from the pending and orphan maps, + // each exactly one time. + pending.delete(frame.id); + orphanedForwards.delete(frame.id); + entry.releaseForwardTokens(); + if (entry.reassembled !== null) { + void entry.reassembled.dispose(); + entry.reassembled = null; + } + }); + }; + + // The request body reassembly failed. A malformed chunk, a size mismatch, an + // overrun, or a truncation is a terminal protocol failure, so the broker fails + // the channel closed. A `channel_closed` error is the broker's own teardown of an + // in-flight body, so it is not a fresh loss. + const onBodyError = (id: string, entry: PendingRequest, error: unknown): void => { + reassembling.delete(id); + // The forward never ran, so release its request tokens and drop the request + // from the pending and orphan maps. The release is idempotent, so a teardown + // that already released the tokens keeps the count correct. + pending.delete(id); + orphanedForwards.delete(id); + entry.releaseForwardTokens(); + if (stopped) return; + if (error instanceof DuplexBodyError && error.code === "channel_closed") return; + recordLoss("protocol_failure", errorMessage(error)); + }; + + // Route one `body_chunk` frame. A chunk for an id under reassembly goes to the + // reassembler, which validates it and appends it. A chunk for a refused or + // duplicate id drains against its recorded byte count and drops. A chunk with no + // matching envelope is a terminal protocol failure. + const handleBodyChunk = (frame: DuplexBodyChunkFrame): void => { + if (reassembling.has(frame.id)) { + const result = receiver.pushChunk(frame); + if (!result.ok) recordLoss("protocol_failure", result.error.message); + return; } + const remaining = draining.get(frame.id); + if (remaining !== undefined) { + // Drain and drop a chunk of a refused body. The broker does not validate a + // drained chunk; it only tracks the byte progress, so it stops draining once + // the refused body ends. A further chunk after that is a chunk with no + // envelope. + const next = remaining - Buffer.byteLength(frame.data, "base64"); + if (next > 0) draining.set(frame.id, next); + else draining.delete(frame.id); + return; + } + recordLoss("protocol_failure", "body_chunk arrived with no matching envelope"); }; const handleFrame = (frame: DuplexFrame): void => { @@ -1111,6 +1314,9 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr case "request": dispatch(frame); return; + case "body_chunk": + handleBodyChunk(frame); + return; case "close": // The gateway asked for an orderly close. The gateway sends this frame on // the agent's orderly completion, so mark the completion on the ordered @@ -1201,6 +1407,9 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr if (closeTimer !== undefined) clearTimeout(closeTimer); recordLoss("close_timeout", errorMessage(error)); } + // Remove the spill directory and every in-flight body. The broker owns the + // reassembler, so the orderly close cleans it up. + await finalizeReceiver(); })(); return closePromise; }; diff --git a/packages/adapter-utils/src/duplex-bridge-local-e2e.test.ts b/packages/adapter-utils/src/duplex-bridge-local-e2e.test.ts index a4e2942c97..91485d7908 100644 --- a/packages/adapter-utils/src/duplex-bridge-local-e2e.test.ts +++ b/packages/adapter-utils/src/duplex-bridge-local-e2e.test.ts @@ -4,6 +4,7 @@ import net from "node:net"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; +import { Readable } from "node:stream"; import { StringDecoder } from "node:string_decoder"; import { afterEach, describe, expect, it } from "vitest"; @@ -14,6 +15,9 @@ import { } from "./sandbox-callback-bridge.js"; import { DuplexFrameDecoder, + DUPLEX_BODY_CHUNK_RAW_BYTES, + DUPLEX_FRAME_VERSION, + encodeDuplexFrame, type DuplexFrame, type DuplexReadyFrame, type DuplexRequestFrame, @@ -23,6 +27,7 @@ import { type DuplexBridgeBroker, type DuplexBrokerForwardResult, } from "./duplex-bridge-broker.js"; +import type { ReassembledBody } from "./duplex-body-spool.js"; import type { CommandManagedDuplexChannel } from "./command-managed-runtime.js"; /** @@ -299,7 +304,7 @@ async function createHarness(options: HarnessOptions = {}): Promise => { forwardedRequests.push(request); // Keep the real route allowlist on the forward seam. The broker forwards @@ -334,12 +339,16 @@ async function createHarness(options: HarnessOptions = {}): Promise; + forwardInit.duplex = "half"; + } + const response = await fetch(target, forwardInit); const body = await response.text(); const outHeaders: Record = {}; response.headers.forEach((value, key) => { @@ -349,7 +358,7 @@ async function createHarness(options: HarnessOptions = {}): Promise undefined, @@ -456,7 +465,7 @@ describe("duplex bridge local end-to-end harness", () => { const ready = harness.observedFrames.find( (frame) => frame.type === "ready", ) as DuplexReadyFrame; - expect(ready.version).toBe(1); + expect(ready.version).toBe(2); expect(ready.nonce).toBe(harness.nonce); // READY is liveness only. It carries no address data. expect((ready as unknown as Record).address).toBeUndefined(); @@ -643,4 +652,90 @@ describe("duplex bridge local end-to-end harness", () => { expect(harness.api.listening()).toBe(false); expect(harness.api.openSocketCount()).toBe(0); }, 20000); + + it("fails a request with a local 502 when the host sends a malformed response chunk", async () => { + const RAW = DUPLEX_BODY_CHUNK_RAW_BYTES; + // Each case declares one malformed body_chunk a broken or hostile host could + // send. The gateway must reject it at once with a local 502. It must not grow + // its response reassembly buffer until the response timeout. `bodyByteCount` + // sets the declared body size on the response envelope. `data` is the base64 + // payload of the one injected chunk. + const cases: Array<{ name: string; bodyByteCount: number; data: string; error: string }> = [ + { name: "an empty chunk", bodyByteCount: RAW, data: "", error: "duplex response body_chunk is empty" }, + { + name: "an undersized non-final chunk", + bodyByteCount: RAW + 8, + data: Buffer.alloc(4, 1).toString("base64"), + error: "duplex response body_chunk has the wrong size", + }, + { + name: "an oversized chunk", + bodyByteCount: RAW + 8, + data: Buffer.alloc(RAW + 4, 1).toString("base64"), + error: "duplex response body_chunk has the wrong size", + }, + { + name: "an overrun past the declared size", + bodyByteCount: 10, + data: Buffer.alloc(200, 1).toString("base64"), + error: "duplex response body overruns the declared size", + }, + { + name: "a non-canonical base64 chunk", + bodyByteCount: 10, + data: "AB==", + error: "duplex response body_chunk is not canonical base64", + }, + ]; + + const harness = await createHarness(); + harnesses.push(harness); + await harness.waitFor(readyPredicate(harness), "the gateway to become ready"); + + // Hold the broker forward open, so only the injected frames answer each + // request. The gateway reassembles the response body itself, so a raw + // malformed chunk on its stdin drives the reject path under test. + harness.setForwardMode("hang"); + + for (const testCase of cases) { + const forwardedBefore = harness.forwardedRequests.length; + const pendingFetch = fetch(`${harness.baseUrl}/api/agents/me`, { + headers: { authorization: `Bearer ${harness.bridgeToken}` }, + }); + void pendingFetch.catch(() => undefined); + await harness.waitFor( + () => harness.forwardedRequests.length > forwardedBefore, + `the broker to receive the forwarded request for ${testCase.name}`, + ); + const id = harness.forwardedRequests[harness.forwardedRequests.length - 1].id; + + // Inject the response envelope, then the one malformed body_chunk, straight + // to the gateway stdin. This bypasses the broker response encoder, so the + // gateway sees the exact bytes a broken or hostile host could send. + harness.child.stdin.write( + encodeDuplexFrame({ + version: DUPLEX_FRAME_VERSION, + type: "response", + id, + status: 200, + headers: {}, + bodyByteCount: testCase.bodyByteCount, + outcome: "completed", + }), + ); + harness.child.stdin.write( + encodeDuplexFrame({ + version: DUPLEX_FRAME_VERSION, + type: "body_chunk", + id, + seq: 0, + data: testCase.data, + }), + ); + + const response = await pendingFetch; + expect(response.status).toBe(502); + await expect(response.json()).resolves.toEqual({ error: testCase.error }); + } + }, 20000); }); diff --git a/packages/adapter-utils/src/duplex-frame-codec.test.ts b/packages/adapter-utils/src/duplex-frame-codec.test.ts index c19008fb55..0fddd5e3cc 100644 --- a/packages/adapter-utils/src/duplex-frame-codec.test.ts +++ b/packages/adapter-utils/src/duplex-frame-codec.test.ts @@ -126,7 +126,7 @@ describe("round-trip", () => { ), ), ); - for (const type of ["request", "response", "ready", "heartbeat", "close", "error"]) { + for (const type of ["request", "response", "body_chunk", "ready", "heartbeat", "close", "error"]) { expect(types).toContain(type); } }); @@ -212,8 +212,8 @@ describe("streaming decoder behavior", () => { method: "POST", path: "/x", query: "", - headers: {}, - body: "😀", + headers: { "x-note": "😀" }, + bodyByteCount: 0, }; const bytes = Buffer.from(encodeDuplexFrame(frame), "utf8"); // Split inside the four-byte emoji, right before a continuation byte. @@ -275,7 +275,7 @@ describe("request id byte bound", () => { path: "/", query: "", headers: {}, - body: "", + bodyByteCount: 0, }); } @@ -314,7 +314,7 @@ describe("request id byte bound", () => { id, status: 200, headers: {}, - body: "", + bodyByteCount: 0, outcome: "completed", }), ); @@ -461,8 +461,8 @@ describe("size-checked encode", () => { type: "response", id: "big", status: 200, - headers: {}, - body: "x".repeat(2_000), + headers: { "x-pad": "x".repeat(2_000) }, + bodyByteCount: 0, outcome: "completed", }; expect(() => encodeDuplexFrameChecked(frame, 1_000)).not.toThrow(); @@ -475,22 +475,25 @@ describe("size-checked encode", () => { // Build a frame whose encoded JSON is exactly N bytes. The guard measures the // JSON without the newline, so N bytes encodes and N+1 bytes fails. This is the // same boundary the decoder applies, so encode and decode agree exactly. - const base: DuplexFrame = { + // Pad a header value to grow the encoded JSON by an exact byte count. The + // "x-pad" value adds one byte per character, so the frame lands on an exact + // size the guard measures without the trailing newline. + const pad = (n: number): DuplexFrame => ({ version: DUPLEX_FRAME_VERSION, type: "response", id: "boundary", status: 200, - headers: {}, - body: "", + headers: { "x-pad": "x".repeat(n) }, + bodyByteCount: 0, outcome: "completed", - }; - const baseBytes = Buffer.byteLength(JSON.stringify(base), "utf8"); + }); + const baseBytes = Buffer.byteLength(JSON.stringify(pad(0)), "utf8"); const limit = baseBytes + 10; - const atLimit: DuplexFrame = { ...base, body: "x".repeat(10) }; + const atLimit = pad(10); expect(Buffer.byteLength(JSON.stringify(atLimit), "utf8")).toBe(limit); const okResult = encodeDuplexFrameChecked(atLimit, limit); expect(okResult.ok).toBe(true); - const overResult = encodeDuplexFrameChecked({ ...base, body: "x".repeat(11) }, limit); + const overResult = encodeDuplexFrameChecked(pad(11), limit); expect(overResult.ok).toBe(false); if (!overResult.ok) expect(overResult.error.code).toBe("frame_too_large"); }); diff --git a/packages/adapter-utils/src/duplex-frame-codec.ts b/packages/adapter-utils/src/duplex-frame-codec.ts index c1723ad763..1013dbf106 100644 --- a/packages/adapter-utils/src/duplex-frame-codec.ts +++ b/packages/adapter-utils/src/duplex-frame-codec.ts @@ -24,7 +24,15 @@ import { } from "./duplex-aggregate-byte-ledger.js"; /** The wire version this codec reads and writes. */ -export const DUPLEX_FRAME_VERSION = 1; +export const DUPLEX_FRAME_VERSION = 2; + +/** + * The fixed raw byte size of one body slice. The sender splits a body into + * `body_chunk` frames of this raw size, except the final chunk, which holds the + * remaining bytes. The base64 text of one full slice is about 349 KiB, so each + * encoded `body_chunk` frame stays well under {@link DEFAULT_MAX_DUPLEX_FRAME_BYTES}. + */ +export const DUPLEX_BODY_CHUNK_RAW_BYTES = 256 * 1024; /** * The default maximum size of one frame, in bytes. The decoder rejects a longer @@ -54,6 +62,7 @@ const EMPTY = Buffer.alloc(0); export const DUPLEX_FRAME_TYPES = { request: "request", response: "response", + body_chunk: "body_chunk", ready: "ready", heartbeat: "heartbeat", close: "close", @@ -63,7 +72,12 @@ export const DUPLEX_FRAME_TYPES = { /** The outcome of a response frame. A loss response carries a non-completed outcome. */ export type DuplexResponseOutcome = "completed" | "indeterminate" | "unavailable"; -/** A request frame. The gateway forwards it to the host API path. */ +/** + * A request envelope frame. The gateway forwards it to the host API path. The + * envelope carries `bodyByteCount`, the raw byte length of the request body. The + * body itself rides one or more `body_chunk` frames that share the same `id`. A + * `bodyByteCount` of 0 means an empty body and no `body_chunk` frame follows. + */ export interface DuplexRequestFrame { version: number; type: "request"; @@ -72,20 +86,43 @@ export interface DuplexRequestFrame { path: string; query: string; headers: Record; - body: string; + bodyByteCount: number; } -/** A response frame. The host returns it for one request id. */ +/** + * A response envelope frame. The host returns it for one request id. The + * envelope carries `bodyByteCount`, the raw byte length of the response body. + * The body itself rides one or more `body_chunk` frames that share the same + * `id`. A `bodyByteCount` of 0 means an empty body and no `body_chunk` frame + * follows. + */ export interface DuplexResponseFrame { version: number; type: "response"; id: string; status: number; headers: Record; - body: string; + bodyByteCount: number; outcome: DuplexResponseOutcome; } +/** + * A body-chunk frame. One frame carries one raw body slice as base64 text in + * `data`. The frames for one body share the `id` of the envelope. `seq` starts + * at 0 and increases by one for each next chunk, with no gap and no repeat. Each + * raw slice is {@link DUPLEX_BODY_CHUNK_RAW_BYTES} bytes, except the final slice, + * which holds the remaining bytes. The receiver reassembles the slices in `seq` + * order and stops when the total raw byte count equals the envelope + * `bodyByteCount`. + */ +export interface DuplexBodyChunkFrame { + version: number; + type: "body_chunk"; + id: string; + seq: number; + data: string; +} + /** * The READY control frame. The gateway sends it one time after it binds the * host-assigned listener port. READY is a liveness signal, not an address @@ -129,6 +166,7 @@ export interface DuplexErrorFrame { export type DuplexFrame = | DuplexRequestFrame | DuplexResponseFrame + | DuplexBodyChunkFrame | DuplexReadyFrame | DuplexHeartbeatFrame | DuplexCloseFrame @@ -194,6 +232,11 @@ function idWithinLimit(id: string): boolean { return Buffer.byteLength(id, "utf8") <= DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES; } +/** Return true when the value is a safe integer that is zero or positive. */ +function isSafeNonNegativeInteger(value: unknown): value is number { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0; +} + /** * Encode one frame to a single line of JSON with a trailing newline. `JSON.stringify` * escapes any newline inside a string value, so the returned line holds no @@ -261,6 +304,8 @@ function validateFrame(frame: Record): DuplexDecodeResult { return validateRequest(frame); case "response": return validateResponse(frame); + case "body_chunk": + return validateBodyChunk(frame); case "ready": return validateReady(frame); case "heartbeat": @@ -280,7 +325,7 @@ function validateRequest(frame: Record): DuplexDecodeResult { typeof frame.method !== "string" || typeof frame.path !== "string" || typeof frame.query !== "string" || - typeof frame.body !== "string" || + !isSafeNonNegativeInteger(frame.bodyByteCount) || !isStringRecord(frame.headers) ) { return fail("malformed_frame", "request frame has a missing or wrong-typed field"); @@ -298,7 +343,7 @@ function validateResponse(frame: Record): DuplexDecodeResult { if ( typeof frame.id !== "string" || typeof frame.status !== "number" || - typeof frame.body !== "string" || + !isSafeNonNegativeInteger(frame.bodyByteCount) || !isStringRecord(frame.headers) || typeof frame.outcome !== "string" || !RESPONSE_OUTCOMES.has(frame.outcome) @@ -313,6 +358,29 @@ function validateResponse(frame: Record): DuplexDecodeResult { return ok(frame as unknown as DuplexResponseFrame); } +/** + * Validate a `body_chunk` frame at the structural level. The codec is stateless, + * so it checks only the per-frame shape here: + * - `id` is a string within {@link DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES}. + * - `seq` is a safe integer that is zero or positive. + * - `data` is a string. + * The receive-side reassembler owns the stateful checks: the base64 canonical + * form, the fixed slice size, the `seq` order, and the total against + * `bodyByteCount`. The codec never decodes `data` here. + */ +function validateBodyChunk(frame: Record): DuplexDecodeResult { + if (typeof frame.id !== "string" || typeof frame.data !== "string") { + return fail("malformed_frame", "body_chunk frame has a missing or wrong-typed field"); + } + if (!isSafeNonNegativeInteger(frame.seq)) { + return fail("malformed_frame", "body_chunk frame has a missing or wrong-typed seq"); + } + if (!idWithinLimit(frame.id)) { + return fail("id_too_large", "body_chunk frame id exceeds the maximum size"); + } + return ok(frame as unknown as DuplexBodyChunkFrame); +} + function validateReady(frame: Record): DuplexDecodeResult { // READY carries a liveness nonce, not an address. The schema is strict: a valid // READY frame holds exactly `version`, `type`, and `nonce`. The decoder rejects diff --git a/packages/adapter-utils/src/duplex-frame-vectors.json b/packages/adapter-utils/src/duplex-frame-vectors.json index 5bedc93081..62d43d1c15 100644 --- a/packages/adapter-utils/src/duplex-frame-vectors.json +++ b/packages/adapter-utils/src/duplex-frame-vectors.json @@ -1,17 +1,17 @@ { - "frameVersion": 1, + "frameVersion": 2, "defaultMaxFrameBytes": 1000000, "description": "Shared wire-compatibility vectors for the duplex frame codec. Every codec copy decodes the same bytes. bytes is a UTF-8 byte stream; splitByteOffsets cuts it into chunks; expected lists the ordered decode results (frame or protocol-error code).", "vectors": [ { "name": "valid-request", "category": "valid", - "bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"body\":\"\"}\n", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"bodyByteCount\":0}\n", "roundTrip": true, "expected": [ { "frame": { - "version": 1, + "version": 2, "type": "request", "id": "req-1", "method": "GET", @@ -21,7 +21,30 @@ "accept": "application/json", "content-type": "application/json" }, - "body": "" + "bodyByteCount": 0 + } + } + ] + }, + { + "name": "valid-request-with-body-count", + "category": "valid", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-body\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"bodyByteCount\":1048576}\n", + "roundTrip": true, + "expected": [ + { + "frame": { + "version": 2, + "type": "request", + "id": "req-body", + "method": "GET", + "path": "/api/issues/PAP-1", + "query": "expand=comments", + "headers": { + "accept": "application/json", + "content-type": "application/json" + }, + "bodyByteCount": 1048576 } } ] @@ -29,19 +52,19 @@ { "name": "valid-response-completed", "category": "valid", - "bytes": "{\"version\":1,\"type\":\"response\",\"id\":\"req-1\",\"status\":200,\"headers\":{\"content-type\":\"application/json\"},\"body\":\"{\\\"ok\\\":true}\",\"outcome\":\"completed\"}\n", + "bytes": "{\"version\":2,\"type\":\"response\",\"id\":\"req-1\",\"status\":200,\"headers\":{\"content-type\":\"application/json\"},\"bodyByteCount\":11,\"outcome\":\"completed\"}\n", "roundTrip": true, "expected": [ { "frame": { - "version": 1, + "version": 2, "type": "response", "id": "req-1", "status": 200, "headers": { "content-type": "application/json" }, - "body": "{\"ok\":true}", + "bodyByteCount": 11, "outcome": "completed" } } @@ -50,19 +73,19 @@ { "name": "valid-response-indeterminate", "category": "valid", - "bytes": "{\"version\":1,\"type\":\"response\",\"id\":\"req-2\",\"status\":409,\"headers\":{\"content-type\":\"application/json\"},\"body\":\"{\\\"error\\\":\\\"outcome_indeterminate\\\"}\",\"outcome\":\"indeterminate\"}\n", + "bytes": "{\"version\":2,\"type\":\"response\",\"id\":\"req-2\",\"status\":409,\"headers\":{\"content-type\":\"application/json\"},\"bodyByteCount\":33,\"outcome\":\"indeterminate\"}\n", "roundTrip": true, "expected": [ { "frame": { - "version": 1, + "version": 2, "type": "response", "id": "req-2", "status": 409, "headers": { "content-type": "application/json" }, - "body": "{\"error\":\"outcome_indeterminate\"}", + "bodyByteCount": 33, "outcome": "indeterminate" } } @@ -71,43 +94,121 @@ { "name": "valid-response-unavailable", "category": "valid", - "bytes": "{\"version\":1,\"type\":\"response\",\"id\":\"req-3\",\"status\":503,\"headers\":{\"content-type\":\"application/json\"},\"body\":\"{\\\"error\\\":\\\"bridge_unavailable\\\"}\",\"outcome\":\"unavailable\"}\n", + "bytes": "{\"version\":2,\"type\":\"response\",\"id\":\"req-3\",\"status\":503,\"headers\":{\"content-type\":\"application/json\"},\"bodyByteCount\":30,\"outcome\":\"unavailable\"}\n", "roundTrip": true, "expected": [ { "frame": { - "version": 1, + "version": 2, "type": "response", "id": "req-3", "status": 503, "headers": { "content-type": "application/json" }, - "body": "{\"error\":\"bridge_unavailable\"}", + "bodyByteCount": 30, "outcome": "unavailable" } } ] }, + { + "name": "valid-body-chunk", + "category": "valid", + "bytes": "{\"version\":2,\"type\":\"body_chunk\",\"id\":\"req-1\",\"seq\":0,\"data\":\"aGVsbG8gYm9keSBjaHVuaw==\"}\n", + "roundTrip": true, + "expected": [ + { + "frame": { + "version": 2, + "type": "body_chunk", + "id": "req-1", + "seq": 0, + "data": "aGVsbG8gYm9keSBjaHVuaw==" + } + } + ] + }, + { + "name": "valid-body-chunk-final", + "category": "valid", + "bytes": "{\"version\":2,\"type\":\"body_chunk\",\"id\":\"req-1\",\"seq\":7,\"data\":\"dGFpbA==\"}\n", + "roundTrip": true, + "expected": [ + { + "frame": { + "version": 2, + "type": "body_chunk", + "id": "req-1", + "seq": 7, + "data": "dGFpbA==" + } + } + ] + }, { "name": "valid-ready", "category": "valid", - "bytes": "{\"version\":1,\"type\":\"ready\",\"nonce\":\"9f8e7d6c5b4a39281706f5e4d3c2b1a0\"}\n", + "bytes": "{\"version\":2,\"type\":\"ready\",\"nonce\":\"9f8e7d6c5b4a39281706f5e4d3c2b1a0\"}\n", "roundTrip": true, "expected": [ { "frame": { - "version": 1, + "version": 2, "type": "ready", "nonce": "9f8e7d6c5b4a39281706f5e4d3c2b1a0" } } ] }, + { + "name": "valid-heartbeat", + "category": "valid", + "bytes": "{\"version\":2,\"type\":\"heartbeat\"}\n", + "roundTrip": true, + "expected": [ + { + "frame": { + "version": 2, + "type": "heartbeat" + } + } + ] + }, + { + "name": "valid-close", + "category": "valid", + "bytes": "{\"version\":2,\"type\":\"close\"}\n", + "roundTrip": true, + "expected": [ + { + "frame": { + "version": 2, + "type": "close" + } + } + ] + }, + { + "name": "valid-error", + "category": "valid", + "bytes": "{\"version\":2,\"type\":\"error\",\"code\":\"malformed_frame\",\"message\":\"peer reported a malformed frame\"}\n", + "roundTrip": true, + "expected": [ + { + "frame": { + "version": 2, + "type": "error", + "code": "malformed_frame", + "message": "peer reported a malformed frame" + } + } + ] + }, { "name": "invalid-ready-missing-nonce", "category": "invalid", - "bytes": "{\"version\":1,\"type\":\"ready\"}\n", + "bytes": "{\"version\":2,\"type\":\"ready\"}\n", "expected": [ { "error": "malformed_frame" @@ -117,7 +218,7 @@ { "name": "invalid-ready-with-address", "category": "invalid", - "bytes": "{\"version\":1,\"type\":\"ready\",\"nonce\":\"9f8e7d6c5b4a39281706f5e4d3c2b1a0\",\"address\":\"http://127.0.0.1:47215\"}\n", + "bytes": "{\"version\":2,\"type\":\"ready\",\"nonce\":\"9f8e7d6c5b4a39281706f5e4d3c2b1a0\",\"address\":\"http://127.0.0.1:47215\"}\n", "expected": [ { "error": "malformed_frame" @@ -127,57 +228,13 @@ { "name": "invalid-ready-with-port", "category": "invalid", - "bytes": "{\"version\":1,\"type\":\"ready\",\"nonce\":\"9f8e7d6c5b4a39281706f5e4d3c2b1a0\",\"port\":47215}\n", + "bytes": "{\"version\":2,\"type\":\"ready\",\"nonce\":\"9f8e7d6c5b4a39281706f5e4d3c2b1a0\",\"port\":47215}\n", "expected": [ { "error": "malformed_frame" } ] }, - { - "name": "valid-heartbeat", - "category": "valid", - "bytes": "{\"version\":1,\"type\":\"heartbeat\"}\n", - "roundTrip": true, - "expected": [ - { - "frame": { - "version": 1, - "type": "heartbeat" - } - } - ] - }, - { - "name": "valid-close", - "category": "valid", - "bytes": "{\"version\":1,\"type\":\"close\"}\n", - "roundTrip": true, - "expected": [ - { - "frame": { - "version": 1, - "type": "close" - } - } - ] - }, - { - "name": "valid-error", - "category": "valid", - "bytes": "{\"version\":1,\"type\":\"error\",\"code\":\"malformed_frame\",\"message\":\"peer reported a malformed frame\"}\n", - "roundTrip": true, - "expected": [ - { - "frame": { - "version": 1, - "type": "error", - "code": "malformed_frame", - "message": "peer reported a malformed frame" - } - } - ] - }, { "name": "invalid-not-json", "category": "invalid", @@ -211,7 +268,7 @@ { "name": "invalid-unknown-type", "category": "invalid", - "bytes": "{\"version\":1,\"type\":\"teleport\"}\n", + "bytes": "{\"version\":2,\"type\":\"teleport\"}\n", "expected": [ { "error": "unknown_type" @@ -221,7 +278,37 @@ { "name": "invalid-missing-id", "category": "invalid", - "bytes": "{\"version\":1,\"type\":\"request\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"body\":\"\"}\n", + "bytes": "{\"version\":2,\"type\":\"request\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"bodyByteCount\":0}\n", + "expected": [ + { + "error": "malformed_frame" + } + ] + }, + { + "name": "invalid-request-body-count-string", + "category": "invalid", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"bodyByteCount\":\"10\"}\n", + "expected": [ + { + "error": "malformed_frame" + } + ] + }, + { + "name": "invalid-request-body-count-negative", + "category": "invalid", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"bodyByteCount\":-1}\n", + "expected": [ + { + "error": "malformed_frame" + } + ] + }, + { + "name": "invalid-request-body-count-fractional", + "category": "invalid", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"bodyByteCount\":1.5}\n", "expected": [ { "error": "malformed_frame" @@ -231,7 +318,47 @@ { "name": "invalid-bad-headers", "category": "invalid", - "bytes": "{\"version\":1,\"type\":\"response\",\"id\":\"req-1\",\"status\":200,\"headers\":{\"accept\":7},\"body\":\"{\\\"ok\\\":true}\",\"outcome\":\"completed\"}\n", + "bytes": "{\"version\":2,\"type\":\"response\",\"id\":\"req-1\",\"status\":200,\"headers\":{\"accept\":7},\"bodyByteCount\":10,\"outcome\":\"completed\"}\n", + "expected": [ + { + "error": "malformed_frame" + } + ] + }, + { + "name": "invalid-body-chunk-negative-seq", + "category": "invalid", + "bytes": "{\"version\":2,\"type\":\"body_chunk\",\"id\":\"req-1\",\"seq\":-1,\"data\":\"eA==\"}\n", + "expected": [ + { + "error": "malformed_frame" + } + ] + }, + { + "name": "invalid-body-chunk-fractional-seq", + "category": "invalid", + "bytes": "{\"version\":2,\"type\":\"body_chunk\",\"id\":\"req-1\",\"seq\":2.5,\"data\":\"eA==\"}\n", + "expected": [ + { + "error": "malformed_frame" + } + ] + }, + { + "name": "invalid-body-chunk-missing-data", + "category": "invalid", + "bytes": "{\"version\":2,\"type\":\"body_chunk\",\"id\":\"req-1\",\"seq\":0}\n", + "expected": [ + { + "error": "malformed_frame" + } + ] + }, + { + "name": "invalid-body-chunk-nonstring-data", + "category": "invalid", + "bytes": "{\"version\":2,\"type\":\"body_chunk\",\"id\":\"req-1\",\"seq\":0,\"data\":42}\n", "expected": [ { "error": "malformed_frame" @@ -241,14 +368,14 @@ { "name": "partial-request-split", "category": "partial", - "bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"body\":\"\"}\n", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"bodyByteCount\":0}\n", "splitByteOffsets": [ - 98 + 102 ], "expected": [ { "frame": { - "version": 1, + "version": 2, "type": "request", "id": "req-1", "method": "GET", @@ -258,7 +385,7 @@ "accept": "application/json", "content-type": "application/json" }, - "body": "" + "bodyByteCount": 0 } } ] @@ -266,23 +393,24 @@ { "name": "partial-utf8-split", "category": "partial", - "bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"req-utf8\",\"method\":\"POST\",\"path\":\"/api/issues/PAP-1/comments\",\"query\":\"\",\"headers\":{\"content-type\":\"application/json\"},\"body\":\"{\\\"text\\\":\\\"héllo 😀 wörld\\\"}\"}\n", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-utf8\",\"method\":\"POST\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"content-type\":\"application/json\",\"x-note\":\"héllo 😀 wörld\"},\"bodyByteCount\":0}\n", "splitByteOffsets": [ - 177 + 172 ], "expected": [ { "frame": { - "version": 1, + "version": 2, "type": "request", "id": "req-utf8", "method": "POST", - "path": "/api/issues/PAP-1/comments", - "query": "", + "path": "/api/issues/PAP-1", + "query": "expand=comments", "headers": { - "content-type": "application/json" + "content-type": "application/json", + "x-note": "héllo 😀 wörld" }, - "body": "{\"text\":\"héllo 😀 wörld\"}" + "bodyByteCount": 0 } } ] @@ -290,16 +418,16 @@ { "name": "partial-two-frames", "category": "partial", - "bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"body\":\"\"}\n{\"version\":1,\"type\":\"response\",\"id\":\"req-1\",\"status\":200,\"headers\":{\"content-type\":\"application/json\"},\"body\":\"{\\\"ok\\\":true}\",\"outcome\":\"completed\"}\n", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"bodyByteCount\":0}\n{\"version\":2,\"type\":\"response\",\"id\":\"req-1\",\"status\":200,\"headers\":{\"content-type\":\"application/json\"},\"bodyByteCount\":11,\"outcome\":\"completed\"}\n", "splitByteOffsets": [ 7, - 199, - 343 + 174, + 344 ], "expected": [ { "frame": { - "version": 1, + "version": 2, "type": "request", "id": "req-1", "method": "GET", @@ -309,19 +437,19 @@ "accept": "application/json", "content-type": "application/json" }, - "body": "" + "bodyByteCount": 0 } }, { "frame": { - "version": 1, + "version": 2, "type": "response", "id": "req-1", "status": 200, "headers": { "content-type": "application/json" }, - "body": "{\"ok\":true}", + "bodyByteCount": 11, "outcome": "completed" } } @@ -330,14 +458,14 @@ { "name": "partial-no-trailing-newline", "category": "partial", - "bytes": "{\"version\":1,\"type\":\"heartbeat\"}", + "bytes": "{\"version\":2,\"type\":\"heartbeat\"}", "expected": [] }, { "name": "oversized-complete-line", "category": "oversized", "maxFrameBytes": 64, - "bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"body\":\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"}\n", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"q=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"bodyByteCount\":0}\n", "expected": [ { "error": "frame_too_large" @@ -351,23 +479,33 @@ "splitByteOffsets": [ 40 ], - "bytes": "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy\n{\"version\":1,\"type\":\"heartbeat\"}\n", + "bytes": "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy\n{\"version\":2,\"type\":\"heartbeat\"}\n", "expected": [ { "error": "frame_too_large" }, { "frame": { - "version": 1, + "version": 2, "type": "heartbeat" } } ] }, + { + "name": "version-mismatch-lower", + "category": "versionMismatch", + "bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"bodyByteCount\":0}\n", + "expected": [ + { + "error": "version_mismatch" + } + ] + }, { "name": "version-mismatch-higher", "category": "versionMismatch", - "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"body\":\"\"}\n", + "bytes": "{\"version\":3,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"bodyByteCount\":0}\n", "expected": [ { "error": "version_mismatch" @@ -387,19 +525,19 @@ { "name": "request-id-at-max-bytes", "category": "idBoundValid", - "bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"body\":\"\"}\n", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"bodyByteCount\":0}\n", "roundTrip": true, "expected": [ { "frame": { - "version": 1, + "version": 2, "type": "request", "id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "method": "GET", "path": "/", "query": "", "headers": {}, - "body": "" + "bodyByteCount": 0 } } ] @@ -407,7 +545,17 @@ { "name": "request-id-over-max-bytes", "category": "idBoundInvalid", - "bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"body\":\"\"}\n", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"bodyByteCount\":0}\n", + "expected": [ + { + "error": "id_too_large" + } + ] + }, + { + "name": "body-chunk-id-over-max-bytes", + "category": "idBoundInvalid", + "bytes": "{\"version\":2,\"type\":\"body_chunk\",\"id\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"seq\":0,\"data\":\"eA==\"}\n", "expected": [ { "error": "id_too_large" @@ -421,14 +569,28 @@ "name": "encode-within-bound", "maxFrameBytes": 200, "frame": { - "version": 1, + "version": 2, "type": "request", "id": "req-enc-1", "method": "GET", "path": "/", "query": "", "headers": {}, - "body": "small" + "bodyByteCount": 5 + }, + "expected": { + "ok": true + } + }, + { + "name": "encode-body-chunk-within-bound", + "maxFrameBytes": 200, + "frame": { + "version": 2, + "type": "body_chunk", + "id": "req-enc-3", + "seq": 0, + "data": "Y2h1bmsgcGF5bG9hZA==" }, "expected": { "ok": true @@ -438,7 +600,7 @@ "name": "encode-control-frame-never-rejected", "maxFrameBytes": 200, "frame": { - "version": 1, + "version": 2, "type": "heartbeat" }, "expected": { @@ -449,14 +611,15 @@ "name": "encode-over-bound", "maxFrameBytes": 200, "frame": { - "version": 1, + "version": 2, "type": "response", "id": "req-enc-2", "status": 200, "headers": { - "content-type": "application/json" + "content-type": "application/json", + "x-pad": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, - "body": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "bodyByteCount": 0, "outcome": "completed" }, "expected": { diff --git a/packages/adapter-utils/src/execution-target-sandbox.test.ts b/packages/adapter-utils/src/execution-target-sandbox.test.ts index 0ecf167102..b5c5c6b148 100644 --- a/packages/adapter-utils/src/execution-target-sandbox.test.ts +++ b/packages/adapter-utils/src/execution-target-sandbox.test.ts @@ -61,6 +61,7 @@ import { type DuplexRequestFrame, type DuplexResponseFrame, } from "./duplex-frame-codec.js"; +import { splitBodyIntoChunkFrames } from "./duplex-body-spool.js"; import { assertNestedDuplexBrokerBudgets, createDuplexBridgeBroker, @@ -92,6 +93,51 @@ import { const execFileAsync = promisify(execFile); +/** + * Emit one duplex request the way the sandbox gateway sends it under the chunked + * body protocol: the envelope frame that carries `bodyByteCount`, then the + * `body_chunk` frames that carry the body. A non-empty body rides one or more + * `body_chunk` frames that share the envelope id; an empty body sends no chunk. + */ +function emitDuplexRequest( + control: { emitData: (chunk: string) => void }, + frame: Omit, + bodyText: string, +): void { + const body = Buffer.from(bodyText, "utf8"); + control.emitData(encodeDuplexFrame({ ...frame, bodyByteCount: body.length })); + for (const chunk of splitBodyIntoChunkFrames(frame.id, body, DUPLEX_FRAME_VERSION)) { + control.emitData(encodeDuplexFrame(chunk)); + } +} + +/** + * Send one duplex response the way the host broker sends it under the chunked + * body protocol: the envelope frame that carries `bodyByteCount`, then the + * `body_chunk` frames that carry the body. The gateway reassembles the body from + * the chunk frames before it writes the HTTP response. + */ +function sendDuplexResponse( + send: (frame: Record) => void, + response: { id: string | undefined; status: number; headers: Record; outcome?: string }, + bodyText: string, +): void { + const id = response.id ?? ""; + const body = Buffer.from(bodyText, "utf8"); + send({ + version: DUPLEX_FRAME_VERSION, + type: "response", + id, + status: response.status, + headers: response.headers, + bodyByteCount: body.length, + ...(response.outcome !== undefined ? { outcome: response.outcome } : {}), + }); + for (const chunk of splitBodyIntoChunkFrames(id, body, DUPLEX_FRAME_VERSION)) { + send(chunk as unknown as Record); + } +} + type RecordedSpan = { name: string; parentName: string | null; ended: boolean }; /** @@ -3140,7 +3186,7 @@ describe("sandbox adapter execution targets", () => { if (onOpen) { onOpen({ nonce, port, emitRaw, emitFrame, emitExit }); } else { - emitFrame({ version: 1, type: "ready", nonce }); + emitFrame({ version: 2, type: "ready", nonce }); } }); }, @@ -3250,8 +3296,9 @@ describe("sandbox adapter execution targets", () => { // The gateway forwards one agent request as a request frame. The broker // forwards it on the host path with the real token and the run id, then // writes one response frame back. - control.emitData( - encodeDuplexFrame({ + emitDuplexRequest( + control, + { version: DUPLEX_FRAME_VERSION, type: "request", id: "req-1", @@ -3259,8 +3306,8 @@ describe("sandbox adapter execution targets", () => { path: "/api/agents/me", query: "", headers: { authorization: "Bearer bridge-token" }, - body: "", - }), + }, + "", ); await waitForCondition( () => control.written.length >= 1, @@ -3304,7 +3351,7 @@ describe("sandbox adapter execution targets", () => { // The flood is larger than the ceiling; the READY frame is far smaller. const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 4096 }); const { runner, control } = makeDuplexSelectionRunner((ctx) => { - ctx.emitFrame({ version: 1, type: "ready", nonce: ctx.nonce }); + ctx.emitFrame({ version: 2, type: "ready", nonce: ctx.nonce }); ctx.emitRaw("x".repeat(64 * 1024)); }); const { recorder, counters } = createRecordingDuplexRecorder(); @@ -3557,15 +3604,15 @@ describe("sandbox adapter execution targets", () => { { name: "a mismatched nonce", onOpen: (ctx: DuplexOpenContext) => - ctx.emitFrame({ version: 1, type: "ready", nonce: "00000000000000000000000000000000" }), + ctx.emitFrame({ version: 2, type: "ready", nonce: "00000000000000000000000000000000" }), }, { name: "an incomplete READY frame", - onOpen: (ctx: DuplexOpenContext) => ctx.emitRaw('{"version":1,"type":"ready"}\n'), + onOpen: (ctx: DuplexOpenContext) => ctx.emitRaw('{"version":2,"type":"ready"}\n'), }, { name: "protocol contamination before READY", - onOpen: (ctx: DuplexOpenContext) => ctx.emitFrame({ version: 1, type: "heartbeat" }), + onOpen: (ctx: DuplexOpenContext) => ctx.emitFrame({ version: 2, type: "heartbeat" }), }, { name: "a gateway bind failure with no READY frame", @@ -3619,12 +3666,12 @@ describe("sandbox adapter execution targets", () => { { name: "an attacker-owned numeric local port", buildReady: (nonce: string, attackerPort: number) => - `{"version":1,"type":"ready","nonce":"${nonce}","port":${attackerPort}}\n`, + `{"version":2,"type":"ready","nonce":"${nonce}","port":${attackerPort}}\n`, }, { name: "a channel-supplied host URL", buildReady: (nonce: string, attackerPort: number) => - `{"version":1,"type":"ready","nonce":"${nonce}","address":"http://127.0.0.1:${attackerPort}"}\n`, + `{"version":2,"type":"ready","nonce":"${nonce}","address":"http://127.0.0.1:${attackerPort}"}\n`, }, ])( "rejects a READY frame that carries $name and never sends the bridge token there", @@ -3724,8 +3771,9 @@ describe("sandbox adapter execution targets", () => { }); try { expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("duplex_v1"); - control.emitData( - encodeDuplexFrame({ + emitDuplexRequest( + control, + { version: DUPLEX_FRAME_VERSION, type: "request", id: "req-forbidden", @@ -3733,8 +3781,8 @@ describe("sandbox adapter execution targets", () => { path: "/api/secret-admin-route", query: "", headers: { authorization: "Bearer bridge-token", "content-type": "application/json" }, - body: JSON.stringify({ escalate: true }), - }), + }, + JSON.stringify({ escalate: true }), ); await waitForCondition( () => control.written.length >= 1, @@ -3850,8 +3898,9 @@ describe("sandbox adapter execution targets", () => { }); try { expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("duplex_v1"); - control.emitData( - encodeDuplexFrame({ + emitDuplexRequest( + control, + { version: DUPLEX_FRAME_VERSION, type: "request", id: "req-obs", @@ -3859,8 +3908,8 @@ describe("sandbox adapter execution targets", () => { path: "/api/agents/me", query: "", headers: { authorization: "Bearer bridge-token" }, - body: "", - }), + }, + "", ); await waitForCondition( () => control.written.length >= 1, @@ -4060,8 +4109,9 @@ describe("sandbox adapter execution targets", () => { try { expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("duplex_v1"); if (dispatchFirst) { - control.emitData( - encodeDuplexFrame({ + emitDuplexRequest( + control, + { version: DUPLEX_FRAME_VERSION, type: "request", id: "req-loss", @@ -4069,8 +4119,8 @@ describe("sandbox adapter execution targets", () => { path: "/api/agents/me", query: "", headers: { authorization: "Bearer bridge-token" }, - body: "", - }), + }, + "", ); await waitForCondition( () => control.written.length >= 1, @@ -4126,8 +4176,9 @@ describe("sandbox adapter execution targets", () => { try { // The throwing recorder never blocked the duplex selection. expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("duplex_v1"); - control.emitData( - encodeDuplexFrame({ + emitDuplexRequest( + control, + { version: DUPLEX_FRAME_VERSION, type: "request", id: "req-guard", @@ -4135,8 +4186,8 @@ describe("sandbox adapter execution targets", () => { path: "/api/agents/me", query: "", headers: { authorization: "Bearer bridge-token" }, - body: "", - }), + }, + "", ); await waitForCondition( () => control.written.length >= 1, @@ -4192,7 +4243,7 @@ describe("sandbox adapter execution targets", () => { onData(listener: (chunk: string) => void): void { dataListener = listener; control.emitData = (chunk) => dataListener?.(chunk); - setImmediate(() => dataListener?.(`${JSON.stringify({ version: 1, type: "ready", nonce })}\n`)); + setImmediate(() => dataListener?.(`${JSON.stringify({ version: 2, type: "ready", nonce })}\n`)); }, onExit(_listener: (exit: { exitCode: number | null }) => void): void {}, stop(): void {}, @@ -4237,8 +4288,9 @@ describe("sandbox adapter execution targets", () => { // A request that carries the sentinel route, query, body, and bridge token. // The response write then fails with the sentinel provider error. - control.emitData( - encodeDuplexFrame({ + emitDuplexRequest( + control, + { version: DUPLEX_FRAME_VERSION, type: "request", id: "req-redact", @@ -4246,8 +4298,8 @@ describe("sandbox adapter execution targets", () => { path: `/api/${ROUTE_SENTINEL}`, query: `secret=${QUERY_SENTINEL}`, headers: { authorization: `Bearer ${BRIDGE_TOKEN_SENTINEL}` }, - body: BODY_SENTINEL, - }), + }, + BODY_SENTINEL, ); // Give the forward and the failing response write time to run and record a loss. await waitForCondition( @@ -4303,8 +4355,9 @@ describe("sandbox adapter execution targets", () => { }); try { expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("duplex_v1"); - control.emitData( - encodeDuplexFrame({ + emitDuplexRequest( + control, + { version: DUPLEX_FRAME_VERSION, type: "request", id: "req-prov", @@ -4312,8 +4365,8 @@ describe("sandbox adapter execution targets", () => { path: "/api/agents/me", query: "", headers: { authorization: "Bearer bridge-token" }, - body: "", - }), + }, + "", ); await waitForCondition( () => spans.some((s) => s.name === DUPLEX_SPAN_REQUEST), @@ -4527,7 +4580,7 @@ describe("sandbox adapter execution targets", () => { const reserveSpy = vi.spyOn(ledger, "reserve"); const { runner, control } = makeDuplexSelectionRunner((ctx) => { ctx.emitRaw("pty-echo-noise"); - ctx.emitFrame({ version: 1, type: "ready", nonce: ctx.nonce }); + ctx.emitFrame({ version: 2, type: "ready", nonce: ctx.nonce }); }); const target: AdapterSandboxExecutionTarget = { kind: "remote", @@ -4644,9 +4697,9 @@ describe("sandbox adapter execution targets", () => { const noisePrefix = "\n".repeat(blankLineCount) + "a non-frame echo line\n"; const { runner, control } = makeDuplexSelectionRunner((ctx) => { ctx.emitRaw(noisePrefix); - ctx.emitFrame({ version: 1, type: "ready", nonce: ctx.nonce }); + ctx.emitFrame({ version: 2, type: "ready", nonce: ctx.nonce }); }); - const readyLine = '{"version":1,"type":"ready","nonce":""}\n'; + const readyLine = '{"version":2,"type":"ready","nonce":""}\n'; const totalBytes = noisePrefix.length + readyLine.length; const { recorder, counters } = createRecordingDuplexRecorder(); const target: AdapterSandboxExecutionTarget = { @@ -4701,8 +4754,8 @@ describe("sandbox adapter execution targets", () => { // must skip the echo line and a partial-JSON line, then accept the READY frame. const { runner, control } = makeDuplexSelectionRunner((ctx) => { ctx.emitRaw("sh -c exec env PAPERCLIP_BRIDGE_NONCE=... node gateway.mjs\n"); - ctx.emitRaw('{"version":1,"type":"ready"}\n'); - ctx.emitFrame({ version: 1, type: "ready", nonce: ctx.nonce }); + ctx.emitRaw('{"version":2,"type":"ready"}\n'); + ctx.emitFrame({ version: 2, type: "ready", nonce: ctx.nonce }); }); const { recorder, counters } = createRecordingDuplexRecorder(); const target: AdapterSandboxExecutionTarget = { @@ -4752,7 +4805,7 @@ describe("sandbox adapter execution targets", () => { // the `ready_nonce_mismatch` reason. const { runner, control } = makeDuplexSelectionRunner((ctx) => { ctx.emitRaw("a non-frame echo line\n"); - ctx.emitFrame({ version: 1, type: "ready", nonce: "00000000000000000000000000000000" }); + ctx.emitFrame({ version: 2, type: "ready", nonce: "00000000000000000000000000000000" }); }); const { recorder, counters } = createRecordingDuplexRecorder(); const target: AdapterSandboxExecutionTarget = { @@ -4808,7 +4861,7 @@ describe("sandbox adapter execution targets", () => { // per-skip cap check stops the bypass. const readinessBufferCapBytes = DEFAULT_MAX_DUPLEX_FRAME_BYTES + 4_096; const { runner, control } = makeDuplexSelectionRunner((ctx) => { - const readyLine = `${JSON.stringify({ version: 1, type: "ready", nonce: ctx.nonce })}\n`; + const readyLine = `${JSON.stringify({ version: 2, type: "ready", nonce: ctx.nonce })}\n`; ctx.emitRaw("\n".repeat(readinessBufferCapBytes + 1) + readyLine); }); const { recorder, counters } = createRecordingDuplexRecorder(); @@ -4860,7 +4913,7 @@ describe("sandbox adapter execution targets", () => { // carries the closed `fallback_reason` dimension, so a reader can group the // failed opens by reason. const { runner } = makeDuplexSelectionRunner((ctx) => - ctx.emitFrame({ version: 1, type: "ready", nonce: "00000000000000000000000000000000" }), + ctx.emitFrame({ version: 2, type: "ready", nonce: "00000000000000000000000000000000" }), ); const { recorder, spans } = createRecordingDuplexRecorder(); const target: AdapterSandboxExecutionTarget = { @@ -4930,8 +4983,9 @@ describe("sandbox adapter execution targets", () => { }); try { expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("duplex_v1"); - control.emitData( - encodeDuplexFrame({ + emitDuplexRequest( + control, + { version: DUPLEX_FRAME_VERSION, type: "request", id: "req-hdr", @@ -4946,8 +5000,8 @@ describe("sandbox adapter execution targets", () => { // A sandbox-supplied auth header the host must replace with the real token. authorization: "Bearer bridge-token", }, - body: "", - }), + }, + "", ); await waitForCondition( () => api.requests.length >= 1, @@ -5000,8 +5054,9 @@ describe("sandbox adapter execution targets", () => { try { // The broker started with derived nested budgets, so the duplex transport serves. expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("duplex_v1"); - control.emitData( - encodeDuplexFrame({ + emitDuplexRequest( + control, + { version: DUPLEX_FRAME_VERSION, type: "request", id: "req-budget", @@ -5009,8 +5064,8 @@ describe("sandbox adapter execution targets", () => { path: "/api/agents/me", query: "", headers: { authorization: "Bearer bridge-token" }, - body: "", - }), + }, + "", ); await waitForCondition( () => control.written.length >= 1, @@ -5120,7 +5175,7 @@ describe("sandbox adapter execution targets", () => { "daytona@212487a7f3c9:~$ exec 2>'/tmp/paperclip-duplex-x.log'; stty raw -echo; " + "exec 'bash' '-c' 'exec env PAPERCLIP_BRIDGE_NONCE=" + ctx.nonce + " node gateway.mjs'\r\n", ); - ctx.emitRaw('{"version":1,"type":"ready","nonce":"' + ctx.nonce + '"}\n'); + ctx.emitRaw('{"version":2,"type":"ready","nonce":"' + ctx.nonce + '"}\n'); }); expect(mode).toBe("duplex_v1"); }, 20000); @@ -5137,7 +5192,7 @@ describe("sandbox adapter execution targets", () => { "stty raw -echo; exec 'bash' '-c' 'exec env node gateway.mjs'\r\n", ); // No newline between the escape sequence and the frame: same line. - ctx.emitRaw('\x1b[?2004l\r{"version":1,"type":"ready","nonce":"' + ctx.nonce + '"}\n'); + ctx.emitRaw('\x1b[?2004l\r{"version":2,"type":"ready","nonce":"' + ctx.nonce + '"}\n'); }); expect(mode).toBe("duplex_v1"); }, 20000); @@ -5146,7 +5201,7 @@ describe("sandbox adapter execution targets", () => { it("PTY replay: accepts READY split across chunk boundaries", async () => { const { mode } = await runReadinessReplay((ctx) => { ctx.emitRaw("prompt$ wrapper-line\r\n"); - const frame = '{"version":1,"type":"ready","nonce":"' + ctx.nonce + '"}\n'; + const frame = '{"version":2,"type":"ready","nonce":"' + ctx.nonce + '"}\n'; ctx.emitRaw(frame.slice(0, 12)); ctx.emitRaw(frame.slice(12)); }); @@ -5159,7 +5214,7 @@ describe("sandbox adapter execution targets", () => { const noise = Buffer.from("prompt ✓ done\r\n", "utf8"); ctx.emitRaw(noise.slice(0, 8).toString("utf8")); ctx.emitRaw(noise.slice(8).toString("utf8")); - ctx.emitRaw('{"version":1,"type":"ready","nonce":"' + ctx.nonce + '"}\n'); + ctx.emitRaw('{"version":2,"type":"ready","nonce":"' + ctx.nonce + '"}\n'); }); expect(mode).toBe("duplex_v1"); }, 20000); @@ -5170,7 +5225,7 @@ describe("sandbox adapter execution targets", () => { const { mode } = await runReadinessReplay((ctx) => { const line = "x".repeat(64) + "\n"; for (let i = 0; i < 80_000; i += 1) ctx.emitRaw(line); - ctx.emitRaw('{"version":1,"type":"ready","nonce":"' + ctx.nonce + '"}\n'); + ctx.emitRaw('{"version":2,"type":"ready","nonce":"' + ctx.nonce + '"}\n'); }); expect(mode).toBe("queue_v1"); }, 30000); @@ -5280,7 +5335,7 @@ describe("sandbox duplex gateway", () => { // process the launch replaced the shell with. await writeFile( stub, - '#!/bin/sh\nprintf \'{"version":1,"type":"ready","nonce":"%s"}\\n\' "$PAPERCLIP_BRIDGE_NONCE"\n', + '#!/bin/sh\nprintf \'{"version":2,"type":"ready","nonce":"%s"}\\n\' "$PAPERCLIP_BRIDGE_NONCE"\n', "utf8", ); const argv = buildDuplexGatewayLaunchArgv({ @@ -5641,15 +5696,16 @@ describe("sandbox duplex gateway", () => { "if-none-match": '"cache-key"', }); - gateway.sendFrame({ - version: 1, - type: "response", - id: requestFrame.id, - status: 200, - headers: { "content-type": "application/json", etag: '"rev-1"', "content-length": "999" }, - body: JSON.stringify({ ok: true }), - outcome: "completed", - }); + sendDuplexResponse( + gateway.sendFrame, + { + id: requestFrame.id, + status: 200, + headers: { "content-type": "application/json", etag: '"rev-1"', "content-length": "999" }, + outcome: "completed", + }, + JSON.stringify({ ok: true }), + ); const response = await responsePromise; expect(response.status).toBe(200); expect(response.headers.get("content-type")).toContain("application/json"); @@ -5666,42 +5722,45 @@ describe("sandbox duplex gateway", () => { const patchFrame = await gateway.waitForFrame( (frame) => frame.type === "request" && frame.id !== requestFrame.id, ); - gateway.sendFrame({ - version: 1, - type: "response", - id: patchFrame.id, - status: 200, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ error: "outcome_indeterminate" }), - outcome: "indeterminate", - }); + sendDuplexResponse( + gateway.sendFrame, + { + id: patchFrame.id, + status: 200, + headers: { "content-type": "application/json" }, + outcome: "indeterminate", + }, + JSON.stringify({ error: "outcome_indeterminate" }), + ); const indeterminate = await indeterminatePromise; expect(indeterminate.status).toBe(409); await gateway.stop(); }, 20000); - it("fails a request that exceeds the frame size bound with a local 413 and keeps the channel open", async () => { + it("fails a request over the body size limit locally and keeps the channel open", async () => { const token = "duplex-token-oversize-request"; - // Raise the body limit above the frame bound, so `readBody` accepts the body - // and the encode guard is the only limit the request meets. The default frame - // bound is 1,000,000 bytes. + // The chunked body protocol splits a large body into fixed-size body_chunk + // frames that each stay under the frame bound, so a body never exceeds the + // frame bound on its own. The operative local guard for a request too large to + // deliver is now the body size limit. Set it to 1,000,000 bytes here. const gateway = await startDuplexGateway({ PAPERCLIP_BRIDGE_TOKEN: token, - PAPERCLIP_BRIDGE_MAX_BODY_BYTES: "3000000", + PAPERCLIP_BRIDGE_MAX_BODY_BYTES: "1000000", }); - // A body over the frame bound makes the request frame exceed the bound. The - // gateway must fail this one local request with a clean 413 and forward no - // frame. + // A body over the body size limit is rejected locally. The gateway must fail + // this one local request cleanly and forward no frame. const oversizeBody = JSON.stringify({ body: "x".repeat(1_100_000) }); const tooLarge = await fetch(`${gateway.baseUrl}/api/issues/issue-1/comments`, { method: "POST", headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, body: oversizeBody, }); - expect(tooLarge.status).toBe(413); - await expect(tooLarge.json()).resolves.toEqual({ error: "request_too_large" }); + expect(tooLarge.status).toBe(502); + await expect(tooLarge.json()).resolves.toEqual({ + error: "Bridge request body exceeded the configured size limit.", + }); // The oversized request forwarded no frame. It never left the gateway. expect(gateway.frames.filter((frame) => frame.type === "request")).toHaveLength(0); @@ -5714,15 +5773,16 @@ describe("sandbox duplex gateway", () => { headers: { authorization: `Bearer ${token}` }, }); const requestFrame = await gateway.waitForFrame((frame) => frame.type === "request"); - gateway.sendFrame({ - version: 1, - type: "response", - id: requestFrame.id, - status: 200, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ ok: true }), - outcome: "completed", - }); + sendDuplexResponse( + gateway.sendFrame, + { + id: requestFrame.id, + status: 200, + headers: { "content-type": "application/json" }, + outcome: "completed", + }, + JSON.stringify({ ok: true }), + ); const okResponse = await okPromise; expect(okResponse.status).toBe(200); await expect(okResponse.json()).resolves.toEqual({ ok: true }); @@ -5859,24 +5919,21 @@ describe("sandbox duplex gateway", () => { // Feed a malformed inbound line and an unknown response id. Both force a // diagnostic path; none of it may reach stdout. gateway.sendRaw("this is not a frame\n"); - gateway.sendFrame({ - version: 1, - type: "response", - id: "unknown-id", - status: 200, - headers: {}, - body: "", - outcome: "completed", - }); - gateway.sendFrame({ - version: 1, - type: "response", - id: requestFrame.id, - status: 200, - headers: { "content-type": "application/json" }, - body: "{}", - outcome: "completed", - }); + sendDuplexResponse( + gateway.sendFrame, + { id: "unknown-id", status: 200, headers: {}, outcome: "completed" }, + "", + ); + sendDuplexResponse( + gateway.sendFrame, + { + id: requestFrame.id, + status: 200, + headers: { "content-type": "application/json" }, + outcome: "completed", + }, + "{}", + ); const response = await responsePromise; expect(response.status).toBe(200); @@ -5932,7 +5989,7 @@ function createFakeDuplexChannel(): { channel: CommandManagedDuplexChannel; emitData: (chunk: string) => void; emitExit: (exit: { exitCode: number | null }) => void; - written: DuplexResponseFrame[]; + written: Array; writtenTypes: string[]; stopped: () => number; setWriteError: (error: Error | null) => void; @@ -5943,16 +6000,43 @@ function createFakeDuplexChannel(): { let writeError: Error | null = null; let closeBehavior: "resolve" | "hang" | "reject" = "resolve"; let stopCount = 0; - const written: DuplexResponseFrame[] = []; + // The broker writes a response as one envelope frame that carries + // `bodyByteCount`, then the `body_chunk` frames that carry the body. Reassemble + // the body so a test reads the response the same way it did with the one-frame + // model. + const written: Array = []; const writtenTypes: string[] = []; + const responseAssembly = new Map< + string, + { frame: DuplexResponseFrame; received: number; chunks: Buffer[] } + >(); const channel: CommandManagedDuplexChannel = { write(data: string): void { if (writeError) throw writeError; const decoded = decodeDuplexLine(data.replace(/\n$/, "")); - if (decoded.ok) { - writtenTypes.push(decoded.frame.type); - if (decoded.frame.type === "response") written.push(decoded.frame); + if (!decoded.ok) return; + const frame = decoded.frame; + writtenTypes.push(frame.type); + if (frame.type === "response") { + if (frame.bodyByteCount === 0) { + written.push({ ...frame, body: "" }); + } else { + responseAssembly.set(frame.id, { frame, received: 0, chunks: [] }); + } + } else if (frame.type === "body_chunk") { + const assembly = responseAssembly.get(frame.id); + if (!assembly) return; + const decodedChunk = Buffer.from(frame.data, "base64"); + assembly.received += decodedChunk.length; + assembly.chunks.push(decodedChunk); + if (assembly.received >= assembly.frame.bodyByteCount) { + responseAssembly.delete(frame.id); + written.push({ + ...assembly.frame, + body: Buffer.concat(assembly.chunks).toString("utf8"), + }); + } } }, onData(listener: (chunk: string) => void): void { @@ -5987,8 +6071,16 @@ function createFakeDuplexChannel(): { }; } -/** Build one request frame line the fake channel can emit. */ -function requestFrameLine(overrides: Partial & { id: string }): string { +/** + * Build one request as its envelope line (carrying `bodyByteCount`) plus its + * `body_chunk` lines, so the fake channel emits a whole request in one write. The + * broker reassembles the body from the chunk frames before it forwards. + */ +function requestFrameLine( + overrides: Partial> & { id: string }, + bodyText: string = JSON.stringify({ body: "hello" }), +): string { + const body = Buffer.from(bodyText, "utf8"); const frame: DuplexRequestFrame = { version: DUPLEX_FRAME_VERSION, type: "request", @@ -5997,9 +6089,13 @@ function requestFrameLine(overrides: Partial & { id: string path: overrides.path ?? "/api/issues/PAP-1/comments", query: overrides.query ?? "", headers: overrides.headers ?? { authorization: "Bearer bridge-token" }, - body: overrides.body ?? JSON.stringify({ body: "hello" }), + bodyByteCount: body.length, }; - return encodeDuplexFrame(frame); + let line = encodeDuplexFrame(frame); + for (const chunk of splitBodyIntoChunkFrames(frame.id, body, DUPLEX_FRAME_VERSION)) { + line += encodeDuplexFrame(chunk); + } + return line; } /** Wait for the pending microtasks and macrotasks to settle. */ @@ -6015,7 +6111,7 @@ describe("createDuplexBridgeBroker", () => { // broker passes the decoded request straight through, so the sandbox request // still carries only the bridge token here, and the handler applies the real // token and the signed run identifier on the existing forward path. - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async (request): Promise => { received.push(request); @@ -6050,7 +6146,7 @@ describe("createDuplexBridgeBroker", () => { it("moves through opening, open, closing, closed in order", async () => { const fake = createFakeDuplexChannel(); const states: DuplexBrokerState[] = []; - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), onStateChange: (state) => states.push(state), @@ -6089,7 +6185,7 @@ describe("createDuplexBridgeBroker", () => { ])("enters lost on $name", async ({ reason, trigger }) => { const fake = createFakeDuplexChannel(); const losses: DuplexBrokerLossRecord[] = []; - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), onLoss: (record) => losses.push(record), @@ -6107,7 +6203,7 @@ describe("createDuplexBridgeBroker", () => { it("enters lost on a close timeout", async () => { const fake = createFakeDuplexChannel(); fake.setCloseBehavior("hang"); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), closeTimeoutMs: 20, @@ -6123,7 +6219,7 @@ describe("createDuplexBridgeBroker", () => { it("stops the heartbeat, marks the run bridge ended, and dispatches nothing after loss", async () => { const fake = createFakeDuplexChannel(); const forwarded: string[] = []; - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async (request) => { forwarded.push(request.id); @@ -6147,7 +6243,7 @@ describe("createDuplexBridgeBroker", () => { it("forwards one request id one time, so a repeated frame never reaches the API twice", async () => { const fake = createFakeDuplexChannel(); const forwarded: string[] = []; - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async (request) => { forwarded.push(request.id); @@ -6171,7 +6267,7 @@ describe("createDuplexBridgeBroker", () => { const fake = createFakeDuplexChannel(); const records: DuplexBrokerRequestRecord[] = []; let clock = 1000; - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), now: () => clock, @@ -6194,7 +6290,7 @@ describe("createDuplexBridgeBroker", () => { it("latches a failure when a loss orders before an orderly completion and names the typed reason", async () => { const fake = createFakeDuplexChannel(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), }); @@ -6227,7 +6323,7 @@ describe("createDuplexBridgeBroker", () => { }, }; const fake = createFakeDuplexChannel(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), telemetry: createDuplexTelemetry({ recorder, providerKey: "daytona" }), @@ -6263,7 +6359,7 @@ describe("createDuplexBridgeBroker", () => { const logLines: string[] = []; const fake = createFakeDuplexChannel(); const sentinel = "SENTINEL-PROVIDER-TEXT-1a2b3c"; - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), telemetry: createDuplexTelemetry({ recorder, providerKey: "daytona" }), @@ -6287,7 +6383,7 @@ describe("createDuplexBridgeBroker", () => { expect(logLines.join("\n")).not.toContain(sentinel); }); - it("rejects a configuration where an inner budget is not smaller than its outer budget", () => { + it("rejects a configuration where an inner budget is not smaller than its outer budget", async () => { expect(() => assertNestedDuplexBrokerBudgets({ forwardTimeoutMs: 32_000, @@ -6302,13 +6398,15 @@ describe("createDuplexBridgeBroker", () => { gatewayWaitMs: 35_000, }), ).toThrow(/response budget/); - expect(() => + // The async broker construction validates budgets before its first await, so + // an invalid budget set surfaces as a rejected promise. + await expect( createDuplexBridgeBroker({ channel: createFakeDuplexChannel().channel, forwardRequest: async () => ({ status: 200 }), budgets: { forwardTimeoutMs: 40_000 }, }), - ).toThrow(/forward budget/); + ).rejects.toThrow(/forward budget/); // The default budget set holds the nested order. expect(() => assertNestedDuplexBrokerBudgets({ @@ -6384,7 +6482,7 @@ describe("sandbox target spec parse: enableSandboxDuplexBridge", () => { describe("settleRunDisposition atomic read and mark", () => { it("marks the orderly completion and reports a success for a healthy channel", async () => { const fake = createFakeDuplexChannel(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), }); @@ -6400,7 +6498,7 @@ describe("settleRunDisposition atomic read and mark", () => { it("reports the failure and does not mark for a latched loss", async () => { const fake = createFakeDuplexChannel(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), }); @@ -6443,8 +6541,8 @@ describe("CLI-lane run-disposition seam", () => { } as AdapterSandboxExecutionTarget; } - function startBroker(fake: ReturnType) { - const broker = createDuplexBridgeBroker({ + async function startBroker(fake: ReturnType) { + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), }); @@ -6454,7 +6552,7 @@ describe("CLI-lane run-disposition seam", () => { it("fails a clean CLI completion closed when the duplex channel was lost mid-turn", async () => { const fake = createFakeDuplexChannel(); - const broker = startBroker(fake); + const broker = await startBroker(fake); // The control channel dies mid-turn, before the CLI process exits. fake.emitExit({ exitCode: 1 }); await flushMacrotasks(); @@ -6478,7 +6576,7 @@ describe("CLI-lane run-disposition seam", () => { it("keeps a clean CLI completion a success when the channel stays healthy, and a teardown loss stays benign", async () => { const fake = createFakeDuplexChannel(); - const broker = startBroker(fake); + const broker = await startBroker(fake); const runner = mockRunner({ ...CLEAN_RESULT }); const result = await runAdapterExecutionTargetProcess("run-cli-ok", sandboxTarget(runner), "agent-cli", [], { @@ -6502,7 +6600,7 @@ describe("CLI-lane run-disposition seam", () => { it("keeps a clean CLI completion a success when the gateway exits during the run-log tail finish", async () => { const fake = createFakeDuplexChannel(); - const broker = startBroker(fake); + const broker = await startBroker(fake); // A run-log tail whose finish emits a gateway exit. This reproduces the // race where the duplex gateway dies after the clean process completion but @@ -6542,7 +6640,7 @@ describe("CLI-lane run-disposition seam", () => { it("cannot clear the loss latch with a later completion", async () => { const fake = createFakeDuplexChannel(); - const broker = startBroker(fake); + const broker = await startBroker(fake); // The loss latches before the CLI process exits. fake.emitExit({ exitCode: 1 }); await flushMacrotasks(); @@ -6565,7 +6663,7 @@ describe("CLI-lane run-disposition seam", () => { it("leaves an already-failed CLI result unchanged and never settles the disposition", async () => { const fake = createFakeDuplexChannel(); - const broker = startBroker(fake); + const broker = await startBroker(fake); // The control channel is lost, but the process itself also exited non-zero. fake.emitExit({ exitCode: 1 }); await flushMacrotasks(); @@ -6662,7 +6760,7 @@ describe("duplex readiness gate replay-buffer reservation", () => { } function readyLine(): string { - return `${JSON.stringify({ version: 1, type: "ready", nonce: READY_NONCE })}\n`; + return `${JSON.stringify({ version: 2, type: "ready", nonce: READY_NONCE })}\n`; } it("charges the post-READY suffix and releases it after the broker handoff", async () => { diff --git a/packages/adapter-utils/src/execution-target.ts b/packages/adapter-utils/src/execution-target.ts index 065a7f8c5b..c21b7d3a87 100644 --- a/packages/adapter-utils/src/execution-target.ts +++ b/packages/adapter-utils/src/execution-target.ts @@ -2,6 +2,7 @@ import fs from "node:fs/promises"; import net from "node:net"; import os from "node:os"; import path from "node:path"; +import { Readable } from "node:stream"; import { randomBytes, randomUUID } from "node:crypto"; import type { SshRemoteExecutionSpec } from "./ssh.js"; import { @@ -59,6 +60,7 @@ import { DEFAULT_MAX_DUPLEX_FRAME_BYTES, type DuplexRequestFrame, } from "./duplex-frame-codec.js"; +import type { ReassembledBody } from "./duplex-body-spool.js"; import { createDuplexTelemetry, type DuplexFallbackReason, @@ -3132,9 +3134,24 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { // file bridge and the duplex broker. The sandbox request carries only the // bridge token; the real agent token never leaves the host. const forwardBridgeRequest = async ( - request: { method: string; path: string; query: string; headers: Record; body: string }, + request: { + method: string; + path: string; + query: string; + headers: Record; + /** The file bridge passes the whole request body here as one string. */ + body?: string; + }, signal?: AbortSignal, - options?: { suppressDebugLog?: boolean }, + options?: { + suppressDebugLog?: boolean; + /** + * The duplex broker passes the reassembled request body here. The forward + * streams it to the host, so a spilled body never loads into memory on the + * receive side. The forward never disposes it; the broker owns its lifecycle. + */ + reassembledBody?: ReassembledBody; + }, ): Promise<{ status: number; headers: Record; body: string }> => { const method = request.method.trim().toUpperCase() || "GET"; // The per-request debug log prints the method, the path, and the query. The @@ -3159,12 +3176,26 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { // the forward budget here, whichever comes first. const timeoutSignal = AbortSignal.timeout(forwardTimeoutMs); const forwardSignal = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal; - const response = await fetch(buildBridgeForwardUrl(hostApiUrl, request), { + // Build the request-body init. A GET or a HEAD carries no body. The duplex + // broker passes the reassembled body: stream it, so a body that spilled to + // disk never loads into memory here. A streamed body needs `duplex: "half"`. + // The file bridge passes the whole body as one string. + const forwardInit: RequestInit & { duplex?: "half" } = { method, headers, - ...(method === "GET" || method === "HEAD" ? {} : { body: request.body }), signal: forwardSignal, - }); + }; + if (method !== "GET" && method !== "HEAD") { + if (options?.reassembledBody) { + forwardInit.body = Readable.toWeb( + options.reassembledBody.createReadStream(), + ) as unknown as ReadableStream; + forwardInit.duplex = "half"; + } else if (typeof request.body === "string") { + forwardInit.body = request.body; + } + } + const response = await fetch(buildBridgeForwardUrl(hostApiUrl, request), forwardInit); if (emitDebugLog) { await onLog( "stdout", @@ -3375,14 +3406,14 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { // token only now, after readiness passed. let broker: DuplexBridgeBroker | null = null; try { - broker = createDuplexBridgeBroker({ + broker = await createDuplexBridgeBroker({ channel: gate.brokerChannel, // Derive the nested budgets from the forward budget, so any forward // budget keeps the nested order the broker asserts at construction. budgets: deriveNestedDuplexBrokerBudgets(forwardTimeoutMs), forwardRequest: async ( request: DuplexRequestFrame, - options: { signal: AbortSignal }, + options: { signal: AbortSignal; body: ReassembledBody }, ): Promise => { const denialReason = authorizeSandboxCallbackBridgeRequestWithRoutes(request); if (denialReason) { @@ -3402,9 +3433,11 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { headers: sanitizeSandboxCallbackBridgeHeaders(request.headers), }; // Suppress the per-request debug log on the duplex path, so no route - // or query rides a log line here. + // or query rides a log line here. Stream the reassembled request + // body, so a spilled body never loads into memory on the host. return forwardBridgeRequest(sanitizedRequest, options.signal, { suppressDebugLog: true, + reassembledBody: options.body, }); }, // The duplex path emits only the fixed transport telemetry. It passes no diff --git a/packages/adapter-utils/src/sandbox-callback-bridge.ts b/packages/adapter-utils/src/sandbox-callback-bridge.ts index 9c3d073919..5d8907ce04 100644 --- a/packages/adapter-utils/src/sandbox-callback-bridge.ts +++ b/packages/adapter-utils/src/sandbox-callback-bridge.ts @@ -1763,7 +1763,8 @@ export async function startSandboxCallbackBridgeServer(input: { * compatible with the host codec. {@link getSandboxDuplexGatewayCodecSource} * returns this exact source so a test can run every fixture vector against it. */ -const DUPLEX_GATEWAY_CODEC_SOURCE = `const DUPLEX_FRAME_VERSION = 1; +const DUPLEX_GATEWAY_CODEC_SOURCE = `const DUPLEX_FRAME_VERSION = 2; +const DUPLEX_BODY_CHUNK_RAW_BYTES = 256 * 1024; const DEFAULT_MAX_DUPLEX_FRAME_BYTES = 1000000; const DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES = 256; const DEFAULT_MAX_DUPLEX_DECODER_BYTES = ${DEFAULT_BRIDGE_MAX_DUPLEX_DECODER_BYTES}; @@ -1792,6 +1793,10 @@ function duplexIsStringRecord(value) { return true; } +function duplexIsSafeNonNegativeInteger(value) { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0; +} + function encodeDuplexFrame(frame) { return JSON.stringify(frame) + "\\n"; } @@ -1828,6 +1833,8 @@ function duplexValidateFrame(frame) { return duplexValidateRequest(frame); case "response": return duplexValidateResponse(frame); + case "body_chunk": + return duplexValidateBodyChunk(frame); case "ready": return duplexValidateReady(frame); case "heartbeat": @@ -1847,7 +1854,7 @@ function duplexValidateRequest(frame) { typeof frame.method !== "string" || typeof frame.path !== "string" || typeof frame.query !== "string" || - typeof frame.body !== "string" || + !duplexIsSafeNonNegativeInteger(frame.bodyByteCount) || !duplexIsStringRecord(frame.headers) ) { return duplexFail("malformed_frame", "request frame has a missing or wrong-typed field"); @@ -1862,7 +1869,7 @@ function duplexValidateResponse(frame) { if ( typeof frame.id !== "string" || typeof frame.status !== "number" || - typeof frame.body !== "string" || + !duplexIsSafeNonNegativeInteger(frame.bodyByteCount) || !duplexIsStringRecord(frame.headers) || typeof frame.outcome !== "string" || !DUPLEX_RESPONSE_OUTCOMES.has(frame.outcome) @@ -1875,6 +1882,19 @@ function duplexValidateResponse(frame) { return duplexOk(frame); } +function duplexValidateBodyChunk(frame) { + if (typeof frame.id !== "string" || typeof frame.data !== "string") { + return duplexFail("malformed_frame", "body_chunk frame has a missing or wrong-typed field"); + } + if (!duplexIsSafeNonNegativeInteger(frame.seq)) { + return duplexFail("malformed_frame", "body_chunk frame has a missing or wrong-typed seq"); + } + if (Buffer.byteLength(frame.id, "utf8") > DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES) { + return duplexFail("id_too_large", "body_chunk frame id exceeds the maximum size"); + } + return duplexOk(frame); +} + function duplexValidateReady(frame) { if (typeof frame.nonce !== "string") { return duplexFail("malformed_frame", "ready frame has a missing or wrong-typed nonce"); @@ -2198,10 +2218,37 @@ async function runFileGateway() { }); } +// Split one whole body buffer into body_chunk frames. The gateway buffers the +// whole source body once, then splits it here. Each frame carries one fixed raw +// slice as base64 text, except the final frame, which carries the remaining +// bytes. A zero-length body yields no frame. Send-side true streaming is a +// separate goal; this whole-body split is acceptable for this gateway. +function splitDuplexBodyIntoChunks(id, body) { + const frames = []; + let seq = 0; + for (let offset = 0; offset < body.length; offset += DUPLEX_BODY_CHUNK_RAW_BYTES) { + const slice = body.subarray(offset, offset + DUPLEX_BODY_CHUNK_RAW_BYTES); + frames.push({ + version: DUPLEX_FRAME_VERSION, + type: "body_chunk", + id: id, + seq: seq, + data: slice.toString("base64"), + }); + seq += 1; + } + return frames; +} + function runDuplexGateway() { // One outstanding local request per id. Each entry holds the HTTP resolver and // the wait-budget timer. const pending = new Map(); + // One in-flight response reassembly per id. The host returns a response as an + // envelope frame that carries bodyByteCount, then the body_chunk frames that + // carry the body. The gateway reassembles the response body in memory here; it + // does not spill, because a production response body stays small. + const responseAssembly = new Map(); let unavailable = false; let lossTriggered = false; let lastInboundAt = Date.now(); @@ -2243,27 +2290,102 @@ function runDuplexGateway() { }); } pending.clear(); + responseAssembly.clear(); const exitTimer = setTimeout(() => process.exit(0), lossExitGraceMs); if (typeof exitTimer.unref === "function") exitTimer.unref(); } + // Fail one outstanding request with a bounded local 502. The gateway calls it + // when a response reassembly breaks: a reordered seq, a base64 that is not + // canonical, or a total that overruns the declared body size. The host is the + // response peer, so this is a defensive local error, not a channel loss. + function failRequest(id, message) { + responseAssembly.delete(id); + const entry = pending.get(id); + if (!entry) return; + pending.delete(id); + clearTimeout(entry.timer); + entry.resolve({ + status: 502, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ error: message }), + }); + } + function handleInboundFrame(frame) { if (frame.type === "response") { const entry = pending.get(frame.id); if (!entry) return; - pending.delete(frame.id); - clearTimeout(entry.timer); // Map an indeterminate outcome to a non-retryable 409, the same contract // the file gateway applies through the outcome header. const statusCode = frame.outcome === "indeterminate" ? 409 : typeof frame.status === "number" ? frame.status : 200; - entry.resolve({ + const headers = frame.headers || {}; + if (frame.bodyByteCount === 0) { + // A zero-length body accepts no body_chunk, so the response completes now. + pending.delete(frame.id); + responseAssembly.delete(frame.id); + clearTimeout(entry.timer); + entry.resolve({ status: statusCode, headers: headers, body: "" }); + return; + } + // The body rides body_chunk frames that share this id. Record the envelope + // and wait for the chunks. + responseAssembly.set(frame.id, { status: statusCode, - headers: frame.headers || {}, - body: typeof frame.body === "string" ? frame.body : "", + headers: headers, + bodyByteCount: frame.bodyByteCount, + received: 0, + nextSeq: 0, + chunks: [], }); return; } + if (frame.type === "body_chunk") { + const asm = responseAssembly.get(frame.id); + if (!asm) return; + if (frame.seq !== asm.nextSeq) { + failRequest(frame.id, "duplex response body_chunk seq is out of order"); + return; + } + const decoded = Buffer.from(frame.data, "base64"); + if (decoded.toString("base64") !== frame.data) { + failRequest(frame.id, "duplex response body_chunk is not canonical base64"); + return; + } + if (decoded.length === 0) { + failRequest(frame.id, "duplex response body_chunk is empty"); + return; + } + if (asm.received + decoded.length > asm.bodyByteCount) { + failRequest(frame.id, "duplex response body overruns the declared size"); + return; + } + const isFinal = asm.received + decoded.length === asm.bodyByteCount; + if ( + decoded.length > DUPLEX_BODY_CHUNK_RAW_BYTES || + (!isFinal && decoded.length !== DUPLEX_BODY_CHUNK_RAW_BYTES) + ) { + failRequest(frame.id, "duplex response body_chunk has the wrong size"); + return; + } + asm.nextSeq += 1; + asm.received += decoded.length; + asm.chunks.push(decoded); + if (asm.received === asm.bodyByteCount) { + const entry = pending.get(frame.id); + responseAssembly.delete(frame.id); + if (!entry) return; + pending.delete(frame.id); + clearTimeout(entry.timer); + entry.resolve({ + status: asm.status, + headers: asm.headers, + body: Buffer.concat(asm.chunks).toString("utf8"), + }); + } + return; + } if (frame.type === "close") { triggerLoss("host sent a close frame"); return; @@ -2320,11 +2442,14 @@ function runDuplexGateway() { return; } const requestId = randomUUID(); - const requestBody = await readBody(req); + const requestBodyBuffer = Buffer.from(await readBody(req), "utf8"); if (unavailable) { writeJsonResponse(res, 503, { error: "bridge_unavailable" }); return; } + // The request body rides body_chunk frames. The envelope carries only the + // raw byte count, so the envelope stays small and the body splits into + // fixed-size slices that each stay under the frame bound. const requestFrame = { version: DUPLEX_FRAME_VERSION, type: "request", @@ -2333,20 +2458,36 @@ function runDuplexGateway() { path: url.pathname, query: url.search, headers: normalizeHeaders(req.headers), - body: requestBody, + bodyByteCount: requestBodyBuffer.length, }; - // Enforce the frame size bound on encode. When the request body makes the - // frame exceed the bound, fail this one local request with a clean 413. Do - // not write the frame and do not trigger loss. The frame never leaves the - // gateway, so no other in-flight request is affected and the channel stays - // open. const encodedRequest = encodeDuplexFrameChecked(requestFrame); if (!encodedRequest.ok) { writeJsonResponse(res, 413, { error: "request_too_large" }); return; } + // Pre-encode every body_chunk frame and enforce the frame size bound on + // each. A fixed raw slice never exceeds the bound, so this guard is + // defensive. On a rejection, fail this one local request with a clean 413. + // The frames never leave the gateway, so no other in-flight request is + // affected and the channel stays open. + const chunkFrames = splitDuplexBodyIntoChunks(requestId, requestBodyBuffer); + const encodedChunks = []; + let chunkTooLarge = false; + for (const chunk of chunkFrames) { + const encodedChunk = encodeDuplexFrameChecked(chunk); + if (!encodedChunk.ok) { + chunkTooLarge = true; + break; + } + encodedChunks.push(encodedChunk.line); + } + if (chunkTooLarge) { + writeJsonResponse(res, 413, { error: "request_too_large" }); + return; + } const response = await new Promise((resolve) => { const timer = setTimeout(() => { + responseAssembly.delete(requestId); if (pending.delete(requestId)) { resolve({ status: 502, @@ -2357,6 +2498,7 @@ function runDuplexGateway() { }, responseTimeoutMs); pending.set(requestId, { resolve: resolve, timer: timer }); process.stdout.write(encodedRequest.line); + for (const line of encodedChunks) process.stdout.write(line); }); res.statusCode = typeof response.status === "number" ? response.status : 200; for (const [key, value] of Object.entries(response.headers || {})) {