diff --git a/doc/observability.md b/doc/observability.md index 8f0d193e29..5da3157eff 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -630,24 +630,6 @@ field. | `sandbox_duplex_loss_total` | One terminal duplex channel loss. The `loss_class` dimension records the phase. | | `sandbox_duplex_session_leak_total` | One leaked provider session at teardown. | -### Aggregate byte ledger metrics - -The host aggregate byte ledger owns one process-scoped gauge and two -process-scoped counters. The ledger bounds the retained bytes across every live -duplex route in one process. It sets the gauge on each reserve and each release. -It increments a counter on a rejected reservation and on an accounting defect. -These records carry no dimension label. The guarded counter store keys each -counter on `(companyId, metric)`, and the gauge reports one process value, so no -dynamic dimension rides them. The code owner is -`packages/adapter-utils/src/duplex-aggregate-byte-ledger.ts`, and the metric -names are literal constants in `duplex-observability.ts`. - -| Metric | Type | Scope | -| --- | --- | --- | -| `sandbox_duplex_aggregate_bytes_in_use` | gauge | The aggregate retained bytes across every live duplex route. The ledger sets it on each reserve and each release. | -| `sandbox_duplex_aggregate_byte_reservation_rejections_total` | counter | One rejected aggregate byte reservation. The ledger increments it when a reservation would pass the aggregate ceiling. | -| `sandbox_duplex_aggregate_byte_accounting_underflow_total` | counter | One aggregate byte accounting defect. The ledger increments it on a double release or on a transfer of a token it does not hold. | - ### Dimension keys Counters carry no dimension labels. The guarded counter store keys each counter @@ -662,10 +644,25 @@ key never reaches a sink by accident. | `provider` | string | no | `daytona`, or `other` for any other plugin key. | | `transport` | string | no | `duplex`, `http2`, or `file`. `duplex` names the retired bespoke frame protocol; `http2` names the Node HTTP/2 session over the sandbox channel; a fallback record uses `file`. | | `outcome` | string | yes | `ok` or `error`. | -| `fallback_reason` | string | yes | `gate_off`, `capability_absent`, `route_busy`, `entrypoint_sync_failed`, `broker_construction_failed`, `channel_open_failed`, `ready_invalid`, `ready_nonce_mismatch`, `ready_timeout`, `contaminated`, `aggregate_bytes_exceeded`, or `preface_missing`. It rides only a fallback record. `route_busy` marks the process-scoped route ceiling full. `entrypoint_sync_failed` and `broker_construction_failed` mark the named build step. `channel_open_failed` marks a failed channel open. `aggregate_bytes_exceeded` marks a readiness handshake, or an `http2` post-preface pre-bind buffer, where the host fell back because the process aggregate byte ceiling had no room. `preface_missing` marks a missing or an invalid HTTP/2 client connection preface inside the bounded readiness buffer: the host found no valid preface after the accepted READY line, aborted the `http2` open, and moved the run to the file bridge (`queue_v1`) one time. | +| `fallback_reason` | string | yes | `gate_off`, `capability_absent`, `route_busy`, `entrypoint_sync_failed`, `broker_construction_failed`, `channel_open_failed`, `ready_invalid`, `ready_nonce_mismatch`, `ready_timeout`, `contaminated`, or `preface_missing`. It rides only a fallback record. `route_busy` marks the process-scoped route ceiling full. `entrypoint_sync_failed` and `broker_construction_failed` mark the named build step. `channel_open_failed` marks a failed channel open. `preface_missing` marks a missing or an invalid HTTP/2 client connection preface inside the bounded readiness buffer: the host found no valid preface after the accepted READY line, aborted the `http2` open, and moved the run to the file bridge (`queue_v1`) one time. | | `loss_class` | string | yes | `pre_dispatch` or `post_dispatch`, relative to the first request dispatch. It rides only a loss record. | | `loss_reason` | string | yes | `stdin_eof`, `provider_exit`, `heartbeat_timeout`, `rpc_failure`, `write_error`, `transport_closed`, or `other`. The host maps every loss cause to one of these values, so no raw provider text reaches a sink. `write_error` marks a rejected host-to-sandbox write. `transport_closed` marks a reason-less provider transport close with no exit data. It rides only a loss record. | To add a name or an enum value, extend the literal constant in `duplex-observability.ts` first, then update the test that asserts the closed set. + +### Known behavior: aggregate retained body bytes + +The HTTP/2 bridge bounds retained body bytes for one route only. Each route +holds up to 8,388,608 bytes (8 MiB) at its own peak (see +`HTTP2_BRIDGE_MAX_CONCURRENT_STREAMS` in `http2-bridge-server.ts`). The host +process admits up to 128 concurrent routes (see +`DEFAULT_MAX_CONCURRENT_DUPLEX_ROUTES` in `plugin-worker-manager.ts`). The +process can therefore retain up to 1,073,741,824 bytes (1 GiB) of body data +across every route at the same time. + +This is accepted, known behavior. The process tracks no aggregate byte +ledger across routes: a per-route bound stops one busy route from starving +another route's own budget, but the host enforces no smaller ceiling on the +sum across every route. Keep every dimension low-cardinality and free of user content. diff --git a/packages/adapter-utils/src/duplex-aggregate-byte-ledger.test.ts b/packages/adapter-utils/src/duplex-aggregate-byte-ledger.test.ts deleted file mode 100644 index 6711bf625b..0000000000 --- a/packages/adapter-utils/src/duplex-aggregate-byte-ledger.test.ts +++ /dev/null @@ -1,223 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { - DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES, - DUPLEX_AGGREGATE_TOKEN_OWNERS, - DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, - DuplexAggregateByteLedger, - MIN_HOST_PROCESS_MEMORY_BYTES_FOR_DUPLEX, - assertDuplexAggregateCeilingBytes, - resolveDuplexAggregateCeilingBytes, - type DuplexAggregateByteLedgerTelemetry, -} from "./duplex-aggregate-byte-ledger.js"; - -// A telemetry surface that counts each fixed record, so a test asserts the exact -// gauge value and the exact counter counts. -function createCountingTelemetry(): DuplexAggregateByteLedgerTelemetry & { - gauge: number; - rejections: number; - underflows: number; -} { - const state = { gauge: 0, rejections: 0, underflows: 0 }; - return { - ...state, - setBytesInUse(bytes: number) { - this.gauge = bytes; - }, - recordReservationRejection() { - this.rejections += 1; - }, - recordAccountingUnderflow() { - this.underflows += 1; - }, - }; -} - -describe("DuplexAggregateByteLedger", () => { - it("reserves under the ceiling and tracks the gauge and the live-token count", () => { - const telemetry = createCountingTelemetry(); - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1000, telemetry }); - - const a = ledger.reserve("buffered_chunk", 400); - const b = ledger.reserve("buffered_chunk", 500); - - expect(a).not.toBeNull(); - expect(b).not.toBeNull(); - expect(ledger.bytesInUse).toBe(900); - expect(ledger.liveTokenCount).toBe(2); - expect(telemetry.gauge).toBe(900); - expect(a?.bytes).toBe(400); - expect(a?.owner).toBe("buffered_chunk"); - expect(a?.state).toBe("held"); - }); - - it("rejects a one-byte-over reservation, retains nothing, and increments the rejection counter", () => { - const telemetry = createCountingTelemetry(); - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1000, telemetry }); - - ledger.reserve("buffered_chunk", 1000); - const over = ledger.reserve("buffered_chunk", 1); - - expect(over).toBeNull(); - expect(ledger.bytesInUse).toBe(1000); - expect(ledger.liveTokenCount).toBe(1); - expect(telemetry.rejections).toBe(1); - // The public rejection marker is the fixed string the caller reports. - expect(DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED).toBe("DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED"); - }); - - it("transfers ownership without a decrement or a re-reserve, so no admission gap opens", () => { - const telemetry = createCountingTelemetry(); - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 100, telemetry }); - - // Reserve the whole ceiling as a pre-bind event, then transfer to buffered. - const token = ledger.reserve("pre_bind_event", 100); - expect(token).not.toBeNull(); - // A second reservation cannot fit while the token stays held. - expect(ledger.reserve("buffered_chunk", 1)).toBeNull(); - - ledger.transfer(token!, "buffered_chunk"); - - // The transfer changed only the owner. The gauge never moved, so no window - // opened where the bytes were released and re-admitted. - expect(ledger.bytesInUse).toBe(100); - expect(ledger.liveTokenCount).toBe(1); - expect(token?.owner).toBe("buffered_chunk"); - expect(telemetry.gauge).toBe(100); - // No reservation fit during the transfer, so no rejection or underflow fired - // beyond the one rejection above. - expect(telemetry.rejections).toBe(1); - expect(telemetry.underflows).toBe(0); - }); - - it("releases a token one time and decrements the gauge", () => { - const telemetry = createCountingTelemetry(); - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1000, telemetry }); - - const token = ledger.reserve("response_body", 250); - expect(token).not.toBeNull(); - ledger.release(token!); - - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - expect(token?.state).toBe("released"); - expect(telemetry.gauge).toBe(0); - expect(telemetry.underflows).toBe(0); - }); - - it("records the underflow counter on a second release and does not clamp the gauge again", () => { - const telemetry = createCountingTelemetry(); - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1000, telemetry }); - - const kept = ledger.reserve("request_frame", 300); - const doubled = ledger.reserve("request_frame", 200); - expect(kept).not.toBeNull(); - expect(doubled).not.toBeNull(); - ledger.release(doubled!); - expect(ledger.bytesInUse).toBe(300); - - // A forced second release of the same token is an accounting defect. - ledger.release(doubled!); - - expect(telemetry.underflows).toBe(1); - // The gauge did not drop a second time, so the defect stays visible. - expect(ledger.bytesInUse).toBe(300); - expect(ledger.liveTokenCount).toBe(1); - }); - - it("records the underflow counter on a transfer of a released token", () => { - const telemetry = createCountingTelemetry(); - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1000, telemetry }); - const token = ledger.reserve("pre_bind_event", 100); - ledger.release(token!); - - ledger.transfer(token!, "buffered_chunk"); - - expect(telemetry.underflows).toBe(1); - expect(ledger.bytesInUse).toBe(0); - }); - - it("throws on a non-integer or negative reservation, so a programming error fails loud", () => { - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1000 }); - expect(() => ledger.reserve("buffered_chunk", -1)).toThrow(); - expect(() => ledger.reserve("buffered_chunk", 1.5)).toThrow(); - expect(() => ledger.reserve("buffered_chunk", Number.NaN)).toThrow(); - }); - - it("admits a zero-byte reservation as a valid held token", () => { - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1000 }); - const token = ledger.reserve("seen_request_id", 0); - expect(token).not.toBeNull(); - expect(token?.bytes).toBe(0); - expect(ledger.liveTokenCount).toBe(1); - }); - - it("exposes the closed owner-label set", () => { - expect([...DUPLEX_AGGREGATE_TOKEN_OWNERS]).toEqual([ - "pre_bind_event", - "buffered_chunk", - "terminal_buffered", - "request_frame", - "request_payload", - "response_body", - "seen_request_id", - "decoder_buffer", - "readiness_buffer", - "readiness_replay", - "http2_preface_scan", - "http2_preface_replay", - "pending_write", - "stdin_write", - ]); - }); -}); - -describe("aggregate ceiling configuration", () => { - it("uses the default when the override is absent", () => { - expect(resolveDuplexAggregateCeilingBytes()).toBe(DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES); - expect(resolveDuplexAggregateCeilingBytes(null)).toBe(DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES); - expect(DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES).toBe(256 * 1024 * 1024); - }); - - it("accepts a valid positive override under the process allocation minus the reserve", () => { - expect(resolveDuplexAggregateCeilingBytes(64 * 1024 * 1024)).toBe(64 * 1024 * 1024); - const maxCeiling = - MIN_HOST_PROCESS_MEMORY_BYTES_FOR_DUPLEX - DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES; - expect(resolveDuplexAggregateCeilingBytes(maxCeiling)).toBe(maxCeiling); - }); - - it("rejects an invalid override to the safe default and reports each rejection", () => { - const overMaximum = MIN_HOST_PROCESS_MEMORY_BYTES_FOR_DUPLEX; - const invalidOverrides = [0, -1, 1.5, Number.POSITIVE_INFINITY, Number.NaN, overMaximum]; - for (const invalid of invalidOverrides) { - const rejected: number[] = []; - // An invalid present override returns the safe default. It never throws, so a - // single bad environment value never fails host startup. - expect(resolveDuplexAggregateCeilingBytes(invalid, (value) => rejected.push(value))).toBe( - DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES, - ); - // The resolver reports the rejection one time with the rejected numeric value. - expect(rejected).toEqual([invalid]); - } - }); - - it("does not report a rejection for a valid or a missing override", () => { - const rejected: number[] = []; - const reporter = (value: number): void => { - rejected.push(value); - }; - expect(resolveDuplexAggregateCeilingBytes(64 * 1024 * 1024, reporter)).toBe(64 * 1024 * 1024); - expect(resolveDuplexAggregateCeilingBytes(undefined, reporter)).toBe( - DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES, - ); - expect(resolveDuplexAggregateCeilingBytes(null, reporter)).toBe( - DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES, - ); - expect(rejected).toEqual([]); - }); - - it("still throws for a direct ceiling assertion on an unsafe integer", () => { - // The ledger constructor asserts its ceiling and must fail loud on a - // programming error, so `assertDuplexAggregateCeilingBytes` keeps throwing. - expect(() => assertDuplexAggregateCeilingBytes(Number.MAX_SAFE_INTEGER + 2)).toThrow(); - }); -}); diff --git a/packages/adapter-utils/src/duplex-aggregate-byte-ledger.ts b/packages/adapter-utils/src/duplex-aggregate-byte-ledger.ts deleted file mode 100644 index 9924659ea8..0000000000 --- a/packages/adapter-utils/src/duplex-aggregate-byte-ledger.ts +++ /dev/null @@ -1,413 +0,0 @@ -/** - * The process-owned aggregate byte ledger for the sandbox duplex channel. - * - * The route-count controller bounds how many concurrent duplex routes stay live. - * It does not bound the aggregate bytes those routes retain. At the maximum route - * count the per-route byte bounds multiply to many gigabytes of retained bytes. - * This ledger closes that gap. One ledger per host server process owns a single - * byte ceiling. Every host-side retention site reserves its exact retained bytes - * against this ledger before it allocates, so the aggregate retained bytes across - * all live routes never pass the ceiling. - * - * The ledger is fail-closed. A reservation that would pass the ceiling returns no - * token. The caller retains nothing and reports the fixed rejection marker - * {@link DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED}. - * - * Ownership is transfer-only. A token holds an immutable byte amount, an owner - * label from a closed enum, and a one-way state: `held` then `released`. The - * ledger moves a token from one representation to the next with - * {@link DuplexAggregateByteLedger.transfer}, which changes only the owner label. - * It never decrements and re-reserves, so no admission gap opens between two - * representations of the same retained bytes. - * - * The ledger holds every live token in one owner registry. Registry removal and - * the release of a token run in one synchronous critical section, before any - * `await` or external callback. A second release of the same token is an - * accounting defect. The ledger records the fixed underflow counter and does not - * clamp the gauge down a second time, so a real defect stays visible. - * - * The ledger carries no payload data and no arbitrary labels. It exposes only the - * byte gauge, the live-token count, and the closed owner enum. - */ - -import { - DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL, - DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL, - DUPLEX_GAUGE_AGGREGATE_BYTES_IN_USE, -} from "./duplex-observability.js"; - -/** - * The public rejection marker. A retention site that cannot reserve its bytes - * reports this fixed string. It carries no payload, no route, and no raw value. - */ -export const DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED = "DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED"; - -/** - * The default aggregate ceiling, in bytes: 256 MiB. This is one eighth of the - * documented minimum host process memory allocation for the sandbox duplex - * transport ({@link MIN_HOST_PROCESS_MEMORY_BYTES_FOR_DUPLEX}, 2 GiB). The - * remaining seven eighths cover the Node runtime, ordinary server work, parser - * and transient headroom, and allocator or garbage-collector variance. - */ -export const DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES = 256 * 1024 * 1024; - -/** - * The documented minimum host process memory allocation to enable the sandbox - * duplex transport, in bytes: 2 GiB. An operator override of the aggregate - * ceiling must not pass this value minus the documented reserve. - */ -export const MIN_HOST_PROCESS_MEMORY_BYTES_FOR_DUPLEX = 2 * 1024 * 1024 * 1024; - -/** - * The closed set of owner labels a reservation token can carry. Each label names - * one host-side representation that retains bytes. The ledger accepts no label - * outside this set, so no arbitrary label ever reaches a token. - * - * - `pre_bind_event`: a held pre-bind duplex event, before route bind. - * - `buffered_chunk`: a buffered data chunk retained for a late listener. - * - `terminal_buffered`: a buffered chunk moved to the terminal registry at - * route terminalization, held for a late listener drain. - * - `request_frame`: a bounded raw request frame the broker retains at dispatch. - * - `request_payload`: a normalized request payload the broker retains at dispatch. - * - `response_body`: a response-body chunk or replacement buffer the reader retains. - * - `seen_request_id`: one entry of the broker no-replay request-id set. - * - `decoder_buffer`: the raw partial-frame bytes the host frame decoder retains - * between chunks, plus the peak replacement buffer it allocates on concat. - * - `readiness_buffer`: the raw untrusted bytes the readiness gate retains before - * the READY frame completes, before the broker binds and the decoder takes over. - * - `readiness_replay`: the post-READY suffix bytes the readiness gate holds in the - * pending replay buffer, from the READY accept until the broker binds and the - * frame decoder charges its own retention. This is a separate retention from - * `readiness_buffer`: the gate drops the whole pre-READY buffer on READY, then - * charges only the retained suffix and each later pre-bind chunk under this owner. - * - `http2_preface_scan`: the raw untrusted bytes the `http2_v1` preface scan - * retains while it searches for the client connection preface, from the - * readiness-replay handoff until the preface match or the scan cap. This - * is a separate retention from `readiness_replay`: it starts only after - * the readiness gate hands its own retained suffix to the preface scan. - * - `http2_preface_replay`: the bytes the `http2_v1` preface scan holds after the - * client connection preface, from the preface match until the HTTP/2 server - * binds its downstream listener. This is a separate retention from - * `http2_preface_scan`: the scan drops its own buffer on the preface match, - * then charges only the retained suffix and each later pre-bind chunk under - * this owner. - * - `pending_write`: the raw host-to-worker write payload a pending duplex write - * RPC retains, from the enqueue seam until the RPC settles. - * - `stdin_write`: the serialized host-to-worker frame the child-stdin transport - * buffer retains, from the write until the stream flushes the chunk, the stream - * errors, the stream closes, or the worker exits. This is a separate retention - * from `pending_write`: the RPC holds the raw payload while the transport buffer - * holds the larger serialized frame, so the two tokens cover the peak of both. - */ -export const DUPLEX_AGGREGATE_TOKEN_OWNERS = [ - "pre_bind_event", - "buffered_chunk", - "terminal_buffered", - "request_frame", - "request_payload", - "response_body", - "seen_request_id", - "decoder_buffer", - "readiness_buffer", - "readiness_replay", - "http2_preface_scan", - "http2_preface_replay", - "pending_write", - "stdin_write", -] as const; - -/** One owner label from the closed {@link DUPLEX_AGGREGATE_TOKEN_OWNERS} set. */ -export type DuplexAggregateTokenOwner = (typeof DUPLEX_AGGREGATE_TOKEN_OWNERS)[number]; - -/** The one-way lifecycle state of a reservation token. */ -export type DuplexReservationTokenState = "held" | "released"; - -/** - * An opaque reservation token. The byte amount is immutable. The owner label - * moves through the closed enum by transfer only. The state moves one way, from - * `held` to `released`. A caller reads these fields but never mutates them; only - * the owning ledger changes the owner and the state. - */ -export interface ReservationToken { - /** The immutable exact byte amount this token reserves. */ - readonly bytes: number; - /** The current owner label. The ledger changes it only through `transfer`. */ - readonly owner: DuplexAggregateTokenOwner; - /** The one-way lifecycle state. The ledger sets it to `released` one time. */ - readonly state: DuplexReservationTokenState; -} - -/** - * The internal mutable token. The ledger owns it and returns it typed as the - * read-only {@link ReservationToken}. Only code in this module changes the owner - * or the state. - */ -class ReservationTokenImpl implements ReservationToken { - readonly bytes: number; - owner: DuplexAggregateTokenOwner; - state: DuplexReservationTokenState = "held"; - - constructor(bytes: number, owner: DuplexAggregateTokenOwner) { - this.bytes = bytes; - this.owner = owner; - } -} - -/** - * The fixed telemetry surface for the aggregate byte ledger. The host binds it to - * the closed duplex telemetry contract. Each record uses only fixed constant - * dimensions. No company, agent, run, route, URL, header, request id, token, byte - * payload, or raw provider value ever reaches a record. The default is a no-op. - */ -export interface DuplexAggregateByteLedgerTelemetry { - /** Set the aggregate-bytes-in-use gauge to the current value. */ - setBytesInUse(bytes: number): void; - /** Increment the reservation-rejection counter one time. */ - recordReservationRejection(): void; - /** Increment the accounting-underflow counter one time. */ - recordAccountingUnderflow(): void; -} - -/** A no-op telemetry surface. Every method does nothing, so the ledger stays inert. */ -export const NOOP_DUPLEX_AGGREGATE_BYTE_LEDGER_TELEMETRY: DuplexAggregateByteLedgerTelemetry = { - setBytesInUse() {}, - recordReservationRejection() {}, - recordAccountingUnderflow() {}, -}; - -/** - * The low-level process metric sink the host binds. It carries the fixed metric - * name and the value only, so no dynamic dimension reaches a sink. The host maps - * the gauge and the two counters to the process metric pipeline and to a warn log - * for a defect. - */ -export interface DuplexAggregateByteLedgerMetricSink { - /** Set a named process gauge to a value. */ - setGauge(name: string, value: number): void; - /** Increment a named process counter one time. */ - incrementCounter(name: string): void; -} - -/** - * Build a {@link DuplexAggregateByteLedgerTelemetry} that maps each ledger event - * to the fixed metric name on the process metric sink. The gauge uses - * {@link DUPLEX_GAUGE_AGGREGATE_BYTES_IN_USE}. The rejection counter uses - * {@link DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL}. The defect - * counter uses {@link DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL}. - * Each call sits in an error swallow, so a sink failure never breaks the reserve - * or the release path. - */ -export function createDuplexAggregateByteLedgerTelemetry( - sink: DuplexAggregateByteLedgerMetricSink, -): DuplexAggregateByteLedgerTelemetry { - return { - setBytesInUse(bytes: number): void { - try { - sink.setGauge(DUPLEX_GAUGE_AGGREGATE_BYTES_IN_USE, bytes); - } catch { - // A telemetry failure never breaks the reserve or the release path. - } - }, - recordReservationRejection(): void { - try { - sink.incrementCounter(DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL); - } catch { - // A telemetry failure never breaks the reserve path. - } - }, - recordAccountingUnderflow(): void { - try { - sink.incrementCounter(DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL); - } catch { - // A telemetry failure never breaks the release path. - } - }, - }; -} - -/** The options for {@link DuplexAggregateByteLedger}. */ -export interface DuplexAggregateByteLedgerOptions { - /** The aggregate ceiling, in bytes. It must be a positive safe integer. */ - ceilingBytes: number; - /** The fixed telemetry surface. The default is the no-op surface. */ - telemetry?: DuplexAggregateByteLedgerTelemetry; -} - -/** - * Assert an aggregate ceiling byte value. The value must be a positive safe - * integer that is no larger than the documented process allocation minus the - * documented reserve. The function throws on any other value, so an invalid - * ceiling fails startup instead of silently disabling the ledger. - * - * The reserve equals the documented process allocation minus the default - * ceiling ({@link DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES}), so the maximum - * accepted ceiling is exactly the documented process allocation minus that - * reserve. - */ -export function assertDuplexAggregateCeilingBytes(value: number): void { - const maxCeiling = maxDuplexAggregateCeilingBytes(); - if (!Number.isSafeInteger(value) || value < 1) { - throw new Error( - `The duplex aggregate byte ceiling must be a positive safe integer; got ${String(value)}.`, - ); - } - if (value > maxCeiling) { - throw new Error( - `The duplex aggregate byte ceiling ${value} must not pass the documented process allocation minus the reserve (${maxCeiling}).`, - ); - } -} - -/** - * The maximum accepted aggregate ceiling, in bytes: the documented process - * allocation minus the documented reserve. Both {@link assertDuplexAggregateCeilingBytes} - * and {@link isValidDuplexAggregateCeilingBytes} read this one bound. - */ -function maxDuplexAggregateCeilingBytes(): number { - return MIN_HOST_PROCESS_MEMORY_BYTES_FOR_DUPLEX - DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES; -} - -/** - * Report whether `value` is a valid aggregate ceiling. A valid ceiling is a - * positive safe integer that is no larger than {@link maxDuplexAggregateCeilingBytes}. - * This is the non-throwing form of {@link assertDuplexAggregateCeilingBytes}. The - * override resolver uses it to reject an invalid override without a throw. - */ -export function isValidDuplexAggregateCeilingBytes(value: number): boolean { - return Number.isSafeInteger(value) && value >= 1 && value <= maxDuplexAggregateCeilingBytes(); -} - -/** - * A reporter the host binds to receive one invalid-override rejection signal. - * {@link resolveDuplexAggregateCeilingBytes} calls it one time when it rejects a - * present invalid override and falls back to the safe default. The signal carries - * only the rejected numeric value. It carries no route, company, run, or payload - * value. - */ -export type DuplexAggregateCeilingOverrideRejectionReporter = (rejectedValue: number) => void; - -/** - * Resolve an aggregate ceiling from an optional operator override. A missing or - * `null` override uses {@link DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES}. A present - * valid value passes through unchanged. - * - * A present invalid override does not fail host startup. The host process is - * multi-tenant, so one invalid environment value must not brick the whole host. - * The resolver rejects the invalid override, reports it through the optional - * `onRejectedOverride` reporter, and returns the conservative safe default. This - * fails loud without failing closed on availability. - */ -export function resolveDuplexAggregateCeilingBytes( - override?: number | null, - onRejectedOverride?: DuplexAggregateCeilingOverrideRejectionReporter, -): number { - if (override === undefined || override === null) { - return DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES; - } - if (!isValidDuplexAggregateCeilingBytes(override)) { - onRejectedOverride?.(override); - return DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES; - } - return override; -} - -/** - * The process-owned aggregate byte ledger. Create one per host server process. - * Inject the same object into every host-side retention site, so one shared gauge - * bounds the aggregate retained bytes across all live duplex routes. - */ -export class DuplexAggregateByteLedger { - private readonly ceiling: number; - private readonly telemetry: DuplexAggregateByteLedgerTelemetry; - private used = 0; - /** The owner registry. It holds every live token. */ - private readonly liveTokens = new Set(); - - constructor(options: DuplexAggregateByteLedgerOptions) { - assertDuplexAggregateCeilingBytes(options.ceilingBytes); - this.ceiling = options.ceilingBytes; - this.telemetry = options.telemetry ?? NOOP_DUPLEX_AGGREGATE_BYTE_LEDGER_TELEMETRY; - } - - /** The configured aggregate ceiling, in bytes. */ - get ceilingBytes(): number { - return this.ceiling; - } - - /** The current aggregate bytes in use across every live token. */ - get bytesInUse(): number { - return this.used; - } - - /** The current number of live tokens in the owner registry. */ - get liveTokenCount(): number { - return this.liveTokens.size; - } - - /** - * Reserve `bytes` for `owner`. Check and increment the gauge synchronously - * before the caller allocates. Return a held token when the reservation fits - * under the ceiling. Return `null` when `used + bytes` would pass the ceiling; - * the caller retains nothing and reports - * {@link DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED}. - * - * The byte amount must be a non-negative safe integer. A caller that passes any - * other value made a programming error, so the ledger throws and fails loud. - */ - reserve(owner: DuplexAggregateTokenOwner, bytes: number): ReservationToken | null { - if (!Number.isSafeInteger(bytes) || bytes < 0) { - throw new Error( - `A duplex byte reservation must be a non-negative safe integer; got ${String(bytes)}.`, - ); - } - if (this.used + bytes > this.ceiling) { - this.telemetry.recordReservationRejection(); - return null; - } - this.used += bytes; - const token = new ReservationTokenImpl(bytes, owner); - this.liveTokens.add(token); - this.telemetry.setBytesInUse(this.used); - return token; - } - - /** - * Transfer a held token to a new owner label. Change only the owner. Never - * decrement and never re-reserve. This is the atomic pre-bind handoff: it moves - * the same reserved bytes from one representation to the next with no admission - * gap. - * - * A transfer of a token the ledger does not hold, or of an already-released - * token, is an accounting defect. The ledger records the fixed underflow - * counter and leaves the gauge unchanged. - */ - transfer(token: ReservationToken, owner: DuplexAggregateTokenOwner): void { - const impl = token as ReservationTokenImpl; - if (impl.state !== "held" || !this.liveTokens.has(impl)) { - this.telemetry.recordAccountingUnderflow(); - return; - } - impl.owner = owner; - } - - /** - * Release a held token. Remove it from the owner registry, mark it `released`, - * and decrement the gauge, all in one synchronous critical section before any - * `await` or external callback. A token releases one time. - * - * A second release of the same token, or a release of a token the ledger does - * not hold, is an accounting defect. The ledger records the fixed underflow - * counter and does not clamp the gauge a second time, so a real defect stays - * visible on the counter. - */ - release(token: ReservationToken): void { - const impl = token as ReservationTokenImpl; - if (impl.state === "released" || !this.liveTokens.delete(impl)) { - this.telemetry.recordAccountingUnderflow(); - return; - } - impl.state = "released"; - this.used -= impl.bytes; - this.telemetry.setBytesInUse(this.used); - } -} diff --git a/packages/adapter-utils/src/duplex-frame-codec.ts b/packages/adapter-utils/src/duplex-frame-codec.ts index b2e848c737..d5f44459cc 100644 --- a/packages/adapter-utils/src/duplex-frame-codec.ts +++ b/packages/adapter-utils/src/duplex-frame-codec.ts @@ -22,9 +22,11 @@ export const DUPLEX_FRAME_VERSION = 2; /** * The default maximum size of one frame, in bytes. The decoder rejects a longer * frame with a `frame_too_large` protocol error. The value matches the per-chunk - * character bound of the host duplex route. + * byte bound of the host duplex route, and the host body limit for one HTTP/2 + * bridge stream ({@link DEFAULT_SANDBOX_CALLBACK_BRIDGE_MAX_BODY_BYTES} in + * `sandbox-callback-bridge.ts`). */ -export const DEFAULT_MAX_DUPLEX_FRAME_BYTES = 1_000_000; +export const DEFAULT_MAX_DUPLEX_FRAME_BYTES = 262_144; /** * The READY control frame. The gateway sends it one time after it binds the @@ -74,8 +76,7 @@ export type DuplexProtocolErrorCode = | "unknown_type" | "version_mismatch" | "frame_too_large" - | "id_too_large" - | "aggregate_bytes_exceeded"; + | "id_too_large"; /** A decode-time protocol error. The read path returns it; it never throws. */ export interface DuplexProtocolError { diff --git a/packages/adapter-utils/src/duplex-frame-vectors.json b/packages/adapter-utils/src/duplex-frame-vectors.json index 99a6762b7a..fb15ba5091 100644 --- a/packages/adapter-utils/src/duplex-frame-vectors.json +++ b/packages/adapter-utils/src/duplex-frame-vectors.json @@ -1,6 +1,6 @@ { "frameVersion": 2, - "defaultMaxFrameBytes": 1000000, + "defaultMaxFrameBytes": 262144, "description": "Shared wire-compatibility vectors for the duplex frame codec. Every codec copy decodes the same bytes. bytes is a UTF-8 byte stream; expected lists the decode result (frame or protocol-error code).", "vectors": [ { diff --git a/packages/adapter-utils/src/duplex-observability.ts b/packages/adapter-utils/src/duplex-observability.ts index 1d144fa60f..db798bbe14 100644 --- a/packages/adapter-utils/src/duplex-observability.ts +++ b/packages/adapter-utils/src/duplex-observability.ts @@ -35,40 +35,6 @@ export const DUPLEX_COUNTER_LOSS_TOTAL = "sandbox_duplex_loss_total"; /** The guarded counter for one leaked provider session on teardown. */ export const DUPLEX_COUNTER_SESSION_LEAK_TOTAL = "sandbox_duplex_session_leak_total"; -/** - * The process-scoped gauge for the aggregate retained bytes across every live - * duplex route. The host aggregate byte ledger sets it on each reserve and each - * release. The record carries no dynamic dimension. - */ -export const DUPLEX_GAUGE_AGGREGATE_BYTES_IN_USE = "sandbox_duplex_aggregate_bytes_in_use"; -/** - * The counter for one rejected aggregate byte reservation. The host ledger - * increments it when a reservation would pass the aggregate ceiling. The record - * carries no dynamic dimension. - */ -export const DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL = - "sandbox_duplex_aggregate_byte_reservation_rejections_total"; -/** - * The counter for one aggregate byte accounting defect. The host ledger - * increments it on a double release or a transfer of a token it does not hold. - * The record carries no dynamic dimension. - */ -export const DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL = - "sandbox_duplex_aggregate_byte_accounting_underflow_total"; - -/** - * The closed set of aggregate byte ledger metric names. A test pins this exact - * set, so a new ledger metric name needs an explicit review. Each record uses - * only closed constant dimensions and no dynamic label. The Observability - * contract documents these metrics under "Aggregate byte ledger metrics" in - * `doc/observability.md`. - */ -export const DUPLEX_AGGREGATE_BYTE_LEDGER_METRIC_NAMES = [ - DUPLEX_GAUGE_AGGREGATE_BYTES_IN_USE, - DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL, - DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL, -] as const; - /** * The closed dimension-key set. Every span attribute, counter label, and event * field uses only these keys. A test asserts the exact set, so a new key never @@ -102,13 +68,10 @@ export type DuplexOutcomeValue = "ok" | "error"; * stage: the process-scoped route ceiling was full (`route_busy`), the entrypoint * sync failed (`entrypoint_sync_failed`), the broker construction failed * (`broker_construction_failed`), or the channel open failed (`channel_open_failed`). - * The `aggregate_bytes_exceeded` reason names a readiness handshake, or an - * `http2` post-preface pre-bind buffer, the host fell back because the - * process aggregate byte ceiling had no room. The `preface_missing` reason - * names a missing or an invalid HTTP/2 client connection preface inside the - * bounded readiness buffer: the host found no valid preface after the - * accepted READY line, aborted the HTTP/2 open, and moved the run to - * `queue_v1` one time. + * The `preface_missing` reason names a missing or an invalid HTTP/2 client + * connection preface inside the bounded readiness buffer: the host found no + * valid preface after the accepted READY line, aborted the HTTP/2 open, and + * moved the run to `queue_v1` one time. */ export type DuplexFallbackReason = | "gate_off" @@ -121,7 +84,6 @@ export type DuplexFallbackReason = | "ready_nonce_mismatch" | "ready_timeout" | "contaminated" - | "aggregate_bytes_exceeded" | "preface_missing"; /** The class of a terminal loss, relative to the first request dispatch. */ diff --git a/packages/adapter-utils/src/execution-target-sandbox.test.ts b/packages/adapter-utils/src/execution-target-sandbox.test.ts index 3a26fbc439..54dea1c5aa 100644 --- a/packages/adapter-utils/src/execution-target-sandbox.test.ts +++ b/packages/adapter-utils/src/execution-target-sandbox.test.ts @@ -42,10 +42,6 @@ import { type StartupTraceContext, type StartupTracer, } from "./acpx-engine/startup-timing.js"; -import { - DuplexAggregateByteLedger, - DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, -} from "./duplex-aggregate-byte-ledger.js"; import { createSandboxRunLogTailFactory, type SandboxRunLogTailFactory } from "./sandbox-run-log-stream.js"; import { runChildProcess } from "./server-utils.js"; import { shellQuote } from "./ssh.js"; @@ -58,14 +54,10 @@ import { } from "./duplex-frame-codec.js"; import { DUPLEX_CHANNEL_LOST_ERROR_CODE } from "./bridge-transport-contract.js"; import { - DUPLEX_AGGREGATE_BYTE_LEDGER_METRIC_NAMES, - DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL, - DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL, DUPLEX_COUNTER_CHANNEL_OPEN_TOTAL, DUPLEX_COUNTER_FALLBACK_TOTAL, DUPLEX_COUNTER_LOSS_TOTAL, DUPLEX_DIMENSION_KEYS, - DUPLEX_GAUGE_AGGREGATE_BYTES_IN_USE, DUPLEX_SPAN_CHANNEL_OPEN, DUPLEX_SPAN_REQUEST, DUPLEX_TRANSPORT_EVENT, @@ -2675,155 +2667,6 @@ describe("sandbox adapter execution targets", () => { } }); - it("charges the response-body bytes against the host aggregate ledger and releases every token on success", async () => { - // The host stamps one process-owned aggregate byte ledger on the sandbox - // target. The forward response-body reader charges its retained bytes against - // that ledger. A successful read charges the chunk bytes and the - // concatenation buffer, then releases every token, so the ledger returns to - // zero after the forward completes. - const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-execution-target-bridge-ledger-ok-")); - cleanupDirs.push(rootDir); - const remoteCwd = path.join(rootDir, "workspace"); - const runtimeRootDir = path.join(remoteCwd, ".paperclip-runtime", "codex"); - await mkdir(runtimeRootDir, { recursive: true }); - - const responseBody = JSON.stringify({ id: "issue-1" }); - const apiServer = createServer((_req, res) => { - res.writeHead(200, { - "content-type": "application/json", - "content-length": String(Buffer.byteLength(responseBody, "utf8")), - }); - res.end(responseBody); - }); - await new Promise((resolve, reject) => { - apiServer.once("error", reject); - apiServer.listen(0, "127.0.0.1", () => resolve()); - }); - const address = apiServer.address(); - if (!address || typeof address === "string") { - throw new Error("Expected the bridge test API server to listen on a TCP port."); - } - - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1024 * 1024 }); - const target: AdapterSandboxExecutionTarget = { - kind: "remote", - transport: "sandbox", - providerKey: "e2b", - environmentId: "env-1", - leaseId: "lease-1", - remoteCwd, - runner: createLocalSandboxRunner(), - timeoutMs: 30_000, - duplexAggregateByteLedger: ledger, - }; - - const bridge = await startAdapterExecutionTargetPaperclipBridge({ - runId: "run-bridge-ledger-ok", - target, - runtimeRootDir, - adapterKey: "codex", - hostApiToken: "real-run-jwt", - hostApiUrl: `http://127.0.0.1:${address.port}`, - maxBodyBytes: 512, - }); - try { - const response = await fetch(`${bridge!.env.PAPERCLIP_API_URL}/api/issues/issue-1`, { - method: "GET", - headers: { - authorization: `Bearer ${bridge!.env.PAPERCLIP_API_KEY}`, - }, - }); - - expect(response.status).toBe(200); - await expect(response.json()).resolves.toEqual({ id: "issue-1" }); - // The reader released every token, so the aggregate gauge and the live-token - // registry both return to zero. - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - } finally { - await bridge?.stop(); - await new Promise((resolve) => apiServer.close(() => resolve())); - } - }); - - it("fails a response-body read closed when the host aggregate ledger has no room and retains no bytes", async () => { - // The aggregate ledger sits at a tiny ceiling, so a response body larger than - // the ceiling cannot reserve its bytes. The reader fails closed: it cancels - // the stream reader, retains nothing, and reports the fixed marker. The safe - // GET maps the marker to a retryable 502. The ledger returns to zero, because - // the reader released the tokens it held before the rejection. - const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-execution-target-bridge-ledger-full-")); - cleanupDirs.push(rootDir); - const remoteCwd = path.join(rootDir, "workspace"); - const runtimeRootDir = path.join(remoteCwd, ".paperclip-runtime", "codex"); - await mkdir(runtimeRootDir, { recursive: true }); - - // The body sits under the per-request size limit but over the aggregate - // ceiling, so the aggregate ledger, not the per-request limit, rejects it. - const responseBody = "x".repeat(256); - const apiServer = createServer((_req, res) => { - res.writeHead(200, { - "content-type": "application/json", - "content-length": String(Buffer.byteLength(responseBody, "utf8")), - }); - res.end(responseBody); - }); - await new Promise((resolve, reject) => { - apiServer.once("error", reject); - apiServer.listen(0, "127.0.0.1", () => resolve()); - }); - const address = apiServer.address(); - if (!address || typeof address === "string") { - throw new Error("Expected the bridge test API server to listen on a TCP port."); - } - - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 8 }); - const target: AdapterSandboxExecutionTarget = { - kind: "remote", - transport: "sandbox", - providerKey: "e2b", - environmentId: "env-1", - leaseId: "lease-1", - remoteCwd, - runner: createLocalSandboxRunner(), - timeoutMs: 30_000, - duplexAggregateByteLedger: ledger, - }; - - const bridge = await startAdapterExecutionTargetPaperclipBridge({ - runId: "run-bridge-ledger-full", - target, - runtimeRootDir, - adapterKey: "codex", - hostApiToken: "real-run-jwt", - hostApiUrl: `http://127.0.0.1:${address.port}`, - maxBodyBytes: 4096, - }); - try { - const response = await fetch(`${bridge!.env.PAPERCLIP_API_URL}/api/issues/issue-1`, { - method: "GET", - headers: { - authorization: `Bearer ${bridge!.env.PAPERCLIP_API_KEY}`, - }, - }); - - // The safe GET maps the aggregate rejection to a retryable 502 with no - // indeterminate marker. The body carries only the fixed rejection marker. - expect(response.status).toBe(502); - expect(response.headers.get("x-paperclip-bridge-outcome")).toBeNull(); - await expect(response.json()).resolves.toEqual({ - error: DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, - }); - // The reader released the tokens it held before the rejection, so the - // aggregate gauge and the live-token registry both return to zero. - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - } finally { - await bridge?.stop(); - await new Promise((resolve) => apiServer.close(() => resolve())); - } - }); - it("forwards the host indeterminate-outcome header so the sandbox server maps the 504 to a non-retryable 409", async () => { // The host marks a possibly-committed mutation with a 504 and the // `x-paperclip-bridge-outcome: indeterminate` header. The forward must keep @@ -3460,66 +3303,6 @@ describe("sandbox adapter execution targets", () => { expect(control.stopCount).toBeGreaterThanOrEqual(1); }, 20000); - it("falls back to the file bridge when a post-READY pre-bind flood exceeds the aggregate ceiling", async () => { - // The gateway sends a valid READY, then floods the channel before the broker - // binds. The pre-READY buffer cap does not bound the post-READY replay buffer, - // so the replay reservation must. The ceiling admits the small READY frame but - // rejects the flood. The host drops the buffer, stops the channel, and selects - // the file bridge with the aggregate marker. No request forwards, and the - // aggregate ledger returns to zero. - const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-duplex-replay-flood-")); - cleanupDirs.push(rootDir); - const remoteCwd = path.join(rootDir, "workspace"); - await mkdir(remoteCwd, { recursive: true }); - const api = await startRecordingApiServer(); - // The flood is larger than the ceiling; the READY frame is far smaller. - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 4096 }); - const { runner, control } = makeDuplexSelectionRunner((ctx) => { - ctx.emitFrame({ version: 2, type: "ready", nonce: ctx.nonce }); - ctx.emitRaw("x".repeat(64 * 1024)); - }); - const { recorder, counters } = createRecordingDuplexRecorder(); - const target: AdapterSandboxExecutionTarget = { - kind: "remote", - transport: "sandbox", - providerKey: "daytona", - remoteCwd, - timeoutMs: 30_000, - runner, - effectiveCapabilities: duplexCapabilities(true), - duplexAggregateByteLedger: ledger, - }; - - const bridge = await startAdapterExecutionTargetPaperclipBridge({ - runId: "run-duplex-replay-flood", - target, - runtimeRootDir: path.join(remoteCwd, ".paperclip-runtime", "codex"), - adapterKey: "codex", - hostApiToken: "real-run-jwt", - hostApiUrl: api.origin, - enableSandboxDuplexBridge: true, - duplexObservabilityRecorder: recorder, - }); - try { - // The file bridge serves, not the duplex transport. - expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("queue_v1"); - // The fallback names the aggregate marker on the file transport. - const fallback = counters.find((c) => c.metric === DUPLEX_COUNTER_FALLBACK_TOTAL); - expect(fallback?.dimensions.fallback_reason).toBe("aggregate_bytes_exceeded"); - expect(fallback?.dimensions.transport).toBe("file"); - // The gate stopped the flooded channel. - expect(control.stopCount).toBeGreaterThanOrEqual(1); - // No request forwarded, because the broker never bound. - expect(api.requests).toHaveLength(0); - // The aggregate ledger returns to zero with no live token. - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - } finally { - await bridge?.stop(); - await api.close(); - } - }, 20000); - it("streams run logs on the http2 path under the same gate and log line as the file path", async () => { const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-http2-runlog-")); cleanupDirs.push(rootDir); @@ -3990,24 +3773,6 @@ describe("sandbox adapter execution targets", () => { ]); }); - it("pins the exact aggregate byte ledger metric names", () => { - // The aggregate byte ledger metric names are closed. This test locks the - // exact set, so a new gauge or counter name needs an explicit change here. - // Each record carries only closed constant dimensions and no dynamic label. - expect([...DUPLEX_AGGREGATE_BYTE_LEDGER_METRIC_NAMES]).toEqual([ - "sandbox_duplex_aggregate_bytes_in_use", - "sandbox_duplex_aggregate_byte_reservation_rejections_total", - "sandbox_duplex_aggregate_byte_accounting_underflow_total", - ]); - expect(DUPLEX_GAUGE_AGGREGATE_BYTES_IN_USE).toBe("sandbox_duplex_aggregate_bytes_in_use"); - expect(DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL).toBe( - "sandbox_duplex_aggregate_byte_reservation_rejections_total", - ); - expect(DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL).toBe( - "sandbox_duplex_aggregate_byte_accounting_underflow_total", - ); - }); - it("records an http2 request span with latency and the fixed dimension keys", async () => { const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-http2-obs-")); cleanupDirs.push(rootDir); @@ -4123,7 +3888,6 @@ describe("sandbox adapter execution targets", () => { "ready_nonce_mismatch", "ready_timeout", "contaminated", - "aggregate_bytes_exceeded", ]; expect(approvedReasons).toContain(fallback?.dimensions.fallback_reason); expect(fallback?.dimensions).toMatchObject({ transport: "file", outcome: "error" }); @@ -4610,129 +4374,6 @@ describe("sandbox adapter execution targets", () => { } }, 20000); - it("fails the readiness handshake closed when the host aggregate ledger has no room for the pre-READY buffer", async () => { - const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-duplex-ready-ledger-full-")); - cleanupDirs.push(rootDir); - const remoteCwd = path.join(rootDir, "workspace"); - await mkdir(remoteCwd, { recursive: true }); - const api = await startRecordingApiServer(); - // The host stamps one process-owned aggregate byte ledger on the sandbox - // target at a tiny ceiling. The fake gateway sends a pre-READY blob larger - // than the ceiling, so the gate cannot reserve the blob bytes. The gate fails - // closed: it retains nothing, records the aggregate fallback reason, and falls - // back to the file bridge. The blob is smaller than the readiness buffer cap, - // so the aggregate ledger, not the buffer cap, drives the failure. - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 256 }); - const preReadyBlob = "x".repeat(4_096); - const { runner, control } = makeDuplexSelectionRunner((ctx) => { - ctx.emitRaw(preReadyBlob); - }); - const { recorder, counters } = createRecordingDuplexRecorder(); - const target: AdapterSandboxExecutionTarget = { - kind: "remote", - transport: "sandbox", - providerKey: "daytona", - remoteCwd, - timeoutMs: 30_000, - runner, - effectiveCapabilities: duplexCapabilities(true), - duplexAggregateByteLedger: ledger, - }; - - const bridge = await startAdapterExecutionTargetPaperclipBridge({ - runId: "run-ready-ledger-full", - target, - runtimeRootDir: path.join(remoteCwd, ".paperclip-runtime", "codex"), - adapterKey: "codex", - hostApiToken: "real-run-jwt", - hostApiUrl: api.origin, - enableSandboxDuplexBridge: true, - // A long readiness timeout, so the aggregate ledger, not the timeout, drives - // the failure. - duplexReadinessTimeoutMs: 5_000, - duplexObservabilityRecorder: recorder, - }); - try { - expect(bridge).not.toBeNull(); - expect(control.openCount).toBe(1); - // The aggregate rejection drove the failure, so the file bridge serves. - expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("queue_v1"); - const fallback = counters.find((c) => c.metric === DUPLEX_COUNTER_FALLBACK_TOTAL); - expect(fallback?.dimensions.fallback_reason).toBe("aggregate_bytes_exceeded"); - // The gate retained nothing after the rejection, so the aggregate gauge and - // the live-token registry both return to zero. - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - expect(control.closeCount + control.stopCount).toBeGreaterThanOrEqual(1); - } finally { - await bridge?.stop(); - await api.close(); - } - }, 20000); - - it("charges the pre-READY buffer against the injected host ledger and releases it when readiness passes", async () => { - const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-http2-ready-ledger-ok-")); - cleanupDirs.push(rootDir); - const remoteCwd = path.join(rootDir, "workspace"); - await mkdir(remoteCwd, { recursive: true }); - const api = await startRecordingApiServer(); - // The host stamps one process-owned aggregate byte ledger on the sandbox - // target at a generous ceiling. The fake gateway sends one pre-READY noise - // line, then the valid READY frame, then a real HTTP/2 client preface. The - // gate charges the noise bytes against the injected ledger, passes - // readiness, and releases the pre-READY tokens. The ledger returns to zero - // once the post-READY replay hands off to the bound HTTP/2 channel. - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1024 * 1024 }); - const reserveSpy = vi.spyOn(ledger, "reserve"); - const { runner, control } = makeHttp2SelectionRunner((ctx) => { - ctx.emitRaw("pty-echo-noise"); - ctx.emitReady(); - ctx.connectHttp2(); - }); - const target: AdapterSandboxExecutionTarget = { - kind: "remote", - transport: "sandbox", - providerKey: "daytona", - remoteCwd, - timeoutMs: 30_000, - runner, - effectiveCapabilities: duplexCapabilities(true), - duplexAggregateByteLedger: ledger, - }; - - const bridge = await startAdapterExecutionTargetPaperclipBridge({ - runId: "run-ready-ledger-ok", - target, - runtimeRootDir: path.join(remoteCwd, ".paperclip-runtime", "codex"), - adapterKey: "codex", - hostApiToken: "real-run-jwt", - hostApiUrl: api.origin, - enableSandboxDuplexBridge: true, - duplexReadinessTimeoutMs: 5_000, - }); - try { - expect(bridge).not.toBeNull(); - // Readiness passed, so the http2 transport serves. - expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("http2_v1"); - // The gate charged the pre-READY noise against the exact injected ledger, so - // the identity holds at this seam. - expect(reserveSpy).toHaveBeenCalledWith("readiness_buffer", expect.any(Number)); - // The gate released every readiness-buffer token on settle, so the aggregate - // gauge and the live-token registry both return to zero. - await waitForCondition( - () => ledger.bytesInUse === 0 && ledger.liveTokenCount === 0, - "the readiness gate to release every pre-READY token", - 4000, - ); - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - } finally { - reserveSpy.mockRestore(); - await bridge?.stop(); - await api.close(); - } - }, 20000); - it("bounds the pre-READY newline-scan work by the bytes received", async () => { const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-duplex-scan-")); cleanupDirs.push(rootDir); @@ -5689,6 +5330,87 @@ describe("sandbox adapter execution targets", () => { } }, 20000); + it("aborts the host forward and its response-body read when the sandbox stream closes", async () => { + // The mock host answers with headers at once, then holds the response + // body open with no further chunk and no end. The read stays pending + // until the outbound fetch itself aborts. If the abort never reaches + // this connection, `hostConnectionClosed` never resolves and the test + // times out instead of failing fast — the assertion is: it does resolve, + // quickly, once the sandbox stream closes. + let sawRequest = false; + let resolveHostConnectionClosed: (() => void) | undefined; + const hostConnectionClosed = new Promise((resolve) => { + resolveHostConnectionClosed = resolve; + }); + const api = createServer((req, res) => { + sawRequest = true; + res.writeHead(200, { "content-type": "application/json" }); + res.on("close", () => resolveHostConnectionClosed!()); + }); + await new Promise((resolve, reject) => { + api.once("error", reject); + api.listen(0, "127.0.0.1", () => resolve()); + }); + const apiAddress = api.address(); + if (!apiAddress || typeof apiAddress === "string") { + throw new Error("Expected the mock host server to listen on a TCP port."); + } + const apiOrigin = `http://127.0.0.1:${apiAddress.port}`; + + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-http2-abort-forward-")); + cleanupDirs.push(rootDir); + const remoteCwd = path.join(rootDir, "workspace"); + await mkdir(remoteCwd, { recursive: true }); + const sessionRef: { current: http2.ClientHttp2Session | null } = { current: null }; + let bridgeToken = ""; + const { runner } = makeHttp2SelectionRunner((ctx) => { + bridgeToken = ctx.bridgeToken; + ctx.emitReady(); + sessionRef.current = ctx.connectHttp2(); + }); + const target: AdapterSandboxExecutionTarget = { + kind: "remote", + transport: "sandbox", + providerKey: "daytona", + remoteCwd, + timeoutMs: 30_000, + runner, + effectiveCapabilities: duplexCapabilities(true), + }; + + const bridge = await startAdapterExecutionTargetPaperclipBridge({ + runId: "run-abort-forward", + target, + runtimeRootDir: path.join(remoteCwd, ".paperclip-runtime", "codex"), + adapterKey: "codex", + hostApiToken: "real-run-jwt", + hostApiUrl: apiOrigin, + enableSandboxDuplexBridge: true, + // Far longer than this test waits, so only the sandbox-side stream + // close — not this ceiling — can end the forward here. + forwardTimeoutMs: 60_000, + }); + try { + expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("http2_v1"); + await waitForCondition(() => sessionRef.current !== null, "the http2 client session to open", 4000); + const clientStream = sessionRef.current!.request({ + ":method": "GET", + ":path": "/api/agents/me", + authorization: `Bearer ${bridgeToken}`, + }); + clientStream.end(); + await waitForCondition(() => sawRequest, "the mock host to receive the forwarded request", 4000); + // The sandbox side closes its stream while the host forward and its + // response-body read are both still in flight. + clientStream.close(http2.constants.NGHTTP2_CANCEL); + await hostConnectionClosed; + } finally { + sessionRef.current?.close(); + await bridge?.stop(); + await new Promise((resolve) => api.close(() => resolve())); + } + }, 20000); + // --------------------------------------------------------------------------- // Real-PTY replay. // @@ -6321,146 +6043,15 @@ describe("duplex readiness gate replay-buffer reservation", () => { return { channel, control }; } - // A ledger that counts the reservation-rejection and the accounting-underflow - // signals, so a test proves the one-owner-one-release invariant holds. - function makeCountingLedger(ceilingBytes: number): { - ledger: DuplexAggregateByteLedger; - counts: { rejections: number; underflows: number }; - } { - const counts = { rejections: 0, underflows: 0 }; - const ledger = new DuplexAggregateByteLedger({ - ceilingBytes, - telemetry: { - setBytesInUse(): void {}, - recordReservationRejection(): void { - counts.rejections += 1; - }, - recordAccountingUnderflow(): void { - counts.underflows += 1; - }, - }, - }); - return { ledger, counts }; - } - function readyLine(): string { return `${JSON.stringify({ version: 2, type: "ready", nonce: READY_NONCE })}\n`; } - it("charges the post-READY suffix and releases it after the broker handoff", async () => { - const { channel, control } = makeFakeReadinessChannel(); - const { ledger, counts } = makeCountingLedger(1024 * 1024); - const gate = __duplexReadinessTesting.createReadinessGate(channel, { - nonce: READY_NONCE, - timeoutMs: 5_000, - ledger, - }); - const suffix = "hello-post-ready-suffix"; - // The READY line and the suffix arrive in one chunk. The gate drops the whole - // pre-READY buffer charge, then charges only the retained suffix. - control.emitData(`${readyLine()}${suffix}`); - const readiness = await gate.ready; - expect(readiness.ok).toBe(true); - expect(gate.replayOverflowed()).toBe(false); - // The gate holds the suffix under one readiness_replay token before the bind. - expect(ledger.bytesInUse).toBe(Buffer.byteLength(suffix, "utf8")); - expect(ledger.liveTokenCount).toBe(1); - // The broker binds and replays the suffix; the gate releases the token after - // the synchronous handoff. - const replayed: string[] = []; - gate.brokerChannel.onData((chunk) => replayed.push(Buffer.from(chunk).toString("utf8"))); - expect(replayed).toEqual([suffix]); - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - expect(counts.underflows).toBe(0); - }); - - it("releases the suffix token on disposePendingReplay without a broker handoff", async () => { - const { channel, control } = makeFakeReadinessChannel(); - const { ledger, counts } = makeCountingLedger(1024 * 1024); - const gate = __duplexReadinessTesting.createReadinessGate(channel, { - nonce: READY_NONCE, - timeoutMs: 5_000, - ledger, - }); - const suffix = "abandoned-suffix"; - control.emitData(`${readyLine()}${suffix}`); - expect((await gate.ready).ok).toBe(true); - expect(ledger.bytesInUse).toBe(Buffer.byteLength(suffix, "utf8")); - // A broker-construction failure abandons the buffer, so the caller disposes it. - gate.disposePendingReplay(); - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - // A second dispose is a no-op and never underflows. - gate.disposePendingReplay(); - expect(ledger.bytesInUse).toBe(0); - expect(counts.underflows).toBe(0); - }); - - it("releases the suffix token after a pre-bind exit then a broker handoff", async () => { - const { channel, control } = makeFakeReadinessChannel(); - const { ledger, counts } = makeCountingLedger(1024 * 1024); - const gate = __duplexReadinessTesting.createReadinessGate(channel, { - nonce: READY_NONCE, - timeoutMs: 5_000, - ledger, - }); - const suffix = "pre-bind-exit-suffix"; - control.emitData(`${readyLine()}${suffix}`); - expect((await gate.ready).ok).toBe(true); - // The channel exits after READY but before the broker binds. The gate holds the - // exit and keeps the pending suffix charged. - control.emitExit({ exitCode: 0 }); - expect(ledger.bytesInUse).toBe(Buffer.byteLength(suffix, "utf8")); - expect(ledger.liveTokenCount).toBe(1); - // The broker binds, replays the suffix and the exit, then the gate releases the - // reservation. - const replayed: string[] = []; - const exits: Array<{ exitCode: number | null }> = []; - gate.brokerChannel.onData((chunk) => replayed.push(Buffer.from(chunk).toString("utf8"))); - gate.brokerChannel.onExit((exit) => exits.push(exit)); - expect(replayed).toEqual([suffix]); - expect(exits).toEqual([{ exitCode: 0 }]); - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - expect(counts.underflows).toBe(0); - }); - - it("fails closed when a post-READY pre-bind chunk floods past the ceiling", async () => { - const { channel, control } = makeFakeReadinessChannel(); - // The ceiling admits the small READY line but not the flood chunk. - const { ledger, counts } = makeCountingLedger(256); - const gate = __duplexReadinessTesting.createReadinessGate(channel, { - nonce: READY_NONCE, - timeoutMs: 5_000, - ledger, - }); - // The READY line arrives alone, so the pending suffix starts empty. - control.emitData(readyLine()); - expect((await gate.ready).ok).toBe(true); - expect(ledger.bytesInUse).toBe(0); - // A post-READY chunk larger than the ceiling floods the replay buffer before - // the broker binds. The gate refuses the reservation and fails closed. - control.emitData("x".repeat(512)); - expect(gate.replayOverflowed()).toBe(true); - expect(control.stopCount).toBe(1); - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - expect(counts.rejections).toBe(1); - expect(counts.underflows).toBe(0); - // Binding the broker replays nothing, because the gate dropped the buffer. - const replayed: string[] = []; - gate.brokerChannel.onData((chunk) => replayed.push(Buffer.from(chunk).toString("utf8"))); - expect(replayed).toEqual([]); - }); - it("drops the retained pre-READY buffer on READY acceptance", async () => { const { channel, control } = makeFakeReadinessChannel(); - const { ledger, counts } = makeCountingLedger(1024 * 1024); const gate = __duplexReadinessTesting.createReadinessGate(channel, { nonce: READY_NONCE, timeoutMs: 5_000, - ledger, }); // A large pre-READY noise line and a large suffix arrive with the READY line // in one chunk. A sandbox controls every byte here. @@ -6469,61 +6060,37 @@ describe("duplex readiness gate replay-buffer reservation", () => { control.emitData(`${noise}${readyLine()}${suffix}`); expect((await gate.ready).ok).toBe(true); // The gate drops the pre-READY buffer, so the process no longer retains the - // noise prefix. Without this, the process holds the full sandbox string while - // the ledger charges only the suffix, so retention passes the ceiling. + // noise prefix. expect(gate.retainedReadinessBufferLength()).toBe(0); - // The ledger charges only the retained suffix, not the dropped prefix. - expect(ledger.bytesInUse).toBe(Buffer.byteLength(suffix, "utf8")); - expect(ledger.liveTokenCount).toBe(1); - // The broker binds, replays the suffix, and the gate releases the token. + // The broker binds and replays the retained suffix. const replayed: string[] = []; gate.brokerChannel.onData((chunk) => replayed.push(Buffer.from(chunk).toString("utf8"))); expect(replayed).toEqual([suffix]); - expect(ledger.bytesInUse).toBe(0); expect(gate.retainedReadinessBufferLength()).toBe(0); - expect(counts.underflows).toBe(0); }); - it("releases the readiness_replay token before the downstream listener runs, so a same-size downstream reservation for the same bytes does not double-book", async () => { + it("caps the post-READY replay buffer without the aggregate ledger", async () => { const { channel, control } = makeFakeReadinessChannel(); - const suffix = "s".repeat(512); - const suffixBytes = Buffer.byteLength(suffix, "utf8"); - // The ceiling admits the READY line plus one reservation of the suffix - // size, but not a second, separate reservation of the suffix on top of - // that. A downstream listener that reserves the same bytes under its own - // owner — the real shape of the HTTP/2 preface scanner — proves the gate - // releases its own `readiness_replay` token first: the downstream - // reservation must still fit. - const { ledger, counts } = makeCountingLedger(Buffer.byteLength(readyLine(), "utf8") + suffixBytes); + const readinessBufferCapBytes = DEFAULT_MAX_DUPLEX_FRAME_BYTES + 4_096; + // No ledger: only the buffer's own direct byte cap can end the channel + // here. This proves the cap holds even when no aggregate ledger is + // present, unlike the ledger-only check this replaces. const gate = __duplexReadinessTesting.createReadinessGate(channel, { nonce: READY_NONCE, timeoutMs: 5_000, - ledger, }); - control.emitData(`${readyLine()}${suffix}`); + // The READY line arrives alone, so the pending suffix starts empty. + control.emitData(readyLine()); expect((await gate.ready).ok).toBe(true); - expect(ledger.bytesInUse).toBe(suffixBytes); - - let peakBytesInUseDuringHandoff = -1; - let downstreamToken: ReturnType = null; - gate.brokerChannel.onData((chunk) => { - // The downstream listener reserves the same bytes under its own owner, - // the same way the preface scanner charges `http2_preface_scan` for the - // replayed chunk. The gate must have released its own token before this - // call runs, so this reservation fits under the tight ceiling. - downstreamToken = ledger.reserve("http2_preface_scan", chunk.byteLength); - peakBytesInUseDuringHandoff = ledger.bytesInUse; - }); - - // The downstream reservation succeeded: the gate's release ran first, so - // only one reservation for these bytes was ever live at once. - expect(downstreamToken).not.toBeNull(); - expect(peakBytesInUseDuringHandoff).toBe(suffixBytes); - expect(counts.rejections).toBe(0); - expect(counts.underflows).toBe(0); - // The gate's own token is gone; only the downstream token remains live. - expect(ledger.liveTokenCount).toBe(1); - expect(ledger.bytesInUse).toBe(suffixBytes); + // A post-READY chunk one byte over the cap floods the replay buffer + // before the broker binds. + control.emitData("x".repeat(readinessBufferCapBytes + 1)); + expect(gate.replayOverflowed()).toBe(true); + expect(control.stopCount).toBe(1); + // Binding the broker replays nothing, because the gate dropped the buffer. + const replayed: string[] = []; + gate.brokerChannel.onData((chunk) => replayed.push(Buffer.from(chunk).toString("utf8"))); + expect(replayed).toEqual([]); }); }); @@ -6562,147 +6129,43 @@ describe("http2 preface scan post-preface replay buffer", () => { return { channel, control }; } - function makeCountingLedger(ceilingBytes: number): { - ledger: DuplexAggregateByteLedger; - counts: { rejections: number }; - } { - const counts = { rejections: 0 }; - const ledger = new DuplexAggregateByteLedger({ - ceilingBytes, - telemetry: { - setBytesInUse(): void {}, - recordReservationRejection(): void { - counts.rejections += 1; - }, - recordAccountingUnderflow(): void {}, - }, - }); - return { ledger, counts }; - } - - it("charges the pre-preface scan buffer while it searches, across many chunks", async () => { - const { channel, control } = makeFakeChannel(); - const { ledger, counts } = makeCountingLedger(1024 * 1024); - const scan = __http2PrefaceScanTesting.scanForHttp2ClientPreface(channel, { - capBytes: 4_096, - timeoutMs: 5_000, - ledger, - }); - // Noise, then the preface, arrive as two separate chunks: the scan - // charges each chunk against the ledger as it grows the buffer, not - // only once the preface is found. - const noise = "not-the-preface-yet"; - control.emitData(Buffer.from(noise)); - expect(ledger.bytesInUse).toBe(Buffer.byteLength(noise, "utf8")); - expect(ledger.liveTokenCount).toBe(1); - control.emitData(PREFACE); - expect(await scan.settled).toBe("found"); - // The preface match releases the whole scan buffer (the noise prefix, - // dropped, plus the preface) and re-charges only the retained preface - // under the replay owner. - expect(ledger.bytesInUse).toBe(PREFACE.byteLength); - expect(ledger.liveTokenCount).toBe(1); - expect(counts.rejections).toBe(0); - }); - - it("fails closed when the aggregate byte ledger refuses the pre-preface scan reservation", async () => { - const { channel, control } = makeFakeChannel(); - // The ceiling admits nothing: even the 24-octet preface itself cannot - // reserve, so the scan fails closed before it ever finds a preface. - const { ledger, counts } = makeCountingLedger(4); - const scan = __http2PrefaceScanTesting.scanForHttp2ClientPreface(channel, { - capBytes: 4_096, - timeoutMs: 5_000, - ledger, - }); - control.emitData(Buffer.concat([PREFACE, Buffer.from("12345678")])); - expect(await scan.settled).toBe("missing"); - expect(scan.replayOverflowed()).toBe(false); - expect(ledger.bytesInUse).toBe(0); - expect(counts.rejections).toBe(1); - }); - it("rejects a single pre-preface chunk over the cap without reserving or concatenating it", async () => { const { channel, control } = makeFakeChannel(); - // The ceiling is far larger than the cap, so a ledger refusal cannot - // explain a rejection here: only the cap check on the chunk's - // prospective length can. - const { ledger, counts } = makeCountingLedger(1024 * 1024); const scan = __http2PrefaceScanTesting.scanForHttp2ClientPreface(channel, { capBytes: 64, timeoutMs: 5_000, - ledger, }); // One chunk, larger than the cap on its own, with no preface inside it. // The scan must reject it on its prospective length before the - // `Buffer.concat` allocation and before the ledger reservation, so - // nothing here ever charges the ledger. + // `Buffer.concat` allocation. control.emitData(Buffer.from("x".repeat(128))); expect(await scan.settled).toBe("missing"); expect(scan.replayOverflowed()).toBe(false); - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - expect(counts.rejections).toBe(0); }); it("rejects a pre-preface chunk that tips an already-buffered scan past the cap", async () => { const { channel, control } = makeFakeChannel(); - const { ledger, counts } = makeCountingLedger(1024 * 1024); const scan = __http2PrefaceScanTesting.scanForHttp2ClientPreface(channel, { capBytes: 64, timeoutMs: 5_000, - ledger, }); - // The first chunk stays under the cap on its own, so the scan buffers - // and charges it while it keeps searching. + // The first chunk stays under the cap on its own, so the scan buffers it + // while it keeps searching. const firstChunk = "n".repeat(40); control.emitData(Buffer.from(firstChunk)); - expect(ledger.bytesInUse).toBe(Buffer.byteLength(firstChunk, "utf8")); // The second chunk, added to the first, passes the cap. The scan must // reject it before the concat that would grow the buffer past the cap, // and it must drop the already-buffered first chunk too. control.emitData(Buffer.from("n".repeat(40))); expect(await scan.settled).toBe("missing"); expect(scan.replayOverflowed()).toBe(false); - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - expect(counts.rejections).toBe(0); - }); - - it("charges the post-preface bytes and releases them after the downstream bind", async () => { - const { channel, control } = makeFakeChannel(); - const { ledger } = makeCountingLedger(1024 * 1024); - const scan = __http2PrefaceScanTesting.scanForHttp2ClientPreface(channel, { - capBytes: 4_096, - timeoutMs: 5_000, - ledger, - }); - const suffix = "one-http2-frame"; - // The preface and the trailing suffix arrive in one chunk. The scan - // delivers every byte from the preface onward, inclusive, so the charged - // and replayed bytes carry the preface itself plus the suffix. - const fromPreface = Buffer.concat([PREFACE, Buffer.from(suffix)]).toString("utf8"); - control.emitData(Buffer.concat([PREFACE, Buffer.from(suffix)])); - expect(await scan.settled).toBe("found"); - expect(scan.replayOverflowed()).toBe(false); - expect(ledger.bytesInUse).toBe(Buffer.byteLength(fromPreface, "utf8")); - expect(ledger.liveTokenCount).toBe(1); - // The HTTP/2 server binds and replays the bytes; the scan releases the - // token after the synchronous handoff. - const replayed: string[] = []; - scan.scanned.onData((chunk) => replayed.push(Buffer.from(chunk).toString("utf8"))); - expect(replayed).toEqual([fromPreface]); - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); }); it("fails closed and stops the channel when the post-preface buffer floods past the cap", async () => { const { channel, control } = makeFakeChannel(); - const { ledger } = makeCountingLedger(1024 * 1024); const scan = __http2PrefaceScanTesting.scanForHttp2ClientPreface(channel, { capBytes: 64, timeoutMs: 5_000, - ledger, }); // The preface arrives alone, so the pending buffer starts empty. control.emitData(PREFACE); @@ -6714,8 +6177,6 @@ describe("http2 preface scan post-preface replay buffer", () => { control.emitData(Buffer.from("x".repeat(128))); expect(scan.replayOverflowed()).toBe(true); expect(control.stopCount).toBe(1); - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); // Binding the downstream listener replays nothing: the scan dropped the // buffer on the overflow. const replayed: string[] = []; @@ -6723,62 +6184,25 @@ describe("http2 preface scan post-preface replay buffer", () => { expect(replayed).toEqual([]); }); - it("fails closed when the aggregate byte ledger refuses the post-preface replay reservation", async () => { + it("settles missing when the readiness timeout elapses with no preface found", async () => { const { channel, control } = makeFakeChannel(); - // The ceiling admits the 24-octet preface itself, but not an 8-byte - // suffix on top of it. - const { ledger, counts } = makeCountingLedger(30); - const scan = __http2PrefaceScanTesting.scanForHttp2ClientPreface(channel, { - capBytes: 4_096, - timeoutMs: 5_000, - ledger, - }); - // The preface arrives alone, so it is the only bytes charged so far. - control.emitData(PREFACE); - expect(await scan.settled).toBe("found"); - expect(scan.replayOverflowed()).toBe(false); - expect(ledger.bytesInUse).toBe(PREFACE.byteLength); - // A post-preface suffix, on top of the retained preface, passes the - // ceiling. The scan drops the buffer, stops the channel, and fails - // closed — the same shape a missing preface fails closed. - control.emitData(Buffer.from("12345678")); - expect(scan.replayOverflowed()).toBe(true); - expect(control.stopCount).toBe(1); - expect(ledger.bytesInUse).toBe(0); - expect(counts.rejections).toBe(1); - }); - - it("releases the scan-buffer tokens when the readiness timeout elapses with no preface found", async () => { - const { channel, control } = makeFakeChannel(); - const { ledger } = makeCountingLedger(1024 * 1024); const scan = __http2PrefaceScanTesting.scanForHttp2ClientPreface(channel, { capBytes: 4_096, // A short bound, so the test does not wait out a production-sized one. timeoutMs: 20, - ledger, }); - // Partial, non-matching data arrives and stays charged while the scan - // keeps searching. No preface ever completes, so nothing else in the - // scan releases this charge — only the timeout path can. + // Partial, non-matching data arrives and the scan keeps searching. No + // preface ever completes, so only the bound timeout settles the scan. const partial = "not-a-preface-and-never-will-be"; control.emitData(Buffer.from(partial)); - expect(ledger.bytesInUse).toBe(Buffer.byteLength(partial, "utf8")); - expect(ledger.liveTokenCount).toBe(1); - // The bound readiness timeout elapses before a preface ever arrives. The - // scan must release its held tokens here — the one terminal path that - // has no cap or ledger refusal of its own to trigger a release. expect(await scan.settled).toBe("missing"); - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); }); it("finds a preface fragmented into many one-byte chunks", async () => { const { channel, control } = makeFakeChannel(); - const { ledger } = makeCountingLedger(1024 * 1024); const scan = __http2PrefaceScanTesting.scanForHttp2ClientPreface(channel, { capBytes: 4_096, timeoutMs: 5_000, - ledger, }); // A slow sandbox socket can deliver the preface one byte at a time. The // scan must still find it and deliver the exact octets, the same as it @@ -6795,7 +6219,6 @@ describe("http2 preface scan post-preface replay buffer", () => { it("bounds the pre-preface scan search and growth-copy work by the bytes received, across many one-byte fragments", async () => { const { channel, control } = makeFakeChannel(); - const { ledger } = makeCountingLedger(1024 * 1024); __http2PrefaceScanTesting.resetScanSearchUnits(); __http2PrefaceScanTesting.resetScanBufferGrowthCopyUnits(); // An adversarial sandbox sends many one-byte fragments, none of them the @@ -6808,7 +6231,6 @@ describe("http2 preface scan post-preface replay buffer", () => { const scan = __http2PrefaceScanTesting.scanForHttp2ClientPreface(channel, { capBytes, timeoutMs: 5_000, - ledger, }); for (let i = 0; i < capBytes; i += 1) { control.emitData(Buffer.from([0x2e])); // '.', never part of the preface @@ -6832,13 +6254,11 @@ describe("http2 preface scan post-preface replay buffer", () => { it("bounds the post-preface replay buffer growth-copy work by the bytes received, across many one-byte fragments, and still enforces the cap", async () => { const { channel, control } = makeFakeChannel(); - const { ledger } = makeCountingLedger(1024 * 1024); __http2PrefaceScanTesting.resetReplayBufferGrowthCopyUnits(); const capBytes = 2_048; const scan = __http2PrefaceScanTesting.scanForHttp2ClientPreface(channel, { capBytes, timeoutMs: 5_000, - ledger, }); control.emitData(PREFACE); expect(await scan.settled).toBe("found"); diff --git a/packages/adapter-utils/src/execution-target.ts b/packages/adapter-utils/src/execution-target.ts index c11463841a..5d08f246fc 100644 --- a/packages/adapter-utils/src/execution-target.ts +++ b/packages/adapter-utils/src/execution-target.ts @@ -63,11 +63,6 @@ import { type DuplexObservabilityRecorder, type Http2TelemetryEventName, } from "./duplex-observability.js"; -import { - DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, - type DuplexAggregateByteLedger, - type ReservationToken, -} from "./duplex-aggregate-byte-ledger.js"; import { createSshCommandManagedRuntimeRunner, parseSshRemoteExecutionSpec, runSshCommand, shellQuote } from "./ssh.js"; import { ensureCommandResolvable, @@ -196,16 +191,6 @@ export interface AdapterSandboxExecutionTarget extends AdapterExecutionTargetWor * duplex observability surface. Absent means the safe no-op default. */ duplexObservabilityRecorder?: DuplexObservabilityRecorder | null; - /** - * The process-owned aggregate byte ledger for the sandbox duplex channel. The - * host stamps this same object on every sandbox target on the same seam as - * `runner`, so one shared gauge bounds the aggregate retained bytes across all - * live duplex routes. The live object stays on the host and never enters the - * sandbox environment. The bridge passes it to the broker, the decoder, and the - * response-body reader. Absent means no host ledger; a non-duplex run keeps the - * bridge inert for this seam. - */ - duplexAggregateByteLedger?: DuplexAggregateByteLedger | null; } export type AdapterExecutionTarget = @@ -450,19 +435,6 @@ export function adapterExecutionTargetDuplexObservabilityRecorder( : null; } -/** - * Read the injected aggregate byte ledger off a target. Only a sandbox target - * with a ledger attached returns it. Every other target returns null, so the - * bridge stays inert for this seam. The reader never makes a fresh ledger, so a - * host duplex run always uses the one process-owned ledger the host stamped. - */ -export function adapterExecutionTargetDuplexAggregateByteLedger( - target: AdapterExecutionTarget | null | undefined, -): DuplexAggregateByteLedger | null { - return target?.kind === "remote" && target.transport === "sandbox" - ? target.duplexAggregateByteLedger ?? null - : null; -} export function adapterExecutionTargetRemoteCwd( target: AdapterExecutionTarget | null | undefined, @@ -1505,26 +1477,15 @@ function bridgeResponseBodyLimitError(maxBodyBytes: number): Error { } /** - * Read the forward response body into a string. The reader bounds the body with - * two controls. The per-request `maxBodyBytes` limit rejects a body larger than - * the configured per-request ceiling. The optional host aggregate byte ledger - * bounds the retained bytes across all live routes. + * Read the forward response body into a string. The per-request `maxBodyBytes` + * limit rejects a body larger than the configured per-request ceiling. * - * The reader charges the ledger for every retained buffer before it allocates - * that buffer. It reserves the exact chunk bytes before it copies a chunk into a - * retained `Buffer`. It reserves the concatenation buffer before it allocates it. - * A reservation that would pass the aggregate ceiling returns no token; the - * reader retains nothing more, cancels the stream reader, and throws the fixed - * marker {@link DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED}. The `finally` releases - * every token exactly one time, so the reader charges the retained bytes only - * while the raw buffers live and never leaves a token held after it returns or - * throws. + * This function reserves no process-wide byte budget: it enforces only the + * one request's own ceiling. See the "Known behavior: aggregate retained + * body bytes" section in `doc/observability.md` for the accepted aggregate + * ceiling this leaves across every concurrent route. */ -async function readBridgeForwardResponseBody( - response: Response, - maxBodyBytes: number, - ledger?: DuplexAggregateByteLedger | null, -): Promise { +async function readBridgeForwardResponseBody(response: Response, maxBodyBytes: number): Promise { const rawContentLength = response.headers.get("content-length"); if (rawContentLength) { const contentLength = Number.parseInt(rawContentLength, 10); @@ -1539,54 +1500,20 @@ async function readBridgeForwardResponseBody( const reader = response.body.getReader(); const chunks: Buffer[] = []; - // Every response-body reservation token the reader holds. The `finally` block - // releases each token one time, so a return, a size error, an aggregate - // rejection, and a read error all release every token. - const tokens: ReservationToken[] = []; let totalBytes = 0; - try { - while (true) { - const { done, value } = await reader.read(); - if (done) break; - if (!value) continue; - const chunkBytes = value.byteLength; - totalBytes += chunkBytes; - if (totalBytes > maxBodyBytes) { - await reader.cancel().catch(() => undefined); - throw bridgeResponseBodyLimitError(maxBodyBytes); - } - // Reserve the exact chunk bytes before the host copies the chunk into a - // retained buffer. A rejection fails closed: cancel the stream reader and - // report the fixed marker; the reader retains nothing more. - if (ledger) { - const token = ledger.reserve("response_body", chunkBytes); - if (!token) { - await reader.cancel().catch(() => undefined); - throw new Error(DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED); - } - tokens.push(token); - } - chunks.push(Buffer.from(value)); - } - // Reserve the concatenation buffer before the reader allocates it. The - // concatenation buffer is a second copy of the body bytes that lives next to - // the chunk buffers during the concatenation, so it is the peak retained - // allocation. A rejection fails closed with the fixed marker. - if (ledger && totalBytes > 0) { - const concatToken = ledger.reserve("response_body", totalBytes); - if (!concatToken) { - throw new Error(DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED); - } - tokens.push(concatToken); - } - return Buffer.concat(chunks, totalBytes).toString("utf8"); - } finally { - if (ledger) { - for (const token of tokens) { - ledger.release(token); - } + while (true) { + const { done, value } = await reader.read(); + if (done) break; + if (!value) continue; + const chunkBytes = value.byteLength; + totalBytes += chunkBytes; + if (totalBytes > maxBodyBytes) { + await reader.cancel().catch(() => undefined); + throw bridgeResponseBodyLimitError(maxBodyBytes); } + chunks.push(Buffer.from(value)); } + return Buffer.concat(chunks, totalBytes).toString("utf8"); } const PROCESS_SESSION_PROXY_SCRIPT = "paperclip-process-session-proxy.mjs"; @@ -3008,12 +2935,7 @@ export function buildDuplexGatewayLaunchArgv(input: { } /** The reason the duplex readiness handshake did not pass. */ -type DuplexReadinessFailure = - | "protocol_contamination" - | "nonce_mismatch" - | "channel_exit" - | "timeout" - | "aggregate_bytes_exceeded"; +type DuplexReadinessFailure = "protocol_contamination" | "nonce_mismatch" | "channel_exit" | "timeout"; /** The outcome of the duplex readiness handshake. */ type DuplexReadinessResult = @@ -3035,8 +2957,6 @@ function duplexReadinessFallbackReason(reason: DuplexReadinessFailure): DuplexFa return "ready_timeout"; case "channel_exit": return "ready_invalid"; - case "aggregate_bytes_exceeded": - return "aggregate_bytes_exceeded"; } } @@ -3187,22 +3107,14 @@ function findPrefaceFrom(buffer: Buffer, from: number): number { * found, it calls `onMissing` exactly one time and stops buffering, so the * caller can abort the open and fall back to `queue_v1`. The function holds * no prologue byte count: it always scans for the fixed 24-octet sequence, - * never a length. The scan buffer holds untrusted, sandbox-controlled bytes - * on the same footing as the readiness gate's own pre-READY buffer, so, when - * the caller supplies a ledger, it charges each received chunk against the - * process aggregate byte ledger under the `http2_preface_scan` owner before - * the chunk grows the buffer. A refusal fails closed the same way the cap - * does: the function drops the buffer and calls `onMissing`. + * never a length. * * The bytes that follow the found preface, before the HTTP/2 server binds a * downstream listener, land in `pendingAfterPreface`. This buffer holds * untrusted bytes on the same footing as the scan buffer, so it carries the - * same {@link DUPLEX_READINESS_BUFFER_CAP_BYTES} cap and, when the caller - * supplies a ledger, charges each retained chunk against it under the - * `http2_preface_replay` owner. A chunk that would pass the cap, or that the - * ledger refuses, fails closed: the function - * drops the buffer, releases its ledger tokens, and stops the channel. The - * caller reads {@link replayOverflowed} after the preface settles and, on + * same {@link DUPLEX_READINESS_BUFFER_CAP_BYTES} cap. A chunk that would pass + * the cap fails closed: the function drops the buffer and stops the channel. + * The caller reads {@link replayOverflowed} after the preface settles and, on * `true`, treats the open the same as a missing preface. */ function createHttp2PrefaceScanningChannel( @@ -3211,14 +3123,12 @@ function createHttp2PrefaceScanningChannel( capBytes: number; onFound: () => void; onMissing: () => void; - ledger?: DuplexAggregateByteLedger | null; }, ): { channel: CommandManagedDuplexChannel; replayOverflowed: () => boolean; disposeScanBuffer: () => void; } { - const ledger = options.ledger ?? null; // The pre-preface scan buffer. `scanBuf.append` grows its backing storage // by doubling, instead of copying the whole retained buffer on every // fragment — see {@link createGrowableByteBuffer}. `scanSearchFrom` is the @@ -3240,42 +3150,14 @@ function createHttp2PrefaceScanningChannel( const pendingAfterPreface = createGrowableByteBuffer((copiedBytes) => { http2PrefaceReplayBufferGrowthCopyUnits += copiedBytes; }); - // Every `http2_preface_replay` reservation token this buffer holds. The - // function releases each token exactly once, on the downstream handoff or - // on an overflow. - const replayTokens: ReservationToken[] = []; - // Every `http2_preface_scan` reservation token the pre-preface scan buffer - // holds. The scan is untrusted, sandbox-controlled input on the same - // footing as the readiness gate's own pre-READY buffer, so it charges the - // ledger the same way: one reservation per received chunk, released in - // full on the terminal scan outcome — the preface found, the cap passed - // with no match, or a ledger refusal. - const scanTokens: ReservationToken[] = []; let replayOverflow = false; - function releaseReplayTokens(): void { - if (!ledger) return; - for (const token of replayTokens) { - ledger.release(token); - } - replayTokens.length = 0; - } - - function releaseScanTokens(): void { - if (!ledger) return; - for (const token of scanTokens) { - ledger.release(token); - } - scanTokens.length = 0; - } - - // Drop the pending buffer, release its tokens, and stop the channel. The - // caller reads `replayOverflowed()` after the preface settles and falls - // back the same way it does for a missing preface. + // Drop the pending buffer and stop the channel. The caller reads + // `replayOverflowed()` after the preface settles and falls back the same + // way it does for a missing preface. function overflowAndStop(): void { replayOverflow = true; pendingAfterPreface.reset(); - releaseReplayTokens(); channel.stop(); } @@ -3289,14 +3171,6 @@ function createHttp2PrefaceScanningChannel( overflowAndStop(); return; } - if (ledger) { - const token = ledger.reserve("http2_preface_replay", chunk.byteLength); - if (!token) { - overflowAndStop(); - return; - } - replayTokens.push(token); - } pendingAfterPreface.append(chunk); } @@ -3318,25 +3192,9 @@ function createHttp2PrefaceScanningChannel( failed = true; scanBuf.reset(); scanSearchFrom = 0; - releaseScanTokens(); options.onMissing(); return; } - // Charge this chunk against the aggregate ledger before it grows the - // scan buffer. A refusal fails closed the same way the cap does: drop - // the buffer and report a missing preface. - if (ledger) { - const token = ledger.reserve("http2_preface_scan", rawChunk.byteLength); - if (!token) { - failed = true; - scanBuf.reset(); - scanSearchFrom = 0; - releaseScanTokens(); - options.onMissing(); - return; - } - scanTokens.push(token); - } scanBuf.append(rawChunk); const scanBuffer = scanBuf.view(); const offset = findPrefaceFrom(scanBuffer, scanSearchFrom); @@ -3358,12 +3216,6 @@ function createHttp2PrefaceScanningChannel( const fromPreface = Buffer.from(scanBuffer.subarray(offset)); scanBuf.reset(); scanSearchFrom = 0; - // Release the scan tokens before `deliver` charges the same bytes under - // `http2_preface_replay`. The two calls run inside one synchronous - // callback with no `await` between them, so no other reservation can - // observe the released state in between; releasing first keeps the - // ledger's momentary peak at the real retained bytes, not double them. - releaseScanTokens(); deliver(fromPreface); }); @@ -3383,15 +3235,6 @@ function createHttp2PrefaceScanningChannel( pendingAfterPreface.reset(); listener(replay); } - // Release every replay token exactly once, after the synchronous - // handoff to the downstream listener. Order is safe here: the - // downstream listener is the bound HTTP/2 server duplex, which holds - // no aggregate-ledger reservation of its own for these bytes, so - // this release cannot overlap a second reservation for the same - // bytes. Contrast the readiness gate's own `readiness_replay` - // handoff, which releases first because its downstream listener (the - // preface scanner) does take its own reservation for the same bytes. - releaseReplayTokens(); }, onExit: (listener: (exit: { exitCode: number | null }) => void) => channel.onExit(listener), stop: () => channel.stop(), @@ -3399,20 +3242,18 @@ function createHttp2PrefaceScanningChannel( }, replayOverflowed: () => replayOverflow, /** - * Release every held `http2_preface_scan` token and drop the scan - * buffer, for a caller-side terminal path this function itself never - * reaches — the bound readiness timeout elapsing while the scan is - * still searching, with no preface found and no cap or ledger refusal - * of its own. A call after the preface already matched, or after the - * cap or the ledger already failed the scan closed, is a no-op: both - * paths already released the scan tokens themselves. + * Drop the scan buffer, for a caller-side terminal path this function + * itself never reaches — the bound readiness timeout elapsing while the + * scan is still searching, with no preface found and no cap refusal of + * its own. A call after the preface already matched, or after the cap + * already failed the scan closed, is a no-op: both paths already reset + * the scan buffer themselves. */ disposeScanBuffer: (): void => { if (sawPreface || failed) return; failed = true; scanBuf.reset(); scanSearchFrom = 0; - releaseScanTokens(); }, }; } @@ -3430,12 +3271,12 @@ type Http2PrefaceScanResult = "found" | "missing"; * Returns the scanning channel alongside the settled result, so the caller * binds the HTTP/2 server to it only on a `found` result. On a `found` * result, the caller must still read `replayOverflowed()`: the post-preface - * buffer can overflow its cap or its ledger reservation after the preface - * settles as `found` and before the caller binds a downstream listener. + * buffer can overflow its cap after the preface settles as `found` and + * before the caller binds a downstream listener. */ function scanForHttp2ClientPreface( channel: CommandManagedDuplexChannel, - options: { capBytes: number; timeoutMs: number; ledger?: DuplexAggregateByteLedger | null }, + options: { capBytes: number; timeoutMs: number }, ): { scanned: CommandManagedDuplexChannel; settled: Promise; @@ -3457,10 +3298,10 @@ function scanForHttp2ClientPreface( settledOnce = true; clearTimeout(timer); // The bound readiness timeout can elapse while the scan still searches, - // with no preface found and no cap or ledger refusal of its own. That - // path holds no other cleanup, so release its `http2_preface_scan` - // tokens here. A `found` result, or a `missing` result the scan itself - // already failed closed, is a no-op inside `disposeScanBuffer`. + // with no preface found and no cap refusal of its own. That path holds + // no other cleanup, so drop the scan buffer here. A `found` result, or a + // `missing` result the scan itself already failed closed, is a no-op + // inside `disposeScanBuffer`. if (result === "missing") disposeScanBuffer(); resolveSettled(result); }; @@ -3470,7 +3311,6 @@ function scanForHttp2ClientPreface( capBytes: options.capBytes, onFound: () => settle("found"), onMissing: () => settle("missing"), - ledger: options.ledger, }); disposeScanBuffer = scan.disposeScanBuffer; return { scanned: scan.channel, settled, replayOverflowed: scan.replayOverflowed }; @@ -3478,13 +3318,13 @@ function scanForHttp2ClientPreface( /** * Test-only surface for {@link scanForHttp2ClientPreface}. A test drives the - * post-preface replay cap and ledger charge across every terminal path - * without the whole bridge. Production code never reads this export. + * post-preface replay cap across every terminal path without the whole + * bridge. Production code never reads this export. */ export const __http2PrefaceScanTesting = { scanForHttp2ClientPreface: ( channel: CommandManagedDuplexChannel, - options: { capBytes: number; timeoutMs: number; ledger?: DuplexAggregateByteLedger | null }, + options: { capBytes: number; timeoutMs: number }, ) => scanForHttp2ClientPreface(channel, options), readScanSearchUnits: (): number => http2PrefaceScanSearchUnits, resetScanSearchUnits: (): void => { @@ -3648,12 +3488,10 @@ export const __duplexReadinessTesting = { duplexReadinessBufferGrowthCopyUnits = 0; }, // Build one readiness gate over a supplied channel, so a test can drive the - // readiness-replay reservation lifecycle across every terminal path without the - // whole bridge. Production code never reads this factory. - createReadinessGate: ( - channel: CommandManagedDuplexChannel, - options: { nonce: string; timeoutMs: number; ledger?: DuplexAggregateByteLedger | null }, - ) => createDuplexReadinessGate(channel, options), + // readiness-replay cap lifecycle across every terminal path without the whole + // bridge. Production code never reads this factory. + createReadinessGate: (channel: CommandManagedDuplexChannel, options: { nonce: string; timeoutMs: number }) => + createDuplexReadinessGate(channel, options), }; interface DuplexReadinessGate { @@ -3667,19 +3505,18 @@ interface DuplexReadinessGate { */ readonly brokerChannel: CommandManagedDuplexChannel; /** - * Report whether a post-READY pre-bind chunk could not reserve its replay bytes - * against the aggregate ledger. On such a refusal the gate drops the pending - * replay buffer and stops the channel. The caller reads this after `ready` - * resolves `ok`, and before it binds the broker. A `true` result means the caller - * must abandon the broker and select the file bridge with the aggregate marker. + * Report whether a post-READY pre-bind chunk tipped the pending replay buffer + * past {@link DUPLEX_READINESS_BUFFER_CAP_BYTES}. On such an overflow the gate + * drops the pending replay buffer and stops the channel. The caller reads this + * after `ready` resolves `ok`, and before it binds the broker. */ replayOverflowed(): boolean; /** - * Release every held readiness-replay reservation exactly once and drop the - * pending replay buffer. The caller runs this on a terminal path that abandons - * the pending replay without a broker handoff: a readiness failure, a replay - * overflow, or a broker-construction failure. The normal handoff releases the - * reservation inside `brokerChannel.onData`, so a later call here is a no-op. + * Drop the pending replay buffer. The caller runs this on a terminal path that + * abandons the pending replay without a broker handoff: a readiness failure, a + * replay overflow, or a broker-construction failure. The normal handoff already + * drops the buffer inside `brokerChannel.onData`, so a later call here is a + * no-op. */ disposePendingReplay(): void; /** @@ -3696,52 +3533,15 @@ function createDuplexReadinessGate( options: { nonce: string; timeoutMs: number; - // The one host-process aggregate byte ledger. The gate charges the untrusted - // pre-READY buffer bytes against it, so a pre-READY flood counts toward the - // aggregate ceiling across all live routes. A gate with no ledger stays inert - // for this seam. The gate holds the same object every other host retention - // site holds, so the aggregate identity holds at this seam. - ledger?: DuplexAggregateByteLedger | null; }, ): DuplexReadinessGate { - const ledger = options.ledger ?? null; let settled = false; let readyOk = false; - // Every readiness-buffer reservation token the gate holds for the pre-READY - // bytes. The gate releases each token one time when it settles or when it - // accepts READY. On a failed handshake the gate drops the buffer, so the release - // frees the untrusted bytes. On READY the gate discards the whole pre-READY - // buffer, then re-charges only the retained suffix under `readiness_replay`. - const retainedTokens: ReservationToken[] = []; - // Every readiness-replay reservation token the gate holds for the post-READY - // suffix and each later pre-bind chunk. The gate releases each token one time - // after the synchronous handoff to the broker, or on a terminal path that - // abandons the pending replay without a broker handoff. - const replayTokens: ReservationToken[] = []; - // The gate sets this when a post-READY pre-bind chunk cannot reserve its replay - // bytes. On that refusal the gate drops the pending buffer and stops the channel. - // The caller reads it through `replayOverflowed` and selects the file bridge. + // The gate sets this when a post-READY pre-bind chunk tips the pending replay + // buffer past the cap. On that overflow the gate drops the pending buffer and + // stops the channel. The caller reads it through `replayOverflowed` and selects + // the file bridge. let replayOverflow = false; - - // Release every readiness-buffer token exactly once and clear the registry. A - // second call is a no-op, because the array is empty. - function releaseReadinessBufferTokens(): void { - if (!ledger) return; - for (const token of retainedTokens) { - ledger.release(token); - } - retainedTokens.length = 0; - } - - // Release every readiness-replay token exactly once and clear the registry. A - // second call is a no-op, because the array is empty. - function releaseReplayTokens(): void { - if (!ledger) return; - for (const token of replayTokens) { - ledger.release(token); - } - replayTokens.length = 0; - } // The raw bytes the host reads before the READY frame completes. `buffer` is // append-only and always a zero-copy view over the used prefix of `storage`, // so the O(1) cap check on `buffer.length` stays valid. @@ -3806,11 +3606,6 @@ function createDuplexReadinessGate( settled = true; clearTimeout(timer); if (result.ok) readyOk = true; - // Release every readiness-buffer token exactly once. The gate no longer owns - // the pre-READY bytes: a failed handshake drops the buffer. The READY-accept - // path already released these tokens and charged the retained suffix under - // `readiness_replay`, so this call is a no-op there. - releaseReadinessBufferTokens(); resolveReady(result); } @@ -3820,23 +3615,20 @@ function createDuplexReadinessGate( return; } if (readyOk) { - // READY already passed; hold the bytes until the broker binds. Reserve the - // exact UTF-8 bytes under `readiness_replay` before the append, so the replay - // buffer counts toward the aggregate ceiling. A refusal fails closed: the gate - // drops the pending buffer, releases the replay tokens, stops the channel, and - // sets the overflow flag. The caller reads the flag and selects the file bridge - // with the aggregate marker, because `ready` already resolved before this - // synchronous post-READY chunk arrived. - if (ledger) { - const token = ledger.reserve("readiness_replay", chunk.byteLength); - if (!token) { - replayOverflow = true; - pending = READINESS_EMPTY_BUFFER; - releaseReplayTokens(); - channel.stop(); - return; - } - replayTokens.push(token); + // READY already passed; hold the bytes until the broker binds. Bound this + // buffer directly against {@link DUPLEX_READINESS_BUFFER_CAP_BYTES}, the + // same way the preface-scan gate's own post-preface replay buffer bounds + // itself: a worker that keeps sending bytes after READY, faster than the + // broker can bind, cannot grow this buffer past the cap. A chunk that + // would pass the cap fails closed: the gate drops the pending buffer, + // stops the channel, and sets the overflow flag. The caller reads the + // flag and selects the file bridge, because `ready` already resolved + // before this synchronous post-READY chunk arrived. + if (pending.length + chunk.byteLength > DUPLEX_READINESS_BUFFER_CAP_BYTES) { + replayOverflow = true; + pending = READINESS_EMPTY_BUFFER; + channel.stop(); + return; } // Copy a first chunk instead of aliasing the caller's `Uint8Array`, so a // channel that reuses its delivered buffer across calls cannot corrupt the @@ -3850,18 +3642,6 @@ function createDuplexReadinessGate( // never grows the buffer after the gate settles. return; } - // Reserve the exact bytes of this chunk against the aggregate ledger before - // the gate retains it. The pre-READY buffer holds untrusted bytes, so a - // flood counts toward the process aggregate ceiling. A rejection fails - // closed: the gate retains nothing more and falls back to the file bridge. - if (ledger) { - const token = ledger.reserve("readiness_buffer", chunk.byteLength); - if (!token) { - finish({ ok: false, reason: "aggregate_bytes_exceeded" }); - return; - } - retainedTokens.push(token); - } // Append the new bytes and continue the newline search from `scanFrom`, the // first index not yet examined. Each byte is read at most one time for the // search, and `appendReadinessBytes` copies at most the incoming chunk, so @@ -3933,40 +3713,21 @@ function createDuplexReadinessGate( return; } // The bytes that follow the READY line become the replay buffer for the - // broker. Drop the whole pre-READY buffer charge first, then reserve the - // retained suffix under `readiness_replay`. The release-before-reserve order - // keeps the transient charge equal to the suffix, not the sum of the dropped - // prefix and the retained suffix. The two steps run in one synchronous - // section, so no other route can take the freed bytes in between. + // broker. // // Copy the suffix instead of slicing it off `buffer`. `buffer` is a view // over `storage`, and `storage`'s capacity can run ahead of the bytes in // use (the doubling growth in `appendReadinessBytes` over-provisions it). - // A slice would keep that whole over-provisioned allocation alive, so the - // process would retain more physical bytes than the ledger charges under - // `readiness_replay`. The copy is exactly `suffix.length` bytes, one time, - // not a per-fragment cost. + // A slice would keep that whole over-provisioned allocation alive for as + // long as the broker replay holds its reference. The copy is exactly + // `suffix.length` bytes, one time, not a per-fragment cost. const suffix = Buffer.from(buffer.subarray(newlineIndex + 1)); // Drop the original pre-READY buffer and its backing storage now. The - // gate keeps only the retained suffix as `pending`, and it charges that - // suffix under `readiness_replay` below. If the gate keeps `storage`, the - // process retains the full sandbox-controlled bytes while the ledger - // counts only the suffix, so aggregate retention passes the ceiling. This - // clear also covers the broker handoff and the replay disposal. Both run - // later and read no buffer bytes. + // gate keeps only the retained suffix as `pending`. This clear also + // covers the broker handoff and the replay disposal. Both run later and + // read no buffer bytes. buffer = READINESS_EMPTY_BUFFER; storage = READINESS_EMPTY_BUFFER; - releaseReadinessBufferTokens(); - if (ledger && suffix.length > 0) { - const token = ledger.reserve("readiness_replay", suffix.byteLength); - if (!token) { - // The retained suffix passes the aggregate ceiling. Fail closed: drop - // the suffix and fall back to the file bridge with the aggregate marker. - finish({ ok: false, reason: "aggregate_bytes_exceeded" }); - return; - } - replayTokens.push(token); - } pending = suffix; finish({ ok: true }); return; @@ -4005,19 +3766,9 @@ function createDuplexReadinessGate( if (pending.length > 0) { const replay = pending; pending = READINESS_EMPTY_BUFFER; - // Release every readiness-replay token before the synchronous handoff - // to the broker, not after. The broker (the HTTP/2 preface scanner) - // charges its own reservation for these same bytes inside - // `listener(replay)` below, under a different owner. Releasing first - // keeps the ledger's momentary peak at the real retained bytes, not - // double them: the release and the broker's reserve both run inside - // this one synchronous call, with no `await` between them, so no - // other route can claim the freed capacity in between. - releaseReplayTokens(); listener(replay); return; } - releaseReplayTokens(); }, onExit: (listener: (exit: { exitCode: number | null }) => void) => { exitSink = listener; @@ -4037,7 +3788,6 @@ function createDuplexReadinessGate( replayOverflowed: () => replayOverflow, disposePendingReplay: () => { pending = READINESS_EMPTY_BUFFER; - releaseReplayTokens(); }, retainedReadinessBufferLength: () => buffer.length, }; @@ -4122,12 +3872,6 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { } const target = input.target; - // The process-owned aggregate byte ledger the host stamped on this sandbox - // target. The forward response-body reader charges its retained bytes against - // this one ledger, so the aggregate retained bytes across all live routes stay - // under the ceiling. A target with no ledger keeps the reader inert for this - // seam. - const duplexAggregateByteLedger = adapterExecutionTargetDuplexAggregateByteLedger(target); const onLog = input.onLog ?? (async () => {}); const hostApiToken = input.hostApiToken?.trim() ?? ""; if (hostApiToken.length === 0) { @@ -4271,11 +4015,7 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { // to a non-retryable 409 for both the file bridge and the duplex broker. let responseBody: string; try { - responseBody = await readBridgeForwardResponseBody( - response, - maxBodyBytes, - duplexAggregateByteLedger, - ); + responseBody = await readBridgeForwardResponseBody(response, maxBodyBytes); } catch (error) { if (isSafeBridgeMethod(method)) { // The method is safe, so a retry cannot double-apply a mutation. Return a @@ -4416,10 +4156,6 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { const gate = createDuplexReadinessGate(channel, { nonce, timeoutMs: readinessTimeoutMs, - // Inject the one host-process aggregate byte ledger, the same object the - // broker and the response-body reader hold. The gate charges the untrusted - // pre-READY buffer against it, so the aggregate identity holds at this seam. - ledger: duplexAggregateByteLedger, }); const readiness = await gate.ready; if (!readiness.ok) { @@ -4435,19 +4171,6 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { "stderr", `[paperclip] Sandbox duplex readiness failed (${readiness.reason}). Using the file bridge.\n`, ); - } else if (gate.replayOverflowed()) { - // Readiness passed, but a post-READY pre-bind chunk passed the aggregate - // byte ceiling. The gate dropped the replay buffer and stopped the channel. - // Release any held replay reservation, close the partial channel inside the - // cleanup budget, and select the file bridge with the aggregate marker. The - // broker never bound, so no request reached the channel or any endpoint. - gate.disposePendingReplay(); - await closeDuplexChannelWithinBudget(channel, DEFAULT_DUPLEX_CLEANUP_BUDGET_MS); - duplexChannelOpen.fallback(duplexReadinessFallbackReason("aggregate_bytes_exceeded")); - await onLog( - "stderr", - "[paperclip] Sandbox duplex readiness replay exceeded the aggregate byte ceiling (aggregate_bytes_exceeded). Using the file bridge.\n", - ); } else { // Readiness passed. The gate retained every byte that followed the // accepted READY line. Scan those retained bytes for the HTTP/2 @@ -4460,32 +4183,20 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { const prefaceScan = scanForHttp2ClientPreface(gate.brokerChannel, { capBytes: DUPLEX_READINESS_BUFFER_CAP_BYTES, timeoutMs: readinessTimeoutMs, - // Inject the same host-process aggregate byte ledger the readiness - // gate charges, so the post-preface pre-bind buffer counts toward - // the same aggregate ceiling. - ledger: duplexAggregateByteLedger, }); const prefaceResult = await prefaceScan.settled; - if (prefaceResult === "missing" || prefaceScan.replayOverflowed()) { + if (prefaceResult === "missing") { // Fail closed, the same shape as a readiness failure: close the // partial channel inside the cleanup budget, then select the file // bridge. No HTTP/2 server ever bound to this channel, so no // request reached it or any endpoint. gate.disposePendingReplay(); await closeDuplexChannelWithinBudget(openedChannel, DEFAULT_DUPLEX_CLEANUP_BUDGET_MS); - if (prefaceResult === "missing") { - duplexChannelOpen.fallback("preface_missing"); - await onLog( - "stderr", - "[paperclip] Sandbox HTTP/2 client preface did not appear inside the bounded readiness buffer (preface_missing). Using the file bridge.\n", - ); - } else { - duplexChannelOpen.fallback("aggregate_bytes_exceeded"); - await onLog( - "stderr", - "[paperclip] Sandbox HTTP/2 post-preface buffer exceeded the aggregate byte ceiling (aggregate_bytes_exceeded). Using the file bridge.\n", - ); - } + duplexChannelOpen.fallback("preface_missing"); + await onLog( + "stderr", + "[paperclip] Sandbox HTTP/2 client preface did not appear inside the bounded readiness buffer (preface_missing). Using the file bridge.\n", + ); } else { // The run disposition latch for the http2_v1 path, in the same // shape the retired duplex_v1 broker exposed. A loss ordered before @@ -4532,7 +4243,7 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { headers: request.headers, body: request.body.toString("utf8"), }, - undefined, + request.signal, { suppressDebugLog: true }, ); duplexObservability.recordRequest({ latencyMs: Date.now() - dispatchStartMs, outcome: "ok" }); diff --git a/packages/adapter-utils/src/http2-bridge-server.test.ts b/packages/adapter-utils/src/http2-bridge-server.test.ts index 113aa212e4..5613620a38 100644 --- a/packages/adapter-utils/src/http2-bridge-server.test.ts +++ b/packages/adapter-utils/src/http2-bridge-server.test.ts @@ -10,6 +10,7 @@ import { createHttp2BridgeServer, parseCanonicalBridgeRequestPath, wrapDuplexChannelAsNodeDuplex, + DEFAULT_HTTP2_BRIDGE_MAX_BUFFERED_READ_BYTES, DEFAULT_HTTP2_BRIDGE_PING_INTERVAL_MS, DEFAULT_HTTP2_BRIDGE_PING_STALL_MS, HTTP2_BRIDGE_ENABLE_PUSH, @@ -28,7 +29,10 @@ import { type Http2BridgeForwardResult, type Http2BridgeGoawayRecord, } from "./http2-bridge-server.js"; -import { createSandboxHttp2BridgeGateway } from "./sandbox-callback-bridge.js"; +import { + createSandboxHttp2BridgeGateway, + DEFAULT_SANDBOX_CALLBACK_BRIDGE_MAX_BODY_BYTES, +} from "./sandbox-callback-bridge.js"; import type { CommandManagedDuplexChannel } from "./command-managed-runtime.js"; /** @@ -346,6 +350,185 @@ describe("createHttp2BridgeServer + createSandboxHttp2BridgeGateway", () => { } }); + it("test_the_server_aborts_the_forward_when_the_client_closes_the_stream", async () => { + let capturedRequest: Http2BridgeForwardRequest | undefined; + let releaseForward: (() => void) | undefined; + const forwardHeld = new Promise((resolve) => { + releaseForward = resolve; + }); + const { handle, bridgeToken, clientSide } = bindTestServer({ + forwardRequest: async (request) => { + capturedRequest = request; + // Hold the handler open, so the test controls exactly when the + // client-side stream close happens relative to the forward. + await forwardHeld; + return { status: 200, body: "{}" }; + }, + }); + const rawClient = connectRawClient(clientSide); + try { + const stream = rawClient.request({ + ":method": "GET", + ":path": "/api/agents/me", + authorization: `Bearer ${bridgeToken}`, + }); + const streamClosed = new Promise((resolve) => { + stream.on("error", () => resolve()); + stream.on("close", () => resolve()); + }); + stream.end(); + // Give the request time to reach the server and enter forwardRequest. + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(capturedRequest).toBeDefined(); + expect(capturedRequest!.signal.aborted).toBe(false); + stream.close(http2.constants.NGHTTP2_CANCEL); + await streamClosed; + // Give the server's own stream `close` listener a turn to run. + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(capturedRequest!.signal.aborted).toBe(true); + releaseForward!(); + } finally { + rawClient.close(); + await handle.close(); + } + }); + + it("test_one_aborted_stream_leaves_the_session_and_the_other_streams_open", async () => { + const signalsByPath = new Map(); + let releaseSurvivor: (() => void) | undefined; + const survivorHeld = new Promise((resolve) => { + releaseSurvivor = resolve; + }); + const { handle, bridgeToken, clientSide } = bindTestServer({ + forwardRequest: async (request) => { + signalsByPath.set(request.pathname, request.signal); + if (request.pathname === "/api/agents/aborted") { + // The RST — not a manual release — is what must end this forward: + // it settles only when the stream's own signal fires, so the test + // proves the signal itself unblocks the handler. + await new Promise((_resolve, reject) => { + request.signal.addEventListener("abort", () => reject(new Error("aborted")), { + once: true, + }); + }).catch(() => undefined); + } else if (request.pathname === "/api/agents/survivor") { + await survivorHeld; + } + return { status: 200, body: JSON.stringify({ path: request.pathname }) }; + }, + }); + const rawClient = connectRawClient(clientSide); + try { + const abortedStream = rawClient.request({ + ":method": "GET", + ":path": "/api/agents/aborted", + authorization: `Bearer ${bridgeToken}`, + }); + const survivorStream = rawClient.request({ + ":method": "GET", + ":path": "/api/agents/survivor", + authorization: `Bearer ${bridgeToken}`, + }); + const abortedStreamClosed = new Promise((resolve) => { + abortedStream.on("error", () => resolve()); + abortedStream.on("close", () => resolve()); + }); + // Capture the survivor stream's own response, so the test proves this + // exact stream — the sibling of the aborted one, on the same session — + // completes normally, not merely that the session admits a fresh one. + const survivorResponse = new Promise<{ status: number; body: string }>((resolve, reject) => { + let status = 0; + let body = ""; + survivorStream.setEncoding("utf8"); + survivorStream.on("response", (h) => { + status = Number(h[":status"]) || 0; + }); + survivorStream.on("data", (chunk) => (body += chunk)); + survivorStream.on("end", () => resolve({ status, body })); + survivorStream.on("error", reject); + }); + abortedStream.end(); + survivorStream.end(); + // Give both requests time to reach the server and enter forwardRequest. + await new Promise((resolve) => setTimeout(resolve, 20)); + abortedStream.close(http2.constants.NGHTTP2_CANCEL); + await abortedStreamClosed; + // Give the server's own stream `close` listener a turn to run. + await new Promise((resolve) => setTimeout(resolve, 20)); + // The one aborted stream's signal fires. The other, still-open stream's + // signal stays untouched: the abort is local to its own stream. + expect(signalsByPath.get("/api/agents/aborted")?.aborted).toBe(true); + expect(signalsByPath.get("/api/agents/survivor")?.aborted).toBe(false); + releaseSurvivor!(); + + const response = await survivorResponse; + expect(response.status).toBe(200); + expect(JSON.parse(response.body)).toEqual({ path: "/api/agents/survivor" }); + } finally { + rawClient.close(); + await handle.close(); + } + }); + + it("test_live_forward_work_never_passes_the_stream_limit", async () => { + // A forward that is never aborted holds open until the caller aborts it + // — this stands in for a forward stuck at its own long timeout. Only the + // per-stream abort binding this file adds can free such a forward before + // that timeout, so this count proves the binding, not merely the HTTP/2 + // session's own stream-slot accounting (a stream's protocol slot frees on + // RST regardless of whether its forward call ever settles). + let liveForwards = 0; + let maxLiveForwards = 0; + const { handle, bridgeToken, clientSide } = bindTestServer({ + forwardRequest: async (request) => { + liveForwards += 1; + maxLiveForwards = Math.max(maxLiveForwards, liveForwards); + try { + await new Promise((resolve, reject) => { + if (request.signal.aborted) { + reject(new Error("aborted")); + return; + } + request.signal.addEventListener("abort", () => reject(new Error("aborted")), { once: true }); + }); + return { status: 200, body: JSON.stringify({ path: request.pathname }) }; + } finally { + liveForwards -= 1; + } + }, + }); + const rawClient = connectRawClient(clientSide); + try { + const openAndCancelOneBatch = async (label: string) => { + const streams = Array.from({ length: HTTP2_BRIDGE_MAX_CONCURRENT_STREAMS }, (_, index) => + rawClient.request({ + ":method": "GET", + ":path": `/api/issues/${label}-${index}`, + authorization: `Bearer ${bridgeToken}`, + }), + ); + for (const stream of streams) stream.end(); + await new Promise((resolve) => setTimeout(resolve, 30)); + for (const stream of streams) stream.close(http2.constants.NGHTTP2_CANCEL); + await new Promise((resolve) => setTimeout(resolve, 30)); + }; + + // One full batch of maximum-concurrency streams, closed while their + // forwards are still held open, then a second full batch opened right + // after. Without the abort binding, the first batch's forwards would + // still be alive when the second batch dispatches, doubling the live- + // forward count past the stream limit. + await openAndCancelOneBatch("first"); + await openAndCancelOneBatch("second"); + + expect(maxLiveForwards).toBeLessThanOrEqual(HTTP2_BRIDGE_MAX_CONCURRENT_STREAMS); + expect(liveForwards).toBe(0); + } finally { + rawClient.close(); + await handle.close(); + } + }); + it("test_a_stalled_request_body_settles_instead_of_hanging_forever", async () => { const forwarderTracker = createForwarderCallTracker(); const { handle, bridgeToken, clientSide } = bindTestServer({ @@ -630,7 +813,7 @@ describe("createHttp2BridgeServer + createSandboxHttp2BridgeGateway", () => { streamResetBurst: HTTP2_BRIDGE_STREAM_RESET_BURST, }); expect(HTTP2_BRIDGE_ENABLE_PUSH).toBe(false); - expect(HTTP2_BRIDGE_MAX_CONCURRENT_STREAMS).toBe(64); + expect(HTTP2_BRIDGE_MAX_CONCURRENT_STREAMS).toBe(4); expect(HTTP2_BRIDGE_MAX_HEADER_LIST_SIZE).toBe(16384); expect(HTTP2_BRIDGE_HEADER_TABLE_SIZE).toBe(4096); expect(HTTP2_BRIDGE_MAX_SESSION_MEMORY).toBe(16); @@ -661,6 +844,20 @@ describe("createHttp2BridgeServer + createSandboxHttp2BridgeGateway", () => { } }); + it("test_the_host_body_budget_matches_the_stream_limit", () => { + // The multiplier counts every retained `Buffer` and string copy of one + // live forward's request and response body: four exact `Buffer` rows, + // plus two string rows. Each string row applies two bytes to each UTF-16 + // code unit of the body limit (`sandbox-callback-bridge.ts`), for an + // accounting peak of eight times the body limit for one live forward. + // `test_live_forward_work_never_passes_the_stream_limit` proves the + // count of live forwards never passes `HTTP2_BRIDGE_MAX_CONCURRENT_STREAMS`, + // so this multiplier bounds live forwards, not merely open streams. + expect( + HTTP2_BRIDGE_MAX_CONCURRENT_STREAMS * 8 * DEFAULT_SANDBOX_CALLBACK_BRIDGE_MAX_BODY_BYTES, + ).toBe(8_388_608); + }); + describe("parseCanonicalBridgeRequestPath", () => { it("parses an origin-form path with a query exactly one time", () => { const result = parseCanonicalBridgeRequestPath({ ":path": "/api/issues/abc?foo=bar" }); @@ -979,6 +1176,36 @@ describe("createHttp2BridgeServer + createSandboxHttp2BridgeGateway", () => { expect(stopped).toBe(true); }); + it("stops the channel when the read queue passes the lowered byte bound", async () => { + let dataListener: ((chunk: Uint8Array) => void) | undefined; + let stopped = false; + const channel: CommandManagedDuplexChannel = { + write: () => undefined, + onData: (listener) => { + dataListener = listener; + }, + onExit: () => undefined, + stop: () => { + stopped = true; + }, + close: async () => undefined, + }; + // No override: this test proves the default bound itself is the direct + // 524,288-byte value, no longer derived from `HTTP2_BRIDGE_MAX_SESSION_MEMORY`. + // No consumer ever attaches, so the readable side never drains. + expect(DEFAULT_HTTP2_BRIDGE_MAX_BUFFERED_READ_BYTES).toBe(524_288); + const duplex = wrapDuplexChannelAsNodeDuplex(channel); + const errored = new Promise((resolve) => duplex.on("error", resolve)); + + dataListener?.(Buffer.alloc(70_000, "a")); // passes the default high-water mark; still pushed directly. + dataListener?.(Buffer.alloc(500_000, "b")); // queues most of the 524,288-byte default cap. + dataListener?.(Buffer.alloc(60_000, "b")); // passes the default cap. + + const error = await errored; + expect(error.message).toMatch(/backpressure/i); + expect(stopped).toBe(true); + }); + it("wrapDuplexChannelAsNodeDuplex fails closed on one inbound chunk larger than the bounded read backpressure buffer, before the queue holds anything to compare it against", async () => { let dataListener: ((chunk: Uint8Array) => void) | undefined; let stopped = false; diff --git a/packages/adapter-utils/src/http2-bridge-server.ts b/packages/adapter-utils/src/http2-bridge-server.ts index a42afa1703..0d07fc5ae9 100644 --- a/packages/adapter-utils/src/http2-bridge-server.ts +++ b/packages/adapter-utils/src/http2-bridge-server.ts @@ -48,8 +48,29 @@ import { /** Server push. The transport never needs it. */ export const HTTP2_BRIDGE_ENABLE_PUSH = false; -/** Open streams. This matches the current broker limit ({@link DEFAULT_DUPLEX_BROKER_MAX_IN_FLIGHT_REQUESTS} in `duplex-bridge-broker.ts`). */ -export const HTTP2_BRIDGE_MAX_CONCURRENT_STREAMS = 64; +/** + * Open streams. The host keeps one forward, its request body, and its + * response body alive for the life of a stream, and — before this file binds + * each forward to its own stream's abort signal — a forward can outlive its + * stream's own HTTP/2 slot until the forward's own timeout runs out. Counting + * every retained `Buffer` and string copy of one stream's request and + * response body against the {@link DEFAULT_SANDBOX_CALLBACK_BRIDGE_MAX_BODY_BYTES} + * body limit (`sandbox-callback-bridge.ts`) gives an accounting peak of eight + * times that limit for one live forward. This bound is the per-route + * in-flight-body budget: `HTTP2_BRIDGE_MAX_CONCURRENT_STREAMS * 8 * + * DEFAULT_SANDBOX_CALLBACK_BRIDGE_MAX_BODY_BYTES` bytes = 4 * 8 * 262,144 + * bytes = 8,388,608 bytes for one route. + * + * Known aggregate behavior: this budget applies to one route only. The host + * process admits up to `DEFAULT_MAX_CONCURRENT_DUPLEX_ROUTES` (128, in + * `plugin-worker-manager.ts`) routes at the same time, and each route holds + * its own 8,388,608-byte peak. The process can therefore retain up to + * 1,073,741,824 bytes (1 GiB) of live body data across every route at once. + * This document accepts that ceiling: the host tracks no process-wide byte + * total, so no single route can starve another route's own budget, but the + * host also enforces no smaller sum across every route. + */ +export const HTTP2_BRIDGE_MAX_CONCURRENT_STREAMS = 4; /** One decompressed header list. The Node default is 65535. */ export const HTTP2_BRIDGE_MAX_HEADER_LIST_SIZE = 16384; /** The header-compression table. This keeps the Node default. */ @@ -105,8 +126,12 @@ export const HTTP2_BRIDGE_SERVER_OPTIONS: http2.ServerOptions = { * bound. This cap also bounds one single chunk: the wrapper checks a chunk's * own size against it before `push()` ever runs, so one oversized chunk * cannot cross the cap on its first delivery, before the queue holds - * anything to compare it against. */ -export const DEFAULT_HTTP2_BRIDGE_MAX_BUFFERED_READ_BYTES = HTTP2_BRIDGE_MAX_SESSION_MEMORY * 1024 * 1024; + * anything to compare it against. This is a fixed share of the fixed + * per-route byte budget the host bounds every duplex retention site + * against; it no longer derives from {@link HTTP2_BRIDGE_MAX_SESSION_MEMORY}, + * which bounds the underlying `Http2Session`'s own memory, not this + * read-side queue. */ +export const DEFAULT_HTTP2_BRIDGE_MAX_BUFFERED_READ_BYTES = 524_288; /** The default bound, in milliseconds, on how long the read-side queue * {@link wrapDuplexChannelAsNodeDuplex} holds can stay non-empty with no * chunk draining from it. The byte cap above bounds how much memory a stuck @@ -505,6 +530,14 @@ export interface Http2BridgeForwardRequest { query: string; headers: Record; body: Buffer; + /** + * The abort signal for this one HTTP/2 stream. `handleStream` aborts it + * when the stream closes, aborts, or errors, so a caller that passes it + * through to its own outbound call (a `fetch`, for example) ends that call + * at once instead of leaving it to run until its own timeout. The signal + * never fires for any other stream or for the session. + */ + signal: AbortSignal; } export type Http2BridgeForwardHandler = ( @@ -759,71 +792,94 @@ export function createHttp2BridgeServer(options: CreateHttp2BridgeServerOptions) stream: http2.ServerHttp2Stream, headers: http2.IncomingHttpHeaders, ): Promise { - // Accepted security fix 4: the constant-time bridge-token compare runs - // before route processing and before header processing. This host check - // is independent of the gateway's own token check on the sandbox side. - if (!compareBridgeTokensConstantTime(options.bridgeToken, readBridgeTokenHeader(headers))) { - denyRequest(stream, 401, { error: "Invalid bridge token." }, bodyBounds); - return; - } - - // Accepted security fix 3: parse `:path` exactly one time; both the route - // allowlist and the forward request below read this one result. - const parsedPath = parseCanonicalBridgeRequestPath(headers); - if (!parsedPath.ok) { - denyRequest(stream, 400, { error: `Invalid request path: ${parsedPath.reason}` }, bodyBounds); - return; - } - const method = normalizeStreamMethod(headers[":method"]); - - const denialReason = authorizeSandboxCallbackBridgeRequestWithRoutes( - { method, path: parsedPath.value.pathname }, - routes, - ); - if (denialReason) { - denyRequest(stream, 403, { error: denialReason }, bodyBounds); - return; - } - - const sanitizedHeaders = sanitizeSandboxCallbackBridgeHeaders( - toOutboundHeaderRecord(headers), - headerAllowlist, - ); - - let body: Buffer; + // One `AbortController` for this one stream. `forwardRequest` below + // receives its signal, so a stream that closes, aborts, or errors ends + // its own forward at once instead of leaving the forward to run until its + // own timeout. The abort stays local to this one stream: it never + // touches the session or any other stream's controller. + const controller = new AbortController(); + const abortForThisStream = (): void => { + if (!controller.signal.aborted) controller.abort(); + }; + stream.once("close", abortForThisStream); + stream.once("aborted", abortForThisStream); + stream.once("error", abortForThisStream); try { - body = await readHttp2StreamBody(stream, bodyBounds); - } catch (error) { - respondJson(stream, 413, { error: error instanceof Error ? error.message : String(error) }); - return; - } + // Accepted security fix 4: the constant-time bridge-token compare runs + // before route processing and before header processing. This host check + // is independent of the gateway's own token check on the sandbox side. + if (!compareBridgeTokensConstantTime(options.bridgeToken, readBridgeTokenHeader(headers))) { + denyRequest(stream, 401, { error: "Invalid bridge token." }, bodyBounds); + return; + } - let result: Http2BridgeForwardResult; - try { - result = await options.forwardRequest({ - method, - pathname: parsedPath.value.pathname, - query: parsedPath.value.query, - headers: sanitizedHeaders, - body, - }); - } catch (error) { - respondJson(stream, 502, { error: error instanceof Error ? error.message : String(error) }); - return; - } + // Accepted security fix 3: parse `:path` exactly one time; both the route + // allowlist and the forward request below read this one result. + const parsedPath = parseCanonicalBridgeRequestPath(headers); + if (!parsedPath.ok) { + denyRequest(stream, 400, { error: `Invalid request path: ${parsedPath.reason}` }, bodyBounds); + return; + } + const method = normalizeStreamMethod(headers[":method"]); - if (stream.destroyed || stream.closed) return; - const responseHeaders: http2.OutgoingHttpHeaders = { ":status": result.status }; - for (const [key, value] of Object.entries(result.headers ?? {})) { - if (key.toLowerCase() === "content-length") continue; - responseHeaders[key] = value; - } - try { - stream.respond(responseHeaders); - stream.end(result.body); - } catch { - // The peer reset the stream (RST_STREAM) between dispatch and response. - // One stream's write fault stays local to that stream. + const denialReason = authorizeSandboxCallbackBridgeRequestWithRoutes( + { method, path: parsedPath.value.pathname }, + routes, + ); + if (denialReason) { + denyRequest(stream, 403, { error: denialReason }, bodyBounds); + return; + } + + const sanitizedHeaders = sanitizeSandboxCallbackBridgeHeaders( + toOutboundHeaderRecord(headers), + headerAllowlist, + ); + + let body: Buffer; + try { + body = await readHttp2StreamBody(stream, bodyBounds); + } catch (error) { + respondJson(stream, 413, { error: error instanceof Error ? error.message : String(error) }); + return; + } + + let result: Http2BridgeForwardResult; + try { + result = await options.forwardRequest({ + method, + pathname: parsedPath.value.pathname, + query: parsedPath.value.query, + headers: sanitizedHeaders, + body, + signal: controller.signal, + }); + } catch (error) { + respondJson(stream, 502, { error: error instanceof Error ? error.message : String(error) }); + return; + } + + if (stream.destroyed || stream.closed) return; + const responseHeaders: http2.OutgoingHttpHeaders = { ":status": result.status }; + for (const [key, value] of Object.entries(result.headers ?? {})) { + if (key.toLowerCase() === "content-length") continue; + responseHeaders[key] = value; + } + try { + stream.respond(responseHeaders); + stream.end(result.body); + } catch { + // The peer reset the stream (RST_STREAM) between dispatch and response. + // One stream's write fault stays local to that stream. + } + } finally { + // Every path above reaches this exactly once: a deny, a body-read + // fault, a forward fault, or a completed response. A completed stream + // must leak no listener; the session, not this one stream, outlives + // the handler. + stream.removeListener("close", abortForThisStream); + stream.removeListener("aborted", abortForThisStream); + stream.removeListener("error", abortForThisStream); } } diff --git a/scripts/general-server-shard-durations.json b/scripts/general-server-shard-durations.json index 9f8ff9e4fe..8f8885be24 100644 --- a/scripts/general-server-shard-durations.json +++ b/scripts/general-server-shard-durations.json @@ -98,7 +98,6 @@ "server/src/__tests__/document-annotations-service.test.ts": 5743, "server/src/__tests__/documents-service.test.ts": 4497, "server/src/__tests__/documents.test.ts": 1104, - "server/src/__tests__/duplex-aggregate-ceiling-env.test.ts": 210, "server/src/__tests__/duplex-observability-recorder.test.ts": 207, "server/src/__tests__/effective-run-config-fingerprints.test.ts": 219, "server/src/__tests__/embedded-postgres-supervisor.test.ts": 206, @@ -267,9 +266,6 @@ "server/src/__tests__/plugin-tenant-isolation.test.ts": 4622, "server/src/__tests__/plugin-tool-dispatcher-pluginDbId.test.ts": 795, "server/src/__tests__/plugin-ui-static.test.ts": 1293, - "server/src/__tests__/plugin-worker-manager-duplex-byte-ledger.test.ts": 1240, - "server/src/__tests__/plugin-worker-manager-duplex-pending-write-ledger.test.ts": 1006, - "server/src/__tests__/plugin-worker-manager-duplex-stdin-write-ledger.test.ts": 3918, "server/src/__tests__/plugin-worker-manager-duplex.test.ts": 2233, "server/src/__tests__/plugin-worker-manager.test.ts": 2669, "server/src/__tests__/private-hostname-guard.test.ts": 300, diff --git a/server/src/__tests__/duplex-aggregate-ceiling-env.test.ts b/server/src/__tests__/duplex-aggregate-ceiling-env.test.ts deleted file mode 100644 index a6f1a50348..0000000000 --- a/server/src/__tests__/duplex-aggregate-ceiling-env.test.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; -import { DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES } from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger"; -import { resolveDuplexAggregateCeilingBytesFromEnv } from "../duplex-aggregate-ceiling-env.js"; - -describe("resolveDuplexAggregateCeilingBytesFromEnv", () => { - it("uses the safe default and reports nothing when the variable is absent", () => { - const onRejectedOverride = vi.fn(); - - const resolved = resolveDuplexAggregateCeilingBytesFromEnv(undefined, onRejectedOverride); - - expect(resolved).toBe(DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES); - expect(onRejectedOverride).not.toHaveBeenCalled(); - }); - - it("treats a whitespace-only value as a present invalid override, not as absent", () => { - const rawOverride = " "; - const onRejectedOverride = vi.fn(); - - // The resolver must continue with the safe default. Startup does not fail. - const resolved = resolveDuplexAggregateCeilingBytesFromEnv(rawOverride, onRejectedOverride); - - expect(resolved).toBe(DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES); - // The rejection reporter fires one time. A blank value is now visible as - // invalid instead of silent as absent. - expect(onRejectedOverride).toHaveBeenCalledTimes(1); - // The reporter receives only the parsed number. The raw whitespace string - // never reaches the reporter, so no operator-supplied text can reach a log - // line. - const reportedValue = onRejectedOverride.mock.calls[0]?.[0]; - expect(typeof reportedValue).toBe("number"); - expect(reportedValue).not.toBe(rawOverride); - expect(Number.isFinite(reportedValue)).toBe(true); - }); - - it("treats an empty-string value as a present invalid override", () => { - const onRejectedOverride = vi.fn(); - - const resolved = resolveDuplexAggregateCeilingBytesFromEnv("", onRejectedOverride); - - expect(resolved).toBe(DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES); - expect(onRejectedOverride).toHaveBeenCalledTimes(1); - }); - - it("rejects a present non-numeric value and uses the safe default", () => { - const onRejectedOverride = vi.fn(); - - const resolved = resolveDuplexAggregateCeilingBytesFromEnv("not-a-number", onRejectedOverride); - - expect(resolved).toBe(DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES); - expect(onRejectedOverride).toHaveBeenCalledTimes(1); - }); - - it("passes a valid override through unchanged", () => { - const onRejectedOverride = vi.fn(); - - const resolved = resolveDuplexAggregateCeilingBytesFromEnv("1048576", onRejectedOverride); - - expect(resolved).toBe(1048576); - expect(onRejectedOverride).not.toHaveBeenCalled(); - }); - - it("accepts a valid override that carries surrounding whitespace", () => { - const onRejectedOverride = vi.fn(); - - const resolved = resolveDuplexAggregateCeilingBytesFromEnv(" 1048576 ", onRejectedOverride); - - expect(resolved).toBe(1048576); - expect(onRejectedOverride).not.toHaveBeenCalled(); - }); -}); diff --git a/server/src/__tests__/environment-execution-target-duplex.test.ts b/server/src/__tests__/environment-execution-target-duplex.test.ts index bc45c9ea58..ba30d3f860 100644 --- a/server/src/__tests__/environment-execution-target-duplex.test.ts +++ b/server/src/__tests__/environment-execution-target-duplex.test.ts @@ -21,11 +21,6 @@ vi.mock("../services/plugin-environment-driver.js", async (importActual) => ({ })); import type { EffectiveExecutionCapabilities } from "@paperclipai/adapter-utils/execution-target"; -import { adapterExecutionTargetDuplexAggregateByteLedger } from "@paperclipai/adapter-utils/execution-target"; -import { - DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES, - DuplexAggregateByteLedger, -} from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger"; import { createSshCommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/ssh"; import type { Environment, EnvironmentLease } from "@paperclipai/shared"; import { resolveEnvironmentExecutionTarget } from "../services/environment-execution-target.js"; @@ -299,50 +294,6 @@ describe("EnvironmentRuntimeService.openDuplexChannel capability gate", () => { }); }); -describe("sandbox execution target aggregate byte ledger stamp", () => { - it("stamps the injected ledger on the target with object identity", async () => { - const ledger = new DuplexAggregateByteLedger({ - ceilingBytes: DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES, - }); - const target = await resolveEnvironmentExecutionTarget({ - db: {} as never, - companyId: "company-1", - adapterType: "codex_local", - environment: { id: "env-1", driver: "sandbox", config: { provider: "daytona" } }, - leaseId: "lease-1", - leaseMetadata: { remoteCwd: "/work" }, - lease: { id: "lease-1", leasePolicy: "reuse_by_environment" } as never, - duplexAggregateByteLedger: ledger, - }); - if (!target || target.kind !== "remote" || target.transport !== "sandbox") { - throw new Error("expected a sandbox execution target"); - } - // The stamped field and the accessor both return the one injected object, so - // one process-owned ledger reaches the host bridge seam. - expect(target.duplexAggregateByteLedger).toBe(ledger); - expect(adapterExecutionTargetDuplexAggregateByteLedger(target)).toBe(ledger); - }); - - it("leaves the ledger absent when the host injects none", async () => { - const target = await resolveEnvironmentExecutionTarget({ - db: {} as never, - companyId: "company-1", - adapterType: "codex_local", - environment: { id: "env-1", driver: "sandbox", config: { provider: "daytona" } }, - leaseId: "lease-1", - leaseMetadata: { remoteCwd: "/work" }, - lease: { id: "lease-1", leasePolicy: "reuse_by_environment" } as never, - }); - if (!target || target.kind !== "remote" || target.transport !== "sandbox") { - throw new Error("expected a sandbox execution target"); - } - // A run with no injected ledger keeps the seam null; the accessor never makes - // a fresh ledger. - expect(target.duplexAggregateByteLedger).toBeNull(); - expect(adapterExecutionTargetDuplexAggregateByteLedger(target)).toBeNull(); - }); -}); - // Build a sandbox execution target with a fixed capability snapshot and a fake // environment runtime whose openDuplexChannel is a spy. The helper returns the // runner and the spy so a test reads the capability-gated member. diff --git a/server/src/__tests__/plugin-worker-manager-duplex-byte-ledger.test.ts b/server/src/__tests__/plugin-worker-manager-duplex-byte-ledger.test.ts deleted file mode 100644 index 12db030a4c..0000000000 --- a/server/src/__tests__/plugin-worker-manager-duplex-byte-ledger.test.ts +++ /dev/null @@ -1,360 +0,0 @@ -import path from "node:path"; -import { fileURLToPath } from "node:url"; -import { describe, expect, it, vi } from "vitest"; -import type { PaperclipPluginManifestV1 } from "@paperclipai/shared"; -import { - DuplexAggregateByteLedger, - type DuplexAggregateByteLedgerTelemetry, -} from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger"; -import { - createDuplexRouteSlotController, - createPluginWorkerHandle, -} from "../services/plugin-worker-manager.js"; - -// This suite proves the plugin worker manager charges every retained duplex route -// representation against the injected aggregate byte ledger, and releases each -// token exactly once through the one cleanup path. The tests drive a real worker -// fixture process, so they exercise the true frame order across the bind boundary. - -const FIXTURES_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures"); -const DUPLEX_CHANNEL_WORKER_ENTRYPOINT = path.join( - FIXTURES_DIR, - "plugin-worker-duplex-channel.cjs", -); - -const TEST_MANIFEST: PaperclipPluginManifestV1 = { - id: "test.plugin", - apiVersion: 1, - version: "1.0.0", - displayName: "Test plugin", - description: "Test plugin", - author: "Paperclip", - categories: ["automation"], - capabilities: [], - entrypoints: { worker: "dist/worker.js" }, -}; - -function makeDuplexHandle(extra?: Record) { - return createPluginWorkerHandle("test.plugin", { - entrypointPath: DUPLEX_CHANNEL_WORKER_ENTRYPOINT, - manifest: TEST_MANIFEST, - config: {}, - instanceInfo: { instanceId: "instance-1", hostVersion: "1.0.0" }, - apiVersion: 1, - hostHandlers: {}, - ...extra, - }); -} - -// The test directive rides in `providerLeaseId`, an opaque field the manager -// forwards to the worker unchanged. -function duplexOpenInput(directive: unknown, companyId = "company-1") { - return { - driverKey: "daytona", - companyId, - environmentId: "env-1", - providerLeaseId: JSON.stringify(directive), - command: "bridge-callback", - }; -} - -// A counting telemetry surface. It records how many times the ledger rejected a -// reservation and how many accounting defects it saw, so a test asserts a -// fail-closed rejection and proves the cleanup never double-releases. -function countingTelemetry(): DuplexAggregateByteLedgerTelemetry & { - rejections: number; - underflows: number; -} { - const state = { - rejections: 0, - underflows: 0, - setBytesInUse() {}, - recordReservationRejection() { - state.rejections += 1; - }, - recordAccountingUnderflow() { - state.underflows += 1; - }, - }; - return state; -} - -describe("plugin worker manager duplex aggregate byte ledger", () => { - it("keeps terminalized buffered bytes charged until a late listener drains them, then a worker exit is harmless", async () => { - const telemetry = countingTelemetry(); - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1 << 20, telemetry }); - // Lower the per-chunk char bound, so the second chunk ends the route while the - // first chunk stays buffered. - const handle = makeDuplexHandle({ - duplexAggregateByteLedger: ledger, - duplexChannelLimits: { maxChunkChars: 4 }, - }); - try { - await handle.start(); - const session = await handle.openDuplexChannel( - // No listener attaches, so "ok" buffers and charges its two raw bytes. The - // "toolong" chunk passes the per-chunk bound, so the route terminalizes and - // moves the still-buffered "ok" token to the terminal registry. - duplexOpenInput({ data: [{ chunk: "ok" }, { chunk: "toolong" }] }), - ); - // The buffered "ok" stays charged after the route leaves the live map. - await vi.waitFor(() => { - expect(ledger.bytesInUse).toBe(2); - expect(ledger.liveTokenCount).toBe(1); - }); - // A late listener drains the terminal buffered record and releases its token. - const chunks: string[] = []; - // The session now streams raw `Uint8Array` chunks. Decode each one back - // to text, so the assertion below still compares the plain-text payload - // the fixture directive scripted. - session.onData((chunk) => chunks.push(new TextDecoder().decode(chunk))); - await vi.waitFor(() => { - expect(chunks).toEqual(["ok"]); - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - }); - // The over-bound chunk retained nothing, so no reservation ever rejected. - expect(telemetry.rejections).toBe(0); - } finally { - await handle.stop().catch(() => undefined); - } - // The worker exit sweep runs on stop. The ledger stays at zero, and the sweep - // records no accounting defect, so the drain and the exit never double-release. - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - expect(telemetry.underflows).toBe(0); - }); - - it("transfers pre-bind held bytes to the buffered representation across the bind, then releases them on drain", async () => { - const telemetry = countingTelemetry(); - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1 << 20, telemetry }); - const handle = makeDuplexHandle({ duplexAggregateByteLedger: ledger }); - try { - await handle.start(); - const session = await handle.openDuplexChannel( - // The worker writes the open reply and the two data frames in one stdout - // write, so both frames arrive before the route binds. The host holds them - // as pre-bind events, then the bind replays and transfers each token to the - // buffered representation. - duplexOpenInput({ batchWithOpenReply: true, data: [{ chunk: "aa" }, { chunk: "bb" }] }), - ); - // The two held events keep their exact reserved bytes across the transfer, so - // the live-token count never grows and the gauge never re-admits. - await vi.waitFor(() => { - expect(ledger.bytesInUse).toBe(4); - expect(ledger.liveTokenCount).toBe(2); - }); - const chunks: string[] = []; - // The session now streams raw `Uint8Array` chunks. Decode each one back - // to text, so the assertion below still compares the plain-text payload - // the fixture directive scripted. - session.onData((chunk) => chunks.push(new TextDecoder().decode(chunk))); - await vi.waitFor(() => { - expect(chunks).toEqual(["aa", "bb"]); - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - }); - } finally { - await handle.stop().catch(() => undefined); - } - expect(ledger.bytesInUse).toBe(0); - expect(telemetry.underflows).toBe(0); - }); - - it("fails closed and retains nothing when a write reservation would pass the ceiling", async () => { - const telemetry = countingTelemetry(); - // A four-byte ceiling. A serialized host-to-worker write frame cannot fit. - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 4, telemetry }); - const handle = makeDuplexHandle({ duplexAggregateByteLedger: ledger }); - try { - await handle.start(); - // Open with no scripted data, so the fixture sends only the open reply. - // The route is bound and live by the time this line resolves. - const session = await handle.openDuplexChannel(duplexOpenInput({})); - // The write happens strictly after the bind, so the rejection is a genuine - // post-bind event, never a frame that races the open reply. The host makes - // two reservations for this write. The one raw payload byte fits the - // ceiling, so the pending-write reservation succeeds. The host then meters - // the serialized frame just before the stdin write. That frame is much - // larger than four bytes, so the transport reservation rejects, the host - // writes nothing, and the route ends fail-closed. - session.write(new TextEncoder().encode("a")); - await vi.waitFor(() => { - expect(telemetry.rejections).toBeGreaterThanOrEqual(1); - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - }); - } finally { - await handle.stop().catch(() => undefined); - } - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - expect(telemetry.underflows).toBe(0); - }); -}); - -// A telemetry surface that also tracks the peak gauge value. A test asserts the -// peak never passes the ceiling, so it proves aggregate admission stops at the -// ceiling instead of overshooting it. -function peakTrackingTelemetry(): DuplexAggregateByteLedgerTelemetry & { - gauge: number; - peak: number; - rejections: number; - underflows: number; -} { - const state = { - gauge: 0, - peak: 0, - rejections: 0, - underflows: 0, - setBytesInUse(bytes: number) { - state.gauge = bytes; - if (bytes > state.peak) state.peak = bytes; - }, - recordReservationRejection() { - state.rejections += 1; - }, - recordAccountingUnderflow() { - state.underflows += 1; - }, - }; - return state; -} - -// The exact raw byte count of one buffered chunk each route retains. -const LOAD_CHUNK_BYTES = 8; -const LOAD_CHUNK = "x".repeat(LOAD_CHUNK_BYTES); - -describe("plugin worker manager duplex aggregate byte ledger load across workers", () => { - it("stops aggregate byte admission at the ceiling across many routes and workers while the route-count controller stays open", async () => { - const telemetry = peakTrackingTelemetry(); - // The ceiling holds exactly six eight-byte buffered chunks. - const fittingRoutes = 6; - const ceilingBytes = fittingRoutes * LOAD_CHUNK_BYTES; - const ledger = new DuplexAggregateByteLedger({ ceilingBytes, telemetry }); - // The route-count controller has ample room, so the byte ledger binds first. - const routeSlots = createDuplexRouteSlotController(100); - const handleCount = 3; - const routesPerHandle = 3; - const totalRoutes = handleCount * routesPerHandle; - const rejectedRoutes = totalRoutes - fittingRoutes; - const handles: Array> = []; - try { - for (let h = 0; h < handleCount; h += 1) { - const handle = makeDuplexHandle({ - duplexAggregateByteLedger: ledger, - duplexRouteSlots: routeSlots, - }); - await handle.start(); - handles.push(handle); - } - // Open every route in order. Each worker batches one data chunk with the open - // reply, so the host holds the chunk as a pre-bind event and reserves its exact - // raw bytes before the route binds. No listener attaches, so a bound route holds - // the chunk. The first six chunks fit the ceiling. Each later reservation passes - // the ceiling, so the ledger rejects it and the route fails closed with the open - // marker (not the route-busy marker), because the route-count controller has room. - let opened = 0; - let failedClosed = 0; - for (let h = 0; h < handleCount; h += 1) { - for (let r = 0; r < routesPerHandle; r += 1) { - try { - await handles[h].openDuplexChannel( - duplexOpenInput({ - workerSessionId: `ws-${h}-${r}`, - batchWithOpenReply: true, - data: [{ chunk: LOAD_CHUNK }], - }), - ); - opened += 1; - } catch (err) { - // A byte-ceiling rejection fails the route closed with the open marker. - // The route-count controller had room, so this is never the route-busy - // marker. - expect(String(err)).toContain("DUPLEX_CHANNEL_OPEN_FAILED"); - expect(String(err)).not.toContain("DUPLEX_CHANNEL_ROUTE_BUSY"); - failedClosed += 1; - } - } - } - // Six routes bound and hold their bytes; three failed closed at the ceiling. - expect(opened).toBe(fittingRoutes); - expect(failedClosed).toBe(rejectedRoutes); - expect(opened + failedClosed).toBe(totalRoutes); - // The aggregate gauge holds at the ceiling and the ledger recorded one rejection - // per over-ceiling route. - await vi.waitFor(() => { - expect(telemetry.rejections).toBe(rejectedRoutes); - expect(ledger.bytesInUse).toBe(ceilingBytes); - expect(ledger.liveTokenCount).toBe(fittingRoutes); - }); - // The gauge never passed the ceiling, so admission stopped at the ceiling and - // never overshot it. - expect(telemetry.peak).toBe(ceilingBytes); - } finally { - for (const handle of handles) await handle.stop().catch(() => undefined); - } - // Every worker exit sweep released its buffered tokens once, so the ledger ends - // at zero with no accounting defect. - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - expect(telemetry.underflows).toBe(0); - }); - - it("still bounds the route count for fixed per-route overhead when the byte ceiling has ample room", async () => { - const telemetry = peakTrackingTelemetry(); - // A large byte ceiling, so retained bytes never bind admission. - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1 << 20, telemetry }); - // The route-count controller holds two slots for the whole process. - const routeSlots = createDuplexRouteSlotController(2); - const handleA = makeDuplexHandle({ - duplexAggregateByteLedger: ledger, - duplexRouteSlots: routeSlots, - }); - const handleB = makeDuplexHandle({ - duplexAggregateByteLedger: ledger, - duplexRouteSlots: routeSlots, - }); - let opened = 0; - let busy = 0; - try { - await handleA.start(); - await handleB.start(); - // Attempt four routes across the two workers. The shared controller admits the - // first two and rejects the last two with the fixed route-busy error. - const attempts: Array<[ReturnType, string]> = [ - [handleA, "ws-a1"], - [handleA, "ws-a2"], - [handleB, "ws-b1"], - [handleB, "ws-b2"], - ]; - for (const [handle, ws] of attempts) { - try { - await handle.openDuplexChannel( - duplexOpenInput({ workerSessionId: ws, data: [{ chunk: "aa" }] }), - ); - opened += 1; - } catch (err) { - expect(String(err)).toContain("DUPLEX_CHANNEL_ROUTE_BUSY"); - busy += 1; - } - } - // The route-count controller admitted two routes and rejected the rest, so it - // still bounds the fixed per-route overhead on its own. - expect(opened).toBe(2); - expect(busy).toBe(2); - // The two open routes buffered four raw bytes, far under the byte ceiling, and - // the byte ledger rejected nothing. - await vi.waitFor(() => { - expect(ledger.bytesInUse).toBe(4); - expect(ledger.liveTokenCount).toBe(2); - }); - expect(telemetry.rejections).toBe(0); - } finally { - await handleA.stop().catch(() => undefined); - await handleB.stop().catch(() => undefined); - } - expect(ledger.bytesInUse).toBe(0); - expect(telemetry.underflows).toBe(0); - }); -}); diff --git a/server/src/__tests__/plugin-worker-manager-duplex-pending-write-ledger.test.ts b/server/src/__tests__/plugin-worker-manager-duplex-pending-write-ledger.test.ts deleted file mode 100644 index 983925cbce..0000000000 --- a/server/src/__tests__/plugin-worker-manager-duplex-pending-write-ledger.test.ts +++ /dev/null @@ -1,252 +0,0 @@ -import path from "node:path"; -import { fileURLToPath } from "node:url"; -import { describe, expect, it, vi } from "vitest"; -import type { PaperclipPluginManifestV1 } from "@paperclipai/shared"; -import { - DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, - DuplexAggregateByteLedger, - type DuplexAggregateByteLedgerTelemetry, -} from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger"; - -// Mock the shared logger, so a test reads the fixed rejection marker the manager -// logs when a pending-write reservation fails. The child logger returns the same -// object, so `log.warn` is this mock's `warn`. -vi.mock("../middleware/logger.js", () => { - const mockLogger: Record = { - trace: vi.fn(), - debug: vi.fn(), - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - fatal: vi.fn(), - child: vi.fn(() => mockLogger), - }; - return { logger: mockLogger, httpLogger: vi.fn() }; -}); - -import { logger } from "../middleware/logger.js"; -import { createPluginWorkerHandle } from "../services/plugin-worker-manager.js"; - -// This suite proves the plugin worker manager charges every host→worker write raw -// payload against the injected aggregate byte ledger under the `pending_write` -// owner, and releases each token one time when the write RPC settles. The child -// reads its stdin here, so the separate `stdin_write` transport token flushes and -// releases at once. Each test waits for that flush, so the assertions isolate the -// raw-payload token. The transport token has its own suite. - -const FIXTURES_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures"); -const DUPLEX_CHANNEL_WORKER_ENTRYPOINT = path.join( - FIXTURES_DIR, - "plugin-worker-duplex-channel.cjs", -); - -const TEST_MANIFEST: PaperclipPluginManifestV1 = { - id: "test.plugin", - apiVersion: 1, - version: "1.0.0", - displayName: "Test plugin", - description: "Test plugin", - author: "Paperclip", - categories: ["automation"], - capabilities: [], - entrypoints: { worker: "dist/worker.js" }, -}; - -function makeDuplexHandle(extra?: Record) { - return createPluginWorkerHandle("test.plugin", { - entrypointPath: DUPLEX_CHANNEL_WORKER_ENTRYPOINT, - manifest: TEST_MANIFEST, - config: {}, - instanceInfo: { instanceId: "instance-1", hostVersion: "1.0.0" }, - apiVersion: 1, - hostHandlers: {}, - ...extra, - }); -} - -// The test directive rides in `providerLeaseId`, an opaque field the manager -// forwards to the worker unchanged. -function duplexOpenInput(directive: unknown, companyId = "company-1") { - return { - driverKey: "daytona", - companyId, - environmentId: "env-1", - providerLeaseId: JSON.stringify(directive), - command: "bridge-callback", - }; -} - -// A telemetry surface that tracks the peak gauge value and the two counters. A -// test asserts the peak never passes the ceiling, so it proves aggregate admission -// stops at the ceiling instead of overshooting it. -function peakTrackingTelemetry(): DuplexAggregateByteLedgerTelemetry & { - gauge: number; - peak: number; - rejections: number; - underflows: number; -} { - const state = { - gauge: 0, - peak: 0, - rejections: 0, - underflows: 0, - setBytesInUse(bytes: number) { - state.gauge = bytes; - if (bytes > state.peak) state.peak = bytes; - }, - recordReservationRejection() { - state.rejections += 1; - }, - recordAccountingUnderflow() { - state.underflows += 1; - }, - }; - return state; -} - -// Return every reason string the manager logged through the mock `warn` sink. -function loggedWarnReasons(): string[] { - const calls = (logger.warn as unknown as { mock: { calls: unknown[][] } }).mock.calls; - return calls - .map((call) => { - const first = call[0]; - return first && typeof first === "object" - ? (first as { reason?: unknown }).reason - : undefined; - }) - .filter((reason): reason is string => typeof reason === "string"); -} - -describe("plugin worker manager duplex pending-write byte ledger", () => { - it("charges each held raw payload and returns to zero after worker exit", async () => { - const telemetry = peakTrackingTelemetry(); - // The ceiling has ample room, so a transient transport reservation never - // rejects. Each write holds its raw payload until the RPC settles. - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1 << 20, telemetry }); - const handle = makeDuplexHandle({ duplexAggregateByteLedger: ledger }); - const writeBytes = 1000; - const writeData = "x".repeat(writeBytes); - try { - await handle.start(); - // The worker reads its stdin but never replies to a write, so each write RPC - // stays pending and holds its raw payload. - const route = await handle.openDuplexChannel( - duplexOpenInput({ workerSessionId: "ws-a", mode: "no-write-reply" }), - ); - const writeBytesEncoded = new TextEncoder().encode(writeData); - route.write(writeBytesEncoded); - route.write(writeBytesEncoded); - route.write(writeBytesEncoded); - // The worker reads its stdin, so each transport token flushes and releases. - // Only the three held raw payloads remain, so the gauge settles at three - // times the payload byte count with three live tokens. - await vi.waitFor(() => { - expect(ledger.liveTokenCount).toBe(3); - expect(ledger.bytesInUse).toBe(3 * writeBytes); - }); - expect(telemetry.peak).toBeLessThanOrEqual(ledger.ceilingBytes); - } finally { - await handle.stop().catch(() => undefined); - } - // The worker exit settles each pending write, so each token releases one time - // and the ledger ends at zero with no accounting defect. - await vi.waitFor(() => { - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - }); - expect(telemetry.underflows).toBe(0); - }); - - it("releases the raw-payload token after a delayed write RPC settles, with no worker exit", async () => { - const telemetry = peakTrackingTelemetry(); - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1 << 20, telemetry }); - const handle = makeDuplexHandle({ duplexAggregateByteLedger: ledger }); - try { - await handle.start(); - // The worker reads its stdin and delays its write reply, so the host holds the - // raw-payload reservation for a measurable time before the RPC settles. - const route = await handle.openDuplexChannel( - duplexOpenInput({ workerSessionId: "ws-d", writeReplyDelayMs: 150 }), - ); - const writeBytes = 1000; - route.write(new TextEncoder().encode("x".repeat(writeBytes))); - // The transport token flushes at once, so one raw-payload token stays held for - // the full write byte count while the RPC is in flight. - await vi.waitFor(() => { - expect(ledger.liveTokenCount).toBe(1); - expect(ledger.bytesInUse).toBe(writeBytes); - }); - // The delayed reply settles the RPC, and the `finally` releases the token. - await vi.waitFor(() => { - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - }); - } finally { - await handle.stop().catch(() => undefined); - } - expect(ledger.bytesInUse).toBe(0); - expect(telemetry.underflows).toBe(0); - }); - - it("reserves the exact UTF-8 byte count of the raw payload, not the character length", async () => { - const telemetry = peakTrackingTelemetry(); - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1 << 20, telemetry }); - const handle = makeDuplexHandle({ duplexAggregateByteLedger: ledger }); - try { - await handle.start(); - const route = await handle.openDuplexChannel( - duplexOpenInput({ workerSessionId: "ws-u", mode: "no-write-reply" }), - ); - // "a€" is two characters but four UTF-8 bytes (one plus three). The raw-payload - // reservation must charge four bytes, so it uses the UTF-8 byte count. - const data = "a€"; - expect(data.length).toBe(2); - expect(Buffer.byteLength(data, "utf8")).toBe(4); - route.write(new TextEncoder().encode(data)); - // The transport token flushes, so one raw-payload token of four bytes remains. - await vi.waitFor(() => { - expect(ledger.liveTokenCount).toBe(1); - expect(ledger.bytesInUse).toBe(4); - }); - } finally { - await handle.stop().catch(() => undefined); - } - await vi.waitFor(() => { - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - }); - expect(telemetry.underflows).toBe(0); - }); - - it("ends an over-ceiling single write fail-closed with the aggregate marker, not the route-busy marker", async () => { - vi.mocked(logger.warn).mockClear(); - const telemetry = peakTrackingTelemetry(); - // The ceiling is smaller than one raw payload, so the raw-payload reservation - // fails before the frame reaches the transport. The manager retains nothing. - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 100, telemetry }); - const handle = makeDuplexHandle({ duplexAggregateByteLedger: ledger }); - try { - await handle.start(); - const route = await handle.openDuplexChannel( - duplexOpenInput({ workerSessionId: "ws-o", mode: "no-write-reply" }), - ); - // The single write is larger than the ceiling. The raw-payload reservation - // fails, so the manager ends the route with the aggregate marker, never the - // route-busy marker, and never writes the frame. - route.write(new TextEncoder().encode("x".repeat(200))); - expect(telemetry.rejections).toBeGreaterThanOrEqual(1); - expect(ledger.bytesInUse).toBe(0); - expect(telemetry.peak).toBeLessThanOrEqual(ledger.ceilingBytes); - const reasons = loggedWarnReasons(); - expect(reasons).toContain(DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED); - expect(reasons).not.toContain("DUPLEX_CHANNEL_ROUTE_BUSY"); - } finally { - await handle.stop().catch(() => undefined); - } - await vi.waitFor(() => { - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - }); - expect(telemetry.underflows).toBe(0); - }); -}); diff --git a/server/src/__tests__/plugin-worker-manager-duplex-stdin-write-ledger.test.ts b/server/src/__tests__/plugin-worker-manager-duplex-stdin-write-ledger.test.ts deleted file mode 100644 index d20d38da32..0000000000 --- a/server/src/__tests__/plugin-worker-manager-duplex-stdin-write-ledger.test.ts +++ /dev/null @@ -1,300 +0,0 @@ -import path from "node:path"; -import { fileURLToPath } from "node:url"; -import { describe, expect, it, vi } from "vitest"; -import type { PaperclipPluginManifestV1 } from "@paperclipai/shared"; -import { - DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, - DuplexAggregateByteLedger, - type DuplexAggregateByteLedgerTelemetry, -} from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger"; - -// Mock the shared logger, so a test reads the fixed rejection marker the manager -// logs when a transport reservation fails. The child logger returns the same -// object, so `log.warn` is this mock's `warn`. -vi.mock("../middleware/logger.js", () => { - const mockLogger: Record = { - trace: vi.fn(), - debug: vi.fn(), - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - fatal: vi.fn(), - child: vi.fn(() => mockLogger), - }; - return { logger: mockLogger, httpLogger: vi.fn() }; -}); - -import { logger } from "../middleware/logger.js"; -import { createPluginWorkerHandle } from "../services/plugin-worker-manager.js"; - -// This suite proves the plugin worker manager charges the child-stdin transport -// buffer for every host→worker duplex write, under the `stdin_write` owner. The -// worker completes the open, then stops reading its stdin. The host write then -// stays in the host stdin write buffer. The tests prove the reservation covers the -// serialized frame, holds until the flush or the worker exit, survives the write -// RPC timeout and the route terminalization, and bounds the aggregate transport -// bytes across several routes and workers. - -const FIXTURES_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures"); -const DUPLEX_CHANNEL_WORKER_ENTRYPOINT = path.join( - FIXTURES_DIR, - "plugin-worker-duplex-channel.cjs", -); - -const TEST_MANIFEST: PaperclipPluginManifestV1 = { - id: "test.plugin", - apiVersion: 1, - version: "1.0.0", - displayName: "Test plugin", - description: "Test plugin", - author: "Paperclip", - categories: ["automation"], - capabilities: [], - entrypoints: { worker: "dist/worker.js" }, -}; - -function makeDuplexHandle(extra?: Record) { - return createPluginWorkerHandle("test.plugin", { - entrypointPath: DUPLEX_CHANNEL_WORKER_ENTRYPOINT, - manifest: TEST_MANIFEST, - config: {}, - instanceInfo: { instanceId: "instance-1", hostVersion: "1.0.0" }, - apiVersion: 1, - hostHandlers: {}, - ...extra, - }); -} - -// The test directive rides in `providerLeaseId`, an opaque field the manager -// forwards to the worker unchanged. -function duplexOpenInput(directive: unknown, companyId = "company-1") { - return { - driverKey: "daytona", - companyId, - environmentId: "env-1", - providerLeaseId: JSON.stringify(directive), - command: "bridge-callback", - }; -} - -// A telemetry surface that tracks the peak gauge value and the two counters. A -// test asserts the peak never passes the ceiling, so it proves aggregate admission -// stops at the ceiling instead of overshooting it. -function peakTrackingTelemetry(): DuplexAggregateByteLedgerTelemetry & { - gauge: number; - peak: number; - rejections: number; - underflows: number; -} { - const state = { - gauge: 0, - peak: 0, - rejections: 0, - underflows: 0, - setBytesInUse(bytes: number) { - state.gauge = bytes; - if (bytes > state.peak) state.peak = bytes; - }, - recordReservationRejection() { - state.rejections += 1; - }, - recordAccountingUnderflow() { - state.underflows += 1; - }, - }; - return state; -} - -// Return every reason string the manager logged through the mock `warn` sink. -function loggedWarnReasons(): string[] { - const calls = (logger.warn as unknown as { mock: { calls: unknown[][] } }).mock.calls; - return calls - .map((call) => { - const first = call[0]; - return first && typeof first === "object" - ? (first as { reason?: unknown }).reason - : undefined; - }) - .filter((reason): reason is string => typeof reason === "string"); -} - -// A frame the host keeps in its stdin write buffer must pass the operating-system -// pipe buffer (about 64 KiB), so the write callback stays pending while the child -// does not read. Each write here is larger than that, so the host holds the -// transport token until the worker exit. -const HELD_WRITE_CHARS = 200_000; - -describe("plugin worker manager duplex stdin transport byte ledger", () => { - it("reserves the encoded serialized-frame size, larger than the raw payload, including the base64 wire inflation", async () => { - const telemetry = peakTrackingTelemetry(); - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 8 << 20, telemetry }); - const handle = makeDuplexHandle({ duplexAggregateByteLedger: ledger }); - try { - await handle.start(); - const route = await handle.openDuplexChannel( - duplexOpenInput({ - workerSessionId: "ws-a", - stopReadingStdinAfterOpen: true, - exitAfterStopMs: 500, - }), - ); - // The JSON-RPC hop carries the write payload as a base64 string, because JSON - // has no binary type (see `ChannelBytesWireValue` in protocol.ts). Base64 - // encodes three raw bytes as four characters, so the serialized frame is - // larger than the raw payload by about that ratio, plus the small fixed cost - // of the surrounding JSON envelope (the method name and the route identifiers). - const rawBytes = 50_000; - const data = new Uint8Array(rawBytes).fill(0x22); // an arbitrary byte value - const encodedLength = Buffer.from(data).toString("base64").length; - route.write(data); - // The raw-payload token and the transport token both hold at once. The worker - // does not read its stdin, so the transport token never flushes. - expect(ledger.liveTokenCount).toBe(2); - const transportBytes = ledger.bytesInUse - rawBytes; - // The transport reservation covers the serialized frame: at least the base64 - // form of the raw payload, plus a bounded JSON envelope around it. - expect(transportBytes).toBeGreaterThan(rawBytes); - expect(transportBytes).toBeGreaterThanOrEqual(encodedLength); - expect(transportBytes).toBeLessThan(encodedLength + 5_000); - } finally { - await handle.stop().catch(() => undefined); - } - // The worker exit discards the stdin buffer, so every token releases one time. - await vi.waitFor(() => { - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - }); - expect(telemetry.underflows).toBe(0); - }); - - it("holds the transport token across a write-RPC timeout and releases it on worker exit", async () => { - const telemetry = peakTrackingTelemetry(); - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 8 << 20, telemetry }); - // The write RPC times out fast; the worker exits later. The gap lets a test - // prove the timeout does not release the transport token. - const handle = makeDuplexHandle({ - duplexAggregateByteLedger: ledger, - duplexChannelLimits: { openTimeoutMs: 150 }, - }); - try { - await handle.start(); - const route = await handle.openDuplexChannel( - duplexOpenInput({ - workerSessionId: "ws-t", - stopReadingStdinAfterOpen: true, - exitAfterStopMs: 900, - }), - ); - const data = "x".repeat(HELD_WRITE_CHARS); - const rawBytes = Buffer.byteLength(data, "utf8"); - route.write(new TextEncoder().encode(data)); - // Both tokens hold at first: the raw payload and the serialized frame. - expect(ledger.liveTokenCount).toBe(2); - const transportBytes = ledger.bytesInUse - rawBytes; - expect(transportBytes).toBeGreaterThan(0); - // The write RPC times out. The RPC settle releases the raw-payload token, so - // one token remains: the transport token. The timeout never releases it. - await vi.waitFor(() => { - expect(ledger.liveTokenCount).toBe(1); - }); - expect(ledger.bytesInUse).toBe(transportBytes); - } finally { - await handle.stop().catch(() => undefined); - } - // The worker exit releases the still-held transport token. - await vi.waitFor(() => { - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - }); - expect(telemetry.underflows).toBe(0); - }); - - it("does not release the transport token when the route terminalizes", async () => { - const telemetry = peakTrackingTelemetry(); - const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 8 << 20, telemetry }); - const handle = makeDuplexHandle({ duplexAggregateByteLedger: ledger }); - try { - await handle.start(); - const route = await handle.openDuplexChannel( - duplexOpenInput({ - workerSessionId: "ws-c", - stopReadingStdinAfterOpen: true, - exitAfterStopMs: 700, - }), - ); - const data = "x".repeat(HELD_WRITE_CHARS); - const rawBytes = Buffer.byteLength(data, "utf8"); - route.write(new TextEncoder().encode(data)); - expect(ledger.liveTokenCount).toBe(2); - const transportBytes = ledger.bytesInUse - rawBytes; - // End the route. Route terminalization releases the route-owned tokens, but it - // must never release the transport token, so at least the serialized frame - // stays charged. - void route.close(); - expect(ledger.bytesInUse).toBeGreaterThanOrEqual(transportBytes); - } finally { - await handle.stop().catch(() => undefined); - } - await vi.waitFor(() => { - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - }); - expect(telemetry.underflows).toBe(0); - }); - - it("bounds the aggregate transport bytes across several routes and workers, ending an over-ceiling near-limit write with the aggregate marker", async () => { - vi.mocked(logger.warn).mockClear(); - const telemetry = peakTrackingTelemetry(); - // The ceiling admits only a couple of held serialized frames, so a near-limit - // write across the routes passes it and rejects. - const ceilingBytes = 600_000; - const ledger = new DuplexAggregateByteLedger({ ceilingBytes, telemetry }); - const handleA = makeDuplexHandle({ duplexAggregateByteLedger: ledger }); - const handleB = makeDuplexHandle({ duplexAggregateByteLedger: ledger }); - try { - await handleA.start(); - await handleB.start(); - const routeA = await handleA.openDuplexChannel( - duplexOpenInput({ - workerSessionId: "ws-a", - stopReadingStdinAfterOpen: true, - exitAfterStopMs: 900, - }), - ); - const routeB = await handleB.openDuplexChannel( - duplexOpenInput({ - workerSessionId: "ws-b", - stopReadingStdinAfterOpen: true, - exitAfterStopMs: 900, - }), - ); - // Each write is near the per-write size. The JSON-RPC hop carries it as - // base64, so the reservation must count the encoded (larger) wire size, not - // the raw payload size. - const nearLimit = new Uint8Array(100_000).fill(0x22); - const routes = [routeA, routeB, routeA, routeB]; - for (const route of routes) { - route.write(nearLimit); - // The aggregate bytes never pass the ceiling. Admission stops at the ceiling. - expect(ledger.bytesInUse).toBeLessThanOrEqual(ceilingBytes); - } - // At least one near-limit write passed the ceiling and rejected. The manager - // ended that route with the aggregate marker, never the route-busy marker. - expect(telemetry.rejections).toBeGreaterThanOrEqual(1); - expect(telemetry.peak).toBeLessThanOrEqual(ceilingBytes); - const reasons = loggedWarnReasons(); - expect(reasons).toContain(DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED); - expect(reasons).not.toContain("DUPLEX_CHANNEL_ROUTE_BUSY"); - } finally { - await handleA.stop().catch(() => undefined); - await handleB.stop().catch(() => undefined); - } - // Every worker exit releases its held transport tokens, so the ledger ends at - // zero with no accounting defect. - await vi.waitFor(() => { - expect(ledger.bytesInUse).toBe(0); - expect(ledger.liveTokenCount).toBe(0); - }); - expect(telemetry.underflows).toBe(0); - }); -}); diff --git a/server/src/__tests__/plugin-worker-manager-duplex.test.ts b/server/src/__tests__/plugin-worker-manager-duplex.test.ts index 40d9c3f552..bf7d01ab57 100644 --- a/server/src/__tests__/plugin-worker-manager-duplex.test.ts +++ b/server/src/__tests__/plugin-worker-manager-duplex.test.ts @@ -423,6 +423,58 @@ describe("plugin worker manager duplex channel route", () => { } }); + it("ends the route when the pending host-to-worker write bytes pass the route bound", async () => { + const handle = makeDuplexHandle({ + duplexChannelLimits: { maxPendingWriteBytes: 10 }, + }); + try { + await handle.start(); + const session = await handle.openDuplexChannel( + duplexOpenInput({ mode: "no-write-reply", workerSessionId: "ws-A" }), + ); + const waitResult = session.wait(); + // The worker never replies to a write, so each write's bytes stay charged + // against the route. The second write brings the cumulative bytes past + // the 10-byte bound and ends the route. + session.write(new TextEncoder().encode("aaaaa")); // 5 bytes → 5, under the bound + session.write(new TextEncoder().encode("bbbbbb")); // 6 bytes → 11 > 10, ends the route + await expect(waitResult).resolves.toEqual({ exitCode: null }); + } finally { + await handle.stop().catch(() => undefined); + } + }); + + it("returns the pending write bytes to the route bound after a failed write", async () => { + const handle = makeDuplexHandle({ + duplexChannelLimits: { maxPendingWriteBytes: 10, openTimeoutMs: 100 }, + }); + try { + await handle.start(); + const session = await handle.openDuplexChannel( + duplexOpenInput({ mode: "no-write-reply", workerSessionId: "ws-A" }), + ); + let routeEnded = false; + session.wait().then(() => { + routeEnded = true; + }); + // The worker never replies, so each write's own request times out and + // rejects. The rejection must release this write's charged bytes, the + // same as a reply would. Wait past the first write's timeout before the + // second write sends. + session.write(new TextEncoder().encode("12345678")); // 8 bytes, under the 10-byte bound + await new Promise((resolve) => setTimeout(resolve, 200)); + // If the first write's bytes had not released on its timeout, this + // second 8-byte write would bring the route to 16 bytes, past the + // 10-byte bound, and end the route at once, synchronously, in this call. + session.write(new TextEncoder().encode("87654321")); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(routeEnded).toBe(false); + await session.close(); + } finally { + await handle.stop().catch(() => undefined); + } + }); + it("ends the route when one host-to-worker write passes the size bound", async () => { const handle = makeDuplexHandle({ duplexChannelLimits: { maxWriteChars: 8 }, @@ -701,6 +753,36 @@ describe("plugin worker manager duplex channel route", () => { } }); + it("ends the route when the pre-bind hold bytes pass the route input bound", async () => { + const handle = makeDuplexHandle({ + duplexChannelLimits: { maxPreBindBufferedChars: 10 }, + }); + try { + await handle.start(); + // The worker batches the three data frames with the open reply, so all + // three frames arrive before the route binds and land in the pre-bind + // hold, not the post-bind buffered queue. The frame-count bound stays + // far above three frames, so only the byte bound can end the route + // here: this proves the hold itself counts bytes, not only frames. The + // hold ends the route before the bind completes, so the open call + // itself fails, the same way a malformed open reply fails it. + await expect( + handle.openDuplexChannel( + duplexOpenInput({ + batchWithOpenReply: true, + data: [ + { chunk: "aaaaa" }, // total 5 → held + { chunk: "bbbbb" }, // total 10 → held + { chunk: "ccccc" }, // total 15 > 10 → end the route in the hold + ], + }), + ), + ).rejects.toThrow("DUPLEX_CHANNEL_OPEN_FAILED"); + } finally { + await handle.stop().catch(() => undefined); + } + }); + it("ends the route on the frame count bound even when a caller raises that bound well past the module default", async () => { // Regression test: the pre-open hold ceiling must track // maxDuplexChannelPreBindFrames, not a fixed value. A fixed ceiling at or diff --git a/server/src/duplex-aggregate-ceiling-env.ts b/server/src/duplex-aggregate-ceiling-env.ts deleted file mode 100644 index a1381c910c..0000000000 --- a/server/src/duplex-aggregate-ceiling-env.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { - resolveDuplexAggregateCeilingBytes, - type DuplexAggregateCeilingOverrideRejectionReporter, -} from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger"; - -/** - * Resolve the aggregate duplex route byte ceiling from the raw operator override - * string. The host reads the override from - * PAPERCLIP_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES. - * - * This helper distinguishes an absent variable from a present blank value. An - * absent variable is `undefined`. The helper then uses the documented default and - * does not report a rejection. A present variable is a string, even when the - * string holds only whitespace. The helper sends every present string through - * `Number`, so a blank or whitespace-only value becomes `0`. The numeric - * validation in {@link resolveDuplexAggregateCeilingBytes} rejects `0`, reports it - * through `onRejectedOverride`, and returns the safe default. So a present blank - * value is visible as invalid instead of silent as absent. - * - * `Number` ignores surrounding whitespace, so a valid value with surrounding - * spaces still parses. The reporter receives only the parsed number, never the raw - * string, so no operator-supplied text reaches a log line. - */ -export function resolveDuplexAggregateCeilingBytesFromEnv( - rawOverride: string | undefined, - onRejectedOverride?: DuplexAggregateCeilingOverrideRejectionReporter, -): number { - return resolveDuplexAggregateCeilingBytes( - rawOverride === undefined ? undefined : Number(rawOverride), - onRejectedOverride, - ); -} diff --git a/server/src/index.ts b/server/src/index.ts index e668adf7a8..9f0dd8e3da 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -87,13 +87,6 @@ import { createFeedbackTraceShareClientFromConfig } from "./services/feedback-sh import { buildRuntimeApiCandidateUrls, choosePrimaryRuntimeApiUrl } from "./runtime-api.js"; import { isLoopbackHost, rewriteLoopbackUrlPort } from "./url-utils.js"; import { createPluginWorkerManager } from "./services/plugin-worker-manager.js"; -import { - createDuplexAggregateByteLedgerTelemetry, - DuplexAggregateByteLedger, - type DuplexAggregateByteLedgerMetricSink, -} from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger"; -import { resolveDuplexAggregateCeilingBytesFromEnv } from "./duplex-aggregate-ceiling-env.js"; -import { DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL } from "@paperclipai/adapter-utils/duplex-observability"; import { createStorageServiceFromConfig } from "./storage/index.js"; import { printStartupBanner } from "./startup-banner.js"; import { getBoardClaimWarningUrl, initializeBoardClaimChallenge } from "./board-claim.js"; @@ -784,57 +777,9 @@ export async function startServer(): Promise { databaseBackupInFlight = false; } }; - // The process-owned aggregate byte ledger for the sandbox duplex channel. One - // ledger per host process bounds the aggregate bytes that all live duplex routes - // retain. The route-count controller bounds only the route count, so without this - // ledger the per-route byte bounds multiply to many gigabytes at the maximum - // route count. The manager injects this same object into every worker handle, so - // one shared gauge bounds every host-side retention site. - // - // The optional operator override reads PAPERCLIP_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES. - // An absent variable uses the documented default. A present invalid, blank, - // whitespace-only, non-finite, zero, negative, non-integer, unsafe, or - // over-maximum value does not fail startup. The host process is multi-tenant, so - // one invalid environment value must not brick the whole host. The helper sends - // the raw string to the resolver, so a present blank value is invalid, not - // absent. The resolver rejects the invalid override and returns the safe default. - // The reporter logs the rejection loudly at error, so the misconfiguration stays - // visible while the host stays up. The log line carries only the rejected numeric - // value, never the raw string. - const duplexAggregateCeilingBytes = resolveDuplexAggregateCeilingBytesFromEnv( - process.env.PAPERCLIP_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES, - (rejectedValue) => { - logger.error( - { rejectedValue }, - "duplex aggregate byte ceiling override rejected; using the safe default", - ); - }, - ); - // The server has no process metric pipeline yet, so the ledger telemetry maps to - // the structured logger. The gauge logs at debug. A reservation rejection logs at - // warn, because it marks an availability limit hit. An accounting-underflow defect - // logs at error, because it marks a real cleanup bug. Each record carries only the - // fixed metric name and the numeric value; no route, company, run, or payload - // value reaches a log line. - const duplexAggregateByteLedgerMetricSink: DuplexAggregateByteLedgerMetricSink = { - setGauge(name, value) { - logger.debug({ metric: name, value }, "duplex aggregate byte ledger gauge"); - }, - incrementCounter(name) { - if (name === DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL) { - logger.error({ metric: name }, "duplex aggregate byte ledger accounting underflow"); - return; - } - logger.warn({ metric: name }, "duplex aggregate byte ledger reservation rejected"); - }, - }; - const duplexAggregateByteLedger = new DuplexAggregateByteLedger({ - ceilingBytes: duplexAggregateCeilingBytes, - telemetry: createDuplexAggregateByteLedgerTelemetry(duplexAggregateByteLedgerMetricSink), - }); - const pluginWorkerManager = createPluginWorkerManager({ duplexAggregateByteLedger }); + const pluginWorkerManager = createPluginWorkerManager(); const heartbeat = config.heartbeatSchedulerEnabled - ? heartbeatService(db as any, { pluginWorkerManager, duplexAggregateByteLedger }) + ? heartbeatService(db as any, { pluginWorkerManager }) : null; const decisionServiceOptions = { wakeOriginAgent: createDecisionWakeOriginAgent(heartbeat?.wakeup ?? null), diff --git a/server/src/services/environment-execution-target.ts b/server/src/services/environment-execution-target.ts index 5ad4642b1b..04c9f66435 100644 --- a/server/src/services/environment-execution-target.ts +++ b/server/src/services/environment-execution-target.ts @@ -6,7 +6,6 @@ import { type AdapterExecutionTarget, } from "@paperclipai/adapter-utils/execution-target"; import type { DuplexObservabilityRecorder } from "@paperclipai/adapter-utils/duplex-observability"; -import type { DuplexAggregateByteLedger } from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger"; import { clampSpanLabel, getActiveStepContext, @@ -208,10 +207,6 @@ export async function resolveEnvironmentExecutionTarget(input: { // enters the sandbox environment. Absent keeps the safe no-op default in the // bridge, so the surface stays inert until the host injects a real recorder. duplexObservabilityRecorder?: DuplexObservabilityRecorder | null; - // The process-owned aggregate byte ledger. The seam stamps it onto the sandbox - // target next to the runner, so the live object stays on the host and never - // enters the sandbox environment. Absent keeps the bridge inert for this seam. - duplexAggregateByteLedger?: DuplexAggregateByteLedger | null; }): Promise { if (input.environment.driver === "local") { return { @@ -340,10 +335,6 @@ export async function resolveEnvironmentExecutionTarget(input: { // binds it to the fixed observability surface. Absent keeps the no-op // default, so the surface stays inert on a run with no injected recorder. duplexObservabilityRecorder: input.duplexObservabilityRecorder ?? null, - // Attach the process-owned aggregate byte ledger next to the runner. The - // bridge passes it to the broker, the decoder, and the response-body reader. - // Absent keeps the bridge inert for this seam. - duplexAggregateByteLedger: input.duplexAggregateByteLedger ?? null, ...(effectiveCapabilities ? { effectiveCapabilities: Object.freeze({ ...effectiveCapabilities }) } : {}), environmentId: input.environment.id ?? null, leaseId: input.leaseId ?? null, diff --git a/server/src/services/environment-run-orchestrator.ts b/server/src/services/environment-run-orchestrator.ts index 9cf984f3b6..ccabc7e51d 100644 --- a/server/src/services/environment-run-orchestrator.ts +++ b/server/src/services/environment-run-orchestrator.ts @@ -44,7 +44,6 @@ import { type AdapterWorkspaceRealization, } from "@paperclipai/adapter-utils/execution-target"; import type { DuplexObservabilityRecorder } from "@paperclipai/adapter-utils/duplex-observability"; -import type { DuplexAggregateByteLedger } from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger"; import { buildWorkspaceRealizationRequest } from "./workspace-realization.js"; import { executionWorkspaceService } from "./execution-workspaces.js"; import { logActivity } from "./activity-log.js"; @@ -156,14 +155,6 @@ export function environmentRunOrchestrator( options: { pluginWorkerManager?: PluginWorkerManager; environmentRuntime?: EnvironmentRuntimeService; - /** - * The process-owned aggregate byte ledger for the sandbox duplex channel. - * The server root creates one ledger per host process and injects the same - * object here. The orchestrator stamps it onto the sandbox execution target, - * so one shared gauge bounds the aggregate retained bytes across all live - * duplex routes. Absent keeps the bridge inert for this seam. - */ - duplexAggregateByteLedger?: DuplexAggregateByteLedger | null; } = {}, ) { const environmentsSvc = environmentService(db); @@ -529,7 +520,6 @@ export function environmentRunOrchestrator( lease, environmentRuntime, duplexObservabilityRecorder: input.duplexObservabilityRecorder ?? null, - duplexAggregateByteLedger: options.duplexAggregateByteLedger ?? null, }); const realizationMode = workspaceRealization.mode === "in_place" ? "in_place" : "copy"; const authoritativeRoot = diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 69419dad9d..ca32a904cf 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -73,7 +73,6 @@ import { import { conflict, HttpError, notFound } from "../errors.js"; import { getStartupTraceContext, getStartupTracer } from "../instrumentation.js"; import { createHostDuplexObservabilityRecorder } from "./duplex-observability-recorder.js"; -import type { DuplexAggregateByteLedger } from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger"; import { incrementToolRuntimeMetricCounter } from "./tool-runtime-metrics.js"; import { logger } from "../middleware/logger.js"; import { @@ -6707,14 +6706,6 @@ export interface HeartbeatServiceOptions { pluginWorkerManager?: PluginWorkerManager; environmentRuntime?: HeartbeatEnvironmentRuntime; runtimeEnv?: Record; - /** - * The process-owned aggregate byte ledger for the sandbox duplex channel. The - * server root creates one ledger per host process and injects the same object - * here. The heartbeat threads it into the environment run orchestrator, which - * stamps it onto the sandbox execution target. Absent keeps the bridge inert - * for this seam. - */ - duplexAggregateByteLedger?: DuplexAggregateByteLedger | null; } type WorkspaceReadyCommentWriter = { @@ -6838,7 +6829,6 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const envOrchestrator = environmentRunOrchestrator(db, { pluginWorkerManager: options.pluginWorkerManager, environmentRuntime, - duplexAggregateByteLedger: options.duplexAggregateByteLedger, }); const workspaceOperationsSvc = workspaceOperationService(db); const liveRunExecutions = { diff --git a/server/src/services/plugin-worker-manager.ts b/server/src/services/plugin-worker-manager.ts index 0b533a3385..9d2d8b8ab3 100644 --- a/server/src/services/plugin-worker-manager.ts +++ b/server/src/services/plugin-worker-manager.ts @@ -59,12 +59,6 @@ import type { InitializeParams, } from "@paperclipai/plugin-sdk"; import { getActiveStepContext } from "@paperclipai/adapter-utils/acpx-engine/startup-timing"; -import { - DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, - type DuplexAggregateByteLedger, - type DuplexAggregateTokenOwner, - type ReservationToken, -} from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger"; import { isLoginCommandKey, validateLoginSessionHome, @@ -73,21 +67,6 @@ import { import { logger } from "../middleware/logger.js"; import { traceparentFromContextToken } from "../instrumentation.js"; -/** - * The host raises this error when the child-stdin transport reservation for a - * duplex write fails against the aggregate byte ledger. The host does not write - * the frame. The write path throws it, `callInternal` rejects the RPC with it - * unwrapped, and the duplex write caller ends the route fail-closed with the - * {@link DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED} marker. Only a duplex write - * meters the transport, so this error never reaches a non-duplex control message. - */ -class DuplexAggregateBytesExceededError extends Error { - constructor() { - super(DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED); - this.name = "DuplexAggregateBytesExceededError"; - } -} - // --------------------------------------------------------------------------- // Constants // --------------------------------------------------------------------------- @@ -193,14 +172,21 @@ const LOGIN_PTY_OPEN_FAILED = "LOGIN_PTY_OPEN_FAILED"; // login pseudo-terminal route, but it carries no command allowlist and adds seven // explicit bounds the pseudo-terminal route lacks. Each bound ends the route when // it passes the limit, so a faulty or hostile worker cannot flood the host. -/** The default maximum characters for one duplex channel data notification. */ -const MAX_DUPLEX_CHANNEL_CHUNK_CHARS = 1_000_000; +/** The default maximum characters for one duplex channel data notification. This + * matches the host duplex frame bound ({@link DEFAULT_MAX_DUPLEX_FRAME_BYTES} in + * `duplex-frame-codec.ts`), so one chunk always fits inside one frame. */ +const MAX_DUPLEX_CHANNEL_CHUNK_CHARS = 262_144; /** - * The default maximum cumulative characters the host buffers for one duplex - * channel route before a data listener attaches. A worker that streams data - * before the consumer binds cannot grow the host buffer without limit. + * The default maximum cumulative bytes the host retains, on one duplex channel + * route, for input the worker has not yet delivered: the pre-bind hold (before + * the worker session id binds), the post-bind buffered queue (bound, before a + * data listener attaches), and the terminal buffer (a route that ended before + * a listener attached). All three representations share this one route-local + * bound, so a worker cannot grow any of them past it. This bound is a fixed + * share of the fixed per-route byte budget the host now bounds every duplex + * retention site against; it holds no measurement of real traffic. */ -const MAX_DUPLEX_CHANNEL_PRE_BIND_CHARS = 8 * 1024 * 1024; +export const DUPLEX_ROUTE_INPUT_MAX_BYTES = 393_216; /** * The default maximum number of data frames the host buffers for one duplex * channel route before a data listener attaches. @@ -229,8 +215,21 @@ const DUPLEX_CHANNEL_PRE_BIND_HOLD_MARGIN_FRAMES = 1; * unbounded number of pending requests. */ const MAX_DUPLEX_CHANNEL_PENDING_REQUESTS = 256; -/** The default maximum characters for one host→worker duplex channel write. */ -const MAX_DUPLEX_CHANNEL_WRITE_CHARS = 1_000_000; +/** The default maximum characters for one host→worker duplex channel write. This + * matches the host duplex frame bound, so one write always fits inside one + * frame. */ +const MAX_DUPLEX_CHANNEL_WRITE_CHARS = 262_144; +/** + * The default maximum cumulative bytes the host retains, on one duplex channel + * route, for a host→worker write the worker has not yet acknowledged: the raw + * write payload and the serialized child-stdin frame the host built from it. + * This bound is route-local and independent of the pending-request count + * bound above; a worker that never replies still cannot make one route hold + * an unbounded number of write bytes, even under a raised pending-request + * count. This bound is a fixed share of the fixed per-route byte budget; it + * holds no measurement of real traffic. + */ +export const DUPLEX_ROUTE_PENDING_WRITE_MAX_BYTES = 393_216; /** * The default maximum number of protocol errors for one duplex channel route. * A protocol error is one malformed or mismatched data frame. The route ends @@ -428,14 +427,6 @@ export interface WorkerStartOptions { * constructs a handle this way). */ duplexRouteSlots?: DuplexRouteSlotController | null; - /** - * The process-owned aggregate byte ledger for the duplex channel. The manager - * injects one shared instance into every worker handle, so one gauge bounds the - * aggregate retained bytes across every route in the process. When it is absent, - * the worker retains duplex bytes unbounded (a unit test constructs a handle - * this way). The manager never makes a fresh per-handle default. - */ - duplexAggregateByteLedger?: DuplexAggregateByteLedger | null; /** * Companies this worker may act on from proactive (no-invocation) worker→host * calls — the plugin's configured companies. Seeded onto the handle at @@ -504,6 +495,9 @@ export interface WorkerStartOptions { maxPendingRequests?: number; /** Max characters for one host→worker duplex channel write. */ maxWriteChars?: number; + /** Max cumulative bytes the host retains, for one route, for a + * host→worker write pending the worker's acknowledgement. */ + maxPendingWriteBytes?: number; /** Max number of protocol errors for one route before the route ends. */ maxProtocolErrors?: number; /** Max cumulative bytes the host forwards for one route over its whole life. */ @@ -935,7 +929,7 @@ export function createPluginWorkerHandle( options.duplexChannelLimits?.maxChunkChars ?? MAX_DUPLEX_CHANNEL_CHUNK_CHARS; const maxDuplexChannelPreBindChars = options.duplexChannelLimits?.maxPreBindBufferedChars ?? - MAX_DUPLEX_CHANNEL_PRE_BIND_CHARS; + DUPLEX_ROUTE_INPUT_MAX_BYTES; const maxDuplexChannelPreBindFrames = options.duplexChannelLimits?.maxPreBindBufferedFrames ?? MAX_DUPLEX_CHANNEL_PRE_BIND_FRAMES; @@ -949,6 +943,8 @@ export function createPluginWorkerHandle( MAX_DUPLEX_CHANNEL_PENDING_REQUESTS; const maxDuplexChannelWriteChars = options.duplexChannelLimits?.maxWriteChars ?? MAX_DUPLEX_CHANNEL_WRITE_CHARS; + const maxDuplexRoutePendingWriteBytes = + options.duplexChannelLimits?.maxPendingWriteBytes ?? DUPLEX_ROUTE_PENDING_WRITE_MAX_BYTES; const maxDuplexChannelProtocolErrors = options.duplexChannelLimits?.maxProtocolErrors ?? MAX_DUPLEX_CHANNEL_PROTOCOL_ERRORS; @@ -1025,42 +1021,11 @@ export function createPluginWorkerHandle( // JSON-RPC message sending // ----------------------------------------------------------------------- - function sendMessage(message: unknown, meterDuplexWrite = false): void { + function sendMessage(message: unknown): void { if (!childProcess?.stdin?.writable) { throw new Error(`Worker process for plugin "${pluginId}" is not writable`); } const serialized = serializeMessage(message as any); - const ledger = duplexAggregateByteLedger; - if (meterDuplexWrite && ledger) { - // Charge the child-stdin transport buffer for a duplex write. The host writes - // the serialized frame to the child stdin without backpressure. When the child - // stops reading its stdin, the frame stays in the host stdin write buffer. - // Reserve the exact serialized-frame byte count. That count includes the JSON - // escaping and the newline framing, so the ledger covers the retained transport - // bytes, not only the raw payload. The RPC separately holds the raw payload - // under a `pending_write` token, so the two tokens cover the peak of both - // retentions at the same time. - const bytes = Buffer.byteLength(serialized); - const token = ledger.reserve("stdin_write", bytes); - if (!token) { - // The reservation would pass the aggregate ceiling. Fail closed before the - // enqueue: do not write the frame. The duplex write caller ends the route. - throw new DuplexAggregateBytesExceededError(); - } - // Hold the token until the stream flushes the chunk. The write callback fires - // when the stream hands the chunk to the operating system, so the bytes then - // leave the host stream buffer. The RPC settle and the RPC timeout never - // release this token. Only the flush, the stream error, the stream close, or - // the worker exit releases it. Release through the outstanding-token set, so a - // later stream-error or worker-exit sweep never double-releases the same token. - pendingStdinWriteTokens.add(token); - childProcess.stdin.write(serialized, () => { - if (pendingStdinWriteTokens.delete(token)) { - ledger.release(token); - } - }); - return; - } childProcess.stdin.write(serialized); } @@ -1784,27 +1749,22 @@ export function createPluginWorkerHandle( // 7. route lifetime — the milliseconds from the open to the terminal end. // One buffered data chunk retained for a late listener drain. It carries the raw - // chunk bytes and the aggregate byte token that reserved its raw bytes. The - // token is `null` when no ledger is injected. + // chunk bytes. interface BufferedDuplexChunk { chunk: Uint8Array; - token: ReservationToken | null; } // One pre-bind data event, normalized to the narrow duplex-event schema. The host - // retains only these bounded scalar fields plus the reservation token, never the - // original arbitrary notification graph. The bind re-resolves the pair from - // `workerSessionId`, so a frame whose pair does not match the bound pair still - // fails closed. + // retains only these bounded scalar fields, never the original arbitrary + // notification graph. The bind re-resolves the pair from `workerSessionId`, so a + // frame whose pair does not match the bound pair still fails closed. interface HeldDuplexEvent { workerSessionId: string; chunk: Uint8Array; - token: ReservationToken | null; } // One pre-bind exit event, normalized to the narrow duplex-event schema. interface HeldDuplexExitEvent { workerSessionId: string; exitCode: number | null; - token: ReservationToken | null; } interface DuplexChannelRoute { @@ -1825,18 +1785,32 @@ export function createPluginWorkerHandle( // lost and a frame whose pair does not match the bound pair still fails closed. // The hold ceiling is `maxDuplexChannelPreBindHoldFrames`, one frame above the // buffered bound, so the replay's buffered-bound check ends the route, not the - // hold. Each held event carries the aggregate byte token that reserved its raw - // bytes; the host never retains the original arbitrary notification graph. + // hold. The host never retains the original arbitrary notification graph. preBind: HeldDuplexEvent[]; + // The cumulative raw bytes `preBind` holds. `DUPLEX_ROUTE_INPUT_MAX_BYTES` + // bounds this route-local counter, so a worker cannot flood the pre-bind + // hold with bytes the frame-count ceiling alone does not catch (a handful + // of large frames, each well under the frame-count limit). This counter + // covers only the hold; the post-bind buffered queue counts its own bytes + // in `bufferedChars`, against the same named bound. + preBindBytes: number; // The single bounded exit event that arrived before the bind. An exit never // consumes a data hold slot, so a worker that batches an exit among enough data // frames to fill the hold cannot crowd out a data frame. The bind replays the // held data events first, then this exit last. preBindExit: HeldDuplexExitEvent | null; - // The authoritative registry of every aggregate byte token this route still - // holds, across the pre-bind, buffered, and terminal-buffered representations. - // The byte cleanup releases every token here exactly once. - retainedTokens: Set; + // The cumulative raw payload bytes of every host→worker write this route + // has sent but the worker has not yet acknowledged. The window this + // counter tracks spans both retained representations of a pending write: + // the raw payload the pending request object holds, and the serialized + // child-stdin frame the host builds from it and writes before the request + // settles. `DUPLEX_ROUTE_PENDING_WRITE_MAX_BYTES` bounds this route-local + // counter, independent of `pendingRequests`: a worker that never replies + // cannot make one route hold an unbounded number of write bytes, even + // under a raised pending-request count. `sendBoundedRequest` increments it + // before the send and decrements it once the request settles, on every + // path: a reply, a rejection, a timeout, a worker exit, or a shutdown. + pendingWriteBytes: number; } // The live duplex routes on this worker, keyed by the exact // `{ hostRouteId, workerSessionId }` pair. The host binds one pair once, at @@ -1862,89 +1836,11 @@ export function createPluginWorkerHandle( // time per route, so a double terminalize never releases two slots. const duplexRouteSlotHolders = new Set(); - // The process-owned aggregate byte ledger. The manager injects it, so one gauge - // bounds the aggregate retained bytes across every route in the process. When it - // is absent, the worker retains duplex bytes unbounded (a unit test constructs it - // this way). - const duplexAggregateByteLedger = options.duplexAggregateByteLedger ?? null; - // The outstanding child-stdin transport tokens for duplex writes. Each token - // covers one serialized frame the host stdin write buffer still retains. The - // write callback releases a token on the flush; a stream error, a stream close, - // or a worker exit releases every remaining token, because each of those - // discards the stdin write buffer. The set is the release guard, so a token - // releases one time across the two paths. - const pendingStdinWriteTokens = new Set(); - // Release every outstanding child-stdin transport token. A stream error, a - // stream close, or a worker exit calls this, because each discards the stdin - // write buffer. The `delete` guard drops each token one time, so a later flush - // callback or a second sweep releases nothing again. - function releaseAllPendingStdinWriteTokens(): void { - if (!duplexAggregateByteLedger) return; - for (const token of pendingStdinWriteTokens) { - duplexAggregateByteLedger.release(token); - } - pendingStdinWriteTokens.clear(); - } // The terminalized routes that still hold buffered bytes for a late listener // drain. A terminalized route leaves the opening and live maps, so this registry - // keeps the worker-exit sweep able to release its still-charged buffered tokens. + // keeps the worker-exit sweep able to clear its still-buffered representations. const terminalDuplexRoutes = new Set(); - // Reserve `bytes` for one route retention against the aggregate ledger. Return - // the held token, or `null` when the reservation would pass the ceiling. When no - // ledger is present, return `"no-ledger"`, so the caller admits the retention - // with no token. The helper adds a real token to `route.retainedTokens`. - function reserveRouteBytes( - route: DuplexChannelRoute, - owner: DuplexAggregateTokenOwner, - bytes: number, - ): ReservationToken | "no-ledger" | null { - if (!duplexAggregateByteLedger) return "no-ledger"; - const token = duplexAggregateByteLedger.reserve(owner, bytes); - if (!token) return null; - route.retainedTokens.add(token); - return token; - } - - // Transfer a held route token to a new owner label. The token identity and the - // reserved bytes stay the same, so no admission gap opens between two - // representations of the same retained bytes. - function transferRouteToken( - token: ReservationToken | null, - owner: DuplexAggregateTokenOwner, - ): void { - if (!token) return; - duplexAggregateByteLedger?.transfer(token, owner); - } - - // Release one held route token and drop it from the route registry, in one - // synchronous step. A `null` token (no ledger) releases nothing. - function releaseRouteToken(route: DuplexChannelRoute, token: ReservationToken | null): void { - if (!token) return; - route.retainedTokens.delete(token); - duplexAggregateByteLedger?.release(token); - } - - // Release every token a route still holds and clear its retained representations, - // exactly once. A later call finds an empty registry and releases nothing, so the - // helper is idempotent. Every terminal-discard, open-failure, bind-replay - // failure, close-acknowledgement failure, worker-exit, and shutdown path calls - // it. Terminal map deletion is never treated as proof that retained bytes are - // gone; this registry is authoritative for byte cleanup. - function discardRouteRetained(route: DuplexChannelRoute): void { - if (duplexAggregateByteLedger) { - for (const token of route.retainedTokens) { - duplexAggregateByteLedger.release(token); - } - } - route.retainedTokens.clear(); - route.preBind = []; - route.preBindExit = null; - route.buffered = []; - route.bufferedChars = 0; - terminalDuplexRoutes.delete(route); - } - // Try to reserve one aggregate route slot for a route. Return true when the // route holds a slot after the call. When no controller is present, the route // always holds a slot. @@ -2042,34 +1938,16 @@ export function createPluginWorkerHandle( route.terminalized = true; route.state = "closed"; route.listener = null; - // Stop admission first. Then move the byte cleanup. Keep the buffered chunks the - // host accepted before the route ended, so a listener that attaches after the - // end still drains them. A frame can end the route during the pre-bind replay, - // before a listener attaches, and the chunks the host accepted before that frame - // are valid data the listener must still receive. The buffered bytes stay - // bounded by the pre-bind buffered bound, and `onData` clears and releases them - // once it drains them. - // - // The buffered records keep their tokens; move each token to the terminal owner - // label and register the route in the terminal registry, so the worker-exit - // sweep can release the still-charged bytes later. Do not release a buffered - // token at map deletion; terminal map deletion is never proof the bytes are - // gone. Release every other token the route still holds (the pre-bind events, - // the held exit, and any stranded token), because they never reach a listener. - const bufferedTokens = new Set(); - for (const record of route.buffered) { - if (record.token) { - bufferedTokens.add(record.token); - transferRouteToken(record.token, "terminal_buffered"); - } - } - for (const token of [...route.retainedTokens]) { - if (!bufferedTokens.has(token)) { - route.retainedTokens.delete(token); - duplexAggregateByteLedger?.release(token); - } - } + // Stop admission first. Keep the buffered chunks the host accepted before the + // route ended, so a listener that attaches after the end still drains them. A + // frame can end the route during the pre-bind replay, before a listener + // attaches, and the chunks the host accepted before that frame are valid data + // the listener must still receive. The buffered bytes stay bounded by the + // pre-bind buffered bound, and `onData` clears them once it drains them. + // Register the route in the terminal registry, so the worker-exit sweep can + // clear the still-buffered chunks later. route.preBind = []; + route.preBindBytes = 0; route.preBindExit = null; if (route.buffered.length > 0) { terminalDuplexRoutes.add(route); @@ -2156,16 +2034,8 @@ export function createPluginWorkerHandle( } // Route one duplex channel data frame. On a live route with a listener, deliver - // the chunk transiently. On a live route with no listener, buffer the chunk and - // charge its raw bytes against the aggregate ledger. The `carried` argument marks - // a bind-replay frame: it carries the pre-bind token the buffered record must - // reuse, so the replay transfers the token to the buffered representation with no - // decrement and no re-reserve. A live frame passes no `carried`, so it reserves a - // fresh buffered token. A `carried.token` of `null` means no ledger is present. - function routeDuplexChannelData( - notification: JsonRpcNotification, - carried?: { token: ReservationToken | null }, - ): void { + // the chunk transiently. On a live route with no listener, buffer the chunk. + function routeDuplexChannelData(notification: JsonRpcNotification): void { const params = isRecord(notification.params) ? notification.params : {}; const hostRouteId = readNonEmptyString(params.hostRouteId); const workerSessionId = readNonEmptyString(params.workerSessionId); @@ -2200,15 +2070,13 @@ export function createPluginWorkerHandle( const chunk = rawChunk instanceof Uint8Array ? rawChunk : decodeChannelBytes(rawChunk); if (chunk === null || chunk.byteLength === 0) { // The exact pair matches, but the chunk is malformed or empty. Count one - // per-route protocol error. Release a carried replay token first. - releaseRouteToken(route, carried?.token ?? null); + // per-route protocol error. recordDuplexChannelProtocolError(route); return; } if (chunk.byteLength > maxDuplexChannelChunkChars) { // One inbound chunk is larger than the per-chunk limit. End the route at // once. Do not count the chunk as a protocol error. - releaseRouteToken(route, carried?.token ?? null); void terminalizeDuplexChannelRoute(route); return; } @@ -2217,16 +2085,13 @@ export function createPluginWorkerHandle( // data past the cap. const chunkBytes = chunk.byteLength; if (route.totalDataBytes + chunkBytes > maxDuplexChannelTotalDataBytes) { - releaseRouteToken(route, carried?.token ?? null); void terminalizeDuplexChannelRoute(route); return; } route.totalDataBytes += chunkBytes; if (route.listener) { - // A listener is attached. Deliver the chunk transiently and release a carried - // replay token after the synchronous delivery boundary. + // A listener is attached. Deliver the chunk transiently. deliverDuplexChannelChunk(route.listener, chunk); - releaseRouteToken(route, carried?.token ?? null); return; } // No listener attached yet. Buffer the frame under the pre-bind bounds. End @@ -2235,28 +2100,10 @@ export function createPluginWorkerHandle( route.buffered.length + 1 > maxDuplexChannelPreBindFrames || route.bufferedChars + chunk.length > maxDuplexChannelPreBindChars ) { - releaseRouteToken(route, carried?.token ?? null); void terminalizeDuplexChannelRoute(route); return; } - let token: ReservationToken | null; - if (carried) { - // The bind replay transfers the pre-bind token to the buffered - // representation. No decrement and no re-reserve, so no admission gap opens. - token = carried.token; - transferRouteToken(token, "buffered_chunk"); - } else { - // A live frame reserves a fresh buffered token for its exact raw bytes. - const reserved = reserveRouteBytes(route, "buffered_chunk", chunkBytes); - if (reserved === null) { - // The aggregate ceiling rejected the reservation. Retain nothing and fail - // closed with the fixed marker. - void terminalizeDuplexChannelRoute(route); - return; - } - token = reserved === "no-ledger" ? null : reserved; - } - route.buffered.push({ chunk, token }); + route.buffered.push({ chunk }); route.bufferedChars += chunk.length; } @@ -2319,23 +2166,10 @@ export function createPluginWorkerHandle( return; } if (notification.method === DUPLEX_CHANNEL_EXIT_NOTIFICATION) { - // Normalize the exit to the narrow duplex-event schema. An exit retains only - // a bounded scalar, so reserve a zero-byte token that carries the record - // through the one cleanup path. Release any earlier held exit token first, so - // a replaced exit never leaks its reservation. + // Normalize the exit to the narrow duplex-event schema. A replaced exit + // simply overwrites the earlier held exit. const exitCode = typeof params.exitCode === "number" ? params.exitCode : null; - if (route.preBindExit) releaseRouteToken(route, route.preBindExit.token); - const reserved = reserveRouteBytes(route, "pre_bind_event", 0); - if (reserved === null) { - // The aggregate ceiling rejected the reservation. Fail closed. - void terminalizeDuplexChannelRoute(route); - return; - } - route.preBindExit = { - workerSessionId, - exitCode, - token: reserved === "no-ledger" ? null : reserved, - }; + route.preBindExit = { workerSessionId, exitCode }; return; } // A data event. Validate and normalize it to the narrow duplex-event schema @@ -2350,20 +2184,16 @@ export function createPluginWorkerHandle( recordDuplexChannelProtocolError(route); return; } - // Reserve the exact retained raw byte count before the host holds the event. - const reserved = reserveRouteBytes(route, "pre_bind_event", chunk.byteLength); - if (reserved === null) { - // The aggregate ceiling rejected the reservation. The caller retains nothing - // and the route fails closed with the fixed marker. - log.warn({ pluginId, reason: DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED }, "duplex pre-bind hold rejected"); + // The route-local input bound covers the pre-bind hold directly. A handful + // of large frames can pass the frame-count ceiling above while still + // flooding the host with bytes, so this route ends at once on the byte + // bound, the same way the post-bind buffered queue already does. + if (route.preBindBytes + chunk.byteLength > maxDuplexChannelPreBindChars) { void terminalizeDuplexChannelRoute(route); return; } - route.preBind.push({ - workerSessionId, - chunk, - token: reserved === "no-ledger" ? null : reserved, - }); + route.preBind.push({ workerSessionId, chunk }); + route.preBindBytes += chunk.byteLength; } // Replay the frames a route held before it bound. The route is live now, so the @@ -2376,36 +2206,29 @@ export function createPluginWorkerHandle( function replayPreBindDuplexFrames(route: DuplexChannelRoute): void { const held = route.preBind; route.preBind = []; + route.preBindBytes = 0; for (const event of held) { if (route.terminalized) { - // The route ended mid-replay. Release the remaining held tokens, so the - // unreplayed events retain nothing. - releaseRouteToken(route, event.token); + // The route ended mid-replay. The remaining held events are dropped. continue; } // Reconstruct a transient data notification for the exact-pair routing. The // host holds only the bounded event, so it builds this notification for the - // routing step and discards it at once. The carried token moves to the - // buffered representation without a decrement or a re-reserve. - routeDuplexChannelData( - { - jsonrpc: "2.0", - method: DUPLEX_CHANNEL_DATA_NOTIFICATION, - params: { - hostRouteId: route.hostRouteId, - workerSessionId: event.workerSessionId, - chunk: event.chunk, - }, + // routing step and discards it at once. + routeDuplexChannelData({ + jsonrpc: "2.0", + method: DUPLEX_CHANNEL_DATA_NOTIFICATION, + params: { + hostRouteId: route.hostRouteId, + workerSessionId: event.workerSessionId, + chunk: event.chunk, }, - { token: event.token }, - ); + }); } const heldExit = route.preBindExit; route.preBindExit = null; if (heldExit) { - // The exit retains only a bounded scalar. Release its placeholder token, then - // resolve the wait through the exact-pair routing when the route still lives. - releaseRouteToken(route, heldExit.token); + // Resolve the wait through the exact-pair routing when the route still lives. if (!route.terminalized) { routeDuplexChannelExit({ jsonrpc: "2.0", @@ -2448,10 +2271,14 @@ export function createPluginWorkerHandle( releaseDuplexRouteSlot(route); settleRouteWait(route, { exitCode: null }); } - // Release every token the route still holds and clear its retained - // representations, exactly once. A route already drained by a late listener - // holds no token, so this is harmless and leaves the ledger at zero. - discardRouteRetained(route); + // Clear the route's retained buffers, exactly once. A route already drained + // by a late listener holds nothing, so this is harmless. + route.preBind = []; + route.preBindBytes = 0; + route.preBindExit = null; + route.buffered = []; + route.bufferedChars = 0; + terminalDuplexRoutes.delete(route); } // Clear the terminal registry. Every route in it was just discarded above. terminalDuplexRoutes.clear(); @@ -2486,8 +2313,9 @@ export function createPluginWorkerHandle( terminalized: false, settleWait, preBind: [], + preBindBytes: 0, preBindExit: null, - retainedTokens: new Set(), + pendingWriteBytes: 0, }; // Reserve one aggregate route slot before any work. When the process-scoped // ceiling is full, reject with the fixed route-busy error and open nothing, so @@ -2579,58 +2407,30 @@ export function createPluginWorkerHandle( void terminalizeDuplexChannelRoute(route); return; } - // Reserve the exact raw byte count of a host→worker write against the - // aggregate ledger before `callInternal` retains the payload. A pending write - // RPC holds `params.data` until it settles, so this reservation bounds the - // aggregate host→worker pending-write bytes across every route. A stop - // request carries no payload, so it reserves nothing. When no ledger is - // present, admit the write with no token (a unit test constructs the handle - // this way). - let pendingWriteToken: ReservationToken | null = null; - if (method === "duplexChannelWrite" && duplexAggregateByteLedger) { - pendingWriteToken = duplexAggregateByteLedger.reserve("pending_write", writeByteCount ?? 0); - if (!pendingWriteToken) { - // The reservation would pass the aggregate ceiling. Retain nothing, do - // not enqueue the RPC, and end the route fail-closed with the aggregate - // marker, not the route-busy marker. - log.warn( - { pluginId, reason: DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED }, - "duplex pending write reservation rejected", - ); - void terminalizeDuplexChannelRoute(route); - return; - } + // The route-local pending-write byte bound: a worker that never replies + // cannot make one route hold an unbounded number of write bytes, even + // under a raised pending-request count. A stop request carries no + // payload, so it never counts against this bound. + const pendingWriteBytesToCharge = method === "duplexChannelWrite" ? writeByteCount ?? 0 : 0; + if (route.pendingWriteBytes + pendingWriteBytesToCharge > maxDuplexRoutePendingWriteBytes) { + void terminalizeDuplexChannelRoute(route); + return; } route.pendingRequests += 1; - // Meter the child-stdin transport buffer only for a duplex write. A stop - // request carries a tiny fixed frame that the host never lets grow, so it - // does not meter or reject. The write path reserves the serialized frame in - // `sendMessage` right before the stdin write. - const meterDuplexWrite = method === "duplexChannelWrite"; - void callInternal(method, params, duplexChannelOpenTimeoutMs, undefined, meterDuplexWrite) - .catch((err: unknown) => { - if (err instanceof DuplexAggregateBytesExceededError) { - // The transport reservation failed. The host did not write the frame. - // End the route fail-closed with the aggregate marker, not the - // route-busy marker. - log.warn( - { pluginId, reason: DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED }, - "duplex stdin write reservation rejected", - ); - void terminalizeDuplexChannelRoute(route); - } - }) + route.pendingWriteBytes += pendingWriteBytesToCharge; + void callInternal(method, params, duplexChannelOpenTimeoutMs, undefined) + // A send failure, an RPC rejection, a timeout, or a worker exit ends up + // here. The route already handles a lost write through its own state + // (a worker exit or a stop rejects every pending request through the + // normal call machinery), so this fire-and-forget call swallows the + // rejection instead of leaving it unhandled. + .catch(() => {}) .finally(() => { route.pendingRequests -= 1; - // Release the pending-write token one time, after the RPC settles on any - // path: success, error, timeout, worker exit, or shutdown. The token is - // not in `route.retainedTokens`, so route terminalization never releases - // it; only this settlement releases it. The separate `stdin_write` token - // covers the serialized frame and releases on the stream flush, the - // stream error, the stream close, or the worker exit, never here. - if (pendingWriteToken) { - duplexAggregateByteLedger?.release(pendingWriteToken); - } + // Release the route-local pending-write bytes one time, after the RPC + // settles on any path: success, error, timeout, worker exit, or + // shutdown. + route.pendingWriteBytes -= pendingWriteBytesToCharge; }); }; @@ -2642,11 +2442,9 @@ export function createPluginWorkerHandle( route.buffered = []; route.bufferedChars = 0; for (const record of pending) { - // Deliver each buffered record, drop its retained buffer, and release - // its exact token before the drain proceeds to the next record. A - // terminal route drains through this same code. + // Deliver each buffered record before the drain proceeds to the next + // record. A terminal route drains through this same code. deliverDuplexChannelChunk(listener, record.chunk); - releaseRouteToken(route, record.token); } // The buffered records are gone, so the route no longer holds terminal // bytes for a late listener. @@ -2664,8 +2462,8 @@ export function createPluginWorkerHandle( } // Encode the raw bytes to the wire-safe base64 form (JSON carries no // binary type) and pass the exact raw byte count separately, so the - // pending-write ledger reservation charges the real payload bytes, not - // the inflated base64 string length. + // pending-write byte bound charges the real payload bytes, not the + // inflated base64 string length. sendBoundedRequest( "duplexChannelWrite", { @@ -2961,16 +2759,11 @@ export function createPluginWorkerHandle( readline.on("line", handleLine); } - // Release the outstanding child-stdin transport tokens when the stdin stream - // errors or closes, because each discards the stdin write buffer. The `error` - // listener also stops an unhandled EPIPE from a child that closed its stdin. + // The `error` listener stops an unhandled EPIPE from a child that closed its + // stdin. if (child.stdin) { - child.stdin.on("error", () => { - releaseAllPendingStdinWriteTokens(); - }); - child.stdin.on("close", () => { - releaseAllPendingStdinWriteTokens(); - }); + child.stdin.on("error", () => {}); + child.stdin.on("close", () => {}); } // Capture stderr for logging @@ -3023,12 +2816,6 @@ export function createPluginWorkerHandle( childProcess = null; startedAt = null; - // The worker exit discards the child-stdin write buffer, so release every - // outstanding transport token. The RPC rejections below never release these - // tokens; only this sweep, a stream flush, a stream error, or a stream close - // releases them. - releaseAllPendingStdinWriteTokens(); - // Reject all pending requests rejectAllPending( new Error(formatWorkerFailureMessage( @@ -3364,7 +3151,6 @@ export function createPluginWorkerHandle( params: HostToWorkerMethods[M][0], timeoutMs?: number, executeLogSink?: ExecuteLogSink, - meterDuplexWrite = false, ): Promise { const rpcPromise = new Promise((resolve, reject) => { if (!childProcess?.stdin?.writable) { @@ -3438,26 +3224,19 @@ export function createPluginWorkerHandle( ...createRequest(method, params, id), ...(invocation ? { paperclipInvocation: invocation } : {}), }; - sendMessage(request, meterDuplexWrite); + sendMessage(request); } catch (err) { clearTimeout(timer); pendingRequests.delete(id); clearInvocation(invocation); clearExecuteRoute(invocation?.id); - if (err instanceof DuplexAggregateBytesExceededError) { - // The transport reservation failed before the write. Reject with the - // typed error unwrapped, so the duplex write caller ends the route - // fail-closed with the aggregate marker. - reject(err); - } else { - reject( - new Error( - `Failed to send "${method}" to worker: ${ - err instanceof Error ? err.message : String(err) - }`, - ), - ); - } + reject( + new Error( + `Failed to send "${method}" to worker: ${ + err instanceof Error ? err.message : String(err) + }`, + ), + ); } }); @@ -3630,20 +3409,20 @@ export interface PluginWorkerManagerOptions { * stays upstream admission only. */ maxConcurrentDuplexRoutes?: number | null; - /** - * The process-owned aggregate byte ledger. The process root creates one ledger - * from validated configuration and passes it here. The manager injects the same - * object into every worker handle, so one gauge bounds the aggregate retained - * bytes across every route in the process. When it is absent, the worker retains - * duplex bytes unbounded. The manager never makes a fresh default ledger. - */ - duplexAggregateByteLedger?: DuplexAggregateByteLedger | null; } /** * The default process-scoped aggregate ceiling for concurrent duplex channel * routes. It caps the manager-wide resource, not one agent's run budget. The host * reports an explicit route-busy outcome when the ceiling is full. + * + * Known aggregate behavior: this ceiling bounds route count only, not + * retained bytes. Each HTTP/2 bridge route bounds its own retained body + * bytes to 8,388,608 bytes (see `HTTP2_BRIDGE_MAX_CONCURRENT_STREAMS` in + * `http2-bridge-server.ts`), so this route ceiling caps the process's + * aggregate retained body bytes at 128 * 8,388,608 = 1,073,741,824 bytes + * (1 GiB). This is accepted, known behavior, not a defect: the process + * tracks no aggregate byte ledger across routes. */ export const DEFAULT_MAX_CONCURRENT_DUPLEX_ROUTES = 128; @@ -3709,11 +3488,6 @@ export function createPluginWorkerManager( const duplexRouteSlots = createDuplexRouteSlotController( managerOptions?.maxConcurrentDuplexRoutes, ); - // The one shared, process-owned aggregate byte ledger. The manager injects the - // same object into every worker handle, so one gauge bounds the aggregate - // retained bytes across every route in the process. It is `null` when the - // process root injected no ledger. - const duplexAggregateByteLedger = managerOptions?.duplexAggregateByteLedger ?? null; return { async startWorker( @@ -3735,11 +3509,9 @@ export function createPluginWorkerManager( } const handle = createPluginWorkerHandle(pluginId, { - // Inject the shared process-scoped route-slot controller and the shared - // process-owned aggregate byte ledger, unless the caller already supplied - // its own (a test may inject its own). + // Inject the shared process-scoped route-slot controller, unless the + // caller already supplied its own (a test may inject its own). duplexRouteSlots, - duplexAggregateByteLedger, ...options, }); workers.set(pluginId, handle);