mirror of https://github.com/garrytan/gstack.git
feat(security): hash-chained egress receipt ledger (core)
Port lib/egress-receipt from the v2 fork as TypeScript: writeReceipt (sync, fail-closed via typed EGRESS_RECEIPT_FAILED), best-effort writeOutcome, readLedger/listReceipts/verifyLedger, GSTACK_HOME -> GSTACK_STATE_DIR -> ~/.gstack resolution, 0600 ledger under a 0700 security dir, and an mkdir spin lock (2.5s budget) with documented >10s-mtime stale-lock reclaim. Changes vs the fork: - lastRawLine tail-reads the final 4KB instead of loading the whole ledger, so appends stay O(1) as the file grows. - WARN-at-size: past 25MB writeReceipt emits one self-explanatory stderr warning per process (what the ledger is, how to inspect it, rotation TODO); verifyLedger gains a sizeWarning field. Rotation TODO carries the chain-genesis sketch (new generation's first record embeds the prior file's tail hash). bin/gstack-egress-receipt is a bun script bridging shell callers: write|outcome subcommands, exit 3 + EGRESS_RECEIPT_FAILED on stderr on failure; --no-payload records sha256:null for git-class ops. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> (cherry picked from commit 619726a3d77d987a2e50151a5727b3faaaf5fc6a)
This commit is contained in:
Garry Tan
parent
4e0725479f
commit
d0dfdf4090
|
|
@ -0,0 +1,84 @@
|
|||
#!/usr/bin/env bun
|
||||
// gstack-egress-receipt — bun script that BRIDGES shell callers (the bash
|
||||
// egress sinks: gstack-telemetry-sync, gstack-update-check, gstack-brain-sync,
|
||||
// and the sourced helpers in gstack-egress-lib.sh) into lib/egress-receipt.ts.
|
||||
//
|
||||
// Usage:
|
||||
// gstack-egress-receipt write --sink S --host H --class C \
|
||||
// (--payload-file F | --no-payload) [--consent "key=value"]
|
||||
// → prints the receipt id on stdout, exit 0.
|
||||
// → exit 3 + "EGRESS_RECEIPT_FAILED: ..." on stderr when the receipt
|
||||
// cannot be written. Fail-closed callers MUST then refuse the send.
|
||||
//
|
||||
// gstack-egress-receipt outcome <receipt-id> <status>
|
||||
// → best-effort response-status record; never blocks anything.
|
||||
//
|
||||
// Home: GSTACK_HOME, legacy GSTACK_STATE_DIR, else ~/.gstack.
|
||||
// The payload file is hashed as-is: pass the SAME file to curl (`-d @file`)
|
||||
// so the receipt hash matches the exact bytes sent (scan-at-sink precedent).
|
||||
|
||||
import fs from 'node:fs';
|
||||
import { EGRESS_RECEIPT_FAILED, sha256Hex, writeOutcome, writeReceipt } from '../lib/egress-receipt';
|
||||
|
||||
function parseArgs(args: string[], valueFlags: string[], boolFlags: string[]) {
|
||||
const values = new Map<string, string>();
|
||||
const flags = new Set<string>();
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const arg = args[index];
|
||||
if (valueFlags.includes(arg)) {
|
||||
const value = args[++index];
|
||||
if (value == null) usage(`${arg} requires a value`);
|
||||
values.set(arg, value);
|
||||
} else if (boolFlags.includes(arg)) {
|
||||
flags.add(arg);
|
||||
} else {
|
||||
usage(`unknown option: ${arg}`);
|
||||
}
|
||||
}
|
||||
return { values, flags };
|
||||
}
|
||||
|
||||
function usage(message: string): never {
|
||||
process.stderr.write(`gstack-egress-receipt: ${message}\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const [command, ...rest] = process.argv.slice(2);
|
||||
|
||||
if (command === 'write') {
|
||||
const { values, flags } = parseArgs(rest,
|
||||
['--sink', '--host', '--class', '--payload-file', '--consent'], ['--no-payload']);
|
||||
const payloadFile = values.get('--payload-file');
|
||||
if (!payloadFile && !flags.has('--no-payload')) usage('write requires --payload-file or --no-payload');
|
||||
try {
|
||||
let bytes = 0;
|
||||
let sha256: string | null = null; // --no-payload = git-class op, subprocess owns the bytes
|
||||
if (payloadFile) {
|
||||
const payload = fs.readFileSync(payloadFile);
|
||||
bytes = payload.byteLength;
|
||||
sha256 = sha256Hex(payload);
|
||||
}
|
||||
const { id } = writeReceipt({
|
||||
sink: values.get('--sink') as string,
|
||||
host: values.get('--host') as string,
|
||||
payloadClass: values.get('--class') as string,
|
||||
bytes,
|
||||
sha256,
|
||||
consent: values.get('--consent') ?? 'unspecified',
|
||||
});
|
||||
process.stdout.write(`${id}\n`);
|
||||
} catch (error) {
|
||||
process.stderr.write(`${EGRESS_RECEIPT_FAILED}: ${(error as Error)?.message ?? error}\n`);
|
||||
process.exit(3);
|
||||
}
|
||||
} else if (command === 'outcome') {
|
||||
const [receipt, status, ...extra] = rest;
|
||||
if (!receipt || !status || extra.length) usage('Usage: gstack-egress-receipt outcome <receipt-id> <status>');
|
||||
try {
|
||||
writeOutcome({ receipt, status });
|
||||
} catch {
|
||||
// Best-effort: the pre-send receipt is the invariant, the outcome is bookkeeping.
|
||||
}
|
||||
} else {
|
||||
usage('Usage: gstack-egress-receipt write|outcome ...');
|
||||
}
|
||||
|
|
@ -0,0 +1,376 @@
|
|||
/**
|
||||
* egress-receipt — hash-chained, content-free receipts for every
|
||||
* gstack-initiated off-machine send (`~/.gstack/security/egress.jsonl`, 0600).
|
||||
*
|
||||
* THREAT MODEL: the egress ledger is forensic observability — it records
|
||||
* ATTEMPTED egress so accidents are auditable; it is not an exfiltration
|
||||
* control. Receipts are written before send, outcomes are best-effort, and
|
||||
* fail-open sinks can send unrecorded with a warning.
|
||||
*
|
||||
* Semantics:
|
||||
* - Receipt-before-send: the receipt line is appended BEFORE the network
|
||||
* call. Fail-closed sinks MUST refuse the send with the typed code
|
||||
* EGRESS_RECEIPT_FAILED when it cannot be written; fail-open sinks warn
|
||||
* on stderr and proceed.
|
||||
* - Content-free: never payload text, never credentials — a sha256 of the
|
||||
* exact bytes sent plus a byte count only (semantic-reviews.jsonl
|
||||
* precedent). Sinks where a subprocess/SDK owns the bytes record
|
||||
* sha256: null.
|
||||
* - Tamper-evident: each line carries `prev` = sha256 of the previous raw
|
||||
* line ("" for line 1). `verifyLedger` recomputes the chain.
|
||||
*
|
||||
* Node builtins only, so bun TS binaries and the compiled browse binary can
|
||||
* both import it (same constraint as browse/src/security.ts: no native
|
||||
* modules).
|
||||
*/
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
|
||||
export const EGRESS_RECEIPT_FAILED = 'EGRESS_RECEIPT_FAILED';
|
||||
|
||||
const SHA256_HEX = /^[0-9a-f]{64}$/;
|
||||
|
||||
/**
|
||||
* WARN-at-size threshold. Above this the ledger still appends (never blocks
|
||||
* on size), but writeReceipt emits one stderr warning per process so the
|
||||
* user learns the file exists and how to inspect it before it gets silly.
|
||||
*/
|
||||
export const LEDGER_WARN_BYTES = 25 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* Tail window for reading the last raw line. Receipt lines are ~300 bytes;
|
||||
* 4KB covers any legal line with an order of magnitude to spare.
|
||||
*/
|
||||
const TAIL_READ_BYTES = 4096;
|
||||
|
||||
// TODO(rotation): ledger rotation. Chain-genesis sketch: when the ledger
|
||||
// exceeds the size threshold, rename it to egress.jsonl.1 and start a new
|
||||
// generation whose FIRST record embeds `genesis: sha256(<tail raw line of
|
||||
// the prior file>)`, so verifyLedger can walk generations end-to-end
|
||||
// (verify each file's internal chain, then check each genesis hash against
|
||||
// the previous generation's last line). Until then we warn at 25MB.
|
||||
|
||||
type Env = Record<string, string | undefined>;
|
||||
|
||||
export interface WriteReceiptOptions {
|
||||
/** gstack home; resolved from env when omitted */
|
||||
home?: string;
|
||||
/** env for home resolution (tests) */
|
||||
env?: Env;
|
||||
/** which gstack component is sending */
|
||||
sink: string;
|
||||
/** destination host[:port] */
|
||||
host: string;
|
||||
/** content-free payload description */
|
||||
payloadClass: string;
|
||||
/** exact byte count sent (0 for bodyless requests) */
|
||||
bytes?: number;
|
||||
/** sha256 hex of the exact bytes sent; null when a subprocess/SDK owns the bytes */
|
||||
sha256?: string | null;
|
||||
/** the consent key+value that authorizes this send */
|
||||
consent: string;
|
||||
}
|
||||
|
||||
export interface WriteOutcomeOptions {
|
||||
home?: string;
|
||||
env?: Env;
|
||||
/** receipt id returned by writeReceipt */
|
||||
receipt: string;
|
||||
status?: string | number;
|
||||
}
|
||||
|
||||
export interface LedgerLine {
|
||||
lineNo: number;
|
||||
raw: string;
|
||||
record: Record<string, unknown> | null;
|
||||
}
|
||||
|
||||
export interface VerifyResult {
|
||||
ok: boolean;
|
||||
count: number;
|
||||
brokenLine: number | null;
|
||||
reason: string | null;
|
||||
/** present when the ledger file exceeds LEDGER_WARN_BYTES */
|
||||
sizeWarning: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Same resolution order as the rest of gstack (shell sinks, selection code):
|
||||
* GSTACK_HOME, legacy GSTACK_STATE_DIR, then $HOME/.gstack.
|
||||
*/
|
||||
export function resolveEgressHome(env: Env = process.env): string {
|
||||
const configured = env.GSTACK_HOME || env.GSTACK_STATE_DIR;
|
||||
if (configured) return path.resolve(configured);
|
||||
return path.join(env.HOME || os.homedir(), '.gstack');
|
||||
}
|
||||
|
||||
export function egressLedgerPath(home: string): string {
|
||||
return path.join(home, 'security', 'egress.jsonl');
|
||||
}
|
||||
|
||||
export function sha256Hex(data: string | Uint8Array): string {
|
||||
return createHash('sha256').update(data).digest('hex');
|
||||
}
|
||||
|
||||
function receiptError(message: string, cause?: unknown): Error & { code: string } {
|
||||
const error = (cause === undefined ? new Error(message) : new Error(message, { cause })) as Error & { code: string };
|
||||
error.code = EGRESS_RECEIPT_FAILED;
|
||||
return error;
|
||||
}
|
||||
|
||||
function requireString(value: unknown, name: string): string {
|
||||
if (typeof value !== 'string' || !value) throw receiptError(`Egress receipt requires a non-empty ${name}`);
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* mkdir spin lock, ~2.5s budget. Egress events are rare (minutes apart); the
|
||||
* lock only protects the read-last-line → append window.
|
||||
*
|
||||
* Stale-lock reclaim: a crashed writer strands the lock dir. Once the spin
|
||||
* budget is exhausted, a lock dir whose mtime is >10s old is stale by
|
||||
* definition (appends take milliseconds), so the waiter removes it and
|
||||
* retries instead of failing. The rmdir/stat races with a concurrent
|
||||
* reclaimer or the owner's own cleanup are harmless — losers just loop.
|
||||
*/
|
||||
function withLedgerLock<T>(ledger: string, callback: () => T): T {
|
||||
const lock = `${ledger}.lock`;
|
||||
const deadline = Date.now() + 2500;
|
||||
for (;;) {
|
||||
try {
|
||||
fs.mkdirSync(lock);
|
||||
break;
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException)?.code !== 'EEXIST') throw error;
|
||||
if (Date.now() > deadline) {
|
||||
try {
|
||||
const age = Date.now() - fs.statSync(lock).mtimeMs;
|
||||
if (age > 10_000) { fs.rmdirSync(lock); continue; }
|
||||
} catch { /* raced with the owner's cleanup — retry */ }
|
||||
throw receiptError(`Egress ledger is locked: ${lock}`);
|
||||
}
|
||||
// Sync sleep (node + bun): the API is sync on purpose so shell, bun,
|
||||
// and node callers all share one implementation.
|
||||
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10);
|
||||
}
|
||||
}
|
||||
try {
|
||||
return callback();
|
||||
} finally {
|
||||
try { fs.rmdirSync(lock); } catch { /* best-effort unlock */ }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Last raw line via a tail read: open the file, read the final
|
||||
* TAIL_READ_BYTES, take the last newline-terminated chunk. Never loads the
|
||||
* whole ledger, so appends stay O(1) as the file grows.
|
||||
*/
|
||||
function lastRawLine(ledger: string): string | null {
|
||||
let fd: number;
|
||||
try {
|
||||
fd = fs.openSync(ledger, 'r');
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException)?.code === 'ENOENT') return null;
|
||||
throw error;
|
||||
}
|
||||
try {
|
||||
const size = fs.fstatSync(fd).size;
|
||||
if (size === 0) return null;
|
||||
const length = Math.min(size, TAIL_READ_BYTES);
|
||||
const buffer = Buffer.alloc(length);
|
||||
fs.readSync(fd, buffer, 0, length, size - length);
|
||||
const tail = buffer.toString('utf8');
|
||||
const lines = tail.split('\n').filter((line) => line.length > 0);
|
||||
return lines.length ? lines[lines.length - 1] : null;
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
}
|
||||
|
||||
let warnedLedgerSize = false;
|
||||
|
||||
/** Test-only: re-arm the once-per-process size warning. */
|
||||
export function resetLedgerSizeWarningForTests(): void {
|
||||
warnedLedgerSize = false;
|
||||
}
|
||||
|
||||
function warnLedgerSizeOnce(ledger: string): void {
|
||||
let size: number;
|
||||
try {
|
||||
size = fs.statSync(ledger).size;
|
||||
} catch {
|
||||
return; // no file yet — nothing to warn about
|
||||
}
|
||||
if (size <= LEDGER_WARN_BYTES || warnedLedgerSize) return;
|
||||
warnedLedgerSize = true;
|
||||
process.stderr.write(ledgerSizeWarning(ledger, size) + '\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Self-explanatory size warning: says what the ledger is (records what
|
||||
* gstack ATTEMPTS to send off-machine), how to inspect it, and that
|
||||
* trimming arrives with rotation.
|
||||
*/
|
||||
export function ledgerSizeWarning(ledger: string, size: number): string {
|
||||
const mb = (size / (1024 * 1024)).toFixed(1);
|
||||
return (
|
||||
`gstack: egress ledger is large (${mb}MB): ${ledger}. ` +
|
||||
`This file records what gstack ATTEMPTS to send off-machine (content-free receipts, for auditing). ` +
|
||||
`Inspect it with 'gstack-egress list'. Trimming arrives with ledger rotation (TODO); until then it only grows.`
|
||||
);
|
||||
}
|
||||
|
||||
function appendChained(
|
||||
homeOrNull: string | null,
|
||||
record: Record<string, unknown>,
|
||||
env?: Env,
|
||||
): { id: string; path: string } {
|
||||
const home = homeOrNull ?? resolveEgressHome(env);
|
||||
const ledger = egressLedgerPath(home);
|
||||
try {
|
||||
fs.mkdirSync(path.dirname(ledger), { recursive: true, mode: 0o700 });
|
||||
return withLedgerLock(ledger, () => {
|
||||
const previous = lastRawLine(ledger);
|
||||
const line = JSON.stringify({ ...record, prev: previous == null ? '' : sha256Hex(previous) });
|
||||
const existed = fs.existsSync(ledger);
|
||||
fs.appendFileSync(ledger, `${line}\n`, { mode: 0o600 });
|
||||
if (!existed) fs.chmodSync(ledger, 0o600); // umask must not weaken the ledger
|
||||
return { id: sha256Hex(line), path: ledger };
|
||||
});
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException)?.code === EGRESS_RECEIPT_FAILED) throw error;
|
||||
throw receiptError(
|
||||
`Egress receipt could not be written to ${ledger}: ${(error as Error)?.message ?? error}`,
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Append one content-free receipt BEFORE a network send.
|
||||
*
|
||||
* @returns id = sha256 of the written line (for writeOutcome)
|
||||
* @throws Error with code EGRESS_RECEIPT_FAILED — fail-closed callers must
|
||||
* refuse the send; fail-open callers warn and proceed
|
||||
*/
|
||||
export function writeReceipt(opts: WriteReceiptOptions): { id: string; path: string } {
|
||||
const sink = requireString(opts.sink, 'sink');
|
||||
const host = requireString(opts.host, 'host');
|
||||
const payloadClass = requireString(opts.payloadClass, 'payloadClass');
|
||||
const consent = requireString(opts.consent, 'consent');
|
||||
const bytes = opts.bytes ?? 0;
|
||||
if (!Number.isSafeInteger(bytes) || bytes < 0) throw receiptError('Egress receipt bytes must be a non-negative integer');
|
||||
const sha256 = opts.sha256 ?? null;
|
||||
if (sha256 !== null && !SHA256_HEX.test(String(sha256))) throw receiptError('Egress receipt sha256 must be 64 lowercase hex chars or null');
|
||||
const home = opts.home ?? resolveEgressHome(opts.env);
|
||||
warnLedgerSizeOnce(egressLedgerPath(home));
|
||||
return appendChained(home, {
|
||||
ts: new Date().toISOString(),
|
||||
type: 'egress',
|
||||
sink,
|
||||
host,
|
||||
payload_class: payloadClass,
|
||||
bytes,
|
||||
sha256,
|
||||
consent,
|
||||
}, opts.env);
|
||||
}
|
||||
|
||||
/**
|
||||
* Append the response status for an earlier receipt (best-effort companion
|
||||
* record — the pre-send receipt is the invariant, the outcome is
|
||||
* bookkeeping). Chained like every other line.
|
||||
*/
|
||||
export function writeOutcome(opts: WriteOutcomeOptions): { id: string; path: string } {
|
||||
const receipt = requireString(opts.receipt, 'receipt id');
|
||||
return appendChained(opts.home ?? null, {
|
||||
ts: new Date().toISOString(),
|
||||
type: 'outcome',
|
||||
receipt,
|
||||
status: String(opts.status ?? 'unknown'),
|
||||
}, opts.env);
|
||||
}
|
||||
|
||||
/** Raw parsed lines: [{lineNo, raw, record|null}]. Missing ledger → []. */
|
||||
export function readLedger(home: string): LedgerLine[] {
|
||||
const ledger = egressLedgerPath(home);
|
||||
let content: string;
|
||||
try {
|
||||
content = fs.readFileSync(ledger, 'utf8');
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException)?.code === 'ENOENT') return [];
|
||||
throw error;
|
||||
}
|
||||
return content.split('\n').filter((line) => line.length > 0).map((raw, index) => {
|
||||
let record: Record<string, unknown> | null = null;
|
||||
try {
|
||||
const parsed = JSON.parse(raw);
|
||||
if (parsed && typeof parsed === 'object') record = parsed;
|
||||
} catch { /* malformed line — verifyLedger reports it */ }
|
||||
return { lineNo: index + 1, raw, record };
|
||||
});
|
||||
}
|
||||
|
||||
export interface Receipt {
|
||||
ts: string;
|
||||
type: 'egress';
|
||||
sink: string;
|
||||
host: string;
|
||||
payload_class: string;
|
||||
bytes: number;
|
||||
sha256: string | null;
|
||||
consent: string;
|
||||
prev: string;
|
||||
id: string;
|
||||
status: string | null;
|
||||
}
|
||||
|
||||
/** Receipts with their joined outcome status (`status: null` = none recorded). */
|
||||
export function listReceipts(home: string): Receipt[] {
|
||||
const lines = readLedger(home);
|
||||
const receipts: Receipt[] = [];
|
||||
const byId = new Map<string, Receipt>();
|
||||
for (const { raw, record } of lines) {
|
||||
if (!record) continue;
|
||||
if (record.type === 'egress') {
|
||||
const entry = { ...(record as unknown as Omit<Receipt, 'id' | 'status'>), id: sha256Hex(raw), status: null };
|
||||
receipts.push(entry);
|
||||
byId.set(entry.id, entry);
|
||||
} else if (record.type === 'outcome' && byId.has(record.receipt as string)) {
|
||||
byId.get(record.receipt as string)!.status = String(record.status);
|
||||
}
|
||||
}
|
||||
return receipts;
|
||||
}
|
||||
|
||||
/**
|
||||
* Recompute the hash chain. `brokenLine` is the 1-indexed first line whose
|
||||
* `prev` no longer matches the sha256 of the previous raw line (or that
|
||||
* fails to parse). `sizeWarning` is set when the ledger exceeds
|
||||
* LEDGER_WARN_BYTES.
|
||||
*/
|
||||
export function verifyLedger(home: string): VerifyResult {
|
||||
const ledger = egressLedgerPath(home);
|
||||
let sizeWarning: string | null = null;
|
||||
try {
|
||||
const size = fs.statSync(ledger).size;
|
||||
if (size > LEDGER_WARN_BYTES) sizeWarning = ledgerSizeWarning(ledger, size);
|
||||
} catch { /* missing ledger — verify of an empty chain below */ }
|
||||
const lines = readLedger(home);
|
||||
let previousRaw: string | null = null;
|
||||
for (const { lineNo, raw, record } of lines) {
|
||||
if (!record || typeof record.prev !== 'string') {
|
||||
return { ok: false, count: lines.length, brokenLine: lineNo, reason: 'unparseable or missing prev', sizeWarning };
|
||||
}
|
||||
const expected = previousRaw == null ? '' : sha256Hex(previousRaw);
|
||||
if (record.prev !== expected) {
|
||||
return { ok: false, count: lines.length, brokenLine: lineNo, reason: 'prev hash does not match previous line', sizeWarning };
|
||||
}
|
||||
previousRaw = raw;
|
||||
}
|
||||
return { ok: true, count: lines.length, brokenLine: null, reason: null, sizeWarning };
|
||||
}
|
||||
|
|
@ -0,0 +1,258 @@
|
|||
/**
|
||||
* Egress receipts — chain, fail-closed, verify, shell bridge. Free tier, no network.
|
||||
*
|
||||
* THREAT MODEL: the egress ledger is forensic observability — it records
|
||||
* ATTEMPTED egress so accidents are auditable; it is not an exfiltration
|
||||
* control.
|
||||
*
|
||||
* Pins the auditor contract:
|
||||
* - receipt-before-send fail-closed (EGRESS_RECEIPT_FAILED, no ledger = no send)
|
||||
* - content-free lines chained by prev = sha256(previous raw line)
|
||||
* - tail-read correctness (last line found without loading the whole file)
|
||||
* - abandoned-lock reclaim (stale lock dir >10s old is removed, not fatal)
|
||||
* - WARN-at-size (one self-explanatory stderr warning per process >25MB)
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { spawnSync } from 'child_process';
|
||||
import {
|
||||
EGRESS_RECEIPT_FAILED,
|
||||
LEDGER_WARN_BYTES,
|
||||
egressLedgerPath,
|
||||
ledgerSizeWarning,
|
||||
listReceipts,
|
||||
resetLedgerSizeWarningForTests,
|
||||
sha256Hex,
|
||||
verifyLedger,
|
||||
writeOutcome,
|
||||
writeReceipt,
|
||||
} from '../lib/egress-receipt';
|
||||
|
||||
const ROOT = path.resolve(new URL(import.meta.url).pathname, '..', '..');
|
||||
|
||||
let home: string;
|
||||
|
||||
beforeEach(() => {
|
||||
home = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-egress-'));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
try { fs.chmodSync(path.join(home, 'security'), 0o700); } catch {} // undo fail-closed fixtures
|
||||
fs.rmSync(home, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('egress receipt library', () => {
|
||||
test('receipts chain: prev = sha256 of the previous raw line, "" for line 1', () => {
|
||||
writeReceipt({ home, sink: 'a', host: 'h1', payloadClass: 'c', bytes: 3, sha256: sha256Hex('abc'), consent: 'k=v' });
|
||||
writeReceipt({ home, sink: 'b', host: 'h2', payloadClass: 'c', bytes: 0, sha256: null, consent: 'k=v' });
|
||||
const lines = fs.readFileSync(egressLedgerPath(home), 'utf-8').trim().split('\n');
|
||||
expect(lines.length).toBe(2);
|
||||
const first = JSON.parse(lines[0]);
|
||||
const second = JSON.parse(lines[1]);
|
||||
expect(first.prev).toBe('');
|
||||
expect(second.prev).toBe(sha256Hex(lines[0]));
|
||||
expect(first.sha256).toBe(sha256Hex('abc'));
|
||||
expect(second.sha256).toBeNull();
|
||||
expect(verifyLedger(home)).toMatchObject({ ok: true, count: 2 });
|
||||
});
|
||||
|
||||
test('ledger is 0600 and the security dir 0700', () => {
|
||||
writeReceipt({ home, sink: 'a', host: 'h', payloadClass: 'c', consent: 'k=v' });
|
||||
const ledger = egressLedgerPath(home);
|
||||
expect(fs.statSync(ledger).mode & 0o777).toBe(0o600);
|
||||
expect(fs.statSync(path.dirname(ledger)).mode & 0o777).toBe(0o700);
|
||||
});
|
||||
|
||||
test('fail-closed: unwritable security dir throws typed EGRESS_RECEIPT_FAILED', () => {
|
||||
if (process.platform === 'win32' || process.getuid?.() === 0) return; // chmod is advisory there
|
||||
writeReceipt({ home, sink: 'a', host: 'h', payloadClass: 'c', consent: 'k=v' });
|
||||
fs.chmodSync(path.join(home, 'security'), 0o500);
|
||||
try {
|
||||
expect(() =>
|
||||
writeReceipt({ home, sink: 'a', host: 'h', payloadClass: 'c', consent: 'k=v' }),
|
||||
).toThrow();
|
||||
try {
|
||||
writeReceipt({ home, sink: 'a', host: 'h', payloadClass: 'c', consent: 'k=v' });
|
||||
} catch (err: any) {
|
||||
expect(err.code).toBe(EGRESS_RECEIPT_FAILED);
|
||||
}
|
||||
} finally {
|
||||
fs.chmodSync(path.join(home, 'security'), 0o700);
|
||||
}
|
||||
});
|
||||
|
||||
test('outcome records join back onto their receipt in listReceipts', () => {
|
||||
const { id } = writeReceipt({ home, sink: 'a', host: 'h', payloadClass: 'c', consent: 'k=v' });
|
||||
writeOutcome({ home, receipt: id, status: 204 });
|
||||
const receipts = listReceipts(home);
|
||||
expect(receipts.length).toBe(1);
|
||||
expect(receipts[0].status).toBe('204');
|
||||
expect(verifyLedger(home)).toMatchObject({ ok: true, count: 2 });
|
||||
});
|
||||
|
||||
test('tampering with a middle line breaks verification at that line', () => {
|
||||
for (let i = 0; i < 3; i += 1) {
|
||||
writeReceipt({ home, sink: `s${i}`, host: 'h', payloadClass: 'c', consent: 'telemetry=community' });
|
||||
}
|
||||
const ledger = egressLedgerPath(home);
|
||||
const lines = fs.readFileSync(ledger, 'utf-8').trim().split('\n');
|
||||
lines[1] = lines[1].replace('community', 'communitX');
|
||||
fs.writeFileSync(ledger, `${lines.join('\n')}\n`);
|
||||
// Line 2's edited bytes no longer hash to line 3's recorded prev.
|
||||
expect(verifyLedger(home)).toMatchObject({ ok: false, brokenLine: 3 });
|
||||
});
|
||||
|
||||
test('validation rejects garbage before touching the ledger', () => {
|
||||
expect(() => writeReceipt({ home, sink: '', host: 'h', payloadClass: 'c', consent: 'k' } as any)).toThrow();
|
||||
expect(() => writeReceipt({ home, sink: 's', host: 'h', payloadClass: 'c', consent: 'k', bytes: -1 })).toThrow();
|
||||
expect(() => writeReceipt({ home, sink: 's', host: 'h', payloadClass: 'c', consent: 'k', sha256: 'nothex' })).toThrow();
|
||||
expect(fs.existsSync(egressLedgerPath(home))).toBe(false);
|
||||
});
|
||||
|
||||
test('abandoned lock: a stale lock dir (>10s-old mtime) is reclaimed by the next writer', () => {
|
||||
// First write creates the security dir so the lock path's parent exists.
|
||||
writeReceipt({ home, sink: 'a', host: 'h', payloadClass: 'c', consent: 'k=v' });
|
||||
const lock = `${egressLedgerPath(home)}.lock`;
|
||||
fs.mkdirSync(lock);
|
||||
const old = new Date(Date.now() - 60_000);
|
||||
fs.utimesSync(lock, old, old);
|
||||
const started = Date.now();
|
||||
const { id } = writeReceipt({ home, sink: 'b', host: 'h', payloadClass: 'c', consent: 'k=v' });
|
||||
expect(id).toMatch(/^[0-9a-f]{64}$/);
|
||||
// Spin budget is 2.5s; reclaim happens right after budget exhaustion.
|
||||
expect(Date.now() - started).toBeLessThan(10_000);
|
||||
expect(fs.existsSync(lock)).toBe(false);
|
||||
expect(verifyLedger(home)).toMatchObject({ ok: true, count: 2 });
|
||||
});
|
||||
|
||||
test('a fresh (recent-mtime) lock held past the budget fails closed instead of being stolen', () => {
|
||||
writeReceipt({ home, sink: 'a', host: 'h', payloadClass: 'c', consent: 'k=v' });
|
||||
const lock = `${egressLedgerPath(home)}.lock`;
|
||||
fs.mkdirSync(lock);
|
||||
// Keep the mtime fresh so the reclaim path never fires: refresh it in the
|
||||
// background while the writer spins out its 2.5s budget.
|
||||
const refresher = setInterval(() => {
|
||||
const now = new Date();
|
||||
try { fs.utimesSync(lock, now, now); } catch { /* test teardown race */ }
|
||||
}, 1000);
|
||||
try {
|
||||
expect(() =>
|
||||
writeReceipt({ home, sink: 'b', host: 'h', payloadClass: 'c', consent: 'k=v' }),
|
||||
).toThrow(/locked/);
|
||||
} finally {
|
||||
clearInterval(refresher);
|
||||
fs.rmdirSync(lock);
|
||||
}
|
||||
}, 15_000);
|
||||
|
||||
test('tail-read: last line is found correctly on a multi-record ledger larger than the tail window', () => {
|
||||
// 30 records ≈ 9KB > the 4KB tail window, so the append path must find
|
||||
// the true last line from a partial read.
|
||||
for (let i = 0; i < 30; i += 1) {
|
||||
writeReceipt({
|
||||
home,
|
||||
sink: `sink-${i}`,
|
||||
host: 'h',
|
||||
payloadClass: `class-${'x'.repeat(200)}-${i}`,
|
||||
consent: 'k=v',
|
||||
});
|
||||
}
|
||||
const ledger = egressLedgerPath(home);
|
||||
expect(fs.statSync(ledger).size).toBeGreaterThan(4096);
|
||||
const result = verifyLedger(home);
|
||||
expect(result).toMatchObject({ ok: true, count: 30 });
|
||||
});
|
||||
|
||||
test('chain verify stays intact across 100+ records', () => {
|
||||
for (let i = 0; i < 120; i += 1) {
|
||||
writeReceipt({ home, sink: `s${i}`, host: 'h', payloadClass: 'c', consent: 'k=v' });
|
||||
if (i % 10 === 0) writeOutcome({ home, receipt: 'f'.repeat(64), status: 200 });
|
||||
}
|
||||
const result = verifyLedger(home);
|
||||
expect(result.ok).toBe(true);
|
||||
expect(result.count).toBe(132);
|
||||
expect(listReceipts(home).length).toBe(120);
|
||||
});
|
||||
|
||||
test('WARN-at-size: >25MB ledger emits one self-explanatory stderr warning per process', () => {
|
||||
const ledger = egressLedgerPath(home);
|
||||
fs.mkdirSync(path.dirname(ledger), { recursive: true, mode: 0o700 });
|
||||
// Grow the file past the threshold with valid-looking filler; the warning
|
||||
// keys off file size only.
|
||||
const filler = `${JSON.stringify({ type: 'egress', pad: 'x'.repeat(1024) })}\n`;
|
||||
const chunk = filler.repeat(1024); // ~1MB
|
||||
const writes = Math.ceil(LEDGER_WARN_BYTES / chunk.length) + 1;
|
||||
for (let i = 0; i < writes; i += 1) fs.appendFileSync(ledger, chunk);
|
||||
expect(fs.statSync(ledger).size).toBeGreaterThan(LEDGER_WARN_BYTES);
|
||||
|
||||
resetLedgerSizeWarningForTests();
|
||||
const captured: string[] = [];
|
||||
const originalWrite = process.stderr.write.bind(process.stderr);
|
||||
(process.stderr as any).write = (chunk: string) => { captured.push(String(chunk)); return true; };
|
||||
try {
|
||||
writeReceipt({ home, sink: 'a', host: 'h', payloadClass: 'c', consent: 'k=v' });
|
||||
writeReceipt({ home, sink: 'b', host: 'h', payloadClass: 'c', consent: 'k=v' });
|
||||
} finally {
|
||||
(process.stderr as any).write = originalWrite;
|
||||
}
|
||||
const warnings = captured.filter((c) => c.includes('egress ledger is large'));
|
||||
expect(warnings.length).toBe(1); // once per process, not per write
|
||||
// Self-explanatory shape: what the ledger is, how to inspect, what's coming.
|
||||
expect(warnings[0]).toContain('ATTEMPTS to send off-machine');
|
||||
expect(warnings[0]).toContain('gstack-egress list');
|
||||
expect(warnings[0]).toContain('rotation');
|
||||
expect(warnings[0]).toContain(ledger);
|
||||
|
||||
// verifyLedger surfaces the same warning as data.
|
||||
const message = ledgerSizeWarning(ledger, fs.statSync(ledger).size);
|
||||
expect(message).toContain('MB');
|
||||
expect(verifyLedger(home).sizeWarning).toBe(message);
|
||||
});
|
||||
});
|
||||
|
||||
describe('gstack-egress-receipt shell bridge', () => {
|
||||
const bin = path.join(ROOT, 'bin', 'gstack-egress-receipt');
|
||||
|
||||
test('write hashes the exact payload file, prints the receipt id; outcome joins', () => {
|
||||
const payload = path.join(home, 'payload.json');
|
||||
fs.writeFileSync(payload, '[{"v":1}]');
|
||||
const write = spawnSync(bin, ['write', '--sink', 'telemetry-sync', '--host', '127.0.0.1:8399',
|
||||
'--class', 'telemetry-events', '--payload-file', payload, '--consent', 'telemetry=community'],
|
||||
{ encoding: 'utf-8', env: { ...process.env, GSTACK_HOME: home } });
|
||||
expect(write.status).toBe(0);
|
||||
const id = write.stdout.trim();
|
||||
expect(id).toMatch(/^[0-9a-f]{64}$/);
|
||||
const outcome = spawnSync(bin, ['outcome', id, '204'],
|
||||
{ encoding: 'utf-8', env: { ...process.env, GSTACK_HOME: home } });
|
||||
expect(outcome.status).toBe(0);
|
||||
const receipts = listReceipts(home);
|
||||
expect(receipts.length).toBe(1);
|
||||
expect(receipts[0].bytes).toBe(9);
|
||||
expect(receipts[0].sha256).toBe(sha256Hex('[{"v":1}]'));
|
||||
expect(receipts[0].status).toBe('204');
|
||||
});
|
||||
|
||||
test('--no-payload records sha256:null (git-class: a subprocess owns the bytes)', () => {
|
||||
const write = spawnSync(bin, ['write', '--sink', 'brain-sync', '--host', 'github.com',
|
||||
'--class', 'git-push', '--no-payload', '--consent', 'artifacts_sync_mode=auto'],
|
||||
{ encoding: 'utf-8', env: { ...process.env, GSTACK_HOME: home } });
|
||||
expect(write.status).toBe(0);
|
||||
const receipts = listReceipts(home);
|
||||
expect(receipts.length).toBe(1);
|
||||
expect(receipts[0].sha256).toBeNull();
|
||||
expect(receipts[0].bytes).toBe(0);
|
||||
});
|
||||
|
||||
test('write exits 3 with EGRESS_RECEIPT_FAILED when the ledger is unwritable', () => {
|
||||
if (process.platform === 'win32' || process.getuid?.() === 0) return;
|
||||
fs.mkdirSync(path.join(home, 'security'), { recursive: true, mode: 0o500 });
|
||||
const write = spawnSync(bin, ['write', '--sink', 's', '--host', 'h', '--class', 'c', '--no-payload'],
|
||||
{ encoding: 'utf-8', env: { ...process.env, GSTACK_HOME: home } });
|
||||
expect(write.status).toBe(3);
|
||||
expect(write.stderr).toContain('EGRESS_RECEIPT_FAILED');
|
||||
fs.chmodSync(path.join(home, 'security'), 0o700);
|
||||
});
|
||||
});
|
||||
Loading…
Reference in New Issue