diff --git a/server/src/services/run-log-store.test.ts b/server/src/services/run-log-store.test.ts new file mode 100644 index 0000000000..029262ee07 --- /dev/null +++ b/server/src/services/run-log-store.test.ts @@ -0,0 +1,168 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { promises as fs } from "node:fs"; +import path from "node:path"; +import os from "node:os"; +import { Readable } from "node:stream"; +import { createDurableRunLogStore } from "./run-log-store.js"; +import type { StorageProvider } from "../storage/types.js"; + +// In-memory StorageProvider stand-in: durable, survives the "pod roll" (local +// dir wipe) the same way Cubbit does. Records calls so we can assert behaviour. +function createMemoryProvider() { + const objects = new Map(); + const calls = { put: 0, get: 0, head: 0 }; + const provider: StorageProvider = { + id: "s3", + async putObject(input) { + calls.put++; + if (Buffer.isBuffer(input.body)) { + objects.set(input.objectKey, Buffer.from(input.body)); + return; + } + const chunks: Buffer[] = []; + for await (const chunk of input.body) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + objects.set(input.objectKey, Buffer.concat(chunks)); + }, + async getObject(input) { + calls.get++; + const buf = objects.get(input.objectKey); + if (!buf) { + const err = new Error("Object not found") as Error & { name: string }; + err.name = "NoSuchKey"; + throw err; + } + const slice = input.range ? buf.subarray(input.range.start, input.range.end + 1) : buf; + return { stream: Readable.from(slice), contentLength: slice.length }; + }, + async headObject(input) { + calls.head++; + const buf = objects.get(input.objectKey); + return buf ? { exists: true, contentLength: buf.length } : { exists: false }; + }, + async deleteObject(input) { + objects.delete(input.objectKey); + }, + }; + return { provider, objects, calls }; +} + +let baseDir: string; +beforeEach(async () => { + baseDir = await fs.mkdtemp(path.join(os.tmpdir(), "run-log-store-test-")); +}); +afterEach(async () => { + await fs.rm(baseDir, { recursive: true, force: true }); +}); + +const begin = { companyId: "co1", agentId: "ag1", runId: "run1" }; + +describe("createDurableRunLogStore", () => { + it("keeps store id 'local_file' so downstream coupling (feedback, casts) is unchanged", async () => { + const { provider } = createMemoryProvider(); + const store = createDurableRunLogStore({ basePath: baseDir, s3: { provider } }); + const handle = await store.begin(begin); + expect(handle.store).toBe("local_file"); + }); + + it("appends locally and reads back during a run WITHOUT hitting S3 (fast live tail)", async () => { + const { provider, calls } = createMemoryProvider(); + const store = createDurableRunLogStore({ basePath: baseDir, s3: { provider } }); + const handle = await store.begin(begin); + await store.append(handle, { stream: "stdout", chunk: "hello", ts: "t1" }); + const res = await store.read(handle); + expect(res.content).toContain("hello"); + expect(calls.get).toBe(0); // local file present -> no S3 read + }); + + it("uploads the complete log to S3 on finalize", async () => { + const { provider, objects, calls } = createMemoryProvider(); + const store = createDurableRunLogStore({ basePath: baseDir, s3: { provider, keyPrefix: "run-logs" } }); + const handle = await store.begin(begin); + await store.append(handle, { stream: "stdout", chunk: "line-A", ts: "t1" }); + await store.append(handle, { stream: "stdout", chunk: "line-B", ts: "t2" }); + const summary = await store.finalize(handle); + expect(calls.put).toBe(1); + expect(summary.bytes).toBeGreaterThan(0); + // keyed by prefix + the handle's logRef so read can find it later + const key = `run-logs/${handle.logRef}`; + expect(objects.has(key)).toBe(true); + expect(objects.get(key)!.toString("utf8")).toContain("line-A"); + expect(objects.get(key)!.toString("utf8")).toContain("line-B"); + }); + + it("falls back to S3 when the local file is gone (the pod-roll case that caused 'Run log not found')", async () => { + const { provider } = createMemoryProvider(); + const store = createDurableRunLogStore({ basePath: baseDir, s3: { provider, keyPrefix: "run-logs" } }); + const handle = await store.begin(begin); + await store.append(handle, { stream: "stdout", chunk: "persisted-line", ts: "t1" }); + await store.finalize(handle); + // Simulate a pod restart wiping the emptyDir. + await fs.rm(baseDir, { recursive: true, force: true }); + const res = await store.read(handle); + expect(res.content).toContain("persisted-line"); + }); + + it("S3 fallback honours offset/limitBytes (range read) and reports nextOffset", async () => { + const { provider } = createMemoryProvider(); + const store = createDurableRunLogStore({ basePath: baseDir, s3: { provider, keyPrefix: "p" } }); + const handle = await store.begin(begin); + // one line; persisted bytes = JSON line + "\n" + await store.append(handle, { stream: "stdout", chunk: "0123456789", ts: "t" }); + await store.finalize(handle); + const full = await store.read(handle); // from local, to learn total size + const total = Buffer.byteLength(full.content, "utf8"); + await fs.rm(baseDir, { recursive: true, force: true }); // force S3 path + const firstHalf = await store.read(handle, { offset: 0, limitBytes: 5 }); + expect(Buffer.byteLength(firstHalf.content, "utf8")).toBe(5); + expect(firstHalf.nextOffset).toBe(5); + const tail = await store.read(handle, { offset: total - 3, limitBytes: 100 }); + expect(Buffer.byteLength(tail.content, "utf8")).toBe(3); + expect(tail.nextOffset).toBeUndefined(); + }); + + it("falls back to S3 when the local file vanishes between stat() and open (TOCTOU race)", async () => { + const { provider } = createMemoryProvider(); + const store = createDurableRunLogStore({ basePath: baseDir, s3: { provider, keyPrefix: "run-logs" } }); + const handle = await store.begin(begin); + await store.append(handle, { stream: "stdout", chunk: "raced-line", ts: "t1" }); + await store.finalize(handle); + // Delete the local file DURING stat(), i.e. after it reports the file + // present but before createReadStream opens it -> the open hits ENOENT. + const realStat = fs.stat.bind(fs); + const statSpy = vi.spyOn(fs, "stat").mockImplementation(async (target, ...rest) => { + const result = await realStat(target as Parameters[0], ...(rest as [])); + if (String(target).endsWith(".ndjson")) { + await fs.rm(target as string, { force: true }); + } + return result; + }); + try { + const res = await store.read(handle); + expect(res.content).toContain("raced-line"); + } finally { + statSpy.mockRestore(); + } + }); + + it("throws notFound when neither local nor S3 has the log (pre-S3 run after a roll)", async () => { + const { provider } = createMemoryProvider(); + const store = createDurableRunLogStore({ basePath: baseDir, s3: { provider } }); + const handle = await store.begin(begin); + await fs.rm(baseDir, { recursive: true, force: true }); // never finalized -> never uploaded + await expect(store.read(handle)).rejects.toThrow(/not found/i); + }); + + it("without S3 configured behaves exactly like the local-only store (safe degrade)", async () => { + const store = createDurableRunLogStore({ basePath: baseDir }); + const handle = await store.begin(begin); + await store.append(handle, { stream: "stdout", chunk: "local-only", ts: "t1" }); + await store.finalize(handle); + const res = await store.read(handle); + expect(res.content).toContain("local-only"); + // and a roll loses it (documented limitation; this is the pre-fix behaviour) + await fs.rm(baseDir, { recursive: true, force: true }); + await expect(store.read(handle)).rejects.toThrow(/not found/i); + }); +}); diff --git a/server/src/services/run-log-store.ts b/server/src/services/run-log-store.ts index 0d3b508c9d..8bf10d1ed4 100644 --- a/server/src/services/run-log-store.ts +++ b/server/src/services/run-log-store.ts @@ -3,6 +3,8 @@ import path from "node:path"; import { createHash } from "node:crypto"; import { notFound } from "../errors.js"; import { resolvePaperclipInstanceRoot } from "../home-paths.js"; +import { createS3StorageProvider } from "../storage/s3-provider.js"; +import type { StorageProvider } from "../storage/types.js"; export type RunLogStoreType = "local_file"; @@ -50,38 +52,100 @@ function resolveWithin(basePath: string, relativePath: string) { return resolved; } -function createLocalFileRunLogStore(basePath: string): RunLogStore { +function normalizeKeyPrefix(prefix: string | undefined): string { + if (!prefix) return ""; + return prefix.trim().replace(/^\/+/, "").replace(/\/+$/, ""); +} + +export interface DurableRunLogStoreOptions { + basePath: string; + // When provided, completed logs are mirrored to object storage on finalize and + // served from there on read whenever the local file is missing (e.g. the pod + // rolled and wiped the emptyDir). When omitted, the store is local-only (the + // historical behaviour: a restart loses the log). + s3?: { provider: StorageProvider; keyPrefix?: string }; +} + +// Run-log store with TRANSPARENT durability. The store id stays "local_file" so +// nothing downstream (feedback.ts, the heartbeat read cast, fixtures) changes; +// the S3 mirror is keyed by the same logRef and is purely an implementation +// detail. Live append/tail stays on the pod-local file (fast, no per-chunk PUT); +// on finalize the complete .ndjson is uploaded to object storage; on read we try +// local first and fall back to S3 when the local file is gone. This is the fix +// for "Run log not found" after a deploy/restart (the /paperclip data dir is an +// emptyDir in cloud_tenant mode -- persistence is disabled to avoid the +// operator's privileged selinux-relabel init container in our hardened ns). +export function createDurableRunLogStore(options: DurableRunLogStoreOptions): RunLogStore { + const { basePath } = options; + const s3 = options.s3; + const s3Prefix = normalizeKeyPrefix(s3?.keyPrefix); + + function s3Key(logRef: string): string { + return s3Prefix ? `${s3Prefix}/${logRef}` : logRef; + } + async function ensureDir(relativeDir: string) { const dir = resolveWithin(basePath, relativeDir); await fs.mkdir(dir, { recursive: true }); } - async function readFileRange(filePath: string, offset: number, limitBytes: number): Promise { + async function readLocalRange( + filePath: string, + offset: number, + limitBytes: number, + ): Promise { const stat = await fs.stat(filePath).catch(() => null); - if (!stat) throw notFound("Run log not found"); - + if (!stat) return null; const start = Math.max(0, Math.min(offset, stat.size)); const end = Math.max(start, Math.min(start + limitBytes - 1, stat.size - 1)); - - if (start > end) { - return { content: "", nextOffset: start }; - } + if (start > end) return { content: "", nextOffset: start }; const chunks: Buffer[] = []; - await new Promise((resolve, reject) => { - const stream = createReadStream(filePath, { start, end }); - stream.on("data", (chunk) => { - chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + try { + await new Promise((resolve, reject) => { + const stream = createReadStream(filePath, { start, end }); + stream.on("data", (chunk) => chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))); + stream.on("error", reject); + stream.on("end", () => resolve()); }); - stream.on("error", reject); - stream.on("end", () => resolve()); - }); - + } catch (err) { + // File deleted between stat() and open (pod-roll cleanup racing a read): + // treat as missing so the caller falls through to the S3 mirror instead + // of surfacing the very "Run log not found" this store exists to prevent. + if ((err as NodeJS.ErrnoException | null)?.code === "ENOENT") return null; + throw err; + } const content = Buffer.concat(chunks).toString("utf8"); const nextOffset = end + 1 < stat.size ? end + 1 : undefined; return { content, nextOffset }; } + async function readS3Range( + logRef: string, + offset: number, + limitBytes: number, + ): Promise { + if (!s3) throw notFound("Run log not found"); + const key = s3Key(logRef); + const head = await s3.provider.headObject({ objectKey: key }); + if (!head.exists) throw notFound("Run log not found"); + const total = head.contentLength ?? 0; + const start = Math.max(0, Math.min(offset, total)); + const end = Math.max(start, Math.min(start + limitBytes - 1, total - 1)); + if (start > end || total === 0) return { content: "", nextOffset: start < total ? start : undefined }; + + const result = await s3.provider.getObject({ objectKey: key, range: { start, end } }); + const chunks: Buffer[] = []; + await new Promise((resolve, reject) => { + result.stream.on("data", (chunk) => chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))); + result.stream.on("error", reject); + result.stream.on("end", () => resolve()); + }); + const content = Buffer.concat(chunks).toString("utf8"); + const nextOffset = end + 1 < total ? end + 1 : undefined; + return { content, nextOffset }; + } + async function sha256File(filePath: string): Promise { return new Promise((resolve, reject) => { const hash = createHash("sha256"); @@ -99,10 +163,8 @@ function createLocalFileRunLogStore(basePath: string): RunLogStore { const relDir = path.join(companyId, agentId); const relPath = path.join(relDir, `${runId}.ndjson`); await ensureDir(relDir); - const absPath = resolveWithin(basePath, relPath); await fs.writeFile(absPath, "", "utf8"); - return { store: "local_file", logRef: relPath }; }, @@ -124,38 +186,79 @@ function createLocalFileRunLogStore(basePath: string): RunLogStore { }, async finalize(handle) { - if (handle.store !== "local_file") { - return { bytes: 0, compressed: false }; - } + if (handle.store !== "local_file") return { bytes: 0, compressed: false }; const absPath = resolveWithin(basePath, handle.logRef); const stat = await fs.stat(absPath).catch(() => null); if (!stat) throw notFound("Run log not found"); - const hash = await sha256File(absPath); - return { - bytes: stat.size, - sha256: hash, - compressed: false, - }; + + // Mirror the completed log to object storage so it survives a pod roll. + // Best-effort upload failures must NOT fail run finalization (which also + // records cost/usage); the local copy still serves reads until the pod + // rolls, and a failed mirror only loses durability for that one run. + if (s3) { + try { + // Stream from disk instead of buffering the whole .ndjson in the + // heap; long agent sessions can produce large logs. The file is + // complete at this point, so stat.size is the exact content length. + await s3.provider.putObject({ + objectKey: s3Key(handle.logRef), + body: createReadStream(absPath), + contentType: "application/x-ndjson", + contentLength: stat.size, + }); + } catch (err) { + // Best-effort: finalization must not break, but a persistently + // failing mirror (bad creds/bucket/endpoint) should be visible to + // operators before a pod roll makes the logs unreadable. + console.warn( + `[run-log-store] Failed to mirror run log to object storage (key: ${s3Key(handle.logRef)}):`, + err, + ); + } + } + + return { bytes: stat.size, sha256: hash, compressed: false }; }, async read(handle, opts) { - if (handle.store !== "local_file") { - throw notFound("Run log not found"); - } + if (handle.store !== "local_file") throw notFound("Run log not found"); const absPath = resolveWithin(basePath, handle.logRef); const offset = opts?.offset ?? 0; const limitBytes = opts?.limitBytes ?? 256_000; - return readFileRange(absPath, offset, limitBytes); + const local = await readLocalRange(absPath, offset, limitBytes); + if (local) return local; + // Local file gone (pod rolled) -> serve from the S3 mirror if configured. + return readS3Range(handle.logRef, offset, limitBytes); }, }; } +// Build the run-log S3 mirror from dedicated RUN_LOG_S3_* env. Deliberately +// separate from PAPERCLIP_STORAGE_PROVIDER so enabling durable run logs does +// NOT redirect the product's workspace/file storage (smaller blast radius). +// Unset RUN_LOG_S3_BUCKET -> no mirror -> local-only (safe degrade). Creds come +// from the standard AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY chain. +function resolveRunLogS3(): { provider: StorageProvider; keyPrefix?: string } | undefined { + const bucket = process.env.RUN_LOG_S3_BUCKET?.trim(); + if (!bucket) return undefined; + const provider = createS3StorageProvider({ + bucket, + region: process.env.RUN_LOG_S3_REGION?.trim() || "us-east-1", + endpoint: process.env.RUN_LOG_S3_ENDPOINT?.trim() || undefined, + prefix: undefined, // prefixing is handled by keyPrefix below (kept off the provider) + forcePathStyle: process.env.RUN_LOG_S3_FORCE_PATH_STYLE + ? process.env.RUN_LOG_S3_FORCE_PATH_STYLE === "true" + : true, // Cubbit (and most S3-compatible endpoints) need path-style + }); + return { provider, keyPrefix: process.env.RUN_LOG_S3_PREFIX?.trim() || "run-logs" }; +} + let cachedStore: RunLogStore | null = null; export function getRunLogStore() { if (cachedStore) return cachedStore; const basePath = process.env.RUN_LOG_BASE_PATH ?? path.resolve(resolvePaperclipInstanceRoot(), "data", "run-logs"); - cachedStore = createLocalFileRunLogStore(basePath); + cachedStore = createDurableRunLogStore({ basePath, s3: resolveRunLogS3() }); return cachedStore; } diff --git a/server/src/storage/types.ts b/server/src/storage/types.ts index efeadd8ba8..dab23fa4c0 100644 --- a/server/src/storage/types.ts +++ b/server/src/storage/types.ts @@ -3,7 +3,9 @@ import type { Readable } from "node:stream"; export interface PutObjectInput { objectKey: string; - body: Buffer; + // Readable bodies stream straight to the backend (contentLength must be the + // exact byte size); Buffer stays supported for small payloads. + body: Buffer | Readable; contentType: string; contentLength: number; }