From c1f18565f2c2a07d46039f42ee32d011e8173486 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?2=20=C2=B7=20CEO=20Codex?= Date: Sat, 12 Sep 2026 06:04:59 +0000 Subject: [PATCH 1/3] fix(health): flag empty database backup archives Detect valid gzip archives that contain no uncompressed data. Verify zero ISIZE trailers without false warnings at the 32-bit wrap boundary, publish the empty result, synchronize OpenAPI, and cover both empty and non-empty cases. Co-Authored-By: Paperclip --- server/src/routes/health.ts | 1 + server/src/routes/openapi.ts | 2 + .../services/database-backup-health.test.ts | 71 +++++++++++++++++++ server/src/services/database-backup-health.ts | 51 +++++++++++-- 4 files changed, 119 insertions(+), 6 deletions(-) create mode 100644 server/src/services/database-backup-health.test.ts diff --git a/server/src/routes/health.ts b/server/src/routes/health.ts index 0e4d954722..04eacc7ed1 100644 --- a/server/src/routes/health.ts +++ b/server/src/routes/health.ts @@ -79,6 +79,7 @@ function hasWorkspaceReadinessToken(providedToken: string | undefined) { function redactedDatabaseBackupWarning(warning: DatabaseBackupHealthWarning): DatabaseBackupHealthWarning { const messages: Record = { database_backup_check_failed: "Database backup health check failed.", + database_backup_empty: "Latest database backup is empty.", database_backup_last_failure: "Database backup failure marker is present.", database_backup_missing: "No recent database backup was found.", database_backup_stale: "Latest database backup is stale.", diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts index 1028baf59a..b0d25d27f7 100644 --- a/server/src/routes/openapi.ts +++ b/server/src/routes/openapi.ts @@ -1749,6 +1749,7 @@ registry.registerPath({ mtime: z.string().datetime(), ageHours: z.number(), sizeBytes: z.number(), + empty: z.boolean(), }) .nullable() .optional(), @@ -1764,6 +1765,7 @@ registry.registerPath({ z.object({ code: z.enum([ "database_backup_check_failed", + "database_backup_empty", "database_backup_last_failure", "database_backup_missing", "database_backup_stale", diff --git a/server/src/services/database-backup-health.test.ts b/server/src/services/database-backup-health.test.ts new file mode 100644 index 0000000000..cda31483ac --- /dev/null +++ b/server/src/services/database-backup-health.test.ts @@ -0,0 +1,71 @@ +import { gzipSync } from "node:zlib"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { inspectDatabaseBackupHealth } from "./database-backup-health.js"; + +describe("inspectDatabaseBackupHealth", () => { + let dir: string | undefined; + + afterEach(() => { + if (dir) rmSync(dir, { recursive: true, force: true }); + dir = undefined; + }); + + // Regression for REVIP-8079: an aborted backup run left a valid but + // empty .sql.gz (gzip of zero bytes) that passed every existing check. + it("warns and reports non-ok status for an empty .sql.gz", () => { + dir = mkdtempSync(join(tmpdir(), "db-backup-health-")); + writeFileSync(join(dir, "paperclip-20260911-135047.sql.gz"), gzipSync(Buffer.from(""))); + + const result = inspectDatabaseBackupHealth({ + enabled: true, + backupDir: dir, + maxAgeHours: 26, + }); + + expect(result.status).toBe("warning"); + expect(result.warnings.map((w) => w.code)).toContain("database_backup_empty"); + expect(result.latestBackup?.empty).toBe(true); + }); + + it("reports ok for a complete, non-empty archive", () => { + dir = mkdtempSync(join(tmpdir(), "db-backup-health-")); + writeFileSync( + join(dir, "paperclip-20260911-125024.sql.gz"), + gzipSync(Buffer.from("CREATE TABLE example (id integer);")), + ); + + const result = inspectDatabaseBackupHealth({ + enabled: true, + backupDir: dir, + maxAgeHours: 26, + }); + + expect(result.status).toBe("ok"); + expect(result.warnings).toEqual([]); + expect(result.latestBackup?.empty).toBe(false); + }); + + it("does not trust a zero ISIZE trailer when the archive contains data", () => { + dir = mkdtempSync(join(tmpdir(), "db-backup-health-")); + writeFileSync( + join(dir, "paperclip-20260911-125024.sql.gz"), + Buffer.concat([ + gzipSync(Buffer.from("CREATE TABLE example (id integer);")), + gzipSync(Buffer.from("")), + ]), + ); + + const result = inspectDatabaseBackupHealth({ + enabled: true, + backupDir: dir, + maxAgeHours: 26, + }); + + expect(result.status).toBe("ok"); + expect(result.warnings).toEqual([]); + expect(result.latestBackup?.empty).toBe(false); + }); +}); diff --git a/server/src/services/database-backup-health.ts b/server/src/services/database-backup-health.ts index efb076db35..1122886a51 100644 --- a/server/src/services/database-backup-health.ts +++ b/server/src/services/database-backup-health.ts @@ -1,8 +1,10 @@ -import { existsSync, readdirSync, readFileSync, statSync } from "node:fs"; +import { closeSync, existsSync, openSync, readdirSync, readFileSync, readSync, statSync } from "node:fs"; import { basename, join, resolve } from "node:path"; +import { gunzipSync } from "node:zlib"; export type DatabaseBackupHealthWarningCode = | "database_backup_check_failed" + | "database_backup_empty" | "database_backup_last_failure" | "database_backup_missing" | "database_backup_stale"; @@ -23,6 +25,7 @@ export type DatabaseBackupHealthStatus = { mtime: string; ageHours: number; sizeBytes: number; + empty: boolean; } | null; lastFailure: { path: string; @@ -78,6 +81,33 @@ function readLastFailure(alertFiles: string[]) { }; } +// gzip stores the uncompressed size mod 2^32 in the last 4 bytes of the +// stream (RFC 1952 ISIZE). A backup that never received real content +// (e.g. an aborted run) is a valid, small gzip stream whose ISIZE is 0 - +// that case is otherwise indistinguishable from a healthy backup by +// looking only at the compressed file size. +function readGzipIsEmpty(filePath: string, compressedSizeBytes: number): boolean { + if (compressedSizeBytes < 18) return false; + const fd = openSync(filePath, "r"); + try { + const trailer = Buffer.alloc(4); + readSync(fd, trailer, 0, 4, compressedSizeBytes - 4); + if (trailer.readUInt32LE(0) !== 0) return false; + } finally { + closeSync(fd); + } + + // ISIZE is stored modulo 2^32. Confirm a zero trailer by inflating at most + // one byte so a non-empty archive whose size wraps to zero is not reported + // as empty. zlib stops as soon as the output limit is exceeded. + try { + return gunzipSync(readFileSync(filePath), { maxOutputLength: 1 }).length === 0; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ERR_BUFFER_TOO_LARGE") return false; + throw error; + } +} + function findLatestBackup(backupDir: string, nowMs: number) { if (!existsSync(backupDir)) return null; @@ -99,6 +129,7 @@ function findLatestBackup(backupDir: string, nowMs: number) { mtime: new Date(latest.stat.mtimeMs).toISOString(), ageHours: roundHours((nowMs - latest.stat.mtimeMs) / 3_600_000), sizeBytes: latest.stat.size, + empty: readGzipIsEmpty(latest.fullPath, latest.stat.size), }; } @@ -121,11 +152,19 @@ export function inspectDatabaseBackupHealth( code: "database_backup_missing", message: `No .sql.gz database backups found in ${opts.backupDir}.`, }); - } else if (latestBackup.ageHours > maxAgeHours) { - warnings.push({ - code: "database_backup_stale", - message: `Latest database backup is ${latestBackup.ageHours}h old, exceeding ${maxAgeHours}h.`, - }); + } else { + if (latestBackup.empty) { + warnings.push({ + code: "database_backup_empty", + message: `Latest database backup ${latestBackup.name} contains no uncompressed data.`, + }); + } + if (latestBackup.ageHours > maxAgeHours) { + warnings.push({ + code: "database_backup_stale", + message: `Latest database backup is ${latestBackup.ageHours}h old, exceeding ${maxAgeHours}h.`, + }); + } } if (lastFailure) { From c77c930846a47428981a5ad02e9d3ba90bfa0031 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?2=20=C2=B7=20CEO=20Codex?= Date: Sat, 12 Sep 2026 06:15:16 +0000 Subject: [PATCH 2/3] fix(health): stream wrapped-ISIZE backup inspection --- server/src/routes/health.ts | 2 +- .../services/database-backup-health.test.ts | 12 ++--- server/src/services/database-backup-health.ts | 54 +++++++++++++------ 3 files changed, 45 insertions(+), 23 deletions(-) diff --git a/server/src/routes/health.ts b/server/src/routes/health.ts index 04eacc7ed1..073fc2cb83 100644 --- a/server/src/routes/health.ts +++ b/server/src/routes/health.ts @@ -375,7 +375,7 @@ export function healthRoutes( : null; const databaseBackup = opts.databaseBackupHealth - ? inspectDatabaseBackupHealth(opts.databaseBackupHealth) + ? await inspectDatabaseBackupHealth(opts.databaseBackupHealth) : undefined; const warnings = databaseBackup?.warnings.length ? databaseBackup.warnings : undefined; const nativeRecovery = exposeFullDetails diff --git a/server/src/services/database-backup-health.test.ts b/server/src/services/database-backup-health.test.ts index cda31483ac..4c87308e19 100644 --- a/server/src/services/database-backup-health.test.ts +++ b/server/src/services/database-backup-health.test.ts @@ -15,11 +15,11 @@ describe("inspectDatabaseBackupHealth", () => { // Regression for REVIP-8079: an aborted backup run left a valid but // empty .sql.gz (gzip of zero bytes) that passed every existing check. - it("warns and reports non-ok status for an empty .sql.gz", () => { + it("warns and reports non-ok status for an empty .sql.gz", async () => { dir = mkdtempSync(join(tmpdir(), "db-backup-health-")); writeFileSync(join(dir, "paperclip-20260911-135047.sql.gz"), gzipSync(Buffer.from(""))); - const result = inspectDatabaseBackupHealth({ + const result = await inspectDatabaseBackupHealth({ enabled: true, backupDir: dir, maxAgeHours: 26, @@ -30,14 +30,14 @@ describe("inspectDatabaseBackupHealth", () => { expect(result.latestBackup?.empty).toBe(true); }); - it("reports ok for a complete, non-empty archive", () => { + it("reports ok for a complete, non-empty archive", async () => { dir = mkdtempSync(join(tmpdir(), "db-backup-health-")); writeFileSync( join(dir, "paperclip-20260911-125024.sql.gz"), gzipSync(Buffer.from("CREATE TABLE example (id integer);")), ); - const result = inspectDatabaseBackupHealth({ + const result = await inspectDatabaseBackupHealth({ enabled: true, backupDir: dir, maxAgeHours: 26, @@ -48,7 +48,7 @@ describe("inspectDatabaseBackupHealth", () => { expect(result.latestBackup?.empty).toBe(false); }); - it("does not trust a zero ISIZE trailer when the archive contains data", () => { + it("does not trust a zero ISIZE trailer when the archive contains data", async () => { dir = mkdtempSync(join(tmpdir(), "db-backup-health-")); writeFileSync( join(dir, "paperclip-20260911-125024.sql.gz"), @@ -58,7 +58,7 @@ describe("inspectDatabaseBackupHealth", () => { ]), ); - const result = inspectDatabaseBackupHealth({ + const result = await inspectDatabaseBackupHealth({ enabled: true, backupDir: dir, maxAgeHours: 26, diff --git a/server/src/services/database-backup-health.ts b/server/src/services/database-backup-health.ts index 1122886a51..077b4d2acc 100644 --- a/server/src/services/database-backup-health.ts +++ b/server/src/services/database-backup-health.ts @@ -1,6 +1,6 @@ -import { closeSync, existsSync, openSync, readdirSync, readFileSync, readSync, statSync } from "node:fs"; +import { closeSync, createReadStream, existsSync, openSync, readdirSync, readFileSync, readSync, statSync } from "node:fs"; import { basename, join, resolve } from "node:path"; -import { gunzipSync } from "node:zlib"; +import { createGunzip } from "node:zlib"; export type DatabaseBackupHealthWarningCode = | "database_backup_check_failed" @@ -86,7 +86,7 @@ function readLastFailure(alertFiles: string[]) { // (e.g. an aborted run) is a valid, small gzip stream whose ISIZE is 0 - // that case is otherwise indistinguishable from a healthy backup by // looking only at the compressed file size. -function readGzipIsEmpty(filePath: string, compressedSizeBytes: number): boolean { +async function readGzipIsEmpty(filePath: string, compressedSizeBytes: number): Promise { if (compressedSizeBytes < 18) return false; const fd = openSync(filePath, "r"); try { @@ -97,18 +97,40 @@ function readGzipIsEmpty(filePath: string, compressedSizeBytes: number): boolean closeSync(fd); } - // ISIZE is stored modulo 2^32. Confirm a zero trailer by inflating at most - // one byte so a non-empty archive whose size wraps to zero is not reported - // as empty. zlib stops as soon as the output limit is exceeded. - try { - return gunzipSync(readFileSync(filePath), { maxOutputLength: 1 }).length === 0; - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ERR_BUFFER_TOO_LARGE") return false; - throw error; - } + // ISIZE is stored modulo 2^32. Confirm a zero trailer by streaming the + // archive and stopping after the first output byte. This keeps large, + // wrapped-ISIZE backups off the synchronous health-request path and avoids + // materializing the compressed archive in memory. + return await new Promise((resolvePromise, reject) => { + const input = createReadStream(filePath); + const gunzip = createGunzip(); + let settled = false; + + const settle = (empty: boolean) => { + if (settled) return; + settled = true; + input.destroy(); + gunzip.destroy(); + resolvePromise(empty); + }; + + const fail = (error: Error) => { + if (settled) return; + settled = true; + input.destroy(); + gunzip.destroy(); + reject(error); + }; + + input.on("error", fail); + gunzip.on("error", fail); + gunzip.once("data", () => settle(false)); + gunzip.once("end", () => settle(true)); + input.pipe(gunzip); + }); } -function findLatestBackup(backupDir: string, nowMs: number) { +async function findLatestBackup(backupDir: string, nowMs: number) { if (!existsSync(backupDir)) return null; const candidates = readdirSync(backupDir) @@ -129,11 +151,11 @@ function findLatestBackup(backupDir: string, nowMs: number) { mtime: new Date(latest.stat.mtimeMs).toISOString(), ageHours: roundHours((nowMs - latest.stat.mtimeMs) / 3_600_000), sizeBytes: latest.stat.size, - empty: readGzipIsEmpty(latest.fullPath, latest.stat.size), + empty: await readGzipIsEmpty(latest.fullPath, latest.stat.size), }; } -export function inspectDatabaseBackupHealth( +export async function inspectDatabaseBackupHealth( opts: InspectDatabaseBackupHealthOptions, ): DatabaseBackupHealthStatus { const warnings: DatabaseBackupHealthWarning[] = []; @@ -144,7 +166,7 @@ export function inspectDatabaseBackupHealth( let lastFailure: DatabaseBackupHealthStatus["lastFailure"] = null; try { - latestBackup = findLatestBackup(opts.backupDir, now.getTime()); + latestBackup = await findLatestBackup(opts.backupDir, now.getTime()); lastFailure = readLastFailure(alertFileCandidates(opts)); if (!latestBackup) { From 6a5be66d44998ea0165d90eb48000ac041c63e0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?2=20=C2=B7=20CEO=20Codex?= Date: Sat, 12 Sep 2026 06:22:25 +0000 Subject: [PATCH 3/3] fix(health): declare async backup inspection result --- server/src/services/database-backup-health.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/services/database-backup-health.ts b/server/src/services/database-backup-health.ts index 077b4d2acc..cf7cba1498 100644 --- a/server/src/services/database-backup-health.ts +++ b/server/src/services/database-backup-health.ts @@ -157,7 +157,7 @@ async function findLatestBackup(backupDir: string, nowMs: number) { export async function inspectDatabaseBackupHealth( opts: InspectDatabaseBackupHealthOptions, -): DatabaseBackupHealthStatus { +): Promise { const warnings: DatabaseBackupHealthWarning[] = []; const now = opts.now ?? new Date(); const maxAgeHours = Math.max(1, opts.maxAgeHours);