diff --git a/server/src/routes/health.ts b/server/src/routes/health.ts index bf7dc4a4a0..46b38c3a2a 100644 --- a/server/src/routes/health.ts +++ b/server/src/routes/health.ts @@ -80,6 +80,7 @@ function hasWorkspaceReadinessToken(providedToken: string | undefined) { function redactedDatabaseBackupWarning(warning: DatabaseBackupHealthWarning): DatabaseBackupHealthWarning { const messages: Record = { database_backup_check_failed: "Database backup health check failed.", + database_backup_empty: "Latest database backup is empty.", database_backup_last_failure: "Database backup failure marker is present.", database_backup_missing: "No recent database backup was found.", database_backup_stale: "Latest database backup is stale.", @@ -375,7 +376,7 @@ export function healthRoutes( : null; const databaseBackup = opts.databaseBackupHealth - ? inspectDatabaseBackupHealth(opts.databaseBackupHealth) + ? await inspectDatabaseBackupHealth(opts.databaseBackupHealth) : undefined; const warnings = databaseBackup?.warnings.length ? databaseBackup.warnings : undefined; const nativeRecovery = exposeFullDetails diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts index 2981e2dcf4..64c8b6e22c 100644 --- a/server/src/routes/openapi.ts +++ b/server/src/routes/openapi.ts @@ -1786,6 +1786,7 @@ registry.registerPath({ mtime: z.string().datetime(), ageHours: z.number(), sizeBytes: z.number(), + empty: z.boolean(), }) .nullable() .optional(), @@ -1801,6 +1802,7 @@ registry.registerPath({ z.object({ code: z.enum([ "database_backup_check_failed", + "database_backup_empty", "database_backup_last_failure", "database_backup_missing", "database_backup_stale", diff --git a/server/src/services/database-backup-health.test.ts b/server/src/services/database-backup-health.test.ts new file mode 100644 index 0000000000..4c87308e19 --- /dev/null +++ b/server/src/services/database-backup-health.test.ts @@ -0,0 +1,71 @@ +import { gzipSync } from "node:zlib"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { inspectDatabaseBackupHealth } from "./database-backup-health.js"; + +describe("inspectDatabaseBackupHealth", () => { + let dir: string | undefined; + + afterEach(() => { + if (dir) rmSync(dir, { recursive: true, force: true }); + dir = undefined; + }); + + // Regression for REVIP-8079: an aborted backup run left a valid but + // empty .sql.gz (gzip of zero bytes) that passed every existing check. + it("warns and reports non-ok status for an empty .sql.gz", async () => { + dir = mkdtempSync(join(tmpdir(), "db-backup-health-")); + writeFileSync(join(dir, "paperclip-20260911-135047.sql.gz"), gzipSync(Buffer.from(""))); + + const result = await inspectDatabaseBackupHealth({ + enabled: true, + backupDir: dir, + maxAgeHours: 26, + }); + + expect(result.status).toBe("warning"); + expect(result.warnings.map((w) => w.code)).toContain("database_backup_empty"); + expect(result.latestBackup?.empty).toBe(true); + }); + + it("reports ok for a complete, non-empty archive", async () => { + dir = mkdtempSync(join(tmpdir(), "db-backup-health-")); + writeFileSync( + join(dir, "paperclip-20260911-125024.sql.gz"), + gzipSync(Buffer.from("CREATE TABLE example (id integer);")), + ); + + const result = await inspectDatabaseBackupHealth({ + enabled: true, + backupDir: dir, + maxAgeHours: 26, + }); + + expect(result.status).toBe("ok"); + expect(result.warnings).toEqual([]); + expect(result.latestBackup?.empty).toBe(false); + }); + + it("does not trust a zero ISIZE trailer when the archive contains data", async () => { + dir = mkdtempSync(join(tmpdir(), "db-backup-health-")); + writeFileSync( + join(dir, "paperclip-20260911-125024.sql.gz"), + Buffer.concat([ + gzipSync(Buffer.from("CREATE TABLE example (id integer);")), + gzipSync(Buffer.from("")), + ]), + ); + + const result = await inspectDatabaseBackupHealth({ + enabled: true, + backupDir: dir, + maxAgeHours: 26, + }); + + expect(result.status).toBe("ok"); + expect(result.warnings).toEqual([]); + expect(result.latestBackup?.empty).toBe(false); + }); +}); diff --git a/server/src/services/database-backup-health.ts b/server/src/services/database-backup-health.ts index efb076db35..cf7cba1498 100644 --- a/server/src/services/database-backup-health.ts +++ b/server/src/services/database-backup-health.ts @@ -1,8 +1,10 @@ -import { existsSync, readdirSync, readFileSync, statSync } from "node:fs"; +import { closeSync, createReadStream, existsSync, openSync, readdirSync, readFileSync, readSync, statSync } from "node:fs"; import { basename, join, resolve } from "node:path"; +import { createGunzip } from "node:zlib"; export type DatabaseBackupHealthWarningCode = | "database_backup_check_failed" + | "database_backup_empty" | "database_backup_last_failure" | "database_backup_missing" | "database_backup_stale"; @@ -23,6 +25,7 @@ export type DatabaseBackupHealthStatus = { mtime: string; ageHours: number; sizeBytes: number; + empty: boolean; } | null; lastFailure: { path: string; @@ -78,7 +81,56 @@ function readLastFailure(alertFiles: string[]) { }; } -function findLatestBackup(backupDir: string, nowMs: number) { +// gzip stores the uncompressed size mod 2^32 in the last 4 bytes of the +// stream (RFC 1952 ISIZE). A backup that never received real content +// (e.g. an aborted run) is a valid, small gzip stream whose ISIZE is 0 - +// that case is otherwise indistinguishable from a healthy backup by +// looking only at the compressed file size. +async function readGzipIsEmpty(filePath: string, compressedSizeBytes: number): Promise { + if (compressedSizeBytes < 18) return false; + const fd = openSync(filePath, "r"); + try { + const trailer = Buffer.alloc(4); + readSync(fd, trailer, 0, 4, compressedSizeBytes - 4); + if (trailer.readUInt32LE(0) !== 0) return false; + } finally { + closeSync(fd); + } + + // ISIZE is stored modulo 2^32. Confirm a zero trailer by streaming the + // archive and stopping after the first output byte. This keeps large, + // wrapped-ISIZE backups off the synchronous health-request path and avoids + // materializing the compressed archive in memory. + return await new Promise((resolvePromise, reject) => { + const input = createReadStream(filePath); + const gunzip = createGunzip(); + let settled = false; + + const settle = (empty: boolean) => { + if (settled) return; + settled = true; + input.destroy(); + gunzip.destroy(); + resolvePromise(empty); + }; + + const fail = (error: Error) => { + if (settled) return; + settled = true; + input.destroy(); + gunzip.destroy(); + reject(error); + }; + + input.on("error", fail); + gunzip.on("error", fail); + gunzip.once("data", () => settle(false)); + gunzip.once("end", () => settle(true)); + input.pipe(gunzip); + }); +} + +async function findLatestBackup(backupDir: string, nowMs: number) { if (!existsSync(backupDir)) return null; const candidates = readdirSync(backupDir) @@ -99,12 +151,13 @@ function findLatestBackup(backupDir: string, nowMs: number) { mtime: new Date(latest.stat.mtimeMs).toISOString(), ageHours: roundHours((nowMs - latest.stat.mtimeMs) / 3_600_000), sizeBytes: latest.stat.size, + empty: await readGzipIsEmpty(latest.fullPath, latest.stat.size), }; } -export function inspectDatabaseBackupHealth( +export async function inspectDatabaseBackupHealth( opts: InspectDatabaseBackupHealthOptions, -): DatabaseBackupHealthStatus { +): Promise { const warnings: DatabaseBackupHealthWarning[] = []; const now = opts.now ?? new Date(); const maxAgeHours = Math.max(1, opts.maxAgeHours); @@ -113,7 +166,7 @@ export function inspectDatabaseBackupHealth( let lastFailure: DatabaseBackupHealthStatus["lastFailure"] = null; try { - latestBackup = findLatestBackup(opts.backupDir, now.getTime()); + latestBackup = await findLatestBackup(opts.backupDir, now.getTime()); lastFailure = readLastFailure(alertFileCandidates(opts)); if (!latestBackup) { @@ -121,11 +174,19 @@ export function inspectDatabaseBackupHealth( code: "database_backup_missing", message: `No .sql.gz database backups found in ${opts.backupDir}.`, }); - } else if (latestBackup.ageHours > maxAgeHours) { - warnings.push({ - code: "database_backup_stale", - message: `Latest database backup is ${latestBackup.ageHours}h old, exceeding ${maxAgeHours}h.`, - }); + } else { + if (latestBackup.empty) { + warnings.push({ + code: "database_backup_empty", + message: `Latest database backup ${latestBackup.name} contains no uncompressed data.`, + }); + } + if (latestBackup.ageHours > maxAgeHours) { + warnings.push({ + code: "database_backup_stale", + message: `Latest database backup is ${latestBackup.ageHours}h old, exceeding ${maxAgeHours}h.`, + }); + } } if (lastFailure) {