diff --git a/server/src/__tests__/cloud-tenant-transient-db-retry.test.ts b/server/src/__tests__/cloud-tenant-transient-db-retry.test.ts new file mode 100644 index 0000000000..70ada7397d --- /dev/null +++ b/server/src/__tests__/cloud-tenant-transient-db-retry.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from "vitest"; + +import { + isTransientDbConnectionError, + retryOnTransientDbConnectionError, +} from "../middleware/auth.ts"; + +/** The shape drizzle produces: a wrapper whose `cause` is the driver error. */ +function driverClosedError(code: string): Error { + const driver = Object.assign(new Error(`write ${code} db.example.internal:5432`), { code }); + return new Error("Failed query: insert into \"companies\" (…)", { cause: driver }); +} + +describe("isTransientDbConnectionError", () => { + it("detects a closed-connection code anywhere on the cause chain", () => { + expect(isTransientDbConnectionError(driverClosedError("CONNECTION_CLOSED"))).toBe(true); + expect(isTransientDbConnectionError(driverClosedError("CONNECTION_ENDED"))).toBe(true); + expect(isTransientDbConnectionError(driverClosedError("CONNECTION_DESTROYED"))).toBe(true); + const bare = Object.assign(new Error("write CONNECTION_CLOSED host:5432"), { + code: "CONNECTION_CLOSED", + }); + expect(isTransientDbConnectionError(bare)).toBe(true); + }); + + it("rejects everything else", () => { + expect(isTransientDbConnectionError(new Error("boom"))).toBe(false); + const unique = Object.assign(new Error("duplicate key"), { code: "23505" }); + expect(isTransientDbConnectionError(unique)).toBe(false); + expect(isTransientDbConnectionError(new Error("outer", { cause: unique }))).toBe(false); + expect(isTransientDbConnectionError("CONNECTION_CLOSED")).toBe(false); + expect(isTransientDbConnectionError(undefined)).toBe(false); + }); +}); + +describe("retryOnTransientDbConnectionError", () => { + it("retries exactly once after a transient closed connection", async () => { + let calls = 0; + const result = await retryOnTransientDbConnectionError(async () => { + calls += 1; + if (calls === 1) throw driverClosedError("CONNECTION_CLOSED"); + return "ok"; + }); + expect(result).toBe("ok"); + expect(calls).toBe(2); + }); + + it("propagates a non-transient failure without retrying", async () => { + let calls = 0; + await expect( + retryOnTransientDbConnectionError(async () => { + calls += 1; + throw new Error("constraint violation"); + }), + ).rejects.toThrow("constraint violation"); + expect(calls).toBe(1); + }); + + it("propagates the second failure when the retry also dies", async () => { + let calls = 0; + await expect( + retryOnTransientDbConnectionError(async () => { + calls += 1; + throw driverClosedError("CONNECTION_CLOSED"); + }), + ).rejects.toThrow("Failed query"); + expect(calls).toBe(2); + }); +}); diff --git a/server/src/middleware/auth.ts b/server/src/middleware/auth.ts index e2e4b84d59..1125cbf4f7 100644 --- a/server/src/middleware/auth.ts +++ b/server/src/middleware/auth.ts @@ -512,9 +512,62 @@ export function cloudActorHeaderSourceFromHeaders( }; } +/** + * postgres.js codes for a connection the server side closed out from under + * an in-flight query — a pooled Postgres endpoint recycling or suspending + * (observed 2026-09-03 with a managed pooler closing the socket mid-INSERT). + * The driver reconnects transparently on the next query; only the statement + * that was on the wire is lost. + */ +const transientDbConnectionCodes = new Set([ + "CONNECTION_CLOSED", + "CONNECTION_ENDED", + "CONNECTION_DESTROYED", +]); + +/** + * True when the error chain (drizzle wraps the driver error as `cause`) + * carries a postgres.js closed-connection code. Exported for tests. + */ +export function isTransientDbConnectionError(error: unknown): boolean { + for (let current: unknown = error; current instanceof Error; current = current.cause) { + const code = (current as { code?: unknown }).code; + if (typeof code === "string" && transientDbConnectionCodes.has(code)) return true; + } + return false; +} + +/** + * Runs `run` and retries it exactly once when it fails on a transient + * closed-connection error. Callers must pass an idempotent operation. + * Exported for tests. + */ +export async function retryOnTransientDbConnectionError(run: () => Promise): Promise { + try { + return await run(); + } catch (error) { + if (!isTransientDbConnectionError(error)) throw error; + return run(); + } +} + +/** + * Trusted-header actor resolution with a single transient-connection retry. + * The tenant sync inside is idempotent end to end — every write is an + * upsert/on-conflict/delete and the write debounce records only after the + * whole sync succeeds — so replaying it after a dropped connection is safe, + * and turns a golden-path authentication 500 into a served request. + */ export async function resolveCloudTenantActor( db: Db, req: CloudActorHeaderSource, +): Promise { + return retryOnTransientDbConnectionError(() => resolveCloudTenantActorOnce(db, req)); +} + +async function resolveCloudTenantActorOnce( + db: Db, + req: CloudActorHeaderSource, ): Promise { const expectedToken = process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN?.trim(); if (!expectedToken) return null;