From 22001bbd2f5cdc6f1475836a0c0b4ece3267d44d Mon Sep 17 00:00:00 2001 From: Nicky Leach Date: Mon, 6 Jul 2026 18:43:12 -0700 Subject: [PATCH] feat(db): add migration safety lint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The `packages/db` module owns all database migrations via a sequential numbering system already validated at build time > - A recent migration introduced an O(n²) batch backfill over a large, unindexed table — it caused the server's `listen` to block for ~5 minutes on databases with millions of rows > - Nothing in the current CI pipeline catches large-table migration risk patterns (DO-loop mutations, batched LIMIT mutations without support indexes, full-table mutations, non-concurrent index creation) before they land > - This PR wires a new static migration-safety checker (`check-migration-safety.ts`) into the existing `check:migrations` gate in `packages/db/package.json`, so risky patterns fail CI before reaching production > - The checker baselines all historical findings already present in the codebase, so the gate fails only on *new* unbaselined risky patterns > - The benefit is that the specific O(n²) backfill shape (and related patterns) will be caught at author time rather than at incident time ## Linked Issues or Issue Description **Feature — static migration safety lint** **Problem or motivation** Migrations against large tables (millions of rows) have caused production startup blocks. The root pattern is a batched `LIMIT`-based backfill iterating via an unindexed column, making each batch a sequential scan — O(n²) overall. No CI gate exists to flag this class of problem before merge. **Proposed solution** A static SQL-level checker that scans new migration files for known dangerous patterns against known-large tables, producing structured findings that are either baselined (suppressed) or fail the build. Patterns detected: `DO $$ loop` mutations on large tables without a same-migration support index, batched `LIMIT` mutations on large tables missing a same-migration support index, unbounded full-table mutations (no `WHERE` clause), and `CREATE INDEX` without `CONCURRENTLY` on large tables. **Alternatives considered** Runtime instrumentation (only catches issues in production), advisory locking in migrations (doesn't prevent the pattern), per-migration code review (doesn't scale consistently). **Roadmap alignment** Defensive infrastructure / operational reliability — keeps migrations from blocking production startups. Not a user-facing feature. ## What Changed - **`packages/db/package.json`** — extended `check:migrations` script to run `check-migration-safety.ts` after the existing numbering check - **`packages/db/src/check-migration-safety.ts`** — new static checker: SQL pattern matching, rule detection for four dangerous patterns, baseline diffing, and structured exit with findings summary - **`packages/db/src/migration-safety-baseline.ts`** — baseline of all existing historical findings (suppressed from failing the gate); new migrations matching these patterns without a baseline entry will fail - **`packages/db/src/table-size-estimates.ts`** — rough table size estimates from the local dev database; drives `isKnownLargeTable()` used by the safety rules - **`packages/db/src/check-migration-safety.test.ts`** — Vitest coverage for the O(n²) backfill failure mode, suppression via baseline, and each rule type ## Verification ```bash # Run the migration safety checker directly cd packages/db tsx src/check-migration-safety.ts # Run tests cd packages/db npx vitest run src/check-migration-safety.test.ts # Run the full migration check gate (numbering + safety) cd packages/db pnpm run check:migrations ``` - Tests cover the core O(n²) backfill pattern (the motivating incident), baseline suppression, and all four rule types - `check:migrations` now exits non-zero for any new unbaselined large-table migration risk pattern ## Risks - **False positives:** Table-size estimates are from a local dev database snapshot — a table small in dev but large in production would be missed. Best-effort heuristic. - **Baseline drift:** If a baselined finding's SQL changes significantly, the baseline ID (content-hash-based) will no longer match and the finding will re-surface. Intentional but may surprise authors doing incremental fixes. - **SQL parsing limitations:** Regex-based pattern matching rather than a full AST parser — complex SQL may not be detected. Acceptable for an initial gate. - **Low risk to existing behavior:** The gate only fails on *new* findings not present in the baseline. All existing migrations are baselined. ## Model Used - **Provider:** Anthropic - **Model ID:** `claude-sonnet-4-6` - **Context window:** 200K - **Tool use:** yes (file reading, bash, git operations) - **Reasoning mode:** standard (no extended thinking) ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [ ] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- packages/db/package.json | 2 +- .../db/src/check-migration-safety.test.ts | 492 ++++++++ packages/db/src/check-migration-safety.ts | 1052 +++++++++++++++++ packages/db/src/migration-safety-baseline.ts | 157 +++ packages/db/src/table-size-estimates.ts | 178 +++ 5 files changed, 1880 insertions(+), 1 deletion(-) create mode 100644 packages/db/src/check-migration-safety.test.ts create mode 100644 packages/db/src/check-migration-safety.ts create mode 100644 packages/db/src/migration-safety-baseline.ts create mode 100644 packages/db/src/table-size-estimates.ts diff --git a/packages/db/package.json b/packages/db/package.json index 90b6edd6c0..1b1aae509a 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -35,7 +35,7 @@ "dist" ], "scripts": { - "check:migrations": "tsx src/check-migration-numbering.ts", + "check:migrations": "tsx src/check-migration-numbering.ts && tsx src/check-migration-safety.ts", "build": "pnpm run check:migrations && tsc && cp -r src/migrations dist/migrations", "clean": "rm -rf dist", "typecheck": "pnpm run check:migrations && tsc --noEmit", diff --git a/packages/db/src/check-migration-safety.test.ts b/packages/db/src/check-migration-safety.test.ts new file mode 100644 index 0000000000..57e3dd220b --- /dev/null +++ b/packages/db/src/check-migration-safety.test.ts @@ -0,0 +1,492 @@ +import { describe, expect, it } from "vitest"; +import { + analyzeMigrationSafety, + type MigrationSafetyInput, +} from "./check-migration-safety.js"; +import { + TABLE_SIZE_ESTIMATE_FACTOR, + TABLE_SIZE_BUCKET_THRESHOLDS, + type TableSizeEstimate, +} from "./table-size-estimates.js"; + +const testEstimates: readonly TableSizeEstimate[] = [ + { + table: "issue_comments", + localRows: 5_034, + estimateFactor: TABLE_SIZE_ESTIMATE_FACTOR, + estimatedRows: 5_034 * TABLE_SIZE_ESTIMATE_FACTOR, + bucket: "large", + }, + { + table: "companies", + localRows: 1, + estimateFactor: TABLE_SIZE_ESTIMATE_FACTOR, + estimatedRows: TABLE_SIZE_ESTIMATE_FACTOR, + bucket: "small", + }, +]; + +function analyze(sql: string) { + const migrations: readonly MigrationSafetyInput[] = [{ fileName: "9999_fixture.sql", sql }]; + return analyzeMigrationSafety(migrations, { baselineIds: [], estimates: testEstimates }); +} + +describe("migration safety check", () => { + it("documents the large table threshold used by the estimates", () => { + expect(TABLE_SIZE_BUCKET_THRESHOLDS.largeRows).toBe(1_000_000); + expect(testEstimates[0]?.estimatedRows).toBeGreaterThanOrEqual( + TABLE_SIZE_BUCKET_THRESHOLDS.largeRows, + ); + }); + + it("fails a 0126-shaped batched loop over a large table without a support index", () => { + const result = analyze(` + DO $$ + DECLARE + last_comment_id uuid := '00000000-0000-0000-0000-000000000000'::uuid; + BEGIN + LOOP + WITH batch AS MATERIALIZED ( + SELECT c."id" + FROM "issue_comments" c + WHERE c."id" > last_comment_id + AND c."author_agent_id" IS NULL + ORDER BY c."id" + LIMIT 5000 + ) + UPDATE "issue_comments" c + SET "derived_author_agent_id" = NULL + FROM batch b + WHERE c."id" = b."id"; + + EXIT WHEN NOT FOUND; + END LOOP; + END $$; + `); + + expect(result.newFindings.map((finding) => finding.rule)).toEqual( + expect.arrayContaining([ + "loop-mutation-large-table", + "batched-mutation-large-table-missing-index", + ]), + ); + expect(result.newFindings[0]?.table).toBe("issue_comments"); + }); + + it("passes the same bounded large-table backfill when a matching concurrent support index exists", () => { + const result = analyze(` + CREATE INDEX CONCURRENTLY IF NOT EXISTS "issue_comments_fixture_backfill_idx" + ON "issue_comments" USING btree ("id") + WHERE "author_agent_id" IS NULL;--> statement-breakpoint + DO $$ + DECLARE + last_comment_id uuid := '00000000-0000-0000-0000-000000000000'::uuid; + BEGIN + LOOP + WITH batch AS MATERIALIZED ( + SELECT c."id" + FROM "issue_comments" c + WHERE c."id" > last_comment_id + AND c."author_agent_id" IS NULL + ORDER BY c."id" + LIMIT 5000 + ) + UPDATE "issue_comments" c + SET "derived_author_agent_id" = NULL + FROM batch b + WHERE c."id" = b."id"; + + EXIT WHEN NOT FOUND; + END LOOP; + END $$; + `); + + expect(result.newFindings).toEqual([]); + }); + + it("does not suppress missing-index finding when a partial support index predicate is incompatible with the batch WHERE", () => { + const result = analyze(` + CREATE INDEX CONCURRENTLY IF NOT EXISTS "issue_comments_fixture_backfill_idx" + ON "issue_comments" USING btree ("id") + WHERE "author_agent_id" IS NOT NULL;--> statement-breakpoint + DO $$ + DECLARE + last_comment_id uuid := '00000000-0000-0000-0000-000000000000'::uuid; + BEGIN + LOOP + WITH batch AS MATERIALIZED ( + SELECT c."id" + FROM "issue_comments" c + WHERE c."id" > last_comment_id + AND c."author_agent_id" IS NULL + ORDER BY c."id" + LIMIT 5000 + ) + UPDATE "issue_comments" c + SET "derived_author_agent_id" = NULL + FROM batch b + WHERE c."id" = b."id"; + + EXIT WHEN NOT FOUND; + END LOOP; + END $$; + `); + + expect(result.newFindings.map((finding) => finding.rule)).toContain( + "batched-mutation-large-table-missing-index", + ); + }); + + it("passes a batched backfill over a small-bucket table", () => { + const result = analyze(` + DO $$ + BEGIN + LOOP + WITH batch AS ( + SELECT "id" + FROM "companies" + ORDER BY "id" + LIMIT 100 + ) + UPDATE "companies" c + SET "description" = c."description" + FROM batch b + WHERE c."id" = b."id"; + + EXIT WHEN NOT FOUND; + END LOOP; + END $$; + `); + + expect(result.newFindings).toEqual([]); + }); + + it("flags UPDATE ... FROM (SELECT ... LIMIT N) subquery batch on a large table", () => { + const result = analyze(` + UPDATE "issue_comments" c + SET "derived_author_agent_id" = NULL + FROM ( + SELECT "id" + FROM "issue_comments" + WHERE "author_agent_id" IS NULL + ORDER BY "id" + LIMIT 5000 + ) batch + WHERE c."id" = batch."id"; + `); + + expect(result.newFindings.map((f) => f.rule)).toContain( + "batched-mutation-large-table-missing-index", + ); + }); + + it("flags UPDATE ... FROM (SELECT ... FETCH FIRST N ROWS ONLY) subquery batch on a large table", () => { + const result = analyze(` + UPDATE "issue_comments" c + SET "derived_author_agent_id" = NULL + FROM ( + SELECT "id" + FROM "issue_comments" + WHERE "author_agent_id" IS NULL + ORDER BY "id" + FETCH FIRST 5000 ROWS ONLY + ) batch + WHERE c."id" = batch."id"; + `); + + expect(result.newFindings.map((f) => f.rule)).toContain( + "batched-mutation-large-table-missing-index", + ); + }); + + it("flags UPDATE ... FROM (SELECT ... FETCH NEXT N ROWS ONLY) subquery batch on a large table", () => { + const result = analyze(` + UPDATE "issue_comments" c + SET "derived_author_agent_id" = NULL + FROM ( + SELECT "id" + FROM "issue_comments" + WHERE "author_agent_id" IS NULL + ORDER BY "id" + FETCH NEXT 5000 ROWS ONLY + ) batch + WHERE c."id" = batch."id"; + `); + + expect(result.newFindings.map((f) => f.rule)).toContain( + "batched-mutation-large-table-missing-index", + ); + }); + + it("flags UPDATE ... FROM (SELECT ... FETCH FIRST N ROWS WITH TIES) subquery batch on a large table", () => { + const result = analyze(` + UPDATE "issue_comments" c + SET "derived_author_agent_id" = NULL + FROM ( + SELECT "id" + FROM "issue_comments" + WHERE "author_agent_id" IS NULL + ORDER BY "id" + FETCH FIRST 5000 ROWS WITH TIES + ) batch + WHERE c."id" = batch."id"; + `); + + expect(result.newFindings.map((f) => f.rule)).toContain( + "batched-mutation-large-table-missing-index", + ); + }); + + it("flags UPDATE ... FROM (SELECT ... FETCH FIRST ROW ONLY) subquery batch on a large table", () => { + const result = analyze(` + UPDATE "issue_comments" c + SET "derived_author_agent_id" = NULL + FROM ( + SELECT "id" + FROM "issue_comments" + WHERE "author_agent_id" IS NULL + ORDER BY "id" + FETCH FIRST ROW ONLY + ) batch + WHERE c."id" = batch."id"; + `); + + expect(result.newFindings.map((f) => f.rule)).toContain( + "batched-mutation-large-table-missing-index", + ); + }); + + it("flags UPDATE ... WHERE IN (SELECT ... LIMIT N) subquery batch on a large table", () => { + const result = analyze(` + UPDATE "issue_comments" + SET "derived_author_agent_id" = NULL + WHERE "id" IN ( + SELECT "id" + FROM "issue_comments" + WHERE "author_agent_id" IS NULL + ORDER BY "id" + LIMIT 5000 + ); + `); + + expect(result.newFindings.map((f) => f.rule)).toContain( + "batched-mutation-large-table-missing-index", + ); + }); + + it("flags a CTE with a selective WHERE when the outer UPDATE has no WHERE clause", () => { + const result = analyze(` + WITH selective AS ( + SELECT "id" FROM "issue_comments" WHERE "author_agent_id" IS NULL + ) + UPDATE "issue_comments" + SET "derived_author_agent_id" = NULL + FROM selective; + `); + + expect(result.newFindings.map((f) => f.rule)).toContain( + "full-table-mutation-large-table", + ); + }); + + it("does not treat WHERE inside a block comment as a selective predicate", () => { + const result = analyze(` + UPDATE "issue_comments" + SET "derived_author_agent_id" = NULL /* ignored + /* nested WHERE "id" > '0' */ + still ignored + */; + `); + + expect(result.newFindings.map((f) => f.rule)).toContain( + "full-table-mutation-large-table", + ); + }); + + it("does not treat WHERE inside an inline line comment as a selective predicate", () => { + const result = analyze(` + UPDATE "issue_comments" + SET "derived_author_agent_id" = NULL -- WHERE "id" > '0' + `); + + expect(result.newFindings.map((f) => f.rule)).toContain( + "full-table-mutation-large-table", + ); + }); + + it("does not treat WHERE inside a string literal as a selective predicate", () => { + const result = analyze(` + UPDATE "issue_comments" + SET "body" = 'WHERE "id" > ''0'''; + `); + + expect(result.newFindings.map((f) => f.rule)).toContain( + "full-table-mutation-large-table", + ); + }); + + it("does not treat WHERE inside a tagged dollar-quoted string as a selective predicate", () => { + const result = analyze(` + UPDATE "issue_comments" + SET "body" = $msg$WHERE "id" > '0'$msg$; + `); + + expect(result.newFindings.map((f) => f.rule)).toContain( + "full-table-mutation-large-table", + ); + }); + + it("does not treat WHERE inside an untagged dollar-quoted string as a selective predicate", () => { + const result = analyze(` + UPDATE "issue_comments" + SET "body" = $$WHERE "id" > '0'$$; + `); + + expect(result.newFindings.map((f) => f.rule)).toContain( + "full-table-mutation-large-table", + ); + }); + + it("still accepts a real selective WHERE clause", () => { + const result = analyze(` + UPDATE "issue_comments" + SET "derived_author_agent_id" = NULL + WHERE "id" > '0'; + `); + + expect(result.newFindings.map((f) => f.rule)).not.toContain( + "full-table-mutation-large-table", + ); + }); + + it("does not suppress full-table finding when WHERE only constrains a joined table", () => { + const result = analyze(` + UPDATE "issue_comments" + SET "derived_author_agent_id" = NULL + FROM "companies" + WHERE "companies"."id" = '00000000-0000-0000-0000-000000000000'; + `); + + expect(result.newFindings.map((f) => f.rule)).toContain( + "full-table-mutation-large-table", + ); + }); + + it("does not suppress full-table finding when WHERE only constrains a joined table via an unquoted alias", () => { + const result = analyze(` + UPDATE "issue_comments" + SET "derived_author_agent_id" = NULL + FROM "companies" c + WHERE c."id" = '00000000-0000-0000-0000-000000000000'; + `); + + expect(result.newFindings.map((f) => f.rule)).toContain( + "full-table-mutation-large-table", + ); + }); + + it("does not suppress full-table finding when WHERE only constrains an unquoted joined table", () => { + const result = analyze(` + UPDATE "issue_comments" + SET "derived_author_agent_id" = NULL + FROM companies + WHERE companies.id = '00000000-0000-0000-0000-000000000000'; + `); + + expect(result.newFindings.map((f) => f.rule)).toContain( + "full-table-mutation-large-table", + ); + }); + + it("does not suppress missing-index finding when support index uses an expression", () => { + const result = analyze(` + CREATE INDEX CONCURRENTLY IF NOT EXISTS "issue_comments_expr_idx" + ON "issue_comments" ((lower("body")));--> statement-breakpoint + UPDATE "issue_comments" c + SET "derived_author_agent_id" = NULL + FROM ( + SELECT "id" + FROM "issue_comments" + ORDER BY "id" + LIMIT 5000 + ) b + WHERE c."id" = b."id"; + `); + + expect(result.newFindings.map((f) => f.rule)).toContain( + "batched-mutation-large-table-missing-index", + ); + }); + + it("flags a batch backfill when the support index does not cover the ORDER BY key", () => { + const result = analyze(` + CREATE INDEX CONCURRENTLY IF NOT EXISTS "issue_comments_author_idx" + ON "issue_comments" ("author_agent_id");--> statement-breakpoint + DO $$ + DECLARE + last_id uuid := '00000000-0000-0000-0000-000000000000'::uuid; + BEGIN + LOOP + WITH batch AS MATERIALIZED ( + SELECT "id" + FROM "issue_comments" + WHERE "id" > last_id + ORDER BY "id" + LIMIT 5000 + ) + UPDATE "issue_comments" c + SET "derived_author_agent_id" = NULL + FROM batch b + WHERE c."id" = b."id"; + + EXIT WHEN NOT FOUND; + END LOOP; + END $$; + `); + + expect(result.newFindings.map((f) => f.rule)).toContain( + "batched-mutation-large-table-missing-index", + ); + }); + + it("flags a batch backfill when the support index only covers a later ORDER BY column", () => { + const result = analyze(` + CREATE INDEX CONCURRENTLY IF NOT EXISTS "issue_comments_created_idx" + ON "issue_comments" ("created_at");--> statement-breakpoint + DO $$ + DECLARE + last_id uuid := '00000000-0000-0000-0000-000000000000'::uuid; + BEGIN + LOOP + WITH batch AS MATERIALIZED ( + SELECT "id" + FROM "issue_comments" + WHERE "id" > last_id + ORDER BY "id", "created_at" + LIMIT 5000 + ) + UPDATE "issue_comments" c + SET "derived_author_agent_id" = NULL + FROM batch b + WHERE c."id" = b."id"; + + EXIT WHEN NOT FOUND; + END LOOP; + END $$; + `); + + expect(result.newFindings.map((f) => f.rule)).toContain( + "batched-mutation-large-table-missing-index", + ); + }); + + it("honors suppressions only when they name a rule and reason", () => { + const result = analyze(` + -- paperclip:migration-safety-ignore full-table-mutation-large-table: one-time metadata reset approved in issue thread + UPDATE "issue_comments" + SET "derived_author_source" = NULL; + `); + + expect(result.newFindings).toEqual([]); + }); +}); diff --git a/packages/db/src/check-migration-safety.ts b/packages/db/src/check-migration-safety.ts new file mode 100644 index 0000000000..1f0e47a9ac --- /dev/null +++ b/packages/db/src/check-migration-safety.ts @@ -0,0 +1,1052 @@ +import { createHash } from "node:crypto"; +import { readdir, readFile } from "node:fs/promises"; +import { basename } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { MIGRATION_SAFETY_BASELINE } from "./migration-safety-baseline.js"; +import { + getTableSizeEstimate, + isKnownLargeTable, + type TableSizeEstimate, +} from "./table-size-estimates.js"; + +const migrationsDir = fileURLToPath(new URL("./migrations", import.meta.url)); + +export type MigrationSafetyRule = + | "loop-mutation-large-table" + | "batched-mutation-large-table-missing-index" + | "full-table-mutation-large-table" + | "large-create-index-not-concurrently"; + +export type MigrationSafetySeverity = "error" | "warning"; + +export type MigrationSafetyFinding = { + readonly id: string; + readonly rule: MigrationSafetyRule; + readonly severity: MigrationSafetySeverity; + readonly migration: string; + readonly table: string; + readonly statement: string; + readonly message: string; +}; + +export type MigrationSafetyInput = { + readonly fileName: string; + readonly sql: string; +}; + +export type MigrationSafetyResult = { + readonly findings: readonly MigrationSafetyFinding[]; + readonly newFindings: readonly MigrationSafetyFinding[]; + readonly baselineFindings: readonly MigrationSafetyFinding[]; + readonly staleBaselineIds: readonly string[]; +}; + +type RuleMetadata = { + readonly severity: MigrationSafetySeverity; + readonly message: string; +}; + +type CreateIndexInfo = { + readonly table: string; + readonly columns: readonly string[]; + readonly predicate: string | null; + readonly predicateColumns: readonly string[]; + readonly concurrently: boolean; + readonly statement: string; +}; + +type MutationInfo = { + readonly table: string; + readonly statementSql: string; + readonly keywordIndex: number; +}; + +const RULE_METADATA: Record = { + "loop-mutation-large-table": { + severity: "error", + message: "DO $$ loop mutates a known-large table without a same-migration support index", + }, + "batched-mutation-large-table-missing-index": { + severity: "error", + message: "Batched LIMIT mutation over a known-large table lacks a same-migration support index", + }, + "full-table-mutation-large-table": { + severity: "error", + message: "Known-large table mutation does not have a selective WHERE clause", + }, + "large-create-index-not-concurrently": { + severity: "warning", + message: "CREATE INDEX on a known-large table is missing CONCURRENTLY", + }, +}; + +const RESERVED_ALIAS_WORDS = new Set([ + "add", + "alter", + "as", + "delete", + "from", + "on", + "returning", + "set", + "using", + "where", + "with", +]); + +function normalizeIdentifier(value: string): string { + return value + .trim() + .replace(/^"public"\s*\.\s*/i, "") + .replace(/^public\s*\.\s*/i, "") + .replace(/^"/, "") + .replace(/"$/, "") + .replaceAll('""', '"'); +} + +function normalizeSql(value: string): string { + return value.replace(/\s+/g, " ").trim(); +} + +function statementExcerpt(statement: string): string { + const normalized = normalizeSql(statement); + if (normalized.length <= 700) return normalized; + return `${normalized.slice(0, 700)}...`; +} + +function findingId( + rule: MigrationSafetyRule, + migration: string, + table: string, + statement: string, +): string { + return createHash("sha256") + .update(`${rule}\0${migration}\0${table}\0${normalizeSql(statement)}`) + .digest("hex") + .slice(0, 16); +} + +function splitSqlStatements(sql: string): string[] { + const breakpointParts = sql + .split(/-->\s*statement-breakpoint/g) + .map((part) => part.trim()) + .filter((part) => part.length > 0); + if (breakpointParts.length > 1) return breakpointParts; + + const statements: string[] = []; + let start = 0; + let singleQuoted = false; + let dollarQuoteTag: string | null = null; + + for (let index = 0; index < sql.length; index += 1) { + if (dollarQuoteTag) { + if (sql.startsWith(dollarQuoteTag, index)) { + index += dollarQuoteTag.length - 1; + dollarQuoteTag = null; + } + continue; + } + + const char = sql[index]; + if (singleQuoted) { + if (char === "'" && sql[index + 1] === "'") { + index += 1; + } else if (char === "'") { + singleQuoted = false; + } + continue; + } + + if (char === "'") { + singleQuoted = true; + continue; + } + + if (char === "$") { + const tag = dollarQuoteTagAt(sql, index); + if (tag) { + dollarQuoteTag = tag; + index += dollarQuoteTag.length - 1; + } + continue; + } + + if (char === ";") { + const statement = sql.slice(start, index + 1).trim(); + if (statement.length > 0) statements.push(statement); + start = index + 1; + } + } + + const tail = sql.slice(start).trim(); + if (tail.length > 0) statements.push(tail); + return statements; +} + +function ignoreRules(statement: string): Set { + const rules = new Set(); + const pattern = /--\s*paperclip:migration-safety-ignore\s+([a-z0-9-]+|all)\s*:\s*(\S.*)$/gim; + for (const match of statement.matchAll(pattern)) { + const rule = match[1]; + const reason = match[2]?.trim(); + if (rule && reason) rules.add(rule); + } + return rules; +} + +function isIgnored(statement: string, rule: MigrationSafetyRule): boolean { + const ignored = ignoreRules(statement); + return ignored.has(rule) || ignored.has("all"); +} + +function skipSingleQuotedLiteral(statement: string, startIndex: number): number { + let index = startIndex + 1; + while (index < statement.length) { + if (statement[index] === "'" && statement[index + 1] === "'") { + index += 2; + continue; + } + if (statement[index] === "'") return index + 1; + index += 1; + } + return index; +} + +function skipDoubleQuotedIdentifier(statement: string, startIndex: number): number { + let index = startIndex + 1; + while (index < statement.length) { + if (statement[index] === '"' && statement[index + 1] === '"') { + index += 2; + continue; + } + if (statement[index] === '"') return index + 1; + index += 1; + } + return index; +} + +function skipLineComment(statement: string, startIndex: number): number { + const newlineIndex = statement.indexOf("\n", startIndex + 2); + return newlineIndex === -1 ? statement.length : newlineIndex; +} + +function skipBlockComment(statement: string, startIndex: number): number { + let depth = 1; + let index = startIndex + 2; + while (index < statement.length && depth > 0) { + if (statement.startsWith("/*", index)) { + depth += 1; + index += 2; + continue; + } + if (statement.startsWith("*/", index)) { + depth -= 1; + index += 2; + continue; + } + index += 1; + } + return index; +} + +function skipDollarQuotedString(statement: string, startIndex: number): number { + const tag = dollarQuoteTagAt(statement, startIndex); + if (!tag) return startIndex; + const closeIndex = statement.indexOf(tag, startIndex + tag.length); + return closeIndex === -1 ? statement.length : closeIndex + tag.length; +} + +function stripSqlComments(statement: string): string { + let stripped = ""; + let index = 0; + + while (index < statement.length) { + const char = statement[index]; + const next = statement[index + 1]; + + if (char === "'") { + const literalEnd = skipSingleQuotedLiteral(statement, index); + stripped += statement.slice(index, literalEnd); + index = literalEnd; + continue; + } + + if (char === '"') { + const identifierEnd = skipDoubleQuotedIdentifier(statement, index); + stripped += statement.slice(index, identifierEnd); + index = identifierEnd; + continue; + } + + if (char === "$") { + const dollarEnd = skipDollarQuotedString(statement, index); + if (dollarEnd !== index) { + stripped += statement.slice(index, dollarEnd); + index = dollarEnd; + continue; + } + } + + if (char === "-" && next === "-") { + stripped += " "; + index = skipLineComment(statement, index); + continue; + } + + if (char === "/" && next === "*") { + stripped += " "; + const commentEnd = skipBlockComment(statement, index); + for (let commentIndex = index; commentIndex < commentEnd; commentIndex += 1) { + if (statement[commentIndex] === "\n") stripped += "\n"; + } + index = commentEnd; + continue; + } + + stripped += char; + index += 1; + } + + return stripped; +} + +function skipSqlTrivia(statement: string, index: number): number { + const char = statement[index]; + const next = statement[index + 1]; + if (char === "'") return skipSingleQuotedLiteral(statement, index); + if (char === '"') return skipDoubleQuotedIdentifier(statement, index); + if (char === "$") { + const end = skipDollarQuotedString(statement, index); + if (end !== index) return end; + } + if (char === "-" && next === "-") return skipLineComment(statement, index); + if (char === "/" && next === "*") return skipBlockComment(statement, index); + return index; +} + +function identifierList(value: string): string[] { + return [...value.matchAll(/"([^"]+)"|(?:^|[\s,(])([A-Za-z_][A-Za-z0-9_]*)/g)] + .map((match) => normalizeIdentifier(match[1] ?? match[2] ?? "")) + .filter((identifier) => identifier.length > 0) + .filter((identifier) => !RESERVED_ALIAS_WORDS.has(identifier.toLowerCase())); +} + +function plainIndexColumns(columnSpec: string): string[] { + // Only include simple column references. Skip expression columns such as + // (id * 2) or lower(col) — PostgreSQL cannot use those to satisfy ORDER BY + // on the plain column, so they must not suppress a missing-index finding. + return splitSqlList(columnSpec).flatMap((part) => { + const trimmed = part.trim(); + if (/\(/.test(trimmed)) return []; + return identifierList(trimmed); + }); +} + +function splitSqlList(value: string): string[] { + const parts: string[] = []; + let start = 0; + let depth = 0; + let singleQuoted = false; + let doubleQuoted = false; + + for (let index = 0; index < value.length; index += 1) { + const char = value[index]; + + if (singleQuoted) { + if (char === "'" && value[index + 1] === "'") { + index += 1; + } else if (char === "'") { + singleQuoted = false; + } + continue; + } + + if (doubleQuoted) { + if (char === '"' && value[index + 1] === '"') { + index += 1; + } else if (char === '"') { + doubleQuoted = false; + } + continue; + } + + if (char === "'") { + singleQuoted = true; + continue; + } + + if (char === '"') { + doubleQuoted = true; + continue; + } + + if (char === "(") { + depth += 1; + continue; + } + + if (char === ")") { + depth = Math.max(0, depth - 1); + continue; + } + + if (char === "," && depth === 0) { + const part = value.slice(start, index).trim(); + if (part) parts.push(part); + start = index + 1; + } + } + + const tail = value.slice(start).trim(); + if (tail) parts.push(tail); + return parts; +} + +function orderByExpressionColumn(value: string): string | null { + const withoutSortModifiers = value + .replace(/\bCOLLATE\s+(?:"[^"]+"|[A-Za-z_][A-Za-z0-9_]*)/gi, " ") + .replace(/\bNULLS\s+(?:FIRST|LAST)\b/gi, " ") + .replace(/\b(?:ASC|DESC)\b/gi, " "); + const identifiers = [ + ...withoutSortModifiers.matchAll(/"([^"]+)"|([A-Za-z_][A-Za-z0-9_]*)/g), + ] + .map((match) => normalizeIdentifier(match[1] ?? match[2] ?? "")) + .filter((identifier) => identifier.length > 0) + .filter((identifier) => !RESERVED_ALIAS_WORDS.has(identifier.toLowerCase())); + return identifiers[identifiers.length - 1] ?? null; +} + +function predicateColumns(statement: string): string[] { + const columns = new Set(); + const sql = stripSqlComments(statement); + const predicatePattern = /\b(?:WHERE|ORDER\s+BY|ON)\b([\s\S]*?)(?=\b(?:LIMIT|RETURNING|GROUP\s+BY|ORDER\s+BY|SET|FROM)\b|$)/gi; + for (const match of sql.matchAll(predicatePattern)) { + for (const identifier of identifierList(match[1] ?? "")) { + columns.add(identifier); + } + } + return [...columns]; +} + +type KeywordOccurrence = { + readonly index: number; + readonly depth: number; + readonly length: number; +}; + +function keywordOccurrenceAt( + sql: string, + index: number, + pattern: RegExp, +): RegExpMatchArray | null { + const previous = sql[index - 1]; + if (previous && /[A-Za-z0-9_]/.test(previous)) return null; + return sql.slice(index).match(pattern); +} + +function keywordOccurrences(sql: string, pattern: RegExp): KeywordOccurrence[] { + const occurrences: KeywordOccurrence[] = []; + let depth = 0; + let index = 0; + + while (index < sql.length) { + const char = sql[index]; + + if (char === "'") { + index = skipSingleQuotedLiteral(sql, index); + continue; + } + + if (char === '"') { + index = skipDoubleQuotedIdentifier(sql, index); + continue; + } + + if (char === "(") { + depth += 1; + index += 1; + continue; + } + + if (char === ")") { + depth = Math.max(0, depth - 1); + index += 1; + continue; + } + + const match = keywordOccurrenceAt(sql, index, pattern); + if (match) { + occurrences.push({ index, depth, length: match[0].length }); + index += match[0].length; + continue; + } + + index += 1; + } + + return occurrences; +} + +function batchWhereClausesBeforeOrderBy(statement: string): string[] { + const sql = stripSqlComments(statement); + const whereOccurrences = keywordOccurrences(sql, /^\bWHERE\b/i); + const orderByOccurrences = keywordOccurrences(sql, /^\bORDER\s+BY\b/i); + const clauses: string[] = []; + + for (const orderBy of orderByOccurrences) { + const where = whereOccurrences + .filter((candidate) => candidate.depth === orderBy.depth && candidate.index < orderBy.index) + .at(-1); + if (!where) continue; + + const clause = sql.slice(where.index + where.length, orderBy.index).trim(); + if (clause) clauses.push(clause); + } + + return clauses; +} + +function hasBalancedOuterParens(value: string): boolean { + if (!value.startsWith("(") || !value.endsWith(")")) return false; + let depth = 0; + + for (let index = 0; index < value.length; index += 1) { + const char = value[index]; + if (char === "'") { + index = skipSingleQuotedLiteral(value, index) - 1; + continue; + } + if (char === '"') { + index = skipDoubleQuotedIdentifier(value, index) - 1; + continue; + } + if (char === "(") depth += 1; + if (char === ")") depth -= 1; + if (depth === 0 && index < value.length - 1) return false; + } + + return depth === 0; +} + +function stripOuterParens(value: string): string { + let stripped = normalizeSql(value).replace(/;$/, "").trim(); + while (hasBalancedOuterParens(stripped)) { + stripped = normalizeSql(stripped.slice(1, -1)); + } + return stripped; +} + +function splitConjunctivePredicate(value: string): string[] { + const terms: string[] = []; + const sql = stripOuterParens(value); + let depth = 0; + let start = 0; + let index = 0; + + while (index < sql.length) { + const char = sql[index]; + + if (char === "'") { + index = skipSingleQuotedLiteral(sql, index); + continue; + } + + if (char === '"') { + index = skipDoubleQuotedIdentifier(sql, index); + continue; + } + + if (char === "(") { + depth += 1; + index += 1; + continue; + } + + if (char === ")") { + depth = Math.max(0, depth - 1); + index += 1; + continue; + } + + const match = depth === 0 ? keywordOccurrenceAt(sql, index, /^\bAND\b/i) : null; + if (match) { + const term = stripOuterParens(sql.slice(start, index)); + if (term) terms.push(term); + start = index + match[0].length; + index = start; + continue; + } + + index += 1; + } + + const tail = stripOuterParens(sql.slice(start)); + if (tail) terms.push(tail); + return terms; +} + +function lowercaseSqlOutsideSingleQuotedLiterals(value: string): string { + let lowered = ""; + let index = 0; + + while (index < value.length) { + if (value[index] === "'") { + const end = skipSingleQuotedLiteral(value, index); + lowered += value.slice(index, end); + index = end; + continue; + } + + lowered += value[index]?.toLowerCase() ?? ""; + index += 1; + } + + return lowered; +} + +function normalizePredicateTerm(term: string): string { + const normalized = stripOuterParens(term) + .replace( + /(?:"[^"]+"|[A-Za-z_][A-Za-z0-9_]*)\s*\.\s*("[^"]+"|[A-Za-z_][A-Za-z0-9_]*)/g, + "$1", + ) + .replace(/"([^"]+)"/g, "$1") + .replace(/\s+/g, " ") + .trim(); + return lowercaseSqlOutsideSingleQuotedLiterals(normalized); +} + +function predicateTerms(value: string): Set { + return new Set(splitConjunctivePredicate(value).map(normalizePredicateTerm)); +} + +function orderByColumns(statement: string): string[] { + const columns: string[] = []; + const sql = stripSqlComments(statement); + const pattern = /\bORDER\s+BY\b([\s\S]*?)(?=\b(?:LIMIT|RETURNING|GROUP\s+BY|WHERE|SET|FROM|END|LOOP)\b|$)/gi; + for (const match of sql.matchAll(pattern)) { + for (const expression of splitSqlList(match[1] ?? "")) { + const column = orderByExpressionColumn(expression); + if (column && !columns.includes(column)) { + columns.push(column); + } + } + } + return columns; +} + +function parseCreateIndexes(statement: string): CreateIndexInfo[] { + const indexes: CreateIndexInfo[] = []; + const sql = stripSqlComments(statement); + const pattern = + /\bCREATE\s+(?:UNIQUE\s+)?INDEX\s+(CONCURRENTLY\s+)?(?:IF\s+NOT\s+EXISTS\s+)?(?:"[^"]+"|[A-Za-z_][A-Za-z0-9_]*)\s+ON\s+(?:(?:"public"|public)\s*\.\s*)?(?:"([^"]+)"|([A-Za-z_][A-Za-z0-9_]*))\s*(?:USING\s+[A-Za-z_][A-Za-z0-9_]*\s*)?\(([\s\S]*?)\)(?:\s+WHERE\s+([\s\S]*))?/gi; + + for (const match of sql.matchAll(pattern)) { + const table = normalizeIdentifier(match[2] ?? match[3] ?? ""); + if (!table) continue; + const predicate = match[5]?.trim().replace(/;$/, "").trim() ?? ""; + indexes.push({ + table, + columns: plainIndexColumns(match[4] ?? ""), + predicate: predicate.length > 0 ? predicate : null, + predicateColumns: predicateColumns(predicate), + concurrently: Boolean(match[1]), + statement, + }); + } + + return indexes; +} + +function parseMutations(statement: string): MutationInfo[] { + const mutations: MutationInfo[] = []; + const sql = stripSqlComments(statement); + const updatePattern = + /\bUPDATE\s+(?:ONLY\s+)?(?:(?:"public"|public)\s*\.\s*)?(?:"([^"]+)"|([A-Za-z_][A-Za-z0-9_]*))(?:\s+(?:AS\s+)?(?:"?([A-Za-z_][A-Za-z0-9_]*)"?))?/gi; + const deletePattern = + /\bDELETE\s+FROM\s+(?:ONLY\s+)?(?:(?:"public"|public)\s*\.\s*)?(?:"([^"]+)"|([A-Za-z_][A-Za-z0-9_]*))(?:\s+(?:AS\s+)?(?:"?([A-Za-z_][A-Za-z0-9_]*)"?))?/gi; + + for (const match of sql.matchAll(updatePattern)) { + const table = normalizeIdentifier(match[1] ?? match[2] ?? ""); + if (table) { + mutations.push({ table, statementSql: sql, keywordIndex: match.index ?? 0 }); + } + } + + for (const match of sql.matchAll(deletePattern)) { + const table = normalizeIdentifier(match[1] ?? match[2] ?? ""); + if (table) { + mutations.push({ table, statementSql: sql, keywordIndex: match.index ?? 0 }); + } + } + + return mutations; +} + +function hasDoLoop(statement: string): boolean { + return /\bDO\s+\$[A-Za-z_]*\$[\s\S]*\bLOOP\b/i.test(statement); +} + +function hasBatchedLimitMutation(statement: string): boolean { + const hasLimit = /\bLIMIT\s+(?:\d+|[A-Za-z_][A-Za-z0-9_]*|\$[0-9]+)\b/i.test(statement); + const hasFetchLimit = + /\bFETCH\s+(?:FIRST|NEXT)(?:\s+(?:\d+|[A-Za-z_][A-Za-z0-9_]*|\$[0-9]+))?\s+ROWS?\s+(?:ONLY|WITH\s+TIES)\b/i.test(statement); + const hasDml = /\b(?:UPDATE|DELETE)\b/i.test(statement); + return (hasLimit || hasFetchLimit) && hasDml; +} + +function dollarQuoteTagAt(statement: string, startIndex: number): string | null { + if (statement[startIndex] !== "$") return null; + const match = statement.slice(startIndex).match(/^\$(?:[A-Za-z_][A-Za-z0-9_]*)?\$/); + return match?.[0] ?? null; +} + +function topLevelWhereClause(statement: string, startIndex: number): string | null { + // Walk character-by-character, tracking paren depth. + // Only consider WHERE keywords at depth 0 after the target mutation. + let depth = 0; + let i = startIndex; + while (i < statement.length) { + const nextIndex = skipSqlTrivia(statement, i); + if (nextIndex !== i) { + i = nextIndex; + continue; + } + + const ch = statement[i]; + if (ch === "(") { + depth++; + i++; + continue; + } + if (ch === ")") { + depth = Math.max(0, depth - 1); + i++; + continue; + } + if (depth === 0) { + const rem = statement.slice(i); + const m = rem.match(/^\bWHERE\b/i); + if (m) return rem.slice(m[0].length); + } + i++; + } + return null; +} + +function hasSelectiveWhere(mutation: MutationInfo): boolean { + const afterWhere = topLevelWhereClause(mutation.statementSql, mutation.keywordIndex); + if (!afterWhere) return false; + + const whereClause = normalizeSql(afterWhere).replace(/;$/, ""); + if (/^(?:true|1\s*=\s*1)$/i.test(whereClause)) return false; + if (!/(?:=|<>|!=|<|>|\bIN\s*\(|\bEXISTS\s*\(|\bLIKE\b|\bIS\s+(?:NOT\s+)?NULL\b)/i.test(whereClause)) + return false; + + // A WHERE that only constrains joined tables is not a filter on the target table. + // Collect every table-qualified column reference (both "tbl"."col" and alias."col"). + const qualRefs = [ + ...whereClause.matchAll(/"([^"]+)"\s*\.\s*(?:"[^"]+"|\w+)/g), + ...whereClause.matchAll(/\b([A-Za-z_][A-Za-z0-9_]*)\s*\."[^"]+"/g), + ...whereClause.matchAll(/\b([A-Za-z_][A-Za-z0-9_]*)\.([A-Za-z_][A-Za-z0-9_]*)\b/g), + ].map((m) => normalizeIdentifier(m[1] ?? "")).filter(Boolean); + + if (qualRefs.length > 0) { + // Resolve unquoted aliases from UPDATE/FROM/JOIN clauses to their real table names. + const aliasMap = new Map(); + const aliasPattern = + /\b(?:UPDATE|FROM|JOIN)\s+(?:"public"\s*\.\s*)?(?:"([^"]+)"|([A-Za-z_][A-Za-z0-9_]*))\s+(?:AS\s+)?(?:"([^"]+)"|([A-Za-z_][A-Za-z0-9_]*))\b/gi; + for (const m of stripSqlComments(mutation.statementSql).matchAll(aliasPattern)) { + const tbl = normalizeIdentifier(m[1] ?? m[2] ?? ""); + const alias = normalizeIdentifier(m[3] ?? m[4] ?? ""); + if (tbl && alias && !RESERVED_ALIAS_WORDS.has(alias.toLowerCase())) aliasMap.set(alias, tbl); + } + const refTables = qualRefs.map((r) => aliasMap.get(r) ?? r); + if (!refTables.some((t) => t === mutation.table)) { + // All refs resolve to other tables. Confirm no bare unqualified identifiers remain. + const noQual = whereClause + .replace(/"[^"]+"\s*\.\s*(?:"[^"]+"|\w+)/g, " ") + .replace(/\b[A-Za-z_][A-Za-z0-9_]*\s*\."[^"]+"/g, " ") + .replace(/\b[A-Za-z_][A-Za-z0-9_]*\.[A-Za-z_][A-Za-z0-9_]*/g, " ") + .replace(/'(?:[^']|'')*'/g, " "); + const SQL_KW = + /\b(?:AND|OR|NOT|IN|EXISTS|LIKE|IS|NULL|TRUE|FALSE|BETWEEN|CASE|WHEN|THEN|END|CAST|AS|ANY|ALL|SOME)\b/gi; + if (!/\b[A-Za-z_][A-Za-z0-9_]*\b/.test(noQual.replace(SQL_KW, " "))) return false; + } + } + + return true; +} + +function hasLeadingOrderPrefix( + supportIndex: CreateIndexInfo, + orderColumns: readonly string[], + statement: string, +): boolean { + const indexedColumns = supportIndex.columns; + const prefixLength = Math.min(indexedColumns.length, orderColumns.length); + if (prefixLength === 0) return false; + for (let index = 0; index < prefixLength; index += 1) { + if (indexedColumns[index] !== orderColumns[index]) return false; + } + if (!supportIndex.predicate) return true; + + const indexPredicateTerms = predicateTerms(supportIndex.predicate); + if (indexPredicateTerms.size === 0) return false; + + return batchWhereClausesBeforeOrderBy(statement).some((whereClause) => { + const batchTerms = predicateTerms(whereClause); + return [...indexPredicateTerms].every((term) => batchTerms.has(term)); + }); +} + +function hasOrderPrefixCompatibleIndex( + supportIndex: CreateIndexInfo, + orderColumns: readonly string[], + statement: string, +): boolean { + return hasLeadingOrderPrefix(supportIndex, orderColumns, statement); +} + +function hasMatchingSupportIndex( + indexes: readonly CreateIndexInfo[], + mutation: MutationInfo, + statement: string, +): boolean { + const matchingIndexes = indexes.filter((index) => index.table === mutation.table); + if (matchingIndexes.length === 0) return false; + + // ORDER BY columns are the batch-progression key. If the statement orders its + // batch, the support index must cover the ordered key prefix using index key + // columns; predicate-only overlap cannot stand in for an unindexed cursor. + const orderCols = orderByColumns(statement); + const allPredicateCols = new Set(predicateColumns(statement)); + if (allPredicateCols.size === 0) return true; + + if (orderCols.length > 0) { + return matchingIndexes.some((index) => + hasOrderPrefixCompatibleIndex(index, orderCols, statement), + ); + } + + return matchingIndexes.some((index) => { + const indexedColumns = new Set([...index.columns, ...index.predicateColumns]); + return [...allPredicateCols].some((col) => indexedColumns.has(col)); + }); +} + +function estimateSuffix(table: string, estimates: ReadonlyMap): string { + const estimate = estimates.get(table) ?? getTableSizeEstimate(table); + if (!estimate) return "bucket=large"; + return `bucket=${estimate.bucket}, localRows=${estimate.localRows}, estimatedRows=${estimate.estimatedRows}`; +} + +function makeFinding( + rule: MigrationSafetyRule, + migration: string, + table: string, + statement: string, + estimates: ReadonlyMap, +): MigrationSafetyFinding { + const metadata = RULE_METADATA[rule]; + return { + id: findingId(rule, migration, table, statement), + rule, + severity: metadata.severity, + migration, + table, + statement: statementExcerpt(statement), + message: `${metadata.message} (${estimateSuffix(table, estimates)})`, + }; +} + +function addFindingOnce( + findings: MigrationSafetyFinding[], + seen: Set, + finding: MigrationSafetyFinding, +): void { + const key = `${finding.rule}:${finding.migration}:${finding.table}:${finding.id}`; + if (seen.has(key)) return; + seen.add(key); + findings.push(finding); +} + +function estimatesByTable( + estimates: readonly TableSizeEstimate[] | undefined, +): ReadonlyMap { + if (!estimates) return new Map(); + return new Map(estimates.map((estimate) => [estimate.table, estimate])); +} + +function tableIsLarge(table: string, estimates: ReadonlyMap): boolean { + if (estimates.size > 0) return estimates.get(table)?.bucket === "large"; + return isKnownLargeTable(table); +} + +export function analyzeMigrationSafety( + migrations: readonly MigrationSafetyInput[], + options: { + readonly baselineIds?: readonly string[]; + readonly estimates?: readonly TableSizeEstimate[]; + } = {}, +): MigrationSafetyResult { + const findings: MigrationSafetyFinding[] = []; + const seen = new Set(); + const estimates = estimatesByTable(options.estimates); + + for (const migration of migrations) { + const statements = splitSqlStatements(migration.sql); + const migrationIndexes = statements.flatMap(parseCreateIndexes); + + for (const statement of statements) { + for (const index of parseCreateIndexes(statement)) { + if ( + tableIsLarge(index.table, estimates) && + !index.concurrently && + !isIgnored(statement, "large-create-index-not-concurrently") + ) { + addFindingOnce( + findings, + seen, + makeFinding( + "large-create-index-not-concurrently", + migration.fileName, + index.table, + statement, + estimates, + ), + ); + } + } + + const mutations = parseMutations(statement) + .filter((mutation) => tableIsLarge(mutation.table, estimates)); + for (const mutation of mutations) { + const hasSupportIndex = hasMatchingSupportIndex(migrationIndexes, mutation, statement); + + if ( + hasDoLoop(statement) && + !hasSupportIndex && + !isIgnored(statement, "loop-mutation-large-table") + ) { + addFindingOnce( + findings, + seen, + makeFinding( + "loop-mutation-large-table", + migration.fileName, + mutation.table, + statement, + estimates, + ), + ); + } + + if ( + hasBatchedLimitMutation(statement) && + !hasSupportIndex && + !isIgnored(statement, "batched-mutation-large-table-missing-index") + ) { + addFindingOnce( + findings, + seen, + makeFinding( + "batched-mutation-large-table-missing-index", + migration.fileName, + mutation.table, + statement, + estimates, + ), + ); + } + + if ( + !hasSelectiveWhere(mutation) && + !isIgnored(statement, "full-table-mutation-large-table") + ) { + addFindingOnce( + findings, + seen, + makeFinding( + "full-table-mutation-large-table", + migration.fileName, + mutation.table, + statement, + estimates, + ), + ); + } + } + } + } + + const baselineIds = new Set(options.baselineIds ?? MIGRATION_SAFETY_BASELINE.map((entry) => entry.id)); + const foundIds = new Set(findings.map((finding) => finding.id)); + const newFindings = findings.filter((finding) => !baselineIds.has(finding.id)); + const baselineFindings = findings.filter((finding) => baselineIds.has(finding.id)); + const staleBaselineIds = [...baselineIds].filter((id) => !foundIds.has(id)); + + return { + findings, + newFindings, + baselineFindings, + staleBaselineIds, + }; +} + +async function readMigrations(): Promise { + const files = (await readdir(migrationsDir)) + .filter((entry) => entry.endsWith(".sql")) + .sort(); + + return Promise.all( + files.map(async (fileName) => ({ + fileName, + sql: await readFile(new URL(`./migrations/${fileName}`, import.meta.url), "utf8"), + })), + ); +} + +function formatFinding(finding: MigrationSafetyFinding): string { + return [ + `[${finding.rule}] ${finding.migration} table=${finding.table} severity=${finding.severity} id=${finding.id}`, + finding.message, + `Statement: ${finding.statement}`, + ].join("\n"); +} + +function formatNewFindings(findings: readonly MigrationSafetyFinding[]): string { + const rendered = findings.map(formatFinding).join("\n\n"); + return [ + `Migration safety check found ${findings.length} new finding(s).`, + "Add a same-migration support index, use CONCURRENTLY where applicable, or add", + "`-- paperclip:migration-safety-ignore : ` next to the statement.", + "", + rendered, + ].join("\n"); +} + +async function main() { + const result = analyzeMigrationSafety(await readMigrations()); + + if (result.newFindings.length > 0) { + throw new Error(formatNewFindings(result.newFindings)); + } + + const staleSuffix = result.staleBaselineIds.length > 0 + ? ` (${result.staleBaselineIds.length} stale baseline id(s) ignored)` + : ""; + console.log( + `Migration safety check passed: ${result.baselineFindings.length} historical finding(s) covered by baseline${staleSuffix}.`, + ); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + await main(); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + console.error(`${basename(process.argv[1])}: ${detail}`); + process.exitCode = 1; + } +} diff --git a/packages/db/src/migration-safety-baseline.ts b/packages/db/src/migration-safety-baseline.ts new file mode 100644 index 0000000000..4c424b5022 --- /dev/null +++ b/packages/db/src/migration-safety-baseline.ts @@ -0,0 +1,157 @@ +export type MigrationSafetyBaselineEntry = { + readonly id: string; + readonly rule: string; + readonly migration: string; + readonly table: string; + readonly reason: string; +}; + +export const MIGRATION_SAFETY_BASELINE = [ + { + id: "2cfa16c89e561306", + rule: "large-create-index-not-concurrently", + migration: "0000_mature_masked_marvel.sql", + table: "activity_log", + reason: "Initial schema history predates the migration-safety guard.", + }, + { + id: "2e21c87a27d0ecf3", + rule: "large-create-index-not-concurrently", + migration: "0000_mature_masked_marvel.sql", + table: "issue_comments", + reason: "Initial schema history predates the migration-safety guard.", + }, + { + id: "3fa7b338f437c89d", + rule: "large-create-index-not-concurrently", + migration: "0000_mature_masked_marvel.sql", + table: "issue_comments", + reason: "Initial schema history predates the migration-safety guard.", + }, + { + id: "26bf13d0e36e3bd0", + rule: "large-create-index-not-concurrently", + migration: "0001_fast_northstar.sql", + table: "agent_wakeup_requests", + reason: "Historical migration predates the migration-safety guard.", + }, + { + id: "da73c844de91f262", + rule: "large-create-index-not-concurrently", + migration: "0001_fast_northstar.sql", + table: "agent_wakeup_requests", + reason: "Historical migration predates the migration-safety guard.", + }, + { + id: "b86e70ea500d5d9e", + rule: "large-create-index-not-concurrently", + migration: "0001_fast_northstar.sql", + table: "agent_wakeup_requests", + reason: "Historical migration predates the migration-safety guard.", + }, + { + id: "d12aeab5a11d37fe", + rule: "large-create-index-not-concurrently", + migration: "0001_fast_northstar.sql", + table: "heartbeat_run_events", + reason: "Historical migration predates the migration-safety guard.", + }, + { + id: "80d2cc53747b47bc", + rule: "large-create-index-not-concurrently", + migration: "0001_fast_northstar.sql", + table: "heartbeat_run_events", + reason: "Historical migration predates the migration-safety guard.", + }, + { + id: "8985fd1ec26c0449", + rule: "large-create-index-not-concurrently", + migration: "0001_fast_northstar.sql", + table: "heartbeat_run_events", + reason: "Historical migration predates the migration-safety guard.", + }, + { + id: "06065c3f2e8bca76", + rule: "large-create-index-not-concurrently", + migration: "0003_shallow_quentin_quire.sql", + table: "activity_log", + reason: "Historical migration predates the migration-safety guard.", + }, + { + id: "d0cb536e5b329013", + rule: "large-create-index-not-concurrently", + migration: "0003_shallow_quentin_quire.sql", + table: "activity_log", + reason: "Historical migration predates the migration-safety guard.", + }, + { + id: "b84802ae05be9943", + rule: "large-create-index-not-concurrently", + migration: "0024_far_beast.sql", + table: "issue_comments", + reason: "Historical migration predates the migration-safety guard.", + }, + { + id: "3eaba6ddfa29a678", + rule: "large-create-index-not-concurrently", + migration: "0024_far_beast.sql", + table: "issue_comments", + reason: "Historical migration predates the migration-safety guard.", + }, + { + id: "2874b94c3f294f53", + rule: "large-create-index-not-concurrently", + migration: "0051_young_korg.sql", + table: "issue_comments", + reason: "Historical migration predates the migration-safety guard.", + }, + { + id: "f0a44a3401b28d62", + rule: "large-create-index-not-concurrently", + migration: "0060_orange_annihilus.sql", + table: "issue_reference_mentions", + reason: "Historical migration predates the migration-safety guard.", + }, + { + id: "f1fbb786a033df8d", + rule: "large-create-index-not-concurrently", + migration: "0060_orange_annihilus.sql", + table: "issue_reference_mentions", + reason: "Historical migration predates the migration-safety guard.", + }, + { + id: "f74aa7dfb0152788", + rule: "large-create-index-not-concurrently", + migration: "0060_orange_annihilus.sql", + table: "issue_reference_mentions", + reason: "Historical migration predates the migration-safety guard.", + }, + { + id: "3c1237481d6ee00d", + rule: "large-create-index-not-concurrently", + migration: "0060_orange_annihilus.sql", + table: "issue_reference_mentions", + reason: "Historical migration predates the migration-safety guard.", + }, + { + id: "21cf0a7bb66a4058", + rule: "large-create-index-not-concurrently", + migration: "0060_orange_annihilus.sql", + table: "issue_reference_mentions", + reason: "Historical migration predates the migration-safety guard.", + }, + { + id: "567b97176f9f06c3", + rule: "large-create-index-not-concurrently", + migration: "0132_issue_comment_derived_attribution_fast.sql", + table: "issue_comments", + reason: "Existing issue-attribution backfill branch uses a temporary support index before this guard landed.", + }, + { + id: "38d8055cc228913d", + rule: "full-table-mutation-large-table", + migration: "0132_issue_comment_derived_attribution_fast.sql", + table: "issue_comments", + reason: "Batched DO-loop backfill with keyset pagination (LIMIT 5000 per batch); reviewed and approved as part of PAP-1505 fix. Already merged to master before this guard landed.", + }, +] as const satisfies readonly MigrationSafetyBaselineEntry[]; diff --git a/packages/db/src/table-size-estimates.ts b/packages/db/src/table-size-estimates.ts new file mode 100644 index 0000000000..8a6d680af4 --- /dev/null +++ b/packages/db/src/table-size-estimates.ts @@ -0,0 +1,178 @@ +export type TableSizeBucket = "large" | "medium" | "small"; + +export type LocalTableRowCount = { + readonly table: string; + readonly localRows: number; +}; + +export type TableSizeEstimate = LocalTableRowCount & { + readonly estimateFactor: number; + readonly estimatedRows: number; + readonly bucket: TableSizeBucket; +}; + +export const TABLE_SIZE_ESTIMATE_SOURCE = { + collectedAt: "2026-07-06", + database: "default local dev embedded Postgres", + method: "SELECT count(*) in a read-only transaction", +} as const; + +export const TABLE_SIZE_ESTIMATE_FACTOR = 250; + +export const TABLE_SIZE_BUCKET_THRESHOLDS = { + largeRows: 1_000_000, + mediumRows: 100_000, +} as const; + +export const LOCAL_TABLE_ROW_COUNTS = [ + { table: "agent_wakeup_requests", localRows: 52_791 }, + { table: "activity_log", localRows: 22_930 }, + { table: "issue_reference_mentions", localRows: 13_218 }, + { table: "heartbeat_run_events", localRows: 10_833 }, + { table: "issue_comments", localRows: 5_034 }, + { table: "document_revisions", localRows: 3_906 }, + { table: "heartbeat_runs", localRows: 3_620 }, + { table: "workspace_operations", localRows: 3_600 }, + { table: "environment_leases", localRows: 3_492 }, + { table: "cost_events", localRows: 3_330 }, + { table: "agent_task_sessions", localRows: 1_743 }, + { table: "documents", localRows: 1_692 }, + { table: "issue_documents", localRows: 1_678 }, + { table: "issues", localRows: 1_609 }, + { table: "secret_access_events", localRows: 933 }, + { table: "issue_read_states", localRows: 608 }, + { table: "issue_thread_interactions", localRows: 455 }, + { table: "issue_relations", localRows: 445 }, + { table: "execution_workspaces", localRows: 414 }, + { table: "agent_config_revisions", localRows: 145 }, + { table: "document_annotation_anchor_snapshots", localRows: 132 }, + { table: "document_annotation_comments", localRows: 122 }, + { table: "document_annotation_threads", localRows: 80 }, + { table: "routine_revisions", localRows: 64 }, + { table: "issue_tree_hold_members", localRows: 62 }, + { table: "routine_runs", localRows: 52 }, + { table: "issue_recovery_actions", localRows: 50 }, + { table: "assets", localRows: 46 }, + { table: "issue_attachments", localRows: 44 }, + { table: "principal_permission_grants", localRows: 25 }, + { table: "company_memberships", localRows: 19 }, + { table: "agent_runtime_state", localRows: 18 }, + { table: "agents", localRows: 18 }, + { table: "routine_triggers", localRows: 16 }, + { table: "company_skills", localRows: 15 }, + { table: "routine_documents", localRows: 14 }, + { table: "routines", localRows: 14 }, + { table: "issue_inbox_archives", localRows: 13 }, + { table: "issue_approvals", localRows: 12 }, + { table: "company_secret_bindings", localRows: 11 }, + { table: "issue_tree_holds", localRows: 10 }, + { table: "approvals", localRows: 9 }, + { table: "session", localRows: 9 }, + { table: "issue_work_products", localRows: 7 }, + { table: "company_skill_versions", localRows: 6 }, + { table: "feedback_exports", localRows: 6 }, + { table: "feedback_votes", localRows: 6 }, + { table: "issue_execution_decisions", localRows: 6 }, + { table: "company_secret_versions", localRows: 3 }, + { table: "company_secrets", localRows: 3 }, + { table: "projects", localRows: 3 }, + { table: "approval_comments", localRows: 2 }, + { table: "goals", localRows: 2 }, + { table: "project_goals", localRows: 2 }, + { table: "project_workspaces", localRows: 2 }, + { table: "account", localRows: 1 }, + { table: "companies", localRows: 1 }, + { table: "environments", localRows: 1 }, + { table: "instance_settings", localRows: 1 }, + { table: "instance_user_roles", localRows: 1 }, + { table: "user", localRows: 1 }, + { table: "agent_api_keys", localRows: 0 }, + { table: "agent_memberships", localRows: 0 }, + { table: "board_api_keys", localRows: 0 }, + { table: "budget_incidents", localRows: 0 }, + { table: "budget_policies", localRows: 0 }, + { table: "cli_auth_challenges", localRows: 0 }, + { table: "cloud_upstream_connections", localRows: 0 }, + { table: "cloud_upstream_runs", localRows: 0 }, + { table: "company_logos", localRows: 0 }, + { table: "company_secret_provider_configs", localRows: 0 }, + { table: "company_skill_comments", localRows: 0 }, + { table: "company_skill_stars", localRows: 0 }, + { table: "company_user_sidebar_preferences", localRows: 0 }, + { table: "environment_custom_image_setup_sessions", localRows: 0 }, + { table: "environment_custom_image_templates", localRows: 0 }, + { table: "external_object_mentions", localRows: 0 }, + { table: "external_objects", localRows: 0 }, + { table: "finance_events", localRows: 0 }, + { table: "heartbeat_run_watchdog_decisions", localRows: 0 }, + { table: "inbox_dismissals", localRows: 0 }, + { table: "invites", localRows: 0 }, + { table: "issue_labels", localRows: 0 }, + { table: "issue_plan_decompositions", localRows: 0 }, + { table: "issue_watchdogs", localRows: 0 }, + { table: "join_requests", localRows: 0 }, + { table: "labels", localRows: 0 }, + { table: "pipeline_automation_executions", localRows: 0 }, + { table: "pipeline_case_blockers", localRows: 0 }, + { table: "pipeline_case_documents", localRows: 0 }, + { table: "pipeline_case_events", localRows: 0 }, + { table: "pipeline_case_issue_links", localRows: 0 }, + { table: "pipeline_cases", localRows: 0 }, + { table: "pipeline_documents", localRows: 0 }, + { table: "pipeline_stages", localRows: 0 }, + { table: "pipeline_transitions", localRows: 0 }, + { table: "pipelines", localRows: 0 }, + { table: "plugin_company_settings", localRows: 0 }, + { table: "plugin_config", localRows: 0 }, + { table: "plugin_database_namespaces", localRows: 0 }, + { table: "plugin_entities", localRows: 0 }, + { table: "plugin_job_runs", localRows: 0 }, + { table: "plugin_jobs", localRows: 0 }, + { table: "plugin_logs", localRows: 0 }, + { table: "plugin_managed_resources", localRows: 0 }, + { table: "plugin_migrations", localRows: 0 }, + { table: "plugin_state", localRows: 0 }, + { table: "plugin_webhook_deliveries", localRows: 0 }, + { table: "plugins", localRows: 0 }, + { table: "project_memberships", localRows: 0 }, + { table: "user_secret_declarations", localRows: 0 }, + { table: "user_secret_definitions", localRows: 0 }, + { table: "user_sidebar_preferences", localRows: 0 }, + { table: "verification", localRows: 0 }, + { table: "workspace_runtime_services", localRows: 0 }, +] as const satisfies readonly LocalTableRowCount[]; + +function bucketForEstimatedRows(estimatedRows: number): TableSizeBucket { + if (estimatedRows >= TABLE_SIZE_BUCKET_THRESHOLDS.largeRows) return "large"; + if (estimatedRows >= TABLE_SIZE_BUCKET_THRESHOLDS.mediumRows) return "medium"; + return "small"; +} + +export const TABLE_SIZE_ESTIMATES: readonly TableSizeEstimate[] = LOCAL_TABLE_ROW_COUNTS.map( + ({ table, localRows }) => { + const estimatedRows = localRows * TABLE_SIZE_ESTIMATE_FACTOR; + return { + table, + localRows, + estimateFactor: TABLE_SIZE_ESTIMATE_FACTOR, + estimatedRows, + bucket: bucketForEstimatedRows(estimatedRows), + }; + }, +); + +export const TABLE_SIZE_ESTIMATES_BY_TABLE: ReadonlyMap = new Map( + TABLE_SIZE_ESTIMATES.map((estimate) => [estimate.table, estimate]), +); + +export function getTableSizeEstimate(table: string): TableSizeEstimate | undefined { + return TABLE_SIZE_ESTIMATES_BY_TABLE.get(table); +} + +export function getTableSizeBucket(table: string): TableSizeBucket { + return getTableSizeEstimate(table)?.bucket ?? "small"; +} + +export function isKnownLargeTable(table: string): boolean { + return getTableSizeBucket(table) === "large"; +}