diff --git a/doc/DATABASE.md b/doc/DATABASE.md index 3f0a2ce57e..5fa9c9487a 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -167,6 +167,14 @@ When authoring migrations or one-time backfills: - Split schema changes, index creation, and data backfill into separate phases so each step has clear locking and rollback behavior. - Treat the `check:migrations` CI gate as the enforcement backstop for these rules. If it flags a migration, rewrite the migration or add a suppression comment with the indexed predicate, batch bound, and reason the remaining scan is safe. +## Migration snapshots + +`drizzle-kit generate` diffs `packages/db/src/schema/` against the newest snapshot in `packages/db/src/migrations/meta/`. That snapshot must describe the schema that every migration produces when they run in order. A snapshot that drifts from the schema makes the *next* migration wrong, because `generate` folds the drift into it. The drift can add a column that an earlier migration already created, which makes that migration fail on a fresh database. It can also drop a column that the schema still uses. + +- Create every migration with `pnpm --filter @paperclipai/db generate`. Do not hand-write a snapshot. +- Do not hand-edit a snapshot to resolve a merge conflict. Renumber your migration and run `generate` again, as `packages/db/.gitattributes` describes. +- `packages/db/src/migration-snapshot-drift.test.ts` is the enforcement backstop. It repeats the diff that `generate` performs and fails when the newest snapshot no longer matches `packages/db/src/schema/`. + ## Resource membership tables Paperclip stores current-user sidebar membership state in: diff --git a/packages/db/src/migration-snapshot-drift.test.ts b/packages/db/src/migration-snapshot-drift.test.ts new file mode 100644 index 0000000000..b67c985698 --- /dev/null +++ b/packages/db/src/migration-snapshot-drift.test.ts @@ -0,0 +1,74 @@ +import { readdir, readFile } from "node:fs/promises"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { generateDrizzleJson, generateMigration } from "drizzle-kit/api"; + +// The newest snapshot in `src/migrations/meta` is the state `drizzle-kit +// generate` diffs the schema against. When it drifts from the schema, the next +// generated migration silently carries the drift: it re-adds a column an +// earlier migration already created (which fails on a fresh database) and drops +// a column the schema never had. This test reproduces the diff `generate` +// performs — schema modules versus newest snapshot — and fails when it is not +// empty, so drift is caught in CI instead of inside someone else's migration. + +const migrationsDir = fileURLToPath(new URL("./migrations", import.meta.url)); +const schemaDir = fileURLToPath(new URL("./schema", import.meta.url)); + +type JournalEntry = { idx: number; tag: string }; + +async function readNewestSnapshot(): Promise<{ file: string; snapshot: Record }> { + const journal = JSON.parse( + await readFile(path.join(migrationsDir, "meta", "_journal.json"), "utf8"), + ) as { entries: JournalEntry[] }; + const newest = journal.entries.at(-1); + if (!newest) throw new Error("migration journal has no entries"); + const file = `${String(newest.idx).padStart(4, "0")}_snapshot.json`; + const snapshot = JSON.parse(await readFile(path.join(migrationsDir, "meta", file), "utf8")) as Record< + string, + unknown + >; + return { file, snapshot }; +} + +// drizzle.config.ts points drizzle-kit at every module in the schema directory, +// so the test imports the same set rather than the hand-maintained barrel — a +// table missing from the barrel must not hide from this check. +async function importSchemaModules(): Promise> { + const files = (await readdir(schemaDir)).filter((file) => file.endsWith(".ts")).sort(); + const exports: Record = {}; + // The barrel re-exports the same table objects the per-table modules export, + // so dedupe by identity: serializing one table twice trips drizzle-kit's + // duplicate-index guard. + const seen = new Set(); + for (const file of files) { + const module = (await import(pathToFileURL(path.join(schemaDir, file)).href)) as Record< + string, + unknown + >; + for (const [name, value] of Object.entries(module)) { + if (typeof value === "object" && value !== null) { + if (seen.has(value)) continue; + seen.add(value); + } + exports[`${file}#${name}`] = value; + } + } + return exports; +} + +describe("migration snapshot drift", () => { + it("keeps the newest snapshot in sync with the drizzle schema", async () => { + const { file, snapshot } = await readNewestSnapshot(); + const current = generateDrizzleJson(await importSchemaModules(), snapshot.id as string); + const statements = await generateMigration( + snapshot as Parameters[0], + current as Parameters[1], + ); + + expect( + statements, + `${file} no longer matches src/schema. Run \`pnpm --filter @paperclipai/db generate\` and commit the migration it emits; do not hand-edit the snapshot.`, + ).toEqual([]); + }); +}); diff --git a/packages/db/src/migrations/meta/0228_snapshot.json b/packages/db/src/migrations/meta/0228_snapshot.json index 9f002ba8ba..17d4d2e41e 100644 --- a/packages/db/src/migrations/meta/0228_snapshot.json +++ b/packages/db/src/migrations/meta/0228_snapshot.json @@ -9471,13 +9471,6 @@ "notNull": true, "default": 0 }, - "error_count": { - "name": "error_count", - "type": "integer", - "primaryKey": false, - "notNull": true, - "default": 0 - }, "last_attempt_at": { "name": "last_attempt_at", "type": "timestamp with time zone", @@ -21193,6 +21186,13 @@ "notNull": true, "default": 0 }, + "error_count": { + "name": "error_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, "last_attempt_at": { "name": "last_attempt_at", "type": "timestamp with time zone", diff --git a/packages/db/src/migrations/meta/0229_snapshot.json b/packages/db/src/migrations/meta/0229_snapshot.json index a548f47cdf..d01518af6c 100644 --- a/packages/db/src/migrations/meta/0229_snapshot.json +++ b/packages/db/src/migrations/meta/0229_snapshot.json @@ -9458,13 +9458,6 @@ "notNull": true, "default": 0 }, - "error_count": { - "name": "error_count", - "type": "integer", - "primaryKey": false, - "notNull": true, - "default": 0 - }, "last_attempt_at": { "name": "last_attempt_at", "type": "timestamp with time zone", @@ -21180,6 +21173,13 @@ "notNull": true, "default": 0 }, + "error_count": { + "name": "error_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, "last_attempt_at": { "name": "last_attempt_at", "type": "timestamp with time zone",