From e591e75b8c5682990babf0b6a47287b7c488f58e Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Thu, 6 Aug 2026 10:25:27 -0500 Subject: [PATCH] perf(db): index context_snapshot/payload issueId lookups used by recovery sweeps (#10969) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - The server runs a recovery sweep every 30 seconds. The sweep makes sure assigned issues do not stall. > - The sweep reads the latest heartbeat run for each candidate issue. It filters `heartbeat_runs` on `context_snapshot ->> 'issueId'`. No index covers this expression. > - Each lookup scans ~26k rows and detoasts each row's JSONB context (~645 ms per query, measured with EXPLAIN ANALYZE). With ~460 candidate issues, one sweep schedules ~500 seconds of database work every 30 seconds. > - The database saturates. Users see the whole server as slow. > - This pull request adds expression indexes so these lookups become index scans. > - The benefit is that the recovery sweep drops from ~500 seconds of database work per tick to milliseconds, and the server becomes responsive again. ## Linked Issues or Issue Description **What happened?** The server became slow for all users. `pg_stat_activity` sampling showed the same two recovery-sweep queries active continuously (15/15 samples). Each `getLatestIssueRun` call filtered `heartbeat_runs` on the unindexed expression `context_snapshot ->> 'issueId'`, scanned ~26k rows, and detoasted a 2.1 GB TOAST region. `heartbeat_runs` accumulated 10.8 billion sequential tuples read. `hasActiveExecutionPath` also scanned `agent_wakeup_requests` (1.8M rows) on the unindexed expression `payload ->> 'issueId'`. **Expected behavior** Per-issue run lookups in the recovery sweep complete in milliseconds. The sweep finishes well inside its 30-second interval. Background maintenance does not degrade interactive latency. **Steps to reproduce** 1. Run a board with several hundred issues in `todo` / `in_progress` / `in_review` and a large `heartbeat_runs` table with big `context_snapshot` payloads. 2. Let the heartbeat scheduler run its 30-second recovery sweep. 3. Observe `pg_stat_activity`: the per-issue `heartbeat_runs` lookups run continuously; `EXPLAIN ANALYZE` shows a filter on `context_snapshot ->> 'issueId'` removing tens of thousands of rows per call. **Paperclip version or commit** master (814cb336) ## What Changed - Add migration `0209_heartbeat_context_snapshot_indexes.sql` with three expression indexes: - `heartbeat_runs (company_id, (context_snapshot ->> 'issueId'), created_at DESC)` - `heartbeat_runs (company_id, (context_snapshot ->> 'taskId'), created_at DESC)` - `agent_wakeup_requests (company_id, (payload ->> 'issueId'))` — with a `large-create-index-not-concurrently` safety pragma and justification, following the migration 0206 precedent. - Mirror the three indexes in the drizzle schema files (`heartbeat_runs.ts`, `agent_wakeup_requests.ts`). - Add `heartbeat-context-snapshot-index-migration.test.ts`. The test boots a fresh embedded Postgres, applies the full migration chain, asserts the indexes exist, and asserts with `EXPLAIN` that the planner selects them for the exact hot query shapes. ## Verification - `pnpm --filter @paperclipai/db exec tsx src/check-migration-numbering.ts` passes. - `pnpm --filter @paperclipai/db exec tsx src/check-migration-safety.ts` passes. - `npx vitest run src/heartbeat-context-snapshot-index-migration.test.ts` passes (fresh embedded Postgres, full chain 0000→0209, planner uses all three indexes). - `npx vitest run src/check-migration-safety.test.ts` passes (25/25). - `tsc --noEmit` clean for `packages/db`. ## Risks - The index builds run inside the transactional migration (no `CONCURRENTLY`). The `heartbeat_runs` build must read its 2.1 GB TOAST once; expect the migration step to add roughly 1–3 minutes to one deploy while writes to the two tables wait. This is a one-time cost at startup, before the server accepts traffic. - Three new indexes add small write overhead to two hot-write tables. The read savings are several orders of magnitude larger. - No query or API behavior changes; the planner simply gains a better access path. ## Model Used Claude Fable 5 (`claude-fable-5`, Anthropic), extended thinking, with tool use (shell, Postgres EXPLAIN/ANALYZE against the live instance for measurement, vitest for verification). ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Fable 5 Co-authored-by: Paperclip --- ...t-context-snapshot-index-migration.test.ts | 65 +++++++++++++++++++ ...209_heartbeat_context_snapshot_indexes.sql | 4 ++ packages/db/src/migrations/meta/_journal.json | 7 ++ .../db/src/schema/agent_wakeup_requests.ts | 4 ++ packages/db/src/schema/heartbeat_runs.ts | 11 ++++ 5 files changed, 91 insertions(+) create mode 100644 packages/db/src/heartbeat-context-snapshot-index-migration.test.ts create mode 100644 packages/db/src/migrations/0209_heartbeat_context_snapshot_indexes.sql diff --git a/packages/db/src/heartbeat-context-snapshot-index-migration.test.ts b/packages/db/src/heartbeat-context-snapshot-index-migration.test.ts new file mode 100644 index 0000000000..e4147bc6dc --- /dev/null +++ b/packages/db/src/heartbeat-context-snapshot-index-migration.test.ts @@ -0,0 +1,65 @@ +import { readFile } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it, afterEach } from "vitest"; +import postgres from "postgres"; +import { + getEmbeddedPostgresTestSupport, + startEmbeddedPostgresTestDatabase, +} from "./test-embedded-postgres.js"; + +const cleanups: Array<() => Promise> = []; +const support = await getEmbeddedPostgresTestSupport(); +const d = support.supported ? describe : describe.skip; + +afterEach(async () => { + while (cleanups.length > 0) await cleanups.pop()?.(); +}); + +d("heartbeat context_snapshot expression index migration", () => { + it("applies full migration chain and uses the new indexes", async () => { + const dbh = await startEmbeddedPostgresTestDatabase("pap16575-idx-"); + cleanups.push(() => dbh.cleanup()); + const sql = postgres(dbh.connectionString, { max: 1 }); + cleanups.push(async () => { await sql.end(); }); + + const idx = await sql`SELECT indexname FROM pg_indexes WHERE tablename IN ('heartbeat_runs','agent_wakeup_requests')`; + const names = idx.map((r) => r.indexname as string); + expect(names).toContain("heartbeat_runs_company_ctx_issue_created_idx"); + expect(names).toContain("heartbeat_runs_company_ctx_task_created_idx"); + expect(names).toContain("agent_wakeup_requests_company_payload_issue_idx"); + + await sql.unsafe("SET enable_seqscan = off"); + const plan = await sql.unsafe( + "EXPLAIN SELECT id FROM heartbeat_runs WHERE company_id = '00000000-0000-0000-0000-000000000001' AND context_snapshot ->> 'issueId' = 'x' ORDER BY created_at DESC, id DESC LIMIT 1", + ); + const planText = plan.map((r) => Object.values(r)[0]).join("\n"); + expect(planText).toContain("heartbeat_runs_company_ctx_issue_created_idx"); + + const taskPlan = await sql.unsafe( + "EXPLAIN SELECT id FROM heartbeat_runs WHERE company_id = '00000000-0000-0000-0000-000000000001' AND context_snapshot ->> 'taskId' = 'x' ORDER BY created_at DESC, id DESC LIMIT 1", + ); + const taskText = taskPlan.map((r) => Object.values(r)[0]).join("\n"); + expect(taskText).toContain("heartbeat_runs_company_ctx_task_created_idx"); + + const wakePlan = await sql.unsafe( + "EXPLAIN SELECT id FROM agent_wakeup_requests WHERE company_id = '00000000-0000-0000-0000-000000000001' AND status = 'deferred_issue_execution' AND payload ->> 'issueId' = 'x' LIMIT 1", + ); + const wakeText = wakePlan.map((r) => Object.values(r)[0]).join("\n"); + expect(wakeText).toContain("agent_wakeup_requests_company_payload_issue_idx"); + + // Idempotency: re-running the migration statements against an already + // migrated database must be a no-op, not an error. + const migrationSql = await readFile( + fileURLToPath(new URL("./migrations/0209_heartbeat_context_snapshot_indexes.sql", import.meta.url)), + "utf8", + ); + const statements = migrationSql + .split("--> statement-breakpoint") + .map((s) => s.trim()) + .filter((s) => s.length > 0); + expect(statements.length).toBeGreaterThan(0); + for (const statement of statements) { + await sql.unsafe(statement); + } + }, 240_000); +}); diff --git a/packages/db/src/migrations/0209_heartbeat_context_snapshot_indexes.sql b/packages/db/src/migrations/0209_heartbeat_context_snapshot_indexes.sql new file mode 100644 index 0000000000..9331d63450 --- /dev/null +++ b/packages/db/src/migrations/0209_heartbeat_context_snapshot_indexes.sql @@ -0,0 +1,4 @@ +CREATE INDEX IF NOT EXISTS "heartbeat_runs_company_ctx_issue_created_idx" ON "heartbeat_runs" USING btree ("company_id", ("context_snapshot" ->> 'issueId'), "created_at" DESC);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "heartbeat_runs_company_ctx_task_created_idx" ON "heartbeat_runs" USING btree ("company_id", ("context_snapshot" ->> 'taskId'), "created_at" DESC);--> statement-breakpoint +-- paperclip:migration-safety-ignore large-create-index-not-concurrently: Drizzle migrations run transactionally, so CONCURRENTLY is unavailable. This expression index removes per-issue full scans of agent_wakeup_requests in recovery sweeps (hasActiveExecutionPath), which currently dominate server load; the one-time build lock is the lesser cost. +CREATE INDEX IF NOT EXISTS "agent_wakeup_requests_company_payload_issue_idx" ON "agent_wakeup_requests" USING btree ("company_id", ("payload" ->> 'issueId')); diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index 07ed0c584e..06f3456dba 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -1450,6 +1450,13 @@ "when": 1785988680096, "tag": "0208_keen_sharon_carter", "breakpoints": true + }, + { + "idx": 209, + "version": "7", + "when": 1786020026023, + "tag": "0209_heartbeat_context_snapshot_indexes", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/schema/agent_wakeup_requests.ts b/packages/db/src/schema/agent_wakeup_requests.ts index 7b0ec64fa9..194e1732a3 100644 --- a/packages/db/src/schema/agent_wakeup_requests.ts +++ b/packages/db/src/schema/agent_wakeup_requests.ts @@ -40,5 +40,9 @@ export const agentWakeupRequests = pgTable( reviewPathRecoveryIdempotencyUq: uniqueIndex("agent_wakeup_requests_review_path_recovery_idempotency_uq") .on(table.companyId, table.idempotencyKey) .where(sql`${table.idempotencyKey} LIKE 'issue_review_path_lost:%' AND ${table.status} <> 'skipped'`), + companyPayloadIssueIdx: index("agent_wakeup_requests_company_payload_issue_idx").on( + table.companyId, + sql`(${table.payload} ->> 'issueId')`, + ), }), ); diff --git a/packages/db/src/schema/heartbeat_runs.ts b/packages/db/src/schema/heartbeat_runs.ts index 0f3590bb0f..49d3dd8f61 100644 --- a/packages/db/src/schema/heartbeat_runs.ts +++ b/packages/db/src/schema/heartbeat_runs.ts @@ -1,3 +1,4 @@ +import { sql } from "drizzle-orm"; import { type AnyPgColumn, pgTable, uuid, text, timestamp, jsonb, index, integer, bigint, boolean } from "drizzle-orm/pg-core"; import { companies } from "./companies.js"; import { agents } from "./agents.js"; @@ -88,5 +89,15 @@ export const heartbeatRuns = pgTable( table.companyId, table.createdAt.desc(), ), + companyCtxIssueCreatedIdx: index("heartbeat_runs_company_ctx_issue_created_idx").on( + table.companyId, + sql`(${table.contextSnapshot} ->> 'issueId')`, + table.createdAt.desc(), + ), + companyCtxTaskCreatedIdx: index("heartbeat_runs_company_ctx_task_created_idx").on( + table.companyId, + sql`(${table.contextSnapshot} ->> 'taskId')`, + table.createdAt.desc(), + ), }), );