diff --git a/server/src/__tests__/recovery-observability.test.ts b/server/src/__tests__/recovery-observability.test.ts new file mode 100644 index 0000000000..1a339934de --- /dev/null +++ b/server/src/__tests__/recovery-observability.test.ts @@ -0,0 +1,392 @@ +import { randomUUID } from "node:crypto"; +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { + agents, + companies, + createDb, + heartbeatRuns, + issueRecoveryActions, + issues, +} from "@paperclipai/db"; +import { + getEmbeddedPostgresTestSupport, + startEmbeddedPostgresTestDatabase, +} from "./helpers/embedded-postgres.js"; +import { + classifyRecoveryHandoff, + evaluateRecoveryRateAlert, + MAX_WINDOW_WEEKS, + recoveryObservabilityService, + type WeeklyRecoveryRate, +} from "../services/recovery-observability.ts"; + +const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; + +if (!embeddedPostgresSupport.supported) { + console.warn( + `Skipping embedded Postgres recovery observability tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`, + ); +} + +function week(runs: number, recoveryActions: number, weekStart: string): WeeklyRecoveryRate { + return { + weekStart, + runs, + recoveryActions, + ratePercent: runs > 0 ? Math.round((recoveryActions / runs) * 10000) / 100 : 0, + }; +} + +describe("evaluateRecoveryRateAlert", () => { + it("fires when a week crosses the 2% threshold", () => { + const weekly = [ + week(1000, 12, "2026-06-01"), // 1.20% + week(2364, 77, "2026-07-06"), // 3.26% — the regression week + week(1000, 10, "2026-07-13"), // 1.00% + ]; + + const alert = evaluateRecoveryRateAlert(weekly, 2); + + expect(alert.breached).toBe(true); + expect(alert.breachedWeeks.map((w) => w.weekStart)).toEqual(["2026-07-06"]); + expect(alert.latestWeek?.weekStart).toBe("2026-07-13"); + expect(alert.latestWeekBreached).toBe(false); + }); + + it("stays quiet when every week is under the threshold", () => { + const weekly = [week(1000, 13, "2026-06-01"), week(1000, 19, "2026-06-08")]; + const alert = evaluateRecoveryRateAlert(weekly, 2); + expect(alert.breached).toBe(false); + expect(alert.breachedWeeks).toHaveLength(0); + expect(alert.latestWeekBreached).toBe(false); + }); + + it("flags the latest week when it is the one that regresses", () => { + const weekly = [week(1000, 10, "2026-07-06"), week(1000, 30, "2026-07-13")]; // 1% then 3% + const alert = evaluateRecoveryRateAlert(weekly, 2); + expect(alert.latestWeekBreached).toBe(true); + }); +}); + +describe("classifyRecoveryHandoff", () => { + const base = { + status: "resolved", + outcome: "restored", + ownerAgentId: "manager", + returnOwnerAgentId: "coder", + finalAssigneeAgentId: "coder", + finalIssueStatus: "done", + }; + + it("marks a manager who kept and completed the work as owner_completed", () => { + expect( + classifyRecoveryHandoff({ ...base, finalAssigneeAgentId: "manager", finalIssueStatus: "done" }), + ).toBe("owner_completed"); + }); + + it("marks work returned to the original assignee as handed_back", () => { + expect(classifyRecoveryHandoff({ ...base, finalAssigneeAgentId: "coder" })).toBe("handed_back"); + }); + + it("marks work delegated to a different specialist as handed_back", () => { + expect(classifyRecoveryHandoff({ ...base, finalAssigneeAgentId: "other-agent" })).toBe( + "handed_back", + ); + }); + + it("marks the original agent recovering its own issue as self_recovery", () => { + expect( + classifyRecoveryHandoff({ ...base, ownerAgentId: "coder", returnOwnerAgentId: "coder" }), + ).toBe("self_recovery"); + }); + + it("treats still-active actions as active regardless of assignee", () => { + expect(classifyRecoveryHandoff({ ...base, status: "active" })).toBe("active"); + }); +}); + +describeEmbeddedPostgres("recovery observability report", () => { + let db!: ReturnType; + let tempDb: Awaited> | null = null; + + const now = new Date("2026-07-15T12:00:00.000Z"); + // Monday of each relevant week (matches Postgres date_trunc('week', ...)). + const regressionWeek = new Date("2026-07-08T12:00:00.000Z"); // in week 2026-07-06 + const latestWeek = new Date("2026-07-14T12:00:00.000Z"); // in week 2026-07-13 + + beforeAll(async () => { + tempDb = await startEmbeddedPostgresTestDatabase("paperclip-recovery-observability-"); + db = createDb(tempDb.connectionString); + }, 20_000); + + afterEach(async () => { + await db.delete(issueRecoveryActions); + await db.delete(heartbeatRuns); + await db.delete(issues); + await db.delete(agents); + await db.delete(companies); + }); + + afterAll(async () => { + await tempDb?.cleanup(); + }); + + async function seedBaseline() { + const companyId = randomUUID(); + const managerId = randomUUID(); + const coderId = randomUUID(); + const otherId = randomUUID(); + + await db.insert(companies).values({ + id: companyId, + name: "Paperclip", + issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`, + requireBoardApprovalForNewAgents: false, + }); + + await db.insert(agents).values( + [managerId, coderId, otherId].map((id, index) => ({ + id, + companyId, + name: `agent-${index}`, + role: "engineer", + status: "running", + adapterType: "codex_local", + adapterConfig: {}, + runtimeConfig: {}, + permissions: {}, + })), + ); + + // 100 runs in the regression week, 100 in the latest week. + await db.insert(heartbeatRuns).values([ + ...Array.from({ length: 100 }, () => ({ + id: randomUUID(), + companyId, + agentId: coderId, + invocationSource: "assignment", + status: "succeeded", + createdAt: regressionWeek, + })), + ...Array.from({ length: 100 }, () => ({ + id: randomUUID(), + companyId, + agentId: coderId, + invocationSource: "assignment", + status: "succeeded", + createdAt: latestWeek, + })), + ]); + + return { companyId, managerId, coderId, otherId }; + } + + async function seedRecoveryAction(input: { + companyId: string; + n: number; + createdAt: Date; + cause: string; + errorCode: string; + status: string; + outcome: string | null; + ownerAgentId: string | null; + returnOwnerAgentId: string | null; + finalAssigneeAgentId: string | null; + finalIssueStatus: string; + }) { + const sourceIssueId = randomUUID(); + await db.insert(issues).values({ + id: sourceIssueId, + companyId: input.companyId, + title: `Source ${input.n}`, + status: input.finalIssueStatus, + priority: "medium", + assigneeAgentId: input.finalAssigneeAgentId, + issueNumber: input.n, + identifier: `SRC-${input.n}`, + }); + await db.insert(issueRecoveryActions).values({ + id: randomUUID(), + companyId: input.companyId, + sourceIssueId, + kind: "stranded_assigned_issue", + status: input.status, + ownerType: input.ownerAgentId ? "agent" : "board", + ownerAgentId: input.ownerAgentId, + returnOwnerAgentId: input.returnOwnerAgentId, + previousOwnerAgentId: input.returnOwnerAgentId, + cause: input.cause, + fingerprint: `fp-${input.n}`, + evidence: { latestRunErrorCode: input.errorCode }, + nextAction: "recover", + outcome: input.outcome, + createdAt: input.createdAt, + updatedAt: input.createdAt, + }); + } + + it("fires the threshold alert on synthetic data crossing 2% of runs", async () => { + const { companyId, managerId, coderId } = await seedBaseline(); + + // Regression week: 3 recovery actions against 100 runs => 3% > 2%. + for (let i = 0; i < 3; i += 1) { + await seedRecoveryAction({ + companyId, + n: i + 1, + createdAt: regressionWeek, + cause: "stranded_assigned_issue", + errorCode: "process_lost", + status: "resolved", + outcome: "restored", + ownerAgentId: managerId, + returnOwnerAgentId: coderId, + finalAssigneeAgentId: managerId, + finalIssueStatus: "done", + }); + } + // Latest week: 1 recovery action against 100 runs => 1% (under threshold). + await seedRecoveryAction({ + companyId, + n: 99, + createdAt: latestWeek, + cause: "stranded_assigned_issue", + errorCode: "process_lost", + status: "resolved", + outcome: "restored", + ownerAgentId: coderId, + returnOwnerAgentId: coderId, + finalAssigneeAgentId: coderId, + finalIssueStatus: "done", + }); + + const report = await recoveryObservabilityService(db).report(companyId, { + now, + weeks: 8, + thresholdPercent: 2, + }); + + expect(report.alert.breached).toBe(true); + expect(report.alert.breachedWeeks.map((w) => w.weekStart)).toEqual(["2026-07-06"]); + const regression = report.weekly.find((w) => w.weekStart === "2026-07-06"); + expect(regression).toMatchObject({ runs: 100, recoveryActions: 3, ratePercent: 3 }); + const latest = report.weekly.find((w) => w.weekStart === "2026-07-13"); + expect(latest).toMatchObject({ runs: 100, recoveryActions: 1, ratePercent: 1 }); + expect(report.alert.latestWeekBreached).toBe(false); + }); + + it("computes the handed_back vs owner_completed ratio and per-cause routing", async () => { + const { companyId, managerId, coderId, otherId } = await seedBaseline(); + + // Two manager takeovers where the manager kept and completed the work. + for (let i = 0; i < 2; i += 1) { + await seedRecoveryAction({ + companyId, + n: i + 1, + createdAt: regressionWeek, + cause: "stranded_assigned_issue", + errorCode: "adapter_failed", + status: "resolved", + outcome: "restored", + ownerAgentId: managerId, + returnOwnerAgentId: coderId, + finalAssigneeAgentId: managerId, + finalIssueStatus: "done", + }); + } + // One takeover handed back to the original assignee. + await seedRecoveryAction({ + companyId, + n: 3, + createdAt: regressionWeek, + cause: "stranded_assigned_issue", + errorCode: "adapter_failed", + status: "resolved", + outcome: "restored", + ownerAgentId: managerId, + returnOwnerAgentId: coderId, + finalAssigneeAgentId: coderId, + finalIssueStatus: "in_progress", + }); + // One handed to a different specialist (still counts as handed back). + await seedRecoveryAction({ + companyId, + n: 4, + createdAt: regressionWeek, + cause: "stranded_assigned_issue", + errorCode: "adapter_failed", + status: "resolved", + outcome: "restored", + ownerAgentId: managerId, + returnOwnerAgentId: coderId, + finalAssigneeAgentId: otherId, + finalIssueStatus: "in_progress", + }); + // Original agent recovered its own process_lost issue (self recovery). + await seedRecoveryAction({ + companyId, + n: 5, + createdAt: regressionWeek, + cause: "process_lost", + errorCode: "process_lost", + status: "resolved", + outcome: "restored", + ownerAgentId: coderId, + returnOwnerAgentId: coderId, + finalAssigneeAgentId: coderId, + finalIssueStatus: "done", + }); + // Escalated actions are still active, but they need their own routing counter. + await seedRecoveryAction({ + companyId, + n: 6, + createdAt: regressionWeek, + cause: "stranded_assigned_issue", + errorCode: "manual_escalation", + status: "escalated", + outcome: null, + ownerAgentId: managerId, + returnOwnerAgentId: coderId, + finalAssigneeAgentId: managerId, + finalIssueStatus: "in_progress", + }); + + const report = await recoveryObservabilityService(db).report(companyId, { now, weeks: 8 }); + + expect(report.handoff.ownerCompleted).toBe(2); + expect(report.handoff.handedBack).toBe(2); + expect(report.handoff.selfRecovery).toBe(1); + expect(report.handoff.resolvedTakeovers).toBe(4); + expect(report.handoff.handedBackRatio).toBe(50); + expect(report.handoff.ownerCompletedRatio).toBe(50); + + const strandedCause = report.byCause.find( + (c) => c.cause === "stranded_assigned_issue" && c.latestRunErrorCode === "adapter_failed", + ); + expect(strandedCause?.count).toBe(4); + + const processLostRouting = report.perCauseRouting.find((r) => r.cause === "process_lost"); + expect(processLostRouting?.retriedByOriginalSucceeded).toBe(1); + const strandedRouting = report.perCauseRouting.find( + (r) => r.cause === "stranded_assigned_issue", + ); + expect(strandedRouting?.ownerCompleted).toBe(2); + expect(strandedRouting?.handedBack).toBe(2); + expect(strandedRouting?.active).toBe(1); + expect(strandedRouting?.escalated).toBe(1); + }); + + it("caps the reporting window so a huge `weeks` value can't over-allocate", async () => { + const { companyId } = await seedBaseline(); + + const report = await recoveryObservabilityService(db).report(companyId, { + now, + weeks: 100_000, + }); + + expect(report.window.weeks).toBe(MAX_WINDOW_WEEKS); + expect(report.window.since).toBe(report.weekly[0]?.weekStart); + expect(report.window.since).not.toContain("T"); + expect(report.weekly).toHaveLength(MAX_WINDOW_WEEKS); + }); +}); diff --git a/server/src/routes/dashboard.ts b/server/src/routes/dashboard.ts index 2a10ca9d0d..b0647cf3f4 100644 --- a/server/src/routes/dashboard.ts +++ b/server/src/routes/dashboard.ts @@ -1,11 +1,28 @@ import { Router } from "express"; import type { Db } from "@paperclipai/db"; import { dashboardService } from "../services/dashboard.js"; +import { + DEFAULT_RECOVERY_RATE_THRESHOLD_PERCENT, + MAX_WINDOW_WEEKS, + recoveryObservabilityService, +} from "../services/recovery-observability.js"; import { assertCompanyAccess } from "./authz.js"; +function parsePositiveNumber( + value: unknown, + fallback: number, + max?: number, +): number { + if (typeof value !== "string") return fallback; + const parsed = Number(value); + if (!Number.isFinite(parsed) || parsed <= 0) return fallback; + return max != null ? Math.min(parsed, max) : parsed; +} + export function dashboardRoutes(db: Db) { const router = Router(); const svc = dashboardService(db); + const recoveryObservability = recoveryObservabilityService(db); router.get("/companies/:companyId/dashboard", async (req, res) => { const companyId = req.params.companyId as string; @@ -14,5 +31,20 @@ export function dashboardRoutes(db: Db) { res.json(summary); }); + router.get("/companies/:companyId/recovery-observability", async (req, res) => { + const companyId = req.params.companyId as string; + assertCompanyAccess(req, companyId); + const weeks = parsePositiveNumber(req.query.weeks, 8, MAX_WINDOW_WEEKS); + const thresholdPercent = parsePositiveNumber( + req.query.threshold, + DEFAULT_RECOVERY_RATE_THRESHOLD_PERCENT, + ); + const report = await recoveryObservability.report(companyId, { + weeks, + thresholdPercent, + }); + res.json(report); + }); + return router; } diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts index bdb8bec2b5..90bf5c2086 100644 --- a/server/src/routes/openapi.ts +++ b/server/src/routes/openapi.ts @@ -2927,6 +2927,21 @@ registry.registerPath({ responses: { 200: r.ok(), 401: r.unauthorized }, }); +registry.registerPath({ + method: "get", + path: "/api/companies/{companyId}/recovery-observability", + tags: ["dashboard"], + summary: "Get recovery observability report", + request: { + params: z.object({ companyId: z.string() }), + query: z.object({ + weeks: z.string().optional(), + threshold: z.string().optional(), + }), + }, + responses: { 200: r.ok(), 401: r.unauthorized }, +}); + // ─── Sidebar ───────────────────────────────────────────────────────────────── registry.registerPath({ diff --git a/server/src/services/recovery-observability.ts b/server/src/services/recovery-observability.ts new file mode 100644 index 0000000000..91fed3f374 --- /dev/null +++ b/server/src/services/recovery-observability.ts @@ -0,0 +1,371 @@ +import { and, eq, gte, sql } from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { heartbeatRuns, issueRecoveryActions, issues } from "@paperclipai/db"; + +// Default alert threshold: the recovery rate that a regression like the 07-06 +// week (3.26% of runs) blew past while nobody noticed by feel. See the plan on +// PAP-14080 (Phase 3) and the retrospective on PAP-14098. +export const DEFAULT_RECOVERY_RATE_THRESHOLD_PERCENT = 2; +const DEFAULT_WINDOW_WEEKS = 8; +// Upper bound on the reporting window. Caps the per-week array allocation so an +// attacker-supplied `weeks` query param can't trigger an unbounded allocation +// (~2 years of weekly buckets is far beyond any real dashboard use). +export const MAX_WINDOW_WEEKS = 104; + +// Weeks are ISO/Monday-anchored to match the retrospective table (2026-06-01 is +// a Monday). Postgres `date_trunc('week', ...)` is Monday-based, so this keeps +// the live report aligned with the numbers in the plan document. + +export type WeeklyRecoveryRate = { + /** Monday (UTC) of the week, `YYYY-MM-DD`. */ + weekStart: string; + runs: number; + recoveryActions: number; + /** recoveryActions / runs, as a percentage (0 when there are no runs). */ + ratePercent: number; +}; + +export type RecoveryRateAlert = { + thresholdPercent: number; + /** True when any complete week in the window exceeded the threshold. */ + breached: boolean; + breachedWeeks: WeeklyRecoveryRate[]; + latestWeek: WeeklyRecoveryRate | null; + latestWeekBreached: boolean; +}; + +export type RecoveryCauseGroup = { + cause: string; + latestRunErrorCode: string; + count: number; + activeCount: number; + resolvedCount: number; + cancelledCount: number; +}; + +export type RecoveryHandoffSummary = { + /** Genuine manager takeovers (recovery owner != original assignee) that resolved. */ + resolvedTakeovers: number; + handedBack: number; + ownerCompleted: number; + otherTakeover: number; + /** Original agent recovered its own issue (owner == original assignee). */ + selfRecovery: number; + boardOwned: number; + activeTakeovers: number; + /** handedBack / (handedBack + ownerCompleted); null when no resolved takeovers. */ + handedBackRatio: number | null; + ownerCompletedRatio: number | null; +}; + +export type RecoveryCauseRouting = { + cause: string; + total: number; + active: number; + /** Original agent recovered its own issue and resolved it. */ + retriedByOriginalSucceeded: number; + handedBack: number; + ownerCompleted: number; + escalated: number; + falsePositive: number; + cancelled: number; +}; + +export type RecoveryObservabilityReport = { + companyId: string; + generatedAt: string; + window: { weeks: number; since: string }; + thresholdPercent: number; + weekly: WeeklyRecoveryRate[]; + alert: RecoveryRateAlert; + byCause: RecoveryCauseGroup[]; + handoff: RecoveryHandoffSummary; + perCauseRouting: RecoveryCauseRouting[]; +}; + +export type HandoffClass = + | "self_recovery" + | "handed_back" + | "owner_completed" + | "board_owned" + | "active" + | "other"; + +type RecoveryActionFacts = { + status: string; + outcome: string | null; + ownerAgentId: string | null; + returnOwnerAgentId: string | null; + finalAssigneeAgentId: string | null; + finalIssueStatus: string | null; +}; + +const ACTIVE_STATUSES = new Set(["active", "escalated"]); +const TERMINAL_ISSUE_STATUSES = new Set(["done", "in_review"]); + +/** + * Classify a recovery action by who ended up owning the deliverable work. + * + * The plan's `handed_back` vs `owner_completed` outcomes were never added to the + * outcome vocabulary (recovery track 1 kept `restored`/`cancelled`/…), so we + * derive the distinction from the durable relationship between the recovery + * owner, the original assignee (`returnOwnerAgentId`), and where the source + * issue actually landed. This directly measures the product goal — "managers + * doing the work becomes rare". + */ +export function classifyRecoveryHandoff(facts: RecoveryActionFacts): HandoffClass { + if (ACTIVE_STATUSES.has(facts.status)) return "active"; + if (!facts.ownerAgentId) return "board_owned"; + // Original agent recovered its own issue — the ideal per-cause routing, not a takeover. + if (facts.ownerAgentId === facts.returnOwnerAgentId) return "self_recovery"; + // Genuine takeover: recovery owner differs from the original assignee. + const landedElsewhere = + facts.finalAssigneeAgentId != null && facts.finalAssigneeAgentId !== facts.ownerAgentId; + if (landedElsewhere) return "handed_back"; + const ownerKept = facts.finalAssigneeAgentId === facts.ownerAgentId; + if (ownerKept && facts.finalIssueStatus && TERMINAL_ISSUE_STATUSES.has(facts.finalIssueStatus)) { + return "owner_completed"; + } + return "other"; +} + +/** + * Pure alert evaluation over weekly rates. Kept independent of the database so + * the 2%-crossing regression case can be tested with synthetic data. + */ +export function evaluateRecoveryRateAlert( + weekly: WeeklyRecoveryRate[], + thresholdPercent: number = DEFAULT_RECOVERY_RATE_THRESHOLD_PERCENT, +): RecoveryRateAlert { + const breachedWeeks = weekly.filter((week) => week.ratePercent > thresholdPercent); + const latestWeek = weekly.length > 0 ? weekly[weekly.length - 1]! : null; + return { + thresholdPercent, + breached: breachedWeeks.length > 0, + breachedWeeks, + latestWeek, + latestWeekBreached: latestWeek != null && latestWeek.ratePercent > thresholdPercent, + }; +} + +function round2(value: number): number { + return Math.round(value * 100) / 100; +} + +function utcWeekStart(now: Date, weeksAgo: number): Date { + const utcMidnight = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()); + const dayOfWeek = new Date(utcMidnight).getUTCDay(); // 0 = Sunday + const mondayOffset = (dayOfWeek + 6) % 7; // days since Monday + const thisMonday = utcMidnight - mondayOffset * 24 * 60 * 60 * 1000; + return new Date(thisMonday - weeksAgo * 7 * 24 * 60 * 60 * 1000); +} + +export function recoveryObservabilityService(db: Db) { + async function report( + companyId: string, + opts?: { now?: Date; weeks?: number; thresholdPercent?: number }, + ): Promise { + const now = opts?.now ?? new Date(); + const weeks = Math.min( + MAX_WINDOW_WEEKS, + Math.max(1, Math.floor(opts?.weeks ?? DEFAULT_WINDOW_WEEKS)), + ); + const thresholdPercent = opts?.thresholdPercent ?? DEFAULT_RECOVERY_RATE_THRESHOLD_PERCENT; + const since = utcWeekStart(now, weeks - 1); + const sinceIso = since.toISOString(); + + // Weekly run volume (denominator) — normalizes the recovery rate per run. + const runRows = (await db.execute(sql` + SELECT + to_char(date_trunc('week', ${heartbeatRuns.createdAt} AT TIME ZONE 'UTC'), 'YYYY-MM-DD') AS week_start, + count(*)::int AS runs + FROM ${heartbeatRuns} + WHERE ${heartbeatRuns.companyId} = ${companyId} + AND ${heartbeatRuns.createdAt} >= ${sinceIso}::timestamptz + GROUP BY week_start + `)) as unknown as Iterable<{ week_start: string; runs: number | string }>; + + // Weekly recovery-action volume (numerator). + const actionRows = (await db.execute(sql` + SELECT + to_char(date_trunc('week', ${issueRecoveryActions.createdAt} AT TIME ZONE 'UTC'), 'YYYY-MM-DD') AS week_start, + count(*)::int AS actions + FROM ${issueRecoveryActions} + WHERE ${issueRecoveryActions.companyId} = ${companyId} + AND ${issueRecoveryActions.createdAt} >= ${sinceIso}::timestamptz + GROUP BY week_start + `)) as unknown as Iterable<{ week_start: string; actions: number | string }>; + + const runsByWeek = new Map(); + for (const row of runRows) runsByWeek.set(String(row.week_start), Number(row.runs)); + const actionsByWeek = new Map(); + for (const row of actionRows) actionsByWeek.set(String(row.week_start), Number(row.actions)); + + const weekly: WeeklyRecoveryRate[] = Array.from({ length: weeks }, (_, index) => { + const weekStart = utcWeekStart(now, weeks - 1 - index).toISOString().slice(0, 10); + const runs = runsByWeek.get(weekStart) ?? 0; + const recoveryActions = actionsByWeek.get(weekStart) ?? 0; + return { + weekStart, + runs, + recoveryActions, + ratePercent: runs > 0 ? round2((recoveryActions / runs) * 100) : 0, + }; + }); + + const alert = evaluateRecoveryRateAlert(weekly, thresholdPercent); + + // Cause + latestRunErrorCode taxonomy (from the action's evidence snapshot). + const causeRows = (await db.execute(sql` + SELECT + ${issueRecoveryActions.cause} AS cause, + coalesce(${issueRecoveryActions.evidence} ->> 'latestRunErrorCode', '(none)') AS error_code, + count(*)::int AS count, + count(*) FILTER (WHERE ${issueRecoveryActions.status} IN ('active', 'escalated'))::int AS active_count, + count(*) FILTER (WHERE ${issueRecoveryActions.status} = 'resolved')::int AS resolved_count, + count(*) FILTER (WHERE ${issueRecoveryActions.status} = 'cancelled')::int AS cancelled_count + FROM ${issueRecoveryActions} + WHERE ${issueRecoveryActions.companyId} = ${companyId} + AND ${issueRecoveryActions.createdAt} >= ${sinceIso}::timestamptz + GROUP BY cause, error_code + ORDER BY count DESC + `)) as unknown as Iterable<{ + cause: string; + error_code: string; + count: number | string; + active_count: number | string; + resolved_count: number | string; + cancelled_count: number | string; + }>; + + const byCause: RecoveryCauseGroup[] = Array.from(causeRows).map((row) => ({ + cause: String(row.cause), + latestRunErrorCode: String(row.error_code), + count: Number(row.count), + activeCount: Number(row.active_count), + resolvedCount: Number(row.resolved_count), + cancelledCount: Number(row.cancelled_count), + })); + + // Hand-back accounting + per-cause routing. Recovery actions are inherently + // low-volume (they are the exception path), so we join to the source issue + // and classify each action in TS rather than encoding the derivation in SQL. + const facts = await db + .select({ + cause: issueRecoveryActions.cause, + status: issueRecoveryActions.status, + outcome: issueRecoveryActions.outcome, + ownerAgentId: issueRecoveryActions.ownerAgentId, + returnOwnerAgentId: issueRecoveryActions.returnOwnerAgentId, + finalAssigneeAgentId: issues.assigneeAgentId, + finalIssueStatus: issues.status, + }) + .from(issueRecoveryActions) + .innerJoin(issues, eq(issues.id, issueRecoveryActions.sourceIssueId)) + .where( + and( + eq(issueRecoveryActions.companyId, companyId), + gte(issueRecoveryActions.createdAt, since), + ), + ); + + const handoff: RecoveryHandoffSummary = { + resolvedTakeovers: 0, + handedBack: 0, + ownerCompleted: 0, + otherTakeover: 0, + selfRecovery: 0, + boardOwned: 0, + activeTakeovers: 0, + handedBackRatio: null, + ownerCompletedRatio: null, + }; + + const routingByCause = new Map(); + const routingFor = (cause: string): RecoveryCauseRouting => { + let entry = routingByCause.get(cause); + if (!entry) { + entry = { + cause, + total: 0, + active: 0, + retriedByOriginalSucceeded: 0, + handedBack: 0, + ownerCompleted: 0, + escalated: 0, + falsePositive: 0, + cancelled: 0, + }; + routingByCause.set(cause, entry); + } + return entry; + }; + + for (const row of facts) { + const klass = classifyRecoveryHandoff(row); + const routing = routingFor(String(row.cause)); + routing.total += 1; + + if (klass === "active") { + routing.active += 1; + if (row.status === "escalated") routing.escalated += 1; + if (row.ownerAgentId && row.ownerAgentId !== row.returnOwnerAgentId) { + handoff.activeTakeovers += 1; + } + continue; + } + + // Routing verification counters (resolved actions). + if (row.status === "escalated") routing.escalated += 1; + if (row.outcome === "false_positive") routing.falsePositive += 1; + if (row.status === "cancelled" && row.outcome !== "false_positive") routing.cancelled += 1; + + switch (klass) { + case "self_recovery": + handoff.selfRecovery += 1; + if (row.status === "resolved") routing.retriedByOriginalSucceeded += 1; + break; + case "handed_back": + handoff.handedBack += 1; + handoff.resolvedTakeovers += 1; + routing.handedBack += 1; + break; + case "owner_completed": + handoff.ownerCompleted += 1; + handoff.resolvedTakeovers += 1; + routing.ownerCompleted += 1; + break; + case "board_owned": + handoff.boardOwned += 1; + break; + default: + if (row.ownerAgentId && row.ownerAgentId !== row.returnOwnerAgentId) { + handoff.otherTakeover += 1; + } + break; + } + } + + const decided = handoff.handedBack + handoff.ownerCompleted; + if (decided > 0) { + handoff.handedBackRatio = round2((handoff.handedBack / decided) * 100); + handoff.ownerCompletedRatio = round2((handoff.ownerCompleted / decided) * 100); + } + + const perCauseRouting = Array.from(routingByCause.values()).sort((a, b) => b.total - a.total); + + return { + companyId, + generatedAt: now.toISOString(), + window: { weeks, since: sinceIso.slice(0, 10) }, + thresholdPercent, + weekly, + alert, + byCause, + handoff, + perCauseRouting, + }; + } + + return { report }; +}