diff --git a/server/src/__tests__/recovery-observability.test.ts b/server/src/__tests__/recovery-observability.test.ts index 53653a83f0..141bc743ae 100644 --- a/server/src/__tests__/recovery-observability.test.ts +++ b/server/src/__tests__/recovery-observability.test.ts @@ -12,8 +12,10 @@ import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; +import { PROVIDER_QUOTA_MONITOR_SERVICE_NAME } from "@paperclipai/shared"; import { classifyRecoveryHandoff, + displayCauseFor, evaluateRecoveryRateAlert, MAX_WINDOW_WEEKS, recoveryObservabilityService, @@ -69,6 +71,17 @@ describe("evaluateRecoveryRateAlert", () => { }); }); +describe("displayCauseFor", () => { + it("aliases provider_quota to the board-facing usage_limit_deferred label", () => { + expect(displayCauseFor("provider_quota")).toBe("usage_limit_deferred"); + }); + + it("passes every other cause through unchanged", () => { + expect(displayCauseFor("process_lost")).toBe("process_lost"); + expect(displayCauseFor("stale_heartbeat")).toBe("stale_heartbeat"); + }); +}); + describe("classifyRecoveryHandoff", () => { const base = { status: "resolved", @@ -232,6 +245,38 @@ describeEmbeddedPostgres("recovery observability report", () => { }); } + async function seedIssueWithMonitor(input: { + companyId: string; + n: number; + status: string; + nextCheckAt: Date; + serviceName?: string; + }) { + const nextCheckAt = input.nextCheckAt.toISOString(); + await db.insert(issues).values({ + id: randomUUID(), + companyId: input.companyId, + title: `Deferred ${input.n}`, + status: input.status, + priority: "medium", + issueNumber: input.n, + identifier: `DEF-${input.n}`, + executionPolicy: { + mode: "normal", + commentRequired: true, + stages: [], + monitor: { + nextCheckAt, + kind: "external_service", + serviceName: input.serviceName ?? PROVIDER_QUOTA_MONITOR_SERVICE_NAME, + scheduledBy: "assignee", + recoveryPolicy: "wake_owner", + }, + }, + monitorNextCheckAt: input.nextCheckAt, + }); + } + it("fires the threshold alert on synthetic data crossing 2% of runs", async () => { const { companyId, managerId, coderId } = await seedBaseline(); @@ -419,4 +464,79 @@ describeEmbeddedPostgres("recovery observability report", () => { active: 1, }); }); + + it("surfaces pending monitor-path provider-quota defers and the fleet banner", async () => { + const { companyId } = await seedBaseline(); + const resetA = new Date(now.getTime() + 45 * 60_000); + const resetB = new Date(now.getTime() + 2 * 60 * 60_000); + await seedIssueWithMonitor({ companyId, n: 201, status: "in_progress", nextCheckAt: resetA }); + await seedIssueWithMonitor({ companyId, n: 202, status: "in_review", nextCheckAt: resetB }); + // Already elapsed: the wake has fired (or is about to), so it is not queued. + await seedIssueWithMonitor({ + companyId, + n: 203, + status: "in_progress", + nextCheckAt: new Date(now.getTime() - 60_000), + }); + // A different external-service monitor is not a quota defer. + await seedIssueWithMonitor({ + companyId, + n: 204, + status: "in_progress", + nextCheckAt: resetB, + serviceName: "CI pipeline", + }); + // A quota monitor on an issue that left the active statuses is not queued. + await seedIssueWithMonitor({ companyId, n: 205, status: "blocked", nextCheckAt: resetB }); + + const report = await recoveryObservabilityService(db).report(companyId, { now, weeks: 8 }); + + expect(report.providerQuotaDeferrals).toEqual({ + pending: 2, + earliestNextCheckAt: resetA.toISOString().replace(/\.\d{3}Z$/, "Z"), + latestNextCheckAt: resetB.toISOString().replace(/\.\d{3}Z$/, "Z"), + }); + expect(report.fleetQuotaBanner).toEqual({ + pausedUntil: report.providerQuotaDeferrals.latestNextCheckAt, + queuedWakes: 2, + }); + }); + + it("reports an empty quota section when nothing is deferred", async () => { + const { companyId } = await seedBaseline(); + + const report = await recoveryObservabilityService(db).report(companyId, { now, weeks: 8 }); + + expect(report.providerQuotaDeferrals).toEqual({ + pending: 0, + earliestNextCheckAt: null, + latestNextCheckAt: null, + }); + expect(report.fleetQuotaBanner).toEqual({ pausedUntil: null, queuedWakes: 0 }); + }); + + it("labels provider_quota recovery actions as usage_limit_deferred in byCause", async () => { + const { companyId, coderId } = await seedBaseline(); + await seedRecoveryAction({ + companyId, + n: 301, + createdAt: latestWeek, + cause: "provider_quota", + errorCode: "provider_quota", + status: "active", + outcome: null, + ownerAgentId: coderId, + returnOwnerAgentId: coderId, + finalAssigneeAgentId: coderId, + finalIssueStatus: "in_progress", + }); + + const report = await recoveryObservabilityService(db).report(companyId, { now, weeks: 8 }); + + expect(report.byCause.find((entry) => entry.cause === "provider_quota")).toMatchObject({ + cause: "provider_quota", + displayCause: "usage_limit_deferred", + count: 1, + }); + }); }); diff --git a/server/src/services/recovery-observability.ts b/server/src/services/recovery-observability.ts index 7e866cdb8a..870214fc8f 100644 --- a/server/src/services/recovery-observability.ts +++ b/server/src/services/recovery-observability.ts @@ -1,6 +1,7 @@ import { and, eq, gte, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { heartbeatRuns, issueRecoveryActions, issues } from "@paperclipai/db"; +import { PROVIDER_QUOTA_MONITOR_SERVICE_NAME } from "@paperclipai/shared"; // Default alert threshold: the recovery rate that a regression like the 07-06 // week (3.26% of runs) blew past while nobody noticed by feel. See the plan on @@ -36,6 +37,8 @@ export type RecoveryRateAlert = { export type RecoveryCauseGroup = { cause: string; + /** Board-facing label; `usage_limit_deferred` for `provider_quota`, otherwise equal to `cause`. */ + displayCause: string; latestRunErrorCode: string; count: number; activeCount: number; @@ -71,6 +74,22 @@ export type RecoveryCauseRouting = { cancelled: number; }; +export type ProviderQuotaDeferrals = { + /** Issues with a provider-quota (usage-limit) monitor defer that has not elapsed yet. */ + pending: number; + /** Earliest pending monitor `nextCheckAt` (ISO 8601, UTC); null when nothing is pending. */ + earliestNextCheckAt: string | null; + /** Latest pending monitor `nextCheckAt` (ISO 8601, UTC); null when nothing is pending. */ + latestNextCheckAt: string | null; +}; + +export type FleetQuotaBanner = { + /** When the last queued quota wake fires (ISO 8601, UTC); null when nothing is paused. */ + pausedUntil: string | null; + /** Number of issues waiting on a provider-quota reset. */ + queuedWakes: number; +}; + export type RecoveryObservabilityReport = { companyId: string; generatedAt: string; @@ -81,6 +100,8 @@ export type RecoveryObservabilityReport = { byCause: RecoveryCauseGroup[]; handoff: RecoveryHandoffSummary; perCauseRouting: RecoveryCauseRouting[]; + providerQuotaDeferrals: ProviderQuotaDeferrals; + fleetQuotaBanner: FleetQuotaBanner; }; export type HandoffClass = @@ -103,6 +124,17 @@ type RecoveryActionFacts = { const ACTIVE_STATUSES = new Set(["active", "escalated"]); const TERMINAL_ISSUE_STATUSES = new Set(["done"]); +// The internal recovery cause enum says `provider_quota`; operators read the +// dashboard in terms of usage limits. Alias at the read boundary so the board +// gets the familiar label without renaming the stored enum. +const CAUSE_DISPLAY_ALIASES: Record = { + provider_quota: "usage_limit_deferred", +}; + +export function displayCauseFor(cause: string): string { + return CAUSE_DISPLAY_ALIASES[cause] ?? cause; +} + /** * Classify a recovery action by who ended up owning the deliverable work. * @@ -240,6 +272,7 @@ export function recoveryObservabilityService(db: Db) { const byCause: RecoveryCauseGroup[] = Array.from(causeRows).map((row) => ({ cause: String(row.cause), + displayCause: displayCauseFor(String(row.cause)), latestRunErrorCode: String(row.error_code), count: Number(row.count), activeCount: Number(row.active_count), @@ -356,6 +389,38 @@ export function recoveryObservabilityService(db: Db) { const perCauseRouting = Array.from(routingByCause.values()).sort((a, b) => b.total - a.total); + // Provider-quota defers taken through the issue-monitor path (the common + // case: the issue stays in_progress/in_review and a monitor is armed for the + // provider reset time) never write an issueRecoveryActions row, so the + // accounting above cannot see them. Read the pending defers straight off + // the denormalized monitor columns instead. + const quotaRows = (await db.execute(sql` + SELECT + count(*)::int AS pending, + to_char(min(${issues.monitorNextCheckAt}) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"') AS earliest, + to_char(max(${issues.monitorNextCheckAt}) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"') AS latest + FROM ${issues} + WHERE ${issues.companyId} = ${companyId} + AND ${issues.monitorNextCheckAt} IS NOT NULL + AND ${issues.monitorNextCheckAt} > ${now.toISOString()}::timestamptz + AND ${issues.executionPolicy} -> 'monitor' ->> 'serviceName' = ${PROVIDER_QUOTA_MONITOR_SERVICE_NAME} + AND ${issues.status} IN ('in_progress', 'in_review') + `)) as unknown as Iterable<{ + pending: number | string | null; + earliest: string | null; + latest: string | null; + }>; + const quota = Array.from(quotaRows)[0]; + const providerQuotaDeferrals: ProviderQuotaDeferrals = { + pending: Number(quota?.pending ?? 0), + earliestNextCheckAt: quota?.earliest ?? null, + latestNextCheckAt: quota?.latest ?? null, + }; + const fleetQuotaBanner: FleetQuotaBanner = { + pausedUntil: providerQuotaDeferrals.latestNextCheckAt, + queuedWakes: providerQuotaDeferrals.pending, + }; + return { companyId, generatedAt: now.toISOString(), @@ -366,6 +431,8 @@ export function recoveryObservabilityService(db: Db) { byCause, handoff, perCauseRouting, + providerQuotaDeferrals, + fleetQuotaBanner, }; }