Merge 0f45902e7c into c9e3bb7ca4

server/src/__tests__/recovery-observability.test.ts

@@ -12,8 +12,10 @@ import {
   getEmbeddedPostgresTestSupport,
   startEmbeddedPostgresTestDatabase,
 } from "./helpers/embedded-postgres.js";
+import { PROVIDER_QUOTA_MONITOR_SERVICE_NAME } from "@paperclipai/shared";
 import {
   classifyRecoveryHandoff,
+  displayCauseFor,
   evaluateRecoveryRateAlert,
   MAX_WINDOW_WEEKS,
   recoveryObservabilityService,
@@ -69,6 +71,17 @@ describe("evaluateRecoveryRateAlert", () => {
   });
 });
 
+describe("displayCauseFor", () => {
+  it("aliases provider_quota to the board-facing usage_limit_deferred label", () => {
+    expect(displayCauseFor("provider_quota")).toBe("usage_limit_deferred");
+  });
+
+  it("passes every other cause through unchanged", () => {
+    expect(displayCauseFor("process_lost")).toBe("process_lost");
+    expect(displayCauseFor("stale_heartbeat")).toBe("stale_heartbeat");
+  });
+});
+
 describe("classifyRecoveryHandoff", () => {
   const base = {
     status: "resolved",
@@ -232,6 +245,38 @@ describeEmbeddedPostgres("recovery observability report", () => {
     });
   }
 
+  async function seedIssueWithMonitor(input: {
+    companyId: string;
+    n: number;
+    status: string;
+    nextCheckAt: Date;
+    serviceName?: string;
+  }) {
+    const nextCheckAt = input.nextCheckAt.toISOString();
+    await db.insert(issues).values({
+      id: randomUUID(),
+      companyId: input.companyId,
+      title: `Deferred ${input.n}`,
+      status: input.status,
+      priority: "medium",
+      issueNumber: input.n,
+      identifier: `DEF-${input.n}`,
+      executionPolicy: {
+        mode: "normal",
+        commentRequired: true,
+        stages: [],
+        monitor: {
+          nextCheckAt,
+          kind: "external_service",
+          serviceName: input.serviceName ?? PROVIDER_QUOTA_MONITOR_SERVICE_NAME,
+          scheduledBy: "assignee",
+          recoveryPolicy: "wake_owner",
+        },
+      },
+      monitorNextCheckAt: input.nextCheckAt,
+    });
+  }
+
   it("fires the threshold alert on synthetic data crossing 2% of runs", async () => {
     const { companyId, managerId, coderId } = await seedBaseline();
 
@@ -419,4 +464,79 @@ describeEmbeddedPostgres("recovery observability report", () => {
       active: 1,
     });
   });
+
+  it("surfaces pending monitor-path provider-quota defers and the fleet banner", async () => {
+    const { companyId } = await seedBaseline();
+    const resetA = new Date(now.getTime() + 45 * 60_000);
+    const resetB = new Date(now.getTime() + 2 * 60 * 60_000);
+    await seedIssueWithMonitor({ companyId, n: 201, status: "in_progress", nextCheckAt: resetA });
+    await seedIssueWithMonitor({ companyId, n: 202, status: "in_review", nextCheckAt: resetB });
+    // Already elapsed: the wake has fired (or is about to), so it is not queued.
+    await seedIssueWithMonitor({
+      companyId,
+      n: 203,
+      status: "in_progress",
+      nextCheckAt: new Date(now.getTime() - 60_000),
+    });
+    // A different external-service monitor is not a quota defer.
+    await seedIssueWithMonitor({
+      companyId,
+      n: 204,
+      status: "in_progress",
+      nextCheckAt: resetB,
+      serviceName: "CI pipeline",
+    });
+    // A quota monitor on an issue that left the active statuses is not queued.
+    await seedIssueWithMonitor({ companyId, n: 205, status: "blocked", nextCheckAt: resetB });
+
+    const report = await recoveryObservabilityService(db).report(companyId, { now, weeks: 8 });
+
+    expect(report.providerQuotaDeferrals).toEqual({
+      pending: 2,
+      earliestNextCheckAt: resetA.toISOString().replace(/\.\d{3}Z$/, "Z"),
+      latestNextCheckAt: resetB.toISOString().replace(/\.\d{3}Z$/, "Z"),
+    });
+    expect(report.fleetQuotaBanner).toEqual({
+      pausedUntil: report.providerQuotaDeferrals.latestNextCheckAt,
+      queuedWakes: 2,
+    });
+  });
+
+  it("reports an empty quota section when nothing is deferred", async () => {
+    const { companyId } = await seedBaseline();
+
+    const report = await recoveryObservabilityService(db).report(companyId, { now, weeks: 8 });
+
+    expect(report.providerQuotaDeferrals).toEqual({
+      pending: 0,
+      earliestNextCheckAt: null,
+      latestNextCheckAt: null,
+    });
+    expect(report.fleetQuotaBanner).toEqual({ pausedUntil: null, queuedWakes: 0 });
+  });
+
+  it("labels provider_quota recovery actions as usage_limit_deferred in byCause", async () => {
+    const { companyId, coderId } = await seedBaseline();
+    await seedRecoveryAction({
+      companyId,
+      n: 301,
+      createdAt: latestWeek,
+      cause: "provider_quota",
+      errorCode: "provider_quota",
+      status: "active",
+      outcome: null,
+      ownerAgentId: coderId,
+      returnOwnerAgentId: coderId,
+      finalAssigneeAgentId: coderId,
+      finalIssueStatus: "in_progress",
+    });
+
+    const report = await recoveryObservabilityService(db).report(companyId, { now, weeks: 8 });
+
+    expect(report.byCause.find((entry) => entry.cause === "provider_quota")).toMatchObject({
+      cause: "provider_quota",
+      displayCause: "usage_limit_deferred",
+      count: 1,
+    });
+  });
 });

server/src/services/recovery-observability.ts

@@ -1,6 +1,7 @@
 import { and, eq, gte, sql } from "drizzle-orm";
 import type { Db } from "@paperclipai/db";
 import { heartbeatRuns, issueRecoveryActions, issues } from "@paperclipai/db";
+import { PROVIDER_QUOTA_MONITOR_SERVICE_NAME } from "@paperclipai/shared";
 
 // Default alert threshold: the recovery rate that a regression like the 07-06
 // week (3.26% of runs) blew past while nobody noticed by feel. See the plan on
@@ -36,6 +37,8 @@ export type RecoveryRateAlert = {
 
 export type RecoveryCauseGroup = {
   cause: string;
+  /** Board-facing label; `usage_limit_deferred` for `provider_quota`, otherwise equal to `cause`. */
+  displayCause: string;
   latestRunErrorCode: string;
   count: number;
   activeCount: number;
@@ -71,6 +74,22 @@ export type RecoveryCauseRouting = {
   cancelled: number;
 };
 
+export type ProviderQuotaDeferrals = {
+  /** Issues with a provider-quota (usage-limit) monitor defer that has not elapsed yet. */
+  pending: number;
+  /** Earliest pending monitor `nextCheckAt` (ISO 8601, UTC); null when nothing is pending. */
+  earliestNextCheckAt: string | null;
+  /** Latest pending monitor `nextCheckAt` (ISO 8601, UTC); null when nothing is pending. */
+  latestNextCheckAt: string | null;
+};
+
+export type FleetQuotaBanner = {
+  /** When the last queued quota wake fires (ISO 8601, UTC); null when nothing is paused. */
+  pausedUntil: string | null;
+  /** Number of issues waiting on a provider-quota reset. */
+  queuedWakes: number;
+};
+
 export type RecoveryObservabilityReport = {
   companyId: string;
   generatedAt: string;
@@ -81,6 +100,8 @@ export type RecoveryObservabilityReport = {
   byCause: RecoveryCauseGroup[];
   handoff: RecoveryHandoffSummary;
   perCauseRouting: RecoveryCauseRouting[];
+  providerQuotaDeferrals: ProviderQuotaDeferrals;
+  fleetQuotaBanner: FleetQuotaBanner;
 };
 
 export type HandoffClass =
@@ -103,6 +124,17 @@ type RecoveryActionFacts = {
 const ACTIVE_STATUSES = new Set(["active", "escalated"]);
 const TERMINAL_ISSUE_STATUSES = new Set(["done"]);
 
+// The internal recovery cause enum says `provider_quota`; operators read the
+// dashboard in terms of usage limits. Alias at the read boundary so the board
+// gets the familiar label without renaming the stored enum.
+const CAUSE_DISPLAY_ALIASES: Record<string, string> = {
+  provider_quota: "usage_limit_deferred",
+};
+
+export function displayCauseFor(cause: string): string {
+  return CAUSE_DISPLAY_ALIASES[cause] ?? cause;
+}
+
 /**
  * Classify a recovery action by who ended up owning the deliverable work.
  *
@@ -240,6 +272,7 @@ export function recoveryObservabilityService(db: Db) {
 
     const byCause: RecoveryCauseGroup[] = Array.from(causeRows).map((row) => ({
       cause: String(row.cause),
+      displayCause: displayCauseFor(String(row.cause)),
       latestRunErrorCode: String(row.error_code),
       count: Number(row.count),
       activeCount: Number(row.active_count),
@@ -356,6 +389,38 @@ export function recoveryObservabilityService(db: Db) {
 
     const perCauseRouting = Array.from(routingByCause.values()).sort((a, b) => b.total - a.total);
 
+    // Provider-quota defers taken through the issue-monitor path (the common
+    // case: the issue stays in_progress/in_review and a monitor is armed for the
+    // provider reset time) never write an issueRecoveryActions row, so the
+    // accounting above cannot see them. Read the pending defers straight off
+    // the denormalized monitor columns instead.
+    const quotaRows = (await db.execute(sql`
+      SELECT
+        count(*)::int AS pending,
+        to_char(min(${issues.monitorNextCheckAt}) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"') AS earliest,
+        to_char(max(${issues.monitorNextCheckAt}) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"') AS latest
+      FROM ${issues}
+      WHERE ${issues.companyId} = ${companyId}
+        AND ${issues.monitorNextCheckAt} IS NOT NULL
+        AND ${issues.monitorNextCheckAt} > ${now.toISOString()}::timestamptz
+        AND ${issues.executionPolicy} -> 'monitor' ->> 'serviceName' = ${PROVIDER_QUOTA_MONITOR_SERVICE_NAME}
+        AND ${issues.status} IN ('in_progress', 'in_review')
+    `)) as unknown as Iterable<{
+      pending: number | string | null;
+      earliest: string | null;
+      latest: string | null;
+    }>;
+    const quota = Array.from(quotaRows)[0];
+    const providerQuotaDeferrals: ProviderQuotaDeferrals = {
+      pending: Number(quota?.pending ?? 0),
+      earliestNextCheckAt: quota?.earliest ?? null,
+      latestNextCheckAt: quota?.latest ?? null,
+    };
+    const fleetQuotaBanner: FleetQuotaBanner = {
+      pausedUntil: providerQuotaDeferrals.latestNextCheckAt,
+      queuedWakes: providerQuotaDeferrals.pending,
+    };
+
     return {
       companyId,
       generatedAt: now.toISOString(),
@@ -366,6 +431,8 @@ export function recoveryObservabilityService(db: Db) {
       byCause,
       handoff,
       perCauseRouting,
+      providerQuotaDeferrals,
+      fleetQuotaBanner,
     };
   }