diff --git a/server/src/__tests__/budgets-service.test.ts b/server/src/__tests__/budgets-service.test.ts index b85050c54b..8a67682726 100644 --- a/server/src/__tests__/budgets-service.test.ts +++ b/server/src/__tests__/budgets-service.test.ts @@ -1,5 +1,20 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { randomUUID } from "node:crypto"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { + agents, + approvals, + budgetIncidents, + budgetPolicies, + companies, + costEvents, + createDb, + projects, +} from "@paperclipai/db"; import { budgetService } from "../services/budgets.ts"; +import { + getEmbeddedPostgresTestSupport, + startEmbeddedPostgresTestDatabase, +} from "./helpers/embedded-postgres.js"; const mockLogActivity = vi.hoisted(() => vi.fn()); @@ -309,3 +324,317 @@ describe("budgetService", () => { ); }); }); + +const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; + +describeEmbeddedPostgres("budgetService release gate enforcement", () => { + let db!: ReturnType; + let tempDb: Awaited> | null = null; + + beforeAll(async () => { + tempDb = await startEmbeddedPostgresTestDatabase("paperclip-budgets-service-"); + db = createDb(tempDb.connectionString); + }, 20_000); + + afterEach(async () => { + await db.delete(budgetIncidents); + await db.delete(approvals); + await db.delete(budgetPolicies); + await db.delete(costEvents); + await db.delete(projects); + await db.delete(agents); + await db.delete(companies); + mockLogActivity.mockClear(); + }); + + afterAll(async () => { + await tempDb?.cleanup(); + }); + + async function createBudgetFixture() { + const companyId = randomUUID(); + const agentId = randomUUID(); + const projectId = randomUUID(); + + await db.insert(companies).values({ + id: companyId, + name: "Paperclip", + issuePrefix: `B${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`, + requireBoardApprovalForNewAgents: false, + }); + await db.insert(agents).values({ + id: agentId, + companyId, + name: "Budget Agent SECRET_TOKEN_SHOULD_NOT_LEAK", + role: "engineer", + status: "active", + adapterType: "codex_local", + adapterConfig: {}, + runtimeConfig: {}, + permissions: {}, + }); + await db.insert(projects).values({ + id: projectId, + companyId, + name: "Budget Project", + status: "in_progress", + }); + + return { companyId, agentId, projectId }; + } + + async function insertCostEvent(input: { + companyId: string; + agentId: string; + projectId?: string | null; + costCents: number; + occurredAt?: Date; + }) { + const [event] = await db + .insert(costEvents) + .values({ + companyId: input.companyId, + agentId: input.agentId, + projectId: input.projectId ?? null, + provider: "openai", + biller: "openai", + billingType: "metered_api", + model: "gpt-5-release-gate", + inputTokens: 100, + cachedInputTokens: 10, + outputTokens: 20, + costCents: input.costCents, + occurredAt: input.occurredAt ?? new Date(), + }) + .returning(); + + return event!; + } + + it("raises one soft incident per window before hard-stopping and safely logging agent telemetry", async () => { + const { companyId, agentId } = await createBudgetFixture(); + const cancelWorkForScope = vi.fn().mockResolvedValue(undefined); + const service = budgetService(db, { cancelWorkForScope }); + const [policy] = await db + .insert(budgetPolicies) + .values({ + companyId, + scopeType: "agent", + scopeId: agentId, + metric: "billed_cents", + windowKind: "calendar_month_utc", + amount: 100, + warnPercent: 80, + hardStopEnabled: true, + notifyEnabled: true, + isActive: true, + }) + .returning(); + + const softEvent = await insertCostEvent({ companyId, agentId, costCents: 80 }); + await service.evaluateCostEvent(softEvent); + await service.evaluateCostEvent(softEvent); + + let incidentRows = await db + .select() + .from(budgetIncidents); + expect(incidentRows.filter((incident) => incident.thresholdType === "soft")).toHaveLength(1); + expect(incidentRows[0]).toMatchObject({ + companyId, + policyId: policy!.id, + scopeType: "agent", + scopeId: agentId, + thresholdType: "soft", + amountLimit: 100, + amountObserved: 80, + approvalId: null, + status: "open", + }); + + const [agentBeforeHardStop] = await db + .select({ status: agents.status, pauseReason: agents.pauseReason }) + .from(agents); + expect(agentBeforeHardStop).toEqual({ status: "active", pauseReason: null }); + + const hardEvent = await insertCostEvent({ companyId, agentId, costCents: 25 }); + await service.evaluateCostEvent(hardEvent); + await service.evaluateCostEvent(hardEvent); + + incidentRows = await db + .select() + .from(budgetIncidents); + expect(incidentRows.filter((incident) => incident.thresholdType === "soft")).toHaveLength(1); + expect(incidentRows.filter((incident) => incident.thresholdType === "hard")).toHaveLength(1); + expect(incidentRows.find((incident) => incident.thresholdType === "soft")).toMatchObject({ + status: "resolved", + }); + expect(incidentRows.find((incident) => incident.thresholdType === "hard")).toMatchObject({ + amountLimit: 100, + amountObserved: 105, + status: "open", + }); + + const [approval] = await db.select().from(approvals); + expect(approval).toMatchObject({ + companyId, + type: "budget_override_required", + status: "pending", + }); + + const [agentAfterHardStop] = await db + .select({ status: agents.status, pauseReason: agents.pauseReason, pausedAt: agents.pausedAt }) + .from(agents); + expect(agentAfterHardStop).toMatchObject({ status: "paused", pauseReason: "budget" }); + expect(agentAfterHardStop?.pausedAt).toBeInstanceOf(Date); + expect(cancelWorkForScope).toHaveBeenCalledTimes(2); + expect(cancelWorkForScope).toHaveBeenCalledWith({ companyId, scopeType: "agent", scopeId: agentId }); + + const block = await service.getInvocationBlock(companyId, agentId); + expect(block).toEqual({ + scopeType: "agent", + scopeId: agentId, + scopeName: "Budget Agent SECRET_TOKEN_SHOULD_NOT_LEAK", + reason: "Agent is paused because its budget hard-stop was reached.", + }); + + const telemetryCalls = mockLogActivity.mock.calls.map(([, input]) => input); + expect(telemetryCalls.filter((call) => call.action === "budget.soft_threshold_crossed")).toHaveLength(1); + expect(telemetryCalls.filter((call) => call.action === "budget.hard_threshold_crossed")).toHaveLength(1); + expect(telemetryCalls).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + action: "budget.soft_threshold_crossed", + entityType: "budget_incident", + details: expect.objectContaining({ + scopeType: "agent", + scopeId: agentId, + amountObserved: 80, + amountLimit: 100, + }), + }), + expect.objectContaining({ + action: "budget.hard_threshold_crossed", + entityType: "budget_incident", + details: expect.objectContaining({ + scopeType: "agent", + scopeId: agentId, + amountObserved: 105, + amountLimit: 100, + approvalId: approval!.id, + }), + }), + ]), + ); + for (const call of telemetryCalls) { + expect(JSON.stringify(call.details)).not.toContain("SECRET_TOKEN_SHOULD_NOT_LEAK"); + expect(call.details).not.toHaveProperty("prompt"); + expect(call.details).not.toHaveProperty("message"); + } + }); + + it("hard-stops project work until a valid budget raise resumes it and overview reconciles ledger spend", async () => { + const { companyId, agentId, projectId } = await createBudgetFixture(); + const cancelWorkForScope = vi.fn().mockResolvedValue(undefined); + const service = budgetService(db, { cancelWorkForScope }); + await db.insert(budgetPolicies).values({ + companyId, + scopeType: "project", + scopeId: projectId, + metric: "billed_cents", + windowKind: "lifetime", + amount: 100, + warnPercent: 75, + hardStopEnabled: true, + notifyEnabled: true, + isActive: true, + }); + + const event = await insertCostEvent({ companyId, agentId, projectId, costCents: 125 }); + await service.evaluateCostEvent(event); + await service.evaluateCostEvent(event); + + const incidentRows = await db + .select() + .from(budgetIncidents); + expect(incidentRows.filter((incident) => incident.thresholdType === "hard")).toHaveLength(1); + const hardIncident = incidentRows.find((incident) => incident.thresholdType === "hard")!; + expect(hardIncident).toMatchObject({ + companyId, + scopeType: "project", + scopeId: projectId, + amountLimit: 100, + amountObserved: 125, + status: "open", + }); + + const [projectAfterHardStop] = await db + .select({ pauseReason: projects.pauseReason, pausedAt: projects.pausedAt }) + .from(projects); + expect(projectAfterHardStop?.pauseReason).toBe("budget"); + expect(projectAfterHardStop?.pausedAt).toBeInstanceOf(Date); + expect(cancelWorkForScope).toHaveBeenCalledWith({ companyId, scopeType: "project", scopeId: projectId }); + + const overviewWhileBlocked = await service.overview(companyId); + expect(overviewWhileBlocked.pausedProjectCount).toBe(1); + expect(overviewWhileBlocked.pendingApprovalCount).toBe(1); + expect(overviewWhileBlocked.policies[0]).toMatchObject({ + scopeType: "project", + scopeId: projectId, + amount: 100, + observedAmount: 125, + remainingAmount: 0, + utilizationPercent: 125, + status: "hard_stop", + paused: true, + pauseReason: "budget", + }); + expect(overviewWhileBlocked.activeIncidents).toHaveLength(1); + + await expect( + service.resolveIncident( + companyId, + hardIncident.id, + { action: "raise_budget_and_resume", amount: 125 }, + "board-user", + ), + ).rejects.toThrow("New budget must exceed current observed spend"); + + expect(await service.getInvocationBlock(companyId, agentId, { projectId })).toEqual({ + scopeType: "project", + scopeId: projectId, + scopeName: "Budget Project", + reason: "Project cannot start work because its budget hard-stop is still exceeded.", + }); + + const resolved = await service.resolveIncident( + companyId, + hardIncident.id, + { action: "raise_budget_and_resume", amount: 175, decisionNote: "Approved release-gate budget raise." }, + "board-user", + ); + expect(resolved).toMatchObject({ status: "resolved", approvalStatus: "approved" }); + + const [projectAfterResume] = await db + .select({ pauseReason: projects.pauseReason, pausedAt: projects.pausedAt }) + .from(projects); + expect(projectAfterResume).toEqual({ pauseReason: null, pausedAt: null }); + expect(await service.getInvocationBlock(companyId, agentId, { projectId })).toBeNull(); + + const overviewAfterResume = await service.overview(companyId); + expect(overviewAfterResume.pausedProjectCount).toBe(0); + expect(overviewAfterResume.pendingApprovalCount).toBe(0); + expect(overviewAfterResume.policies[0]).toMatchObject({ + scopeType: "project", + scopeId: projectId, + amount: 175, + observedAmount: 125, + remainingAmount: 50, + utilizationPercent: expect.closeTo(71.43, 2), + status: "ok", + paused: false, + pauseReason: null, + }); + expect(overviewAfterResume.activeIncidents).toHaveLength(0); + }); +}); diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index a340df5059..22154cb9b3 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -917,6 +917,40 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { return { companyId, agentId, issueId }; } + async function seedIdleTimerAgentFixture() { + const companyId = randomUUID(); + const agentId = randomUUID(); + const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`; + + await db.insert(companies).values({ + id: companyId, + name: "Paperclip", + issuePrefix, + requireBoardApprovalForNewAgents: false, + }); + + await db.insert(agents).values({ + id: agentId, + companyId, + name: "CodexCoder", + role: "engineer", + status: "idle", + adapterType: "codex_local", + adapterConfig: {}, + runtimeConfig: { + heartbeat: { + enabled: true, + intervalSec: 60, + wakeOnDemand: true, + skipTimerWhenNoActionableWork: true, + }, + }, + permissions: {}, + }); + + return { companyId, agentId }; + } + async function expectSourceScopedStrandedRecoveryAction(input: { companyId: string; agentId: string; @@ -1156,6 +1190,46 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(wakeup?.status).toBe("claimed"); }); + it("skips generic timer wakes without invoking an adapter when no assigned work is actionable", async () => { + const { companyId, agentId } = await seedIdleTimerAgentFixture(); + const heartbeat = heartbeatService(db); + + const run = await heartbeat.wakeup(agentId, { + source: "timer", + triggerDetail: "system", + reason: "heartbeat_timer", + requestedByActorType: "system", + requestedByActorId: "heartbeat_scheduler", + contextSnapshot: { + source: "scheduler", + reason: "interval_elapsed", + now: "2026-03-19T00:00:00.000Z", + }, + }); + + expect(run).toBeNull(); + expect(mockAdapterExecute).not.toHaveBeenCalled(); + + const requests = await db + .select() + .from(agentWakeupRequests) + .where(eq(agentWakeupRequests.agentId, agentId)); + expect(requests).toHaveLength(1); + expect(requests[0]).toMatchObject({ + companyId, + source: "timer", + reason: "heartbeat.timer.no_actionable_work", + status: "skipped", + error: null, + }); + + const runs = await db + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.agentId, agentId)); + expect(runs).toHaveLength(0); + }); + it("queues exactly one retry when the recorded local pid is dead", async () => { const { agentId, runId, issueId } = await seedRunFixture({ agentStatus: "idle", diff --git a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts index b560d82f22..32611ab4fa 100644 --- a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts +++ b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts @@ -394,7 +394,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => { expect(countExecuteCallsForRun(run!.id)).toBe(1); }); - it("runs generic timer wakes by default for proactive agents without assigned issue work", async () => { + it("allows legacy generic timer wakes by default when no skip policy is set", async () => { const { agentId } = await seedCompanyAndAgent({ heartbeatConfig: { enabled: true, @@ -408,7 +408,24 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => { expect(run).not.toBeNull(); await waitForCondition(async () => countExecuteCallsForRun(run!.id) > 0); + expect(countExecuteCallsForRun(run!.id)).toBe(1); + }); + it("allows explicit proactive generic timer wakes without assigned issue work", async () => { + const { agentId } = await seedCompanyAndAgent({ + heartbeatConfig: { + enabled: true, + skipTimerWhenNoActionableWork: false, + }, + }); + + const run = await heartbeat.wakeup(agentId, { + source: "timer", + triggerDetail: "schedule", + }); + + expect(run).not.toBeNull(); + await waitForCondition(async () => countExecuteCallsForRun(run!.id) > 0); expect(countExecuteCallsForRun(run!.id)).toBe(1); }); diff --git a/server/src/__tests__/issue-stale-execution-lock-routes.test.ts b/server/src/__tests__/issue-stale-execution-lock-routes.test.ts index 1f91fa278b..944948ec03 100644 --- a/server/src/__tests__/issue-stale-execution-lock-routes.test.ts +++ b/server/src/__tests__/issue-stale-execution-lock-routes.test.ts @@ -290,6 +290,67 @@ describeEmbeddedPostgres("stale issue execution lock routes", () => { expect(res.body?.error).toBe("Issue run ownership conflict"); }); + it("preserves live checkout ownership on checkout conflicts without retry side effects", async () => { + const { companyId, agentId, currentRunId } = await seedCompanyAgentAndRuns(); + const contenderRunId = randomUUID(); + const issueId = randomUUID(); + await db.insert(heartbeatRuns).values({ + id: contenderRunId, + companyId, + agentId, + status: "running", + invocationSource: "assignment", + startedAt: new Date(), + }); + await db.insert(issues).values({ + id: issueId, + companyId, + title: "Live checkout race", + status: "in_progress", + priority: "high", + assigneeAgentId: agentId, + checkoutRunId: currentRunId, + executionRunId: currentRunId, + executionAgentNameKey: "codexcoder", + executionLockedAt: new Date(), + }); + + const res = await request(createApp(agentActor(companyId, agentId, contenderRunId))) + .post(`/api/issues/${issueId}/checkout`) + .send({ + agentId, + expectedStatuses: ["todo", "backlog", "blocked", "in_review"], + }); + + expect(res.status, JSON.stringify(res.body)).toBe(409); + expect(res.body).toMatchObject({ + error: "Issue checkout conflict", + }); + + const row = await db + .select({ + status: issues.status, + assigneeAgentId: issues.assigneeAgentId, + checkoutRunId: issues.checkoutRunId, + executionRunId: issues.executionRunId, + }) + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => rows[0]); + expect(row).toEqual({ + status: "in_progress", + assigneeAgentId: agentId, + checkoutRunId: currentRunId, + executionRunId: currentRunId, + }); + + const checkoutActivity = await db + .select() + .from(activityLog) + .where(eq(activityLog.action, "issue.checked_out")); + expect(checkoutActivity).toHaveLength(0); + }); + it("restricts admin force-release to board users with company access and writes an audit event", async () => { const { companyId, agentId, failedRunId, currentRunId } = await seedCompanyAgentAndRuns(); const issueId = randomUUID(); diff --git a/server/src/services/budgets.ts b/server/src/services/budgets.ts index 84d6663e12..e281dcb51f 100644 --- a/server/src/services/budgets.ts +++ b/server/src/services/budgets.ts @@ -365,7 +365,7 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) { ), ) .then((rows) => rows[0] ?? null); - if (existing) return existing; + if (existing) return { incident: existing, created: false }; const scope = await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId); const payload = buildApprovalPayload({ @@ -392,7 +392,7 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) { .then((rows) => rows[0] ?? null) : null; - return db + const incident = await db .insert(budgetIncidents) .values({ companyId: policy.companyId, @@ -411,6 +411,7 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) { }) .returning() .then((rows) => rows[0] ?? null); + return incident ? { incident, created: true } : null; } async function resolveOpenSoftIncidents(policyId: string) { @@ -671,14 +672,14 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) { if (policy.notifyEnabled && observedAmount >= softThreshold) { const softIncident = await createIncidentIfNeeded(policy, "soft", observedAmount); - if (softIncident) { + if (softIncident?.created) { await logActivity(db, { companyId: policy.companyId, actorType: "system", actorId: "budget_service", action: "budget.soft_threshold_crossed", entityType: "budget_incident", - entityId: softIncident.id, + entityId: softIncident.incident.id, details: { scopeType: policy.scopeType, scopeId: policy.scopeId, @@ -693,20 +694,20 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) { await resolveOpenSoftIncidents(policy.id); const hardIncident = await createIncidentIfNeeded(policy, "hard", observedAmount); await pauseAndCancelScopeForBudget(policy); - if (hardIncident) { + if (hardIncident?.created) { await logActivity(db, { companyId: policy.companyId, actorType: "system", actorId: "budget_service", action: "budget.hard_threshold_crossed", entityType: "budget_incident", - entityId: hardIncident.id, + entityId: hardIncident.incident.id, details: { scopeType: policy.scopeType, scopeId: policy.scopeId, amountObserved: observedAmount, amountLimit: policy.amount, - approvalId: hardIncident.approvalId ?? null, + approvalId: hardIncident.incident.approvalId ?? null, }, }); }