[codex] Enforce backend execution release gates (#9089)
## Thinking Path > - Paperclip is the open source control plane people use to manage AI agents for work. > - Backend execution safety is part of the control plane contract: agents must stop at budget hard limits, stale execution paths must not create duplicate live work, and checkout ownership must remain authoritative. > - The recovery branch bundled these release-gate checks with broader unrelated work. > - Reviewers need a narrow PR that isolates only the backend safety behavior and regression coverage. > - This pull request keeps budget incident creation idempotent so repeated evaluation does not duplicate release-gate telemetry or approvals. > - It also adds focused coverage for idle timer skips, stale queued-run behavior, and live checkout conflict preservation. > - The benefit is a smaller, reviewable release-gate slice for budget hard stops, stale execution recovery, and ownership-safe issue mutation. ## Linked Issues or Issue Description Refs #8866 This PR extracts a focused backend safety slice from the closed broad recovery PR. The underlying problem is that release-gate behavior needs direct regression coverage before review: budget hard stops should not duplicate incidents/logging on repeated evaluation, timer wakes should respect the no-actionable-work skip policy, stale queued runs should remain invalidated, and active checkout ownership must survive conflicting checkout attempts without side effects. ## What Changed - Made budget incident creation report whether an incident was newly created, so soft/hard threshold activity logs are emitted once per incident window. - Added embedded Postgres budget release-gate tests covering soft incident idempotency, hard-stop pause/cancel behavior, budget override resume behavior, and telemetry redaction. - Added heartbeat coverage for skipping generic timer wakes when the agent opts into `skipTimerWhenNoActionableWork`, while preserving legacy/proactive timer behavior. - Added stale execution lock route coverage proving a conflicting checkout returns `409` without overwriting live checkout or execution ownership and without writing checkout activity. ## Verification - `./node_modules/.bin/vitest run server/src/__tests__/budgets-service.test.ts server/src/__tests__/heartbeat-process-recovery.test.ts server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts server/src/__tests__/issue-stale-execution-lock-routes.test.ts --no-file-parallelism --maxWorkers=1` - First run: 3 files passed, 98 tests passed; `issue-stale-execution-lock-routes.test.ts` failed during import because the isolated worktree initially lacked dev dependency links for `supertest`. - `CI=true NODE_ENV=development pnpm install --frozen-lockfile --ignore-scripts` - Recreated worktree dev dependency links; emitted unrelated plugin SDK bin warnings because plugin SDK dist files were not built under `--ignore-scripts`. - `./node_modules/.bin/vitest run server/src/__tests__/issue-stale-execution-lock-routes.test.ts --no-file-parallelism --maxWorkers=1` - Passed: 1 file, 7 tests. - `git diff --check` - Passed. ## Risks Low to medium risk. The production code change is intentionally small and only suppresses duplicate threshold activity logging for already-open budget incidents, but it affects budget release-gate observability. The new tests use embedded Postgres and should catch regressions in budget hard stops, timer wake gating, stale queue invalidation, and checkout conflict preservation. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used OpenAI Codex, GPT-5 coding agent, tool-use enabled. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta
GitHub
parent
dfc256a543
commit
ef617bee5c
|
|
@ -1,5 +1,20 @@
|
|||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
agents,
|
||||
approvals,
|
||||
budgetIncidents,
|
||||
budgetPolicies,
|
||||
companies,
|
||||
costEvents,
|
||||
createDb,
|
||||
projects,
|
||||
} from "@paperclipai/db";
|
||||
import { budgetService } from "../services/budgets.ts";
|
||||
import {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
} from "./helpers/embedded-postgres.js";
|
||||
|
||||
const mockLogActivity = vi.hoisted(() => vi.fn());
|
||||
|
||||
|
|
@ -309,3 +324,317 @@ describe("budgetService", () => {
|
|||
);
|
||||
});
|
||||
});
|
||||
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
describeEmbeddedPostgres("budgetService release gate enforcement", () => {
|
||||
let db!: ReturnType<typeof createDb>;
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-budgets-service-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
}, 20_000);
|
||||
|
||||
afterEach(async () => {
|
||||
await db.delete(budgetIncidents);
|
||||
await db.delete(approvals);
|
||||
await db.delete(budgetPolicies);
|
||||
await db.delete(costEvents);
|
||||
await db.delete(projects);
|
||||
await db.delete(agents);
|
||||
await db.delete(companies);
|
||||
mockLogActivity.mockClear();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
async function createBudgetFixture() {
|
||||
const companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
const projectId = randomUUID();
|
||||
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Paperclip",
|
||||
issuePrefix: `B${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
await db.insert(agents).values({
|
||||
id: agentId,
|
||||
companyId,
|
||||
name: "Budget Agent SECRET_TOKEN_SHOULD_NOT_LEAK",
|
||||
role: "engineer",
|
||||
status: "active",
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
runtimeConfig: {},
|
||||
permissions: {},
|
||||
});
|
||||
await db.insert(projects).values({
|
||||
id: projectId,
|
||||
companyId,
|
||||
name: "Budget Project",
|
||||
status: "in_progress",
|
||||
});
|
||||
|
||||
return { companyId, agentId, projectId };
|
||||
}
|
||||
|
||||
async function insertCostEvent(input: {
|
||||
companyId: string;
|
||||
agentId: string;
|
||||
projectId?: string | null;
|
||||
costCents: number;
|
||||
occurredAt?: Date;
|
||||
}) {
|
||||
const [event] = await db
|
||||
.insert(costEvents)
|
||||
.values({
|
||||
companyId: input.companyId,
|
||||
agentId: input.agentId,
|
||||
projectId: input.projectId ?? null,
|
||||
provider: "openai",
|
||||
biller: "openai",
|
||||
billingType: "metered_api",
|
||||
model: "gpt-5-release-gate",
|
||||
inputTokens: 100,
|
||||
cachedInputTokens: 10,
|
||||
outputTokens: 20,
|
||||
costCents: input.costCents,
|
||||
occurredAt: input.occurredAt ?? new Date(),
|
||||
})
|
||||
.returning();
|
||||
|
||||
return event!;
|
||||
}
|
||||
|
||||
it("raises one soft incident per window before hard-stopping and safely logging agent telemetry", async () => {
|
||||
const { companyId, agentId } = await createBudgetFixture();
|
||||
const cancelWorkForScope = vi.fn().mockResolvedValue(undefined);
|
||||
const service = budgetService(db, { cancelWorkForScope });
|
||||
const [policy] = await db
|
||||
.insert(budgetPolicies)
|
||||
.values({
|
||||
companyId,
|
||||
scopeType: "agent",
|
||||
scopeId: agentId,
|
||||
metric: "billed_cents",
|
||||
windowKind: "calendar_month_utc",
|
||||
amount: 100,
|
||||
warnPercent: 80,
|
||||
hardStopEnabled: true,
|
||||
notifyEnabled: true,
|
||||
isActive: true,
|
||||
})
|
||||
.returning();
|
||||
|
||||
const softEvent = await insertCostEvent({ companyId, agentId, costCents: 80 });
|
||||
await service.evaluateCostEvent(softEvent);
|
||||
await service.evaluateCostEvent(softEvent);
|
||||
|
||||
let incidentRows = await db
|
||||
.select()
|
||||
.from(budgetIncidents);
|
||||
expect(incidentRows.filter((incident) => incident.thresholdType === "soft")).toHaveLength(1);
|
||||
expect(incidentRows[0]).toMatchObject({
|
||||
companyId,
|
||||
policyId: policy!.id,
|
||||
scopeType: "agent",
|
||||
scopeId: agentId,
|
||||
thresholdType: "soft",
|
||||
amountLimit: 100,
|
||||
amountObserved: 80,
|
||||
approvalId: null,
|
||||
status: "open",
|
||||
});
|
||||
|
||||
const [agentBeforeHardStop] = await db
|
||||
.select({ status: agents.status, pauseReason: agents.pauseReason })
|
||||
.from(agents);
|
||||
expect(agentBeforeHardStop).toEqual({ status: "active", pauseReason: null });
|
||||
|
||||
const hardEvent = await insertCostEvent({ companyId, agentId, costCents: 25 });
|
||||
await service.evaluateCostEvent(hardEvent);
|
||||
await service.evaluateCostEvent(hardEvent);
|
||||
|
||||
incidentRows = await db
|
||||
.select()
|
||||
.from(budgetIncidents);
|
||||
expect(incidentRows.filter((incident) => incident.thresholdType === "soft")).toHaveLength(1);
|
||||
expect(incidentRows.filter((incident) => incident.thresholdType === "hard")).toHaveLength(1);
|
||||
expect(incidentRows.find((incident) => incident.thresholdType === "soft")).toMatchObject({
|
||||
status: "resolved",
|
||||
});
|
||||
expect(incidentRows.find((incident) => incident.thresholdType === "hard")).toMatchObject({
|
||||
amountLimit: 100,
|
||||
amountObserved: 105,
|
||||
status: "open",
|
||||
});
|
||||
|
||||
const [approval] = await db.select().from(approvals);
|
||||
expect(approval).toMatchObject({
|
||||
companyId,
|
||||
type: "budget_override_required",
|
||||
status: "pending",
|
||||
});
|
||||
|
||||
const [agentAfterHardStop] = await db
|
||||
.select({ status: agents.status, pauseReason: agents.pauseReason, pausedAt: agents.pausedAt })
|
||||
.from(agents);
|
||||
expect(agentAfterHardStop).toMatchObject({ status: "paused", pauseReason: "budget" });
|
||||
expect(agentAfterHardStop?.pausedAt).toBeInstanceOf(Date);
|
||||
expect(cancelWorkForScope).toHaveBeenCalledTimes(2);
|
||||
expect(cancelWorkForScope).toHaveBeenCalledWith({ companyId, scopeType: "agent", scopeId: agentId });
|
||||
|
||||
const block = await service.getInvocationBlock(companyId, agentId);
|
||||
expect(block).toEqual({
|
||||
scopeType: "agent",
|
||||
scopeId: agentId,
|
||||
scopeName: "Budget Agent SECRET_TOKEN_SHOULD_NOT_LEAK",
|
||||
reason: "Agent is paused because its budget hard-stop was reached.",
|
||||
});
|
||||
|
||||
const telemetryCalls = mockLogActivity.mock.calls.map(([, input]) => input);
|
||||
expect(telemetryCalls.filter((call) => call.action === "budget.soft_threshold_crossed")).toHaveLength(1);
|
||||
expect(telemetryCalls.filter((call) => call.action === "budget.hard_threshold_crossed")).toHaveLength(1);
|
||||
expect(telemetryCalls).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
action: "budget.soft_threshold_crossed",
|
||||
entityType: "budget_incident",
|
||||
details: expect.objectContaining({
|
||||
scopeType: "agent",
|
||||
scopeId: agentId,
|
||||
amountObserved: 80,
|
||||
amountLimit: 100,
|
||||
}),
|
||||
}),
|
||||
expect.objectContaining({
|
||||
action: "budget.hard_threshold_crossed",
|
||||
entityType: "budget_incident",
|
||||
details: expect.objectContaining({
|
||||
scopeType: "agent",
|
||||
scopeId: agentId,
|
||||
amountObserved: 105,
|
||||
amountLimit: 100,
|
||||
approvalId: approval!.id,
|
||||
}),
|
||||
}),
|
||||
]),
|
||||
);
|
||||
for (const call of telemetryCalls) {
|
||||
expect(JSON.stringify(call.details)).not.toContain("SECRET_TOKEN_SHOULD_NOT_LEAK");
|
||||
expect(call.details).not.toHaveProperty("prompt");
|
||||
expect(call.details).not.toHaveProperty("message");
|
||||
}
|
||||
});
|
||||
|
||||
it("hard-stops project work until a valid budget raise resumes it and overview reconciles ledger spend", async () => {
|
||||
const { companyId, agentId, projectId } = await createBudgetFixture();
|
||||
const cancelWorkForScope = vi.fn().mockResolvedValue(undefined);
|
||||
const service = budgetService(db, { cancelWorkForScope });
|
||||
await db.insert(budgetPolicies).values({
|
||||
companyId,
|
||||
scopeType: "project",
|
||||
scopeId: projectId,
|
||||
metric: "billed_cents",
|
||||
windowKind: "lifetime",
|
||||
amount: 100,
|
||||
warnPercent: 75,
|
||||
hardStopEnabled: true,
|
||||
notifyEnabled: true,
|
||||
isActive: true,
|
||||
});
|
||||
|
||||
const event = await insertCostEvent({ companyId, agentId, projectId, costCents: 125 });
|
||||
await service.evaluateCostEvent(event);
|
||||
await service.evaluateCostEvent(event);
|
||||
|
||||
const incidentRows = await db
|
||||
.select()
|
||||
.from(budgetIncidents);
|
||||
expect(incidentRows.filter((incident) => incident.thresholdType === "hard")).toHaveLength(1);
|
||||
const hardIncident = incidentRows.find((incident) => incident.thresholdType === "hard")!;
|
||||
expect(hardIncident).toMatchObject({
|
||||
companyId,
|
||||
scopeType: "project",
|
||||
scopeId: projectId,
|
||||
amountLimit: 100,
|
||||
amountObserved: 125,
|
||||
status: "open",
|
||||
});
|
||||
|
||||
const [projectAfterHardStop] = await db
|
||||
.select({ pauseReason: projects.pauseReason, pausedAt: projects.pausedAt })
|
||||
.from(projects);
|
||||
expect(projectAfterHardStop?.pauseReason).toBe("budget");
|
||||
expect(projectAfterHardStop?.pausedAt).toBeInstanceOf(Date);
|
||||
expect(cancelWorkForScope).toHaveBeenCalledWith({ companyId, scopeType: "project", scopeId: projectId });
|
||||
|
||||
const overviewWhileBlocked = await service.overview(companyId);
|
||||
expect(overviewWhileBlocked.pausedProjectCount).toBe(1);
|
||||
expect(overviewWhileBlocked.pendingApprovalCount).toBe(1);
|
||||
expect(overviewWhileBlocked.policies[0]).toMatchObject({
|
||||
scopeType: "project",
|
||||
scopeId: projectId,
|
||||
amount: 100,
|
||||
observedAmount: 125,
|
||||
remainingAmount: 0,
|
||||
utilizationPercent: 125,
|
||||
status: "hard_stop",
|
||||
paused: true,
|
||||
pauseReason: "budget",
|
||||
});
|
||||
expect(overviewWhileBlocked.activeIncidents).toHaveLength(1);
|
||||
|
||||
await expect(
|
||||
service.resolveIncident(
|
||||
companyId,
|
||||
hardIncident.id,
|
||||
{ action: "raise_budget_and_resume", amount: 125 },
|
||||
"board-user",
|
||||
),
|
||||
).rejects.toThrow("New budget must exceed current observed spend");
|
||||
|
||||
expect(await service.getInvocationBlock(companyId, agentId, { projectId })).toEqual({
|
||||
scopeType: "project",
|
||||
scopeId: projectId,
|
||||
scopeName: "Budget Project",
|
||||
reason: "Project cannot start work because its budget hard-stop is still exceeded.",
|
||||
});
|
||||
|
||||
const resolved = await service.resolveIncident(
|
||||
companyId,
|
||||
hardIncident.id,
|
||||
{ action: "raise_budget_and_resume", amount: 175, decisionNote: "Approved release-gate budget raise." },
|
||||
"board-user",
|
||||
);
|
||||
expect(resolved).toMatchObject({ status: "resolved", approvalStatus: "approved" });
|
||||
|
||||
const [projectAfterResume] = await db
|
||||
.select({ pauseReason: projects.pauseReason, pausedAt: projects.pausedAt })
|
||||
.from(projects);
|
||||
expect(projectAfterResume).toEqual({ pauseReason: null, pausedAt: null });
|
||||
expect(await service.getInvocationBlock(companyId, agentId, { projectId })).toBeNull();
|
||||
|
||||
const overviewAfterResume = await service.overview(companyId);
|
||||
expect(overviewAfterResume.pausedProjectCount).toBe(0);
|
||||
expect(overviewAfterResume.pendingApprovalCount).toBe(0);
|
||||
expect(overviewAfterResume.policies[0]).toMatchObject({
|
||||
scopeType: "project",
|
||||
scopeId: projectId,
|
||||
amount: 175,
|
||||
observedAmount: 125,
|
||||
remainingAmount: 50,
|
||||
utilizationPercent: expect.closeTo(71.43, 2),
|
||||
status: "ok",
|
||||
paused: false,
|
||||
pauseReason: null,
|
||||
});
|
||||
expect(overviewAfterResume.activeIncidents).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -917,6 +917,40 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
return { companyId, agentId, issueId };
|
||||
}
|
||||
|
||||
async function seedIdleTimerAgentFixture() {
|
||||
const companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
|
||||
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Paperclip",
|
||||
issuePrefix,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
|
||||
await db.insert(agents).values({
|
||||
id: agentId,
|
||||
companyId,
|
||||
name: "CodexCoder",
|
||||
role: "engineer",
|
||||
status: "idle",
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
runtimeConfig: {
|
||||
heartbeat: {
|
||||
enabled: true,
|
||||
intervalSec: 60,
|
||||
wakeOnDemand: true,
|
||||
skipTimerWhenNoActionableWork: true,
|
||||
},
|
||||
},
|
||||
permissions: {},
|
||||
});
|
||||
|
||||
return { companyId, agentId };
|
||||
}
|
||||
|
||||
async function expectSourceScopedStrandedRecoveryAction(input: {
|
||||
companyId: string;
|
||||
agentId: string;
|
||||
|
|
@ -1156,6 +1190,46 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
expect(wakeup?.status).toBe("claimed");
|
||||
});
|
||||
|
||||
it("skips generic timer wakes without invoking an adapter when no assigned work is actionable", async () => {
|
||||
const { companyId, agentId } = await seedIdleTimerAgentFixture();
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
const run = await heartbeat.wakeup(agentId, {
|
||||
source: "timer",
|
||||
triggerDetail: "system",
|
||||
reason: "heartbeat_timer",
|
||||
requestedByActorType: "system",
|
||||
requestedByActorId: "heartbeat_scheduler",
|
||||
contextSnapshot: {
|
||||
source: "scheduler",
|
||||
reason: "interval_elapsed",
|
||||
now: "2026-03-19T00:00:00.000Z",
|
||||
},
|
||||
});
|
||||
|
||||
expect(run).toBeNull();
|
||||
expect(mockAdapterExecute).not.toHaveBeenCalled();
|
||||
|
||||
const requests = await db
|
||||
.select()
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.agentId, agentId));
|
||||
expect(requests).toHaveLength(1);
|
||||
expect(requests[0]).toMatchObject({
|
||||
companyId,
|
||||
source: "timer",
|
||||
reason: "heartbeat.timer.no_actionable_work",
|
||||
status: "skipped",
|
||||
error: null,
|
||||
});
|
||||
|
||||
const runs = await db
|
||||
.select()
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.agentId, agentId));
|
||||
expect(runs).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("queues exactly one retry when the recorded local pid is dead", async () => {
|
||||
const { agentId, runId, issueId } = await seedRunFixture({
|
||||
agentStatus: "idle",
|
||||
|
|
|
|||
|
|
@ -394,7 +394,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
|||
expect(countExecuteCallsForRun(run!.id)).toBe(1);
|
||||
});
|
||||
|
||||
it("runs generic timer wakes by default for proactive agents without assigned issue work", async () => {
|
||||
it("allows legacy generic timer wakes by default when no skip policy is set", async () => {
|
||||
const { agentId } = await seedCompanyAndAgent({
|
||||
heartbeatConfig: {
|
||||
enabled: true,
|
||||
|
|
@ -408,7 +408,24 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
|||
|
||||
expect(run).not.toBeNull();
|
||||
await waitForCondition(async () => countExecuteCallsForRun(run!.id) > 0);
|
||||
expect(countExecuteCallsForRun(run!.id)).toBe(1);
|
||||
});
|
||||
|
||||
it("allows explicit proactive generic timer wakes without assigned issue work", async () => {
|
||||
const { agentId } = await seedCompanyAndAgent({
|
||||
heartbeatConfig: {
|
||||
enabled: true,
|
||||
skipTimerWhenNoActionableWork: false,
|
||||
},
|
||||
});
|
||||
|
||||
const run = await heartbeat.wakeup(agentId, {
|
||||
source: "timer",
|
||||
triggerDetail: "schedule",
|
||||
});
|
||||
|
||||
expect(run).not.toBeNull();
|
||||
await waitForCondition(async () => countExecuteCallsForRun(run!.id) > 0);
|
||||
expect(countExecuteCallsForRun(run!.id)).toBe(1);
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -290,6 +290,67 @@ describeEmbeddedPostgres("stale issue execution lock routes", () => {
|
|||
expect(res.body?.error).toBe("Issue run ownership conflict");
|
||||
});
|
||||
|
||||
it("preserves live checkout ownership on checkout conflicts without retry side effects", async () => {
|
||||
const { companyId, agentId, currentRunId } = await seedCompanyAgentAndRuns();
|
||||
const contenderRunId = randomUUID();
|
||||
const issueId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: contenderRunId,
|
||||
companyId,
|
||||
agentId,
|
||||
status: "running",
|
||||
invocationSource: "assignment",
|
||||
startedAt: new Date(),
|
||||
});
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId,
|
||||
title: "Live checkout race",
|
||||
status: "in_progress",
|
||||
priority: "high",
|
||||
assigneeAgentId: agentId,
|
||||
checkoutRunId: currentRunId,
|
||||
executionRunId: currentRunId,
|
||||
executionAgentNameKey: "codexcoder",
|
||||
executionLockedAt: new Date(),
|
||||
});
|
||||
|
||||
const res = await request(createApp(agentActor(companyId, agentId, contenderRunId)))
|
||||
.post(`/api/issues/${issueId}/checkout`)
|
||||
.send({
|
||||
agentId,
|
||||
expectedStatuses: ["todo", "backlog", "blocked", "in_review"],
|
||||
});
|
||||
|
||||
expect(res.status, JSON.stringify(res.body)).toBe(409);
|
||||
expect(res.body).toMatchObject({
|
||||
error: "Issue checkout conflict",
|
||||
});
|
||||
|
||||
const row = await db
|
||||
.select({
|
||||
status: issues.status,
|
||||
assigneeAgentId: issues.assigneeAgentId,
|
||||
checkoutRunId: issues.checkoutRunId,
|
||||
executionRunId: issues.executionRunId,
|
||||
})
|
||||
.from(issues)
|
||||
.where(eq(issues.id, issueId))
|
||||
.then((rows) => rows[0]);
|
||||
expect(row).toEqual({
|
||||
status: "in_progress",
|
||||
assigneeAgentId: agentId,
|
||||
checkoutRunId: currentRunId,
|
||||
executionRunId: currentRunId,
|
||||
});
|
||||
|
||||
const checkoutActivity = await db
|
||||
.select()
|
||||
.from(activityLog)
|
||||
.where(eq(activityLog.action, "issue.checked_out"));
|
||||
expect(checkoutActivity).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("restricts admin force-release to board users with company access and writes an audit event", async () => {
|
||||
const { companyId, agentId, failedRunId, currentRunId } = await seedCompanyAgentAndRuns();
|
||||
const issueId = randomUUID();
|
||||
|
|
|
|||
|
|
@ -365,7 +365,7 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
|
|||
),
|
||||
)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (existing) return existing;
|
||||
if (existing) return { incident: existing, created: false };
|
||||
|
||||
const scope = await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId);
|
||||
const payload = buildApprovalPayload({
|
||||
|
|
@ -392,7 +392,7 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
|
|||
.then((rows) => rows[0] ?? null)
|
||||
: null;
|
||||
|
||||
return db
|
||||
const incident = await db
|
||||
.insert(budgetIncidents)
|
||||
.values({
|
||||
companyId: policy.companyId,
|
||||
|
|
@ -411,6 +411,7 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
|
|||
})
|
||||
.returning()
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return incident ? { incident, created: true } : null;
|
||||
}
|
||||
|
||||
async function resolveOpenSoftIncidents(policyId: string) {
|
||||
|
|
@ -671,14 +672,14 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
|
|||
|
||||
if (policy.notifyEnabled && observedAmount >= softThreshold) {
|
||||
const softIncident = await createIncidentIfNeeded(policy, "soft", observedAmount);
|
||||
if (softIncident) {
|
||||
if (softIncident?.created) {
|
||||
await logActivity(db, {
|
||||
companyId: policy.companyId,
|
||||
actorType: "system",
|
||||
actorId: "budget_service",
|
||||
action: "budget.soft_threshold_crossed",
|
||||
entityType: "budget_incident",
|
||||
entityId: softIncident.id,
|
||||
entityId: softIncident.incident.id,
|
||||
details: {
|
||||
scopeType: policy.scopeType,
|
||||
scopeId: policy.scopeId,
|
||||
|
|
@ -693,20 +694,20 @@ export function budgetService(db: Db, hooks: BudgetServiceHooks = {}) {
|
|||
await resolveOpenSoftIncidents(policy.id);
|
||||
const hardIncident = await createIncidentIfNeeded(policy, "hard", observedAmount);
|
||||
await pauseAndCancelScopeForBudget(policy);
|
||||
if (hardIncident) {
|
||||
if (hardIncident?.created) {
|
||||
await logActivity(db, {
|
||||
companyId: policy.companyId,
|
||||
actorType: "system",
|
||||
actorId: "budget_service",
|
||||
action: "budget.hard_threshold_crossed",
|
||||
entityType: "budget_incident",
|
||||
entityId: hardIncident.id,
|
||||
entityId: hardIncident.incident.id,
|
||||
details: {
|
||||
scopeType: policy.scopeType,
|
||||
scopeId: policy.scopeId,
|
||||
amountObserved: observedAmount,
|
||||
amountLimit: policy.amount,
|
||||
approvalId: hardIncident.approvalId ?? null,
|
||||
approvalId: hardIncident.incident.approvalId ?? null,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue