refactor(server): move the release half of the deferred wake state machine into a wake-queue module (#13132)
## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The heartbeat service releases deferred issue wakes and promotes the next run > - The release logic sat in a large service function, which made its decisions and database effects hard to test > - A promotion race could reopen an issue without creating the promoted run > - Company checks did not protect every read and write, and some readers used a second transaction connection > - This pull request moves the release logic into a layered wake-queue module and closes these race and company-scope defects > - The benefit is clearer decisions, safer writes, and focused tests while existing callers keep the same entry point ## Linked Issues or Issue Description Refs: #10195 ## What Changed - Move the release half of deferred issue execution from `heartbeat.ts` into `server/src/modules/wake-queue/`. - Add pure policy decisions with table-driven tests. - Claim a wake before the reopen write and advance to the next wake when the claim fails. - Add company predicates to guarded reads and writes. - Pass the transaction-scoped issue snapshot to reader ports. - Keep `releaseIssueExecutionAndPromote` as the public wrapper. ## Verification - Run `pnpm check:module-boundaries`. - Run `pnpm exec tsc --noEmit` inside `server/` and compare the result with the known baseline. - Run the wake-queue unit and adapter tests in continuous integration. - Check the promotion-claim ordering, guarded writes, transaction-scoped reads, and cross-company outcomes. - Search the pull request diff for internal issue identifiers. ## Risks - The refactor changes the transaction path for deferred wake release. - A stale or lost wake claim now skips that wake and continues with the next queued wake. - The public wrapper keeps its name and signature, which limits caller risk. - Continuous integration must confirm the full server test suite and build. ## Model Used OpenAI Codex, GPT-5, exact runtime model version not exposed, context window not exposed, with shell, Git, and GitHub tool use. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Nicky Leach
GitHub
parent
018ca5daaf
commit
6dd48cad43
|
|
@ -619,152 +619,6 @@ describeEmbeddedPostgres("heartbeat comment wake batching", () => {
|
|||
}
|
||||
}, 120_000);
|
||||
|
||||
it("cancels an empty deferred comment wake instead of promoting deleted input", async () => {
|
||||
const gateway = await createControlledGatewayServer();
|
||||
const companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
const issueId = randomUUID();
|
||||
const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
try {
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Paperclip",
|
||||
issuePrefix,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
defaultResponsibleUserId: "responsible-user",
|
||||
});
|
||||
await db.insert(agents).values({
|
||||
id: agentId,
|
||||
companyId,
|
||||
name: "Gateway Agent",
|
||||
role: "engineer",
|
||||
status: "idle",
|
||||
adapterType: "openclaw_gateway",
|
||||
adapterConfig: {
|
||||
url: gateway.url,
|
||||
headers: { "x-openclaw-token": "gateway-token" },
|
||||
payloadTemplate: { message: "wake now" },
|
||||
waitTimeoutMs: 2_000,
|
||||
},
|
||||
runtimeConfig: {},
|
||||
permissions: {},
|
||||
});
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId,
|
||||
title: "Discard deferred follow-up",
|
||||
status: "todo",
|
||||
priority: "medium",
|
||||
responsibleUserId: "responsible-user",
|
||||
assigneeAgentId: agentId,
|
||||
issueNumber: 1,
|
||||
identifier: `${issuePrefix}-1`,
|
||||
});
|
||||
const firstComment = await db
|
||||
.insert(issueComments)
|
||||
.values({
|
||||
companyId,
|
||||
issueId,
|
||||
authorUserId: "user-1",
|
||||
body: "First comment",
|
||||
})
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
const firstRun = await heartbeat.wakeup(agentId, {
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_commented",
|
||||
payload: { issueId, commentId: firstComment.id },
|
||||
contextSnapshot: {
|
||||
issueId,
|
||||
taskId: issueId,
|
||||
commentId: firstComment.id,
|
||||
wakeReason: "issue_commented",
|
||||
},
|
||||
requestedByActorType: "user",
|
||||
requestedByActorId: "user-1",
|
||||
});
|
||||
expect(firstRun).not.toBeNull();
|
||||
await waitFor(async () => {
|
||||
const current = await db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, firstRun!.id))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return current?.status === "running";
|
||||
});
|
||||
|
||||
const discardedComment = await db
|
||||
.insert(issueComments)
|
||||
.values({
|
||||
companyId,
|
||||
issueId,
|
||||
authorUserId: "user-1",
|
||||
body: "Delete this before the current turn finishes",
|
||||
})
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
expect(await heartbeat.wakeup(agentId, {
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_commented",
|
||||
payload: { issueId, commentId: discardedComment.id },
|
||||
contextSnapshot: {
|
||||
issueId,
|
||||
taskId: issueId,
|
||||
commentId: discardedComment.id,
|
||||
wakeReason: "issue_commented",
|
||||
},
|
||||
requestedByActorType: "user",
|
||||
requestedByActorId: "user-1",
|
||||
})).toBeNull();
|
||||
await waitFor(async () => db
|
||||
.select({ id: agentWakeupRequests.id })
|
||||
.from(agentWakeupRequests)
|
||||
.where(and(
|
||||
eq(agentWakeupRequests.companyId, companyId),
|
||||
eq(agentWakeupRequests.agentId, agentId),
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
))
|
||||
.then((rows) => Boolean(rows[0])));
|
||||
const deferredWake = await db
|
||||
.select()
|
||||
.from(agentWakeupRequests)
|
||||
.where(and(
|
||||
eq(agentWakeupRequests.companyId, companyId),
|
||||
eq(agentWakeupRequests.agentId, agentId),
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
))
|
||||
.then((rows) => rows[0]);
|
||||
if (!deferredWake) throw new Error("Expected a deferred comment wake");
|
||||
await db.delete(issueComments).where(eq(issueComments.id, discardedComment.id));
|
||||
|
||||
gateway.releaseFirstWait();
|
||||
await waitFor(async () => {
|
||||
const wake = await db
|
||||
.select({ status: agentWakeupRequests.status })
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.id, deferredWake.id))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return wake?.status === "cancelled";
|
||||
}, 90_000);
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
|
||||
expect(gateway.getAgentPayloads()).toHaveLength(1);
|
||||
const runs = await db
|
||||
.select({ id: heartbeatRuns.id })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.agentId, agentId));
|
||||
expect(runs.map((run) => run.id)).toEqual([firstRun!.id]);
|
||||
} finally {
|
||||
gateway.releaseFirstWait();
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
await gateway.close();
|
||||
}
|
||||
}, 120_000);
|
||||
|
||||
it("retains deferred comments for reconciliation after cancelling an unknown provider outcome", async () => {
|
||||
const gateway = await createControlledGatewayServer();
|
||||
const companyId = randomUUID();
|
||||
|
|
@ -1312,363 +1166,6 @@ describeEmbeddedPostgres("heartbeat comment wake batching", () => {
|
|||
}
|
||||
}, 120_000);
|
||||
|
||||
it("cancels a deferred wake containing only a comment authored by the closing run", async () => {
|
||||
const gateway = await createControlledGatewayServer();
|
||||
const companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
const issueId = randomUUID();
|
||||
const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
try {
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Paperclip",
|
||||
issuePrefix,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
defaultResponsibleUserId: "responsible-user",
|
||||
});
|
||||
|
||||
await db.insert(agents).values({
|
||||
id: agentId,
|
||||
companyId,
|
||||
name: "Local CLI Agent",
|
||||
role: "engineer",
|
||||
status: "idle",
|
||||
adapterType: "openclaw_gateway",
|
||||
adapterConfig: {
|
||||
url: gateway.url,
|
||||
headers: {
|
||||
"x-openclaw-token": "gateway-token",
|
||||
},
|
||||
payloadTemplate: {
|
||||
message: "wake now",
|
||||
},
|
||||
waitTimeoutMs: 2_000,
|
||||
},
|
||||
runtimeConfig: {},
|
||||
permissions: {},
|
||||
});
|
||||
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId,
|
||||
title: "Self-comment must not reopen",
|
||||
status: "todo",
|
||||
priority: "medium",
|
||||
responsibleUserId: "responsible-user",
|
||||
assigneeAgentId: agentId,
|
||||
issueNumber: 1,
|
||||
identifier: `${issuePrefix}-1`,
|
||||
});
|
||||
|
||||
const firstRun = await heartbeat.wakeup(agentId, {
|
||||
source: "assignment",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_assigned",
|
||||
payload: { issueId },
|
||||
contextSnapshot: {
|
||||
issueId,
|
||||
taskId: issueId,
|
||||
wakeReason: "issue_assigned",
|
||||
},
|
||||
requestedByActorType: "system",
|
||||
requestedByActorId: null,
|
||||
});
|
||||
|
||||
expect(firstRun).not.toBeNull();
|
||||
await waitFor(async () => {
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, firstRun!.id))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return run?.status === "running";
|
||||
});
|
||||
await waitFor(() => gateway.getAgentPayloads().length === 1);
|
||||
|
||||
// Local-CLI agents post comments under user auth, but stamp the heartbeat
|
||||
// run id on each comment via createdByRunId. Simulate that here: a "user"
|
||||
// comment that was actually authored by the run that is about to close
|
||||
// the issue. Without the Path A guard this would trigger a reopen.
|
||||
const selfComment = await db
|
||||
.insert(issueComments)
|
||||
.values({
|
||||
companyId,
|
||||
issueId,
|
||||
authorUserId: "local-cli-user",
|
||||
createdByRunId: firstRun?.id ?? null,
|
||||
body: "Closing comment from the same run",
|
||||
})
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
|
||||
const deferredRun = await heartbeat.wakeup(agentId, {
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_commented",
|
||||
payload: { issueId, commentId: selfComment.id },
|
||||
contextSnapshot: {
|
||||
issueId,
|
||||
taskId: issueId,
|
||||
commentId: selfComment.id,
|
||||
wakeCommentId: selfComment.id,
|
||||
wakeReason: "issue_commented",
|
||||
},
|
||||
requestedByActorType: "user",
|
||||
requestedByActorId: "local-cli-user",
|
||||
});
|
||||
|
||||
expect(deferredRun).toBeNull();
|
||||
|
||||
await waitFor(async () => {
|
||||
const deferred = await db
|
||||
.select()
|
||||
.from(agentWakeupRequests)
|
||||
.where(
|
||||
and(
|
||||
eq(agentWakeupRequests.companyId, companyId),
|
||||
eq(agentWakeupRequests.agentId, agentId),
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
),
|
||||
)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return Boolean(deferred);
|
||||
});
|
||||
|
||||
// Running records admission. Wait for provider acceptance before
|
||||
// simulating completion by that provider, or startup correctly rejects
|
||||
// the already-closed task before this scenario reaches its follow-up.
|
||||
await waitFor(() => gateway.getAgentPayloads().length >= 1);
|
||||
await db
|
||||
.update(issues)
|
||||
.set({
|
||||
status: "done",
|
||||
completedAt: new Date(),
|
||||
executionRunId: null,
|
||||
executionAgentNameKey: null,
|
||||
executionLockedAt: null,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(issues.id, issueId));
|
||||
|
||||
gateway.releaseFirstWait();
|
||||
|
||||
await waitFor(async () => {
|
||||
const run = await db
|
||||
.select()
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, firstRun!.id))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
const deferred = await db
|
||||
.select()
|
||||
.from(agentWakeupRequests)
|
||||
.where(
|
||||
and(
|
||||
eq(agentWakeupRequests.companyId, companyId),
|
||||
eq(agentWakeupRequests.agentId, agentId),
|
||||
),
|
||||
)
|
||||
.then((rows) => rows.find((request) => request.status === "cancelled") ?? null);
|
||||
return (
|
||||
run?.status === "succeeded" &&
|
||||
deferred?.error ===
|
||||
"Deferred wake contained only comments authored by the finishing run"
|
||||
);
|
||||
}, 90_000);
|
||||
|
||||
expect(gateway.getAgentPayloads()).toHaveLength(1);
|
||||
const runs = await db
|
||||
.select()
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.agentId, agentId));
|
||||
expect(runs).toHaveLength(1);
|
||||
|
||||
const issueAfterPromotion = await db
|
||||
.select({
|
||||
status: issues.status,
|
||||
completedAt: issues.completedAt,
|
||||
})
|
||||
.from(issues)
|
||||
.where(eq(issues.id, issueId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
|
||||
expect(issueAfterPromotion).toMatchObject({
|
||||
status: "done",
|
||||
});
|
||||
expect(issueAfterPromotion?.completedAt).not.toBeNull();
|
||||
} finally {
|
||||
gateway.releaseFirstWait();
|
||||
await gateway.close();
|
||||
}
|
||||
}, 120_000);
|
||||
|
||||
it("promotes an interaction continuation after removing a coalesced self-authored comment", async () => {
|
||||
const gateway = await createControlledGatewayServer();
|
||||
const companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
const issueId = randomUUID();
|
||||
const interactionId = randomUUID();
|
||||
const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
try {
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Paperclip",
|
||||
issuePrefix,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
defaultResponsibleUserId: "responsible-user",
|
||||
});
|
||||
|
||||
await db.insert(agents).values({
|
||||
id: agentId,
|
||||
companyId,
|
||||
name: "Local CLI Agent",
|
||||
role: "engineer",
|
||||
status: "idle",
|
||||
adapterType: "openclaw_gateway",
|
||||
adapterConfig: {
|
||||
url: gateway.url,
|
||||
headers: {
|
||||
"x-openclaw-token": "gateway-token",
|
||||
},
|
||||
payloadTemplate: {
|
||||
message: "wake now",
|
||||
},
|
||||
waitTimeoutMs: 2_000,
|
||||
},
|
||||
runtimeConfig: {},
|
||||
permissions: {},
|
||||
});
|
||||
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId,
|
||||
title: "Interaction continuation survives self-comment filtering",
|
||||
status: "todo",
|
||||
priority: "medium",
|
||||
responsibleUserId: "responsible-user",
|
||||
assigneeAgentId: agentId,
|
||||
issueNumber: 1,
|
||||
identifier: `${issuePrefix}-1`,
|
||||
});
|
||||
|
||||
const firstRun = await heartbeat.wakeup(agentId, {
|
||||
source: "assignment",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_assigned",
|
||||
payload: { issueId },
|
||||
contextSnapshot: {
|
||||
issueId,
|
||||
taskId: issueId,
|
||||
wakeReason: "issue_assigned",
|
||||
},
|
||||
requestedByActorType: "system",
|
||||
requestedByActorId: null,
|
||||
});
|
||||
|
||||
expect(firstRun).not.toBeNull();
|
||||
await waitFor(async () => {
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, firstRun!.id))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return run?.status === "running";
|
||||
});
|
||||
|
||||
const selfComment = await db
|
||||
.insert(issueComments)
|
||||
.values({
|
||||
companyId,
|
||||
issueId,
|
||||
authorUserId: "local-cli-user",
|
||||
createdByRunId: firstRun!.id,
|
||||
body: "Completion note from the source run",
|
||||
})
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
|
||||
expect(await heartbeat.wakeup(agentId, {
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_commented",
|
||||
payload: { issueId, commentId: selfComment.id },
|
||||
contextSnapshot: {
|
||||
issueId,
|
||||
taskId: issueId,
|
||||
commentId: selfComment.id,
|
||||
wakeCommentId: selfComment.id,
|
||||
wakeReason: "issue_commented",
|
||||
},
|
||||
requestedByActorType: "user",
|
||||
requestedByActorId: "local-cli-user",
|
||||
})).toBeNull();
|
||||
|
||||
expect(await heartbeat.wakeup(agentId, {
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_commented",
|
||||
payload: {
|
||||
issueId,
|
||||
interactionId,
|
||||
interactionKind: "request_confirmation",
|
||||
interactionStatus: "accepted",
|
||||
mutation: "interaction",
|
||||
},
|
||||
contextSnapshot: {
|
||||
issueId,
|
||||
taskId: issueId,
|
||||
interactionId,
|
||||
interactionKind: "request_confirmation",
|
||||
interactionStatus: "accepted",
|
||||
wakeReason: "issue_commented",
|
||||
source: "issue.interaction.respond",
|
||||
},
|
||||
requestedByActorType: "user",
|
||||
requestedByActorId: "user-1",
|
||||
})).toBeNull();
|
||||
|
||||
gateway.releaseFirstWait();
|
||||
|
||||
await waitFor(async () => {
|
||||
const runs = await db
|
||||
.select()
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.agentId, agentId))
|
||||
.orderBy(asc(heartbeatRuns.createdAt));
|
||||
return (
|
||||
runs.length === 2 &&
|
||||
runs[0]?.status === "succeeded" &&
|
||||
runs[1]?.status === "succeeded"
|
||||
);
|
||||
}, 90_000);
|
||||
|
||||
const promotedRun = await db
|
||||
.select()
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.agentId, agentId))
|
||||
.orderBy(asc(heartbeatRuns.createdAt))
|
||||
.then((runs) => runs[1] ?? null);
|
||||
expect(promotedRun?.contextSnapshot).toMatchObject({
|
||||
interactionId,
|
||||
interactionKind: "request_confirmation",
|
||||
interactionStatus: "accepted",
|
||||
});
|
||||
expect(promotedRun?.contextSnapshot).not.toMatchObject({
|
||||
wakeCommentIds: expect.anything(),
|
||||
});
|
||||
expect(promotedRun?.contextSnapshot).not.toMatchObject({
|
||||
commentId: selfComment.id,
|
||||
});
|
||||
expect(gateway.getAgentPayloads()).toHaveLength(2);
|
||||
} finally {
|
||||
gateway.releaseFirstWait();
|
||||
await gateway.close();
|
||||
}
|
||||
}, 120_000);
|
||||
|
||||
it("still reopens a finished issue when a deferred batch mixes self-authored and human comments", async () => {
|
||||
const gateway = await createControlledGatewayServer();
|
||||
const companyId = randomUUID();
|
||||
|
|
@ -1886,6 +1383,377 @@ describeEmbeddedPostgres("heartbeat comment wake batching", () => {
|
|||
}
|
||||
}, 120_000);
|
||||
|
||||
it("cancels a deferred comment wake when its only queued comment is deleted before promotion", async () => {
|
||||
const gateway = await createControlledGatewayServer();
|
||||
const companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
const issueId = randomUUID();
|
||||
const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
try {
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Paperclip",
|
||||
issuePrefix,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
defaultResponsibleUserId: "responsible-user",
|
||||
});
|
||||
|
||||
await db.insert(agents).values({
|
||||
id: agentId,
|
||||
companyId,
|
||||
name: "Gateway Agent",
|
||||
role: "engineer",
|
||||
status: "idle",
|
||||
adapterType: "openclaw_gateway",
|
||||
adapterConfig: {
|
||||
url: gateway.url,
|
||||
headers: {
|
||||
"x-openclaw-token": "gateway-token",
|
||||
},
|
||||
payloadTemplate: {
|
||||
message: "wake now",
|
||||
},
|
||||
waitTimeoutMs: 2_000,
|
||||
},
|
||||
runtimeConfig: {},
|
||||
permissions: {},
|
||||
});
|
||||
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId,
|
||||
title: "Deleted follow-up must not reopen",
|
||||
status: "todo",
|
||||
priority: "medium",
|
||||
responsibleUserId: "responsible-user",
|
||||
assigneeAgentId: agentId,
|
||||
issueNumber: 1,
|
||||
identifier: `${issuePrefix}-1`,
|
||||
});
|
||||
|
||||
const firstRun = await heartbeat.wakeup(agentId, {
|
||||
source: "assignment",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_assigned",
|
||||
payload: { issueId },
|
||||
contextSnapshot: {
|
||||
issueId,
|
||||
taskId: issueId,
|
||||
wakeReason: "issue_assigned",
|
||||
},
|
||||
requestedByActorType: "system",
|
||||
requestedByActorId: null,
|
||||
});
|
||||
|
||||
expect(firstRun).not.toBeNull();
|
||||
await waitFor(async () => {
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, firstRun!.id))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return run?.status === "running";
|
||||
});
|
||||
await waitFor(() => gateway.getAgentPayloads().length === 1);
|
||||
|
||||
const queuedComment = await db
|
||||
.insert(issueComments)
|
||||
.values({
|
||||
companyId,
|
||||
issueId,
|
||||
authorUserId: "user-1",
|
||||
body: "Please look at this once you finish",
|
||||
})
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
|
||||
const deferredRun = await heartbeat.wakeup(agentId, {
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_commented",
|
||||
payload: { issueId, commentId: queuedComment.id },
|
||||
contextSnapshot: {
|
||||
issueId,
|
||||
taskId: issueId,
|
||||
commentId: queuedComment.id,
|
||||
wakeCommentId: queuedComment.id,
|
||||
wakeReason: "issue_commented",
|
||||
},
|
||||
requestedByActorType: "user",
|
||||
requestedByActorId: "user-1",
|
||||
});
|
||||
|
||||
expect(deferredRun).toBeNull();
|
||||
|
||||
await waitFor(async () => {
|
||||
const deferred = await db
|
||||
.select()
|
||||
.from(agentWakeupRequests)
|
||||
.where(
|
||||
and(
|
||||
eq(agentWakeupRequests.companyId, companyId),
|
||||
eq(agentWakeupRequests.agentId, agentId),
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
),
|
||||
)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return Boolean(deferred);
|
||||
});
|
||||
|
||||
const deferredWake = await db
|
||||
.select({ id: agentWakeupRequests.id })
|
||||
.from(agentWakeupRequests)
|
||||
.where(
|
||||
and(
|
||||
eq(agentWakeupRequests.companyId, companyId),
|
||||
eq(agentWakeupRequests.agentId, agentId),
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
),
|
||||
)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
const deferredWakeId = deferredWake!.id;
|
||||
|
||||
// The author retracts the comment before the first run finishes, so the
|
||||
// real comment-liveness query must find zero live comments left in the
|
||||
// queued batch.
|
||||
await db.update(issueComments).set({ deletedAt: new Date() }).where(eq(issueComments.id, queuedComment.id));
|
||||
|
||||
// Running records admission. Wait for provider acceptance before
|
||||
// simulating completion by that provider, or startup correctly rejects
|
||||
// the already-closed task before this scenario reaches its follow-up.
|
||||
await waitFor(() => gateway.getAgentPayloads().length >= 1);
|
||||
await db
|
||||
.update(issues)
|
||||
.set({
|
||||
status: "done",
|
||||
completedAt: new Date(),
|
||||
executionRunId: null,
|
||||
executionAgentNameKey: null,
|
||||
executionLockedAt: null,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(issues.id, issueId));
|
||||
|
||||
gateway.releaseFirstWait();
|
||||
|
||||
await waitFor(async () => {
|
||||
const [wake] = await db
|
||||
.select({ status: agentWakeupRequests.status })
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.id, deferredWakeId));
|
||||
return wake?.status === "cancelled";
|
||||
}, 90_000);
|
||||
|
||||
const [cancelledWake] = await db
|
||||
.select({ status: agentWakeupRequests.status, error: agentWakeupRequests.error })
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.id, deferredWakeId));
|
||||
expect(cancelledWake).toMatchObject({
|
||||
status: "cancelled",
|
||||
error: "Queued messages were discarded before promotion",
|
||||
});
|
||||
|
||||
// No live comment remained, so the queue must not promote a second run
|
||||
// and the issue must stay in the state the first run's completion left it in.
|
||||
expect(gateway.getAgentPayloads()).toHaveLength(1);
|
||||
const issueAfterCompletion = await db
|
||||
.select({ status: issues.status, completedAt: issues.completedAt })
|
||||
.from(issues)
|
||||
.where(eq(issues.id, issueId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
expect(issueAfterCompletion).toMatchObject({ status: "done" });
|
||||
expect(issueAfterCompletion?.completedAt).not.toBeNull();
|
||||
} finally {
|
||||
gateway.releaseFirstWait();
|
||||
await gateway.close();
|
||||
}
|
||||
}, 120_000);
|
||||
|
||||
it("cancels a deferred comment wake when its only queued comment was authored by the finishing run", async () => {
|
||||
const gateway = await createControlledGatewayServer();
|
||||
const companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
const issueId = randomUUID();
|
||||
const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
try {
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Paperclip",
|
||||
issuePrefix,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
defaultResponsibleUserId: "responsible-user",
|
||||
});
|
||||
|
||||
await db.insert(agents).values({
|
||||
id: agentId,
|
||||
companyId,
|
||||
name: "Local CLI Agent",
|
||||
role: "engineer",
|
||||
status: "idle",
|
||||
adapterType: "openclaw_gateway",
|
||||
adapterConfig: {
|
||||
url: gateway.url,
|
||||
headers: {
|
||||
"x-openclaw-token": "gateway-token",
|
||||
},
|
||||
payloadTemplate: {
|
||||
message: "wake now",
|
||||
},
|
||||
waitTimeoutMs: 2_000,
|
||||
},
|
||||
runtimeConfig: {},
|
||||
permissions: {},
|
||||
});
|
||||
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId,
|
||||
title: "Self-authored note must not reopen",
|
||||
status: "todo",
|
||||
priority: "medium",
|
||||
responsibleUserId: "responsible-user",
|
||||
assigneeAgentId: agentId,
|
||||
issueNumber: 1,
|
||||
identifier: `${issuePrefix}-1`,
|
||||
});
|
||||
|
||||
const firstRun = await heartbeat.wakeup(agentId, {
|
||||
source: "assignment",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_assigned",
|
||||
payload: { issueId },
|
||||
contextSnapshot: {
|
||||
issueId,
|
||||
taskId: issueId,
|
||||
wakeReason: "issue_assigned",
|
||||
},
|
||||
requestedByActorType: "system",
|
||||
requestedByActorId: null,
|
||||
});
|
||||
|
||||
expect(firstRun).not.toBeNull();
|
||||
await waitFor(async () => {
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, firstRun!.id))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return run?.status === "running";
|
||||
});
|
||||
|
||||
const selfComment = await db
|
||||
.insert(issueComments)
|
||||
.values({
|
||||
companyId,
|
||||
issueId,
|
||||
authorUserId: "local-cli-user",
|
||||
createdByRunId: firstRun?.id ?? null,
|
||||
body: "Closing note from the same run",
|
||||
})
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
|
||||
const deferredRun = await heartbeat.wakeup(agentId, {
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_commented",
|
||||
payload: { issueId, commentId: selfComment.id },
|
||||
contextSnapshot: {
|
||||
issueId,
|
||||
taskId: issueId,
|
||||
commentId: selfComment.id,
|
||||
wakeCommentId: selfComment.id,
|
||||
wakeReason: "issue_commented",
|
||||
},
|
||||
requestedByActorType: "user",
|
||||
requestedByActorId: "local-cli-user",
|
||||
});
|
||||
|
||||
expect(deferredRun).toBeNull();
|
||||
|
||||
await waitFor(async () => {
|
||||
const deferred = await db
|
||||
.select()
|
||||
.from(agentWakeupRequests)
|
||||
.where(
|
||||
and(
|
||||
eq(agentWakeupRequests.companyId, companyId),
|
||||
eq(agentWakeupRequests.agentId, agentId),
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
),
|
||||
)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return Boolean(deferred);
|
||||
});
|
||||
|
||||
const queuedWake = await db
|
||||
.select({ id: agentWakeupRequests.id })
|
||||
.from(agentWakeupRequests)
|
||||
.where(
|
||||
and(
|
||||
eq(agentWakeupRequests.companyId, companyId),
|
||||
eq(agentWakeupRequests.agentId, agentId),
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
),
|
||||
)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
|
||||
// Running records admission. Wait for provider acceptance before
|
||||
// simulating completion by that provider, or startup correctly rejects
|
||||
// the already-closed task before this scenario reaches its follow-up.
|
||||
await waitFor(() => gateway.getAgentPayloads().length >= 1);
|
||||
await db
|
||||
.update(issues)
|
||||
.set({
|
||||
status: "done",
|
||||
completedAt: new Date(),
|
||||
executionRunId: null,
|
||||
executionAgentNameKey: null,
|
||||
executionLockedAt: null,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(issues.id, issueId));
|
||||
|
||||
gateway.releaseFirstWait();
|
||||
|
||||
await waitFor(async () => {
|
||||
const [wake] = await db
|
||||
.select({ status: agentWakeupRequests.status })
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.id, queuedWake!.id));
|
||||
return wake?.status === "cancelled";
|
||||
}, 90_000);
|
||||
|
||||
const [cancelledWake] = await db
|
||||
.select({ status: agentWakeupRequests.status, error: agentWakeupRequests.error })
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.id, queuedWake!.id));
|
||||
expect(cancelledWake).toMatchObject({
|
||||
status: "cancelled",
|
||||
error: "Deferred wake contained only comments authored by the finishing run",
|
||||
});
|
||||
|
||||
// The only queued comment came from the run that just finished, so the
|
||||
// queue must not promote a second run for the same agent to re-read its
|
||||
// own note, and the issue must stay in the state the run's completion left it in.
|
||||
expect(gateway.getAgentPayloads()).toHaveLength(1);
|
||||
const issueAfterCompletion = await db
|
||||
.select({ status: issues.status, completedAt: issues.completedAt })
|
||||
.from(issues)
|
||||
.where(eq(issues.id, issueId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
expect(issueAfterCompletion).toMatchObject({ status: "done" });
|
||||
expect(issueAfterCompletion?.completedAt).not.toBeNull();
|
||||
} finally {
|
||||
gateway.releaseFirstWait();
|
||||
await gateway.close();
|
||||
}
|
||||
}, 120_000);
|
||||
|
||||
it("queues exactly one follow-up run when an issue-bound run exits without a comment", async () => {
|
||||
const gateway = await createControlledGatewayServer();
|
||||
const companyId = randomUUID();
|
||||
|
|
|
|||
|
|
@ -0,0 +1,439 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
||||
import {
|
||||
agentWakeupRequests,
|
||||
agents,
|
||||
companies,
|
||||
createDb,
|
||||
heartbeatRuns,
|
||||
issueComments,
|
||||
issues,
|
||||
} from "@paperclipai/db";
|
||||
import {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
} from "../../../__tests__/helpers/embedded-postgres.js";
|
||||
import { createPostgresWakeQueueAdapter } from "./postgres.js";
|
||||
import type { WakeQueuePostgresAdapterDeps } from "./postgres.js";
|
||||
|
||||
// Proves the atomicity and company-scope properties the security review
|
||||
// requires: every mutation names `companyId` in its own SQL `WHERE` clause,
|
||||
// a foreign-company row is invisible to a read, and a deferred-status
|
||||
// compare-and-set that affects no row leaves no other trace. The decision
|
||||
// branching itself is proven against plain facts in `domain/policy.test.ts`.
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
if (!embeddedPostgresSupport.supported) {
|
||||
console.warn(
|
||||
`Skipping embedded Postgres wake-queue adapter tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
|
||||
);
|
||||
}
|
||||
|
||||
describeEmbeddedPostgres("wake-queue postgres adapter", () => {
|
||||
let db!: ReturnType<typeof createDb>;
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
|
||||
const stubDeps: WakeQueuePostgresAdapterDeps = {
|
||||
resolveResponsibleUserId: async () => "responsible-user",
|
||||
getRoutineEnv: async () => ({ routineId: null, env: null, responsibleUserId: null }),
|
||||
resolveSessionBeforeForWakeup: async () => null,
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-wake-queue-postgres-adapter-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
}, 20_000);
|
||||
|
||||
afterEach(async () => {
|
||||
await db.delete(issueComments);
|
||||
// `heartbeat_runs.wakeup_request_id` references `agent_wakeup_requests.id`,
|
||||
// so the run row must go first.
|
||||
await db.delete(heartbeatRuns);
|
||||
await db.delete(agentWakeupRequests);
|
||||
await db.delete(issues);
|
||||
await db.delete(agents);
|
||||
await db.delete(companies);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
async function seedCompany(): Promise<string> {
|
||||
const companyId = randomUUID();
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Paperclip",
|
||||
issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
defaultResponsibleUserId: "responsible-user",
|
||||
});
|
||||
return companyId;
|
||||
}
|
||||
|
||||
async function seedAgent(input: { companyId: string; name?: string }): Promise<string> {
|
||||
const agentId = randomUUID();
|
||||
await db.insert(agents).values({
|
||||
id: agentId,
|
||||
companyId: input.companyId,
|
||||
name: input.name ?? "CodexCoder",
|
||||
role: "engineer",
|
||||
status: "active",
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } },
|
||||
permissions: {},
|
||||
});
|
||||
return agentId;
|
||||
}
|
||||
|
||||
async function seedIssue(input: {
|
||||
companyId: string;
|
||||
issueId?: string;
|
||||
status?: string;
|
||||
assigneeAgentId?: string | null;
|
||||
executionRunId?: string | null;
|
||||
checkoutRunId?: string | null;
|
||||
}): Promise<string> {
|
||||
const issueId = input.issueId ?? randomUUID();
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId: input.companyId,
|
||||
title: "Wake-queue adapter fixture issue",
|
||||
status: input.status ?? "in_progress",
|
||||
priority: "medium",
|
||||
assigneeAgentId: input.assigneeAgentId ?? null,
|
||||
executionRunId: input.executionRunId ?? null,
|
||||
checkoutRunId: input.checkoutRunId ?? null,
|
||||
});
|
||||
return issueId;
|
||||
}
|
||||
|
||||
async function seedRun(input: {
|
||||
companyId: string;
|
||||
agentId: string;
|
||||
status?: string;
|
||||
contextSnapshot?: Record<string, unknown>;
|
||||
errorCode?: string | null;
|
||||
runtimeMode?: string;
|
||||
}): Promise<string> {
|
||||
const runId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: runId,
|
||||
companyId: input.companyId,
|
||||
agentId: input.agentId,
|
||||
invocationSource: "on_demand",
|
||||
status: input.status ?? "failed",
|
||||
contextSnapshot: input.contextSnapshot ?? {},
|
||||
errorCode: input.errorCode ?? null,
|
||||
});
|
||||
return runId;
|
||||
}
|
||||
|
||||
async function seedDeferredWake(input: {
|
||||
companyId: string;
|
||||
agentId: string;
|
||||
issueId: string;
|
||||
requestedByActorType?: string;
|
||||
requestedByActorId?: string | null;
|
||||
payload?: Record<string, unknown>;
|
||||
}): Promise<string> {
|
||||
const id = randomUUID();
|
||||
await db.insert(agentWakeupRequests).values({
|
||||
id,
|
||||
companyId: input.companyId,
|
||||
agentId: input.agentId,
|
||||
source: "automation",
|
||||
reason: "issue_commented",
|
||||
status: "deferred_issue_execution",
|
||||
requestedByActorType: input.requestedByActorType ?? "user",
|
||||
requestedByActorId: input.requestedByActorId ?? null,
|
||||
payload: { issueId: input.issueId, ...(input.payload ?? {}) },
|
||||
});
|
||||
return id;
|
||||
}
|
||||
|
||||
// Review test (a): a foreign-company agent id produces the current failed
|
||||
// wake status and the current error text, and creates no run.
|
||||
it("fails a deferred wake whose agent belongs to a different company, without creating a run", async () => {
|
||||
const companyId = await seedCompany();
|
||||
const otherCompanyId = await seedCompany();
|
||||
const foreignAgentId = await seedAgent({ companyId: otherCompanyId });
|
||||
const finishingAgentId = await seedAgent({ companyId });
|
||||
const issueId = await seedIssue({ companyId, assigneeAgentId: finishingAgentId, status: "in_progress" });
|
||||
// A finishing run status other than the legacy-reconciliation set (failed,
|
||||
// timed_out, interrupted, cancelled) reaches the module's own drain logic.
|
||||
const runId = await seedRun({ companyId, agentId: finishingAgentId, contextSnapshot: { issueId }, status: "succeeded" });
|
||||
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId));
|
||||
const wakeId = await seedDeferredWake({ companyId, agentId: foreignAgentId, issueId });
|
||||
|
||||
const adapter = createPostgresWakeQueueAdapter(db, stubDeps);
|
||||
const result = await adapter.withIssueExecutionLock({ companyId, runId, now: new Date() }, async (locked, ports) => {
|
||||
const candidate = await ports.writer.claimNextDeferredWake({ companyId, issueId: locked.primaryIssue.id });
|
||||
expect(candidate?.id).toBe(wakeId);
|
||||
const agent = await ports.reader.findInvokableAgent({ companyId, agentId: foreignAgentId });
|
||||
expect(agent).toBeNull();
|
||||
const failed = await ports.writer.failDeferredWake({ companyId, wakeId: candidate!.id, now: new Date() });
|
||||
expect(failed).toBe(true);
|
||||
return { outcome: { kind: "released" as const }, postCommitEffects: [] };
|
||||
});
|
||||
expect(result.outcome.kind).toBe("released");
|
||||
|
||||
const wakeRow = (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeId)))[0];
|
||||
expect(wakeRow?.status).toBe("failed");
|
||||
expect(wakeRow?.error).toBe("Deferred wake could not be promoted: agent is not invokable");
|
||||
expect(wakeRow?.runId).toBeNull();
|
||||
const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId));
|
||||
expect(runs).toHaveLength(1);
|
||||
expect(runs[0]!.id).toBe(runId);
|
||||
});
|
||||
|
||||
// Review test (b): each release adapter mutation with a foreign company
|
||||
// affects no row.
|
||||
it("scopes every release mutation to its own company and affects no row across a company boundary", async () => {
|
||||
const companyId = await seedCompany();
|
||||
const otherCompanyId = await seedCompany();
|
||||
const agentId = await seedAgent({ companyId });
|
||||
const issueId = await seedIssue({ companyId, assigneeAgentId: agentId, status: "blocked" });
|
||||
await db.update(issues).set({ executionState: { phase: "running" } }).where(eq(issues.id, issueId));
|
||||
const wakeId = await seedDeferredWake({ companyId, agentId, issueId });
|
||||
const runId = await seedRun({ companyId, agentId, contextSnapshot: { issueId }, status: "succeeded" });
|
||||
|
||||
const adapter = createPostgresWakeQueueAdapter(db, stubDeps);
|
||||
await adapter.withIssueExecutionLock(
|
||||
{ companyId, runId, now: new Date() },
|
||||
async (_locked, ports) => {
|
||||
const cancelledUnderWrongCompany = await ports.writer.cancelDeferredWake({
|
||||
companyId: otherCompanyId,
|
||||
wakeId,
|
||||
reason: "cross-company cancel attempt",
|
||||
now: new Date(),
|
||||
});
|
||||
expect(cancelledUnderWrongCompany).toBe(false);
|
||||
|
||||
const failedUnderWrongCompany = await ports.writer.failDeferredWake({
|
||||
companyId: otherCompanyId,
|
||||
wakeId,
|
||||
now: new Date(),
|
||||
});
|
||||
expect(failedUnderWrongCompany).toBe(false);
|
||||
|
||||
const normalizedUnderWrongCompany = await ports.writer.normalizeDeferredWakeCommentIds({
|
||||
companyId: otherCompanyId,
|
||||
wakeId,
|
||||
payload: { issueId },
|
||||
liveCommentIds: ["c1"],
|
||||
now: new Date(),
|
||||
});
|
||||
expect(normalizedUnderWrongCompany).toBeNull();
|
||||
|
||||
const reopenedUnderWrongCompany = await ports.writer.reopenIssue({
|
||||
companyId: otherCompanyId,
|
||||
issueId,
|
||||
runId,
|
||||
});
|
||||
expect(reopenedUnderWrongCompany).toBeNull();
|
||||
|
||||
return { outcome: { kind: "released" as const }, postCommitEffects: [] };
|
||||
},
|
||||
);
|
||||
|
||||
const wakeRow = (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeId)))[0];
|
||||
expect(wakeRow?.status).toBe("deferred_issue_execution");
|
||||
expect(wakeRow?.error).toBeNull();
|
||||
const issueRow = (await db.select().from(issues).where(eq(issues.id, issueId)))[0];
|
||||
expect(issueRow?.status).toBe("blocked");
|
||||
expect(issueRow?.executionState).toEqual({ phase: "running" });
|
||||
});
|
||||
|
||||
// Review defect: the reopen path must carry the company into every read,
|
||||
// lock, and write. A check before the write is not a boundary, because
|
||||
// `issues.company_id` can change between that check and the write.
|
||||
it("refuses to reopen an issue for a company that does not own it, and leaves the issue untouched", async () => {
|
||||
const companyId = await seedCompany();
|
||||
const otherCompanyId = await seedCompany();
|
||||
const agentId = await seedAgent({ companyId });
|
||||
const issueId = await seedIssue({ companyId, assigneeAgentId: agentId, status: "blocked" });
|
||||
await db.update(issues).set({ executionState: { phase: "running" } }).where(eq(issues.id, issueId));
|
||||
const runId = await seedRun({ companyId, agentId, contextSnapshot: { issueId }, status: "succeeded" });
|
||||
|
||||
const adapter = createPostgresWakeQueueAdapter(db, stubDeps);
|
||||
const captured: { reopened: { status: string; executionState: Record<string, unknown> | null } | null } = {
|
||||
reopened: null,
|
||||
};
|
||||
await adapter.withIssueExecutionLock({ companyId, runId, now: new Date() }, async (_locked, ports) => {
|
||||
captured.reopened = await ports.writer.reopenIssue({ companyId: otherCompanyId, issueId, runId });
|
||||
return { outcome: { kind: "released" as const }, postCommitEffects: [] };
|
||||
});
|
||||
expect(captured.reopened).toBeNull();
|
||||
|
||||
const issueRow = (await db.select().from(issues).where(eq(issues.id, issueId)))[0];
|
||||
expect(issueRow?.status).toBe("blocked");
|
||||
expect(issueRow?.executionState).toEqual({ phase: "running" });
|
||||
});
|
||||
|
||||
it("reopens an issue for the company that owns it, and clears the execution state", async () => {
|
||||
const companyId = await seedCompany();
|
||||
const agentId = await seedAgent({ companyId });
|
||||
const issueId = await seedIssue({ companyId, assigneeAgentId: agentId, status: "blocked" });
|
||||
await db.update(issues).set({ executionState: { phase: "running" } }).where(eq(issues.id, issueId));
|
||||
const runId = await seedRun({ companyId, agentId, contextSnapshot: { issueId }, status: "succeeded" });
|
||||
|
||||
const adapter = createPostgresWakeQueueAdapter(db, stubDeps);
|
||||
const captured: { reopened: { status: string; executionState: Record<string, unknown> | null } | null } = {
|
||||
reopened: null,
|
||||
};
|
||||
await adapter.withIssueExecutionLock({ companyId, runId, now: new Date() }, async (_locked, ports) => {
|
||||
captured.reopened = await ports.writer.reopenIssue({ companyId, issueId, runId });
|
||||
return { outcome: { kind: "released" as const }, postCommitEffects: [] };
|
||||
});
|
||||
expect(captured.reopened?.status).toBe("todo");
|
||||
expect(captured.reopened?.executionState).toBeNull();
|
||||
|
||||
const issueRow = (await db.select().from(issues).where(eq(issues.id, issueId)))[0];
|
||||
expect(issueRow?.status).toBe("todo");
|
||||
expect(issueRow?.executionState).toBeNull();
|
||||
});
|
||||
|
||||
// Review test (c): a deferred-status compare-and-set that affects no row
|
||||
// claims nothing, and no other write in the promotion path ever runs.
|
||||
it("fails the promotion claim when the deferred-status compare-and-set loses the race, before any other write", async () => {
|
||||
const companyId = await seedCompany();
|
||||
const agentId = await seedAgent({ companyId });
|
||||
const issueId = await seedIssue({ companyId, assigneeAgentId: agentId });
|
||||
const wakeId = await seedDeferredWake({ companyId, agentId, issueId });
|
||||
// A concurrent finalization already claimed this wake before the promotion claim runs.
|
||||
await db.update(agentWakeupRequests).set({ status: "cancelled" }).where(eq(agentWakeupRequests.id, wakeId));
|
||||
|
||||
const adapter = createPostgresWakeQueueAdapter(db, stubDeps);
|
||||
const runId = await seedRun({ companyId, agentId, contextSnapshot: { issueId }, status: "succeeded" });
|
||||
const result = await adapter.withIssueExecutionLock({ companyId, runId, now: new Date() }, async (_locked, ports) => {
|
||||
const claimed = await ports.writer.claimDeferredWakeForPromotion({ companyId, wakeId, now: new Date() });
|
||||
expect(claimed).toBe(false);
|
||||
return { outcome: { kind: "released" as const }, postCommitEffects: [] };
|
||||
});
|
||||
expect(result.outcome.kind).toBe("released");
|
||||
|
||||
const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId));
|
||||
expect(runs).toHaveLength(1);
|
||||
expect(runs[0]!.id).toBe(runId);
|
||||
const issueRow = (await db.select().from(issues).where(eq(issues.id, issueId)))[0];
|
||||
expect(issueRow?.executionRunId).toBeNull();
|
||||
const wakeRow = (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeId)))[0];
|
||||
expect(wakeRow?.status).toBe("cancelled");
|
||||
});
|
||||
|
||||
// `finalizePromotedWake`'s own writes guard against clobbering state a
|
||||
// concurrent write already changed: it never takes the issue's execution
|
||||
// lock away from a run that already holds it, and it never overwrites a
|
||||
// `runId` a wake row already carries. Both guards only matter as defense
|
||||
// in depth today (the release drain calls this at most once per
|
||||
// transaction), so this drives the port directly to prove the SQL itself,
|
||||
// independent of that call pattern.
|
||||
it("guards finalizePromotedWake's own writes against clobbering an already-set execution lock or runId", async () => {
|
||||
const companyId = await seedCompany();
|
||||
const agentId = await seedAgent({ companyId });
|
||||
const issueId = await seedIssue({ companyId, assigneeAgentId: agentId });
|
||||
const wakeIdA = await seedDeferredWake({ companyId, agentId, issueId });
|
||||
const wakeIdB = await seedDeferredWake({ companyId, agentId, issueId });
|
||||
const runId = await seedRun({ companyId, agentId, contextSnapshot: { issueId }, status: "succeeded" });
|
||||
const deferredAgent = { id: agentId, companyId, name: "CodexCoder", invokable: true };
|
||||
|
||||
const finalizedRunIds: string[] = [];
|
||||
const adapter = createPostgresWakeQueueAdapter(db, stubDeps);
|
||||
await adapter.withIssueExecutionLock({ companyId, runId, now: new Date() }, async (locked, ports) => {
|
||||
const finalize = async (wakeId: string) => {
|
||||
const promoted = await ports.writer.finalizePromotedWake({
|
||||
companyId,
|
||||
wakeId,
|
||||
deferredAgent,
|
||||
issue: locked.primaryIssue,
|
||||
finishingRun: locked.run,
|
||||
contextSnapshot: { issueId },
|
||||
reason: "issue_execution_promoted",
|
||||
source: "automation",
|
||||
triggerDetail: null,
|
||||
payload: {},
|
||||
responsibleUserId: "responsible-user",
|
||||
sessionBefore: null,
|
||||
now: new Date(),
|
||||
});
|
||||
finalizedRunIds.push(promoted.id);
|
||||
};
|
||||
// The issue's execution lock is free; this call takes it.
|
||||
await finalize(wakeIdA);
|
||||
// The issue's execution lock is already held by the first call's run,
|
||||
// so this call's issue-lock write must no-op even though a run is
|
||||
// still created.
|
||||
await finalize(wakeIdB);
|
||||
// Repeating the same wake must not overwrite its now-set `runId`.
|
||||
await finalize(wakeIdA);
|
||||
return { outcome: { kind: "released" as const }, postCommitEffects: [] };
|
||||
});
|
||||
const [runA, runB, runC] = finalizedRunIds;
|
||||
|
||||
const wakeRowA = (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeIdA)))[0];
|
||||
const wakeRowB = (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeIdB)))[0];
|
||||
expect(wakeRowA?.runId).toBe(runA);
|
||||
expect(wakeRowB?.runId).toBe(runB);
|
||||
const issueRow = (await db.select().from(issues).where(eq(issues.id, issueId)))[0];
|
||||
expect(issueRow?.executionRunId).toBe(runA);
|
||||
expect(issueRow?.executionRunId).not.toBe(runB);
|
||||
expect(issueRow?.executionRunId).not.toBe(runC);
|
||||
const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId));
|
||||
// All three finalize calls each still insert their own run row.
|
||||
expect(runs.map((run) => run.id).sort()).toEqual([runId, runA, runB, runC].sort());
|
||||
});
|
||||
|
||||
it("locks the context issue and every sibling issue in id order, and two concurrent releases do not deadlock", async () => {
|
||||
const companyId = await seedCompany();
|
||||
const agentId = await seedAgent({ companyId });
|
||||
const issueA = await seedIssue({ companyId, assigneeAgentId: agentId });
|
||||
const issueB = await seedIssue({ companyId, assigneeAgentId: agentId });
|
||||
const runA = await seedRun({ companyId, agentId, contextSnapshot: { issueId: issueA } });
|
||||
const runB = await seedRun({ companyId, agentId, contextSnapshot: { issueId: issueB } });
|
||||
await db.update(issues).set({ executionRunId: runA, checkoutRunId: runB }).where(eq(issues.id, issueA));
|
||||
await db.update(issues).set({ executionRunId: runB, checkoutRunId: runA }).where(eq(issues.id, issueB));
|
||||
|
||||
const adapterA = createPostgresWakeQueueAdapter(db, stubDeps);
|
||||
const adapterB = createPostgresWakeQueueAdapter(db, stubDeps);
|
||||
const releaseA = adapterA.withIssueExecutionLock({ companyId, runId: runA, now: new Date() }, async (_locked, _ports) => {
|
||||
await new Promise((resolve) => setTimeout(resolve, 25));
|
||||
return { outcome: { kind: "released" as const }, postCommitEffects: [] };
|
||||
});
|
||||
const releaseB = adapterB.withIssueExecutionLock({ companyId, runId: runB, now: new Date() }, async (_locked, _ports) => {
|
||||
await new Promise((resolve) => setTimeout(resolve, 25));
|
||||
return { outcome: { kind: "released" as const }, postCommitEffects: [] };
|
||||
});
|
||||
|
||||
await expect(Promise.all([releaseA, releaseB])).resolves.toBeDefined();
|
||||
|
||||
const rows = await db.select().from(issues).where(eq(issues.companyId, companyId));
|
||||
for (const row of rows) {
|
||||
expect(row.executionRunId).toBeNull();
|
||||
expect(row.checkoutRunId).toBeNull();
|
||||
}
|
||||
});
|
||||
|
||||
it("clears both lock columns on every sibling and keeps a transferred executionRunId", async () => {
|
||||
const companyId = await seedCompany();
|
||||
const agentId = await seedAgent({ companyId });
|
||||
const finishingRunId = await seedRun({ companyId, agentId, contextSnapshot: {} });
|
||||
const retryRunId = await seedRun({ companyId, agentId, contextSnapshot: {}, status: "queued" });
|
||||
const issueId = await seedIssue({ companyId, assigneeAgentId: agentId, executionRunId: retryRunId, checkoutRunId: finishingRunId });
|
||||
|
||||
const adapter = createPostgresWakeQueueAdapter(db, stubDeps);
|
||||
const result = await adapter.withIssueExecutionLock({ companyId, runId: finishingRunId, now: new Date() }, async () => ({
|
||||
outcome: { kind: "released" as const },
|
||||
postCommitEffects: [],
|
||||
}));
|
||||
expect(result.outcome.kind).toBe("released");
|
||||
|
||||
const issueRow = (await db.select().from(issues).where(eq(issues.id, issueId)))[0];
|
||||
// executionRunId already pointed at the retry, not the finishing run, so it must survive.
|
||||
expect(issueRow?.executionRunId).toBe(retryRunId);
|
||||
expect(issueRow?.checkoutRunId).toBeNull();
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,817 @@
|
|||
import { and, asc, eq, inArray, isNull, notInArray, or, sql } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import {
|
||||
agentWakeupRequests,
|
||||
agents,
|
||||
heartbeatRuns,
|
||||
issueComments,
|
||||
issueRecoveryActions,
|
||||
issueRelations,
|
||||
issues,
|
||||
nativeRunFinalizations,
|
||||
} from "@paperclipai/db";
|
||||
import { legacyExecutionNeedsReconciliation } from "../../../services/legacy-execution-recovery.js";
|
||||
import { evaluateAgentInvokability } from "../../../services/agent-invokability.js";
|
||||
import { issueTreeControlService, isVerifiedIssueTreeControlInteractionWake } from "../../../services/issue-tree-control.js";
|
||||
import { isAutomaticRecoverySuppressedByPauseHold } from "../../../services/recovery/pause-hold-guard.js";
|
||||
import { issueService } from "../../../services/issues.js";
|
||||
import { issueRecoveryActionService } from "../../../services/issue-recovery-actions.js";
|
||||
import { readContinuationAttempt } from "../../../services/recovery/run-liveness-continuations.js";
|
||||
import { withRecoveryContext } from "../../../services/recovery/status-only-context.js";
|
||||
import { parseIssueExecutionState } from "../../../services/issue-execution-policy.js";
|
||||
import {
|
||||
buildConfigurationIncompleteRecoveryNoticeSeed,
|
||||
buildExecutionReviewParticipantRecoveryNoticeSeed,
|
||||
buildImmediateExecutionPathRecoveryNoticeSeed,
|
||||
buildWorkspaceValidationRecoveryNoticeSeed,
|
||||
} from "../../../services/recovery/stranded-notice.js";
|
||||
import {
|
||||
queuedCommentIdsFromWakePayload,
|
||||
withQueuedCommentIdsInWakePayload,
|
||||
} from "../../../services/issue-queued-comment-queue.js";
|
||||
import { extractWakeCommentIds } from "../../run-dispatch/index.js";
|
||||
import { hasInteractionContinuationWakeContext } from "../domain/context.js";
|
||||
import type {
|
||||
DeferredWakeCandidate,
|
||||
InvokableAgentSnapshot,
|
||||
IssueLockWriter,
|
||||
IssueSnapshot,
|
||||
LockedIssueExecution,
|
||||
ReleaseTransactionResult,
|
||||
RunSnapshot,
|
||||
WakeQueueReader,
|
||||
WakeQueueWriter,
|
||||
} from "../application/ports.js";
|
||||
import type { RunSummary } from "../application/types.js";
|
||||
import { WakeQueueApplicationError } from "../application/types.js";
|
||||
|
||||
const DEFERRED_WAKE_STATUS = "deferred_issue_execution";
|
||||
const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext";
|
||||
const WORKSPACE_VALIDATION_FAILURE_CODE = "workspace_validation_failed";
|
||||
const CONFIGURATION_INCOMPLETE_FAILURE_CODE = "configuration_incomplete";
|
||||
const WORKSPACE_VALIDATION_RECOVERY_CAUSE = "workspace_validation_failed";
|
||||
const CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE = "configuration_incomplete";
|
||||
const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE = "execution_review_participant_recovery";
|
||||
const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON = "execution_review_participant_recovery";
|
||||
const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON = "execution_review_participant_recovery";
|
||||
const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const;
|
||||
|
||||
type HeartbeatRunRow = typeof heartbeatRuns.$inferSelect;
|
||||
type IssueRow = typeof issues.$inferSelect;
|
||||
|
||||
function parseObject(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value)
|
||||
? (value as Record<string, unknown>)
|
||||
: {};
|
||||
}
|
||||
|
||||
function readNonEmptyString(value: unknown): string | null {
|
||||
return typeof value === "string" && value.trim().length > 0 ? value : null;
|
||||
}
|
||||
|
||||
function normalizeAgentNameKey(value: string | null | undefined): string | null {
|
||||
if (typeof value !== "string") return null;
|
||||
const normalized = value.trim().toLowerCase();
|
||||
return normalized.length > 0 ? normalized : null;
|
||||
}
|
||||
|
||||
function isWorkspaceValidationFailedRun(run: Pick<HeartbeatRunRow, "errorCode">): boolean {
|
||||
return run.errorCode === WORKSPACE_VALIDATION_FAILURE_CODE;
|
||||
}
|
||||
|
||||
function isConfigurationIncompleteFailedRun(run: Pick<HeartbeatRunRow, "errorCode">): boolean {
|
||||
return run.errorCode === CONFIGURATION_INCOMPLETE_FAILURE_CODE || run.errorCode === "model_not_found";
|
||||
}
|
||||
|
||||
function toRunSnapshot(row: HeartbeatRunRow): RunSnapshot {
|
||||
const configurationIncompletePayload = parseObject(parseObject(row.resultJson).configurationIncomplete);
|
||||
return {
|
||||
id: row.id,
|
||||
companyId: row.companyId,
|
||||
agentId: row.agentId,
|
||||
status: row.status,
|
||||
runtimeMode: row.runtimeMode,
|
||||
errorCode: row.errorCode,
|
||||
responsibleUserId: row.responsibleUserId,
|
||||
contextSnapshot: parseObject(row.contextSnapshot),
|
||||
configurationIncompletePayload: Object.keys(configurationIncompletePayload).length > 0 ? configurationIncompletePayload : null,
|
||||
};
|
||||
}
|
||||
|
||||
function toIssueSnapshot(row: IssueRow): IssueSnapshot {
|
||||
return {
|
||||
id: row.id,
|
||||
companyId: row.companyId,
|
||||
identifier: row.identifier ?? "",
|
||||
status: row.status,
|
||||
assigneeAgentId: row.assigneeAgentId,
|
||||
assigneeUserId: row.assigneeUserId,
|
||||
hiddenAt: row.hiddenAt,
|
||||
originKind: row.originKind,
|
||||
monitorNextCheckAt: row.monitorNextCheckAt,
|
||||
executionState: (row.executionState as Record<string, unknown> | null) ?? null,
|
||||
responsibleUserId: row.responsibleUserId,
|
||||
parentId: row.parentId,
|
||||
originId: row.originId,
|
||||
originRunId: row.originRunId,
|
||||
};
|
||||
}
|
||||
|
||||
function toRunSummary(row: HeartbeatRunRow): RunSummary {
|
||||
return {
|
||||
id: row.id,
|
||||
companyId: row.companyId,
|
||||
agentId: row.agentId,
|
||||
invocationSource: row.invocationSource,
|
||||
triggerDetail: row.triggerDetail,
|
||||
wakeupRequestId: row.wakeupRequestId,
|
||||
};
|
||||
}
|
||||
|
||||
function toDeferredWakeCandidate(row: typeof agentWakeupRequests.$inferSelect): DeferredWakeCandidate {
|
||||
const payload = parseObject(row.payload);
|
||||
const queuedCommentIds = queuedCommentIdsFromWakePayload(payload);
|
||||
const deferredContextSeed = parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]);
|
||||
const deferredCommentIds = extractWakeCommentIds(deferredContextSeed);
|
||||
const wakeReason = readNonEmptyString(deferredContextSeed.wakeReason);
|
||||
const queuedReason = wakeReason ?? readNonEmptyString(row.reason);
|
||||
const queuedWakeIsCommentOnly =
|
||||
!queuedReason ||
|
||||
queuedReason === "issue_commented" ||
|
||||
queuedReason === "issue_reopened_via_comment" ||
|
||||
queuedReason === "issue_comment_mentioned";
|
||||
const preservesIndependentContinuation =
|
||||
hasInteractionContinuationWakeContext(deferredContextSeed) ||
|
||||
deferredContextSeed.resumeIntent === true ||
|
||||
!queuedWakeIsCommentOnly;
|
||||
|
||||
return {
|
||||
id: row.id,
|
||||
companyId: row.companyId,
|
||||
agentId: row.agentId,
|
||||
reason: row.reason,
|
||||
source: row.source,
|
||||
triggerDetail: row.triggerDetail,
|
||||
requestedByActorType: row.requestedByActorType,
|
||||
requestedByActorId: row.requestedByActorId,
|
||||
payload,
|
||||
queuedCommentIds,
|
||||
preservesIndependentContinuation,
|
||||
deferredContextSeed,
|
||||
deferredCommentIds,
|
||||
wakeReason,
|
||||
};
|
||||
}
|
||||
|
||||
export type WakeQueuePostgresAdapterDeps = {
|
||||
resolveResponsibleUserId: WakeQueueReader["resolveResponsibleUserId"];
|
||||
getRoutineEnv: WakeQueueReader["getRoutineEnv"];
|
||||
resolveSessionBeforeForWakeup: WakeQueueReader["resolveSessionBeforeForWakeup"];
|
||||
};
|
||||
|
||||
function buildReader(tx: Db, deps: WakeQueuePostgresAdapterDeps): WakeQueueReader {
|
||||
return {
|
||||
async findInvokableAgent({ companyId, agentId }): Promise<InvokableAgentSnapshot | null> {
|
||||
const agent = await tx
|
||||
.select()
|
||||
.from(agents)
|
||||
.where(and(eq(agents.id, agentId), eq(agents.companyId, companyId)))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (!agent) return null;
|
||||
const companyAgents = await tx
|
||||
.select({ id: agents.id, companyId: agents.companyId, name: agents.name, reportsTo: agents.reportsTo, status: agents.status })
|
||||
.from(agents)
|
||||
.where(eq(agents.companyId, companyId));
|
||||
const invokability = evaluateAgentInvokability(agent, companyAgents);
|
||||
return { id: agent.id, companyId: agent.companyId, name: agent.name, invokable: invokability.invokable };
|
||||
},
|
||||
resolveResponsibleUserId: deps.resolveResponsibleUserId,
|
||||
getRoutineEnv: deps.getRoutineEnv,
|
||||
resolveSessionBeforeForWakeup: deps.resolveSessionBeforeForWakeup,
|
||||
};
|
||||
}
|
||||
|
||||
function buildWriter(tx: Db, deps: WakeQueuePostgresAdapterDeps): WakeQueueWriter {
|
||||
const treeControlSvc = issueTreeControlService(tx);
|
||||
const issuesSvc = issueService(tx);
|
||||
|
||||
return {
|
||||
async claimNextDeferredWake({ companyId, issueId }) {
|
||||
const row = await tx
|
||||
.select()
|
||||
.from(agentWakeupRequests)
|
||||
.where(
|
||||
and(
|
||||
eq(agentWakeupRequests.companyId, companyId),
|
||||
eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS),
|
||||
sql`${agentWakeupRequests.payload} ->> 'issueId' = ${issueId}`,
|
||||
),
|
||||
)
|
||||
.orderBy(asc(agentWakeupRequests.requestedAt))
|
||||
.limit(1)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return row ? toDeferredWakeCandidate(row) : null;
|
||||
},
|
||||
|
||||
async getQueuedCommentLiveness({ companyId, issueId, wakeAgentId, finishingRunId, finishingRunAgentId, queuedCommentIds }) {
|
||||
const rows = await tx
|
||||
.select({ id: issueComments.id, deletedAt: issueComments.deletedAt, createdByRunId: issueComments.createdByRunId })
|
||||
.from(issueComments)
|
||||
.where(and(eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId), inArray(issueComments.id, queuedCommentIds)));
|
||||
const targetsFinishingRunAgent = wakeAgentId === finishingRunAgentId;
|
||||
const liveNonSelfCommentIds = queuedCommentIds.filter((commentId) => {
|
||||
const row = rows.find((candidate) => candidate.id === commentId);
|
||||
return Boolean(row && !row.deletedAt && (!targetsFinishingRunAgent || row.createdByRunId !== finishingRunId));
|
||||
});
|
||||
const containedSelfAuthoredComment = rows.some(
|
||||
(row) => targetsFinishingRunAgent && !row.deletedAt && row.createdByRunId === finishingRunId,
|
||||
);
|
||||
return { liveNonSelfCommentIds, containedSelfAuthoredComment };
|
||||
},
|
||||
|
||||
async cancelDeferredWake({ companyId, wakeId, reason, now }) {
|
||||
const rows = await tx
|
||||
.update(agentWakeupRequests)
|
||||
.set({ status: "cancelled", finishedAt: now, error: reason, updatedAt: now })
|
||||
.where(
|
||||
and(
|
||||
eq(agentWakeupRequests.id, wakeId),
|
||||
eq(agentWakeupRequests.companyId, companyId),
|
||||
eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS),
|
||||
),
|
||||
)
|
||||
.returning({ id: agentWakeupRequests.id });
|
||||
return rows.length > 0;
|
||||
},
|
||||
|
||||
async normalizeDeferredWakeCommentIds({ companyId, wakeId, payload, liveCommentIds, now }) {
|
||||
const rows = await tx
|
||||
.update(agentWakeupRequests)
|
||||
.set({ payload: withQueuedCommentIdsInWakePayload(payload, liveCommentIds), updatedAt: now })
|
||||
.where(
|
||||
and(
|
||||
eq(agentWakeupRequests.id, wakeId),
|
||||
eq(agentWakeupRequests.companyId, companyId),
|
||||
eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS),
|
||||
),
|
||||
)
|
||||
.returning();
|
||||
const row = rows[0];
|
||||
return row ? toDeferredWakeCandidate(row) : null;
|
||||
},
|
||||
|
||||
async failDeferredWake({ companyId, wakeId, now }) {
|
||||
const rows = await tx
|
||||
.update(agentWakeupRequests)
|
||||
.set({
|
||||
status: "failed",
|
||||
finishedAt: now,
|
||||
error: "Deferred wake could not be promoted: agent is not invokable",
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(
|
||||
and(
|
||||
eq(agentWakeupRequests.id, wakeId),
|
||||
eq(agentWakeupRequests.companyId, companyId),
|
||||
eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS),
|
||||
),
|
||||
)
|
||||
.returning({ id: agentWakeupRequests.id });
|
||||
return rows.length > 0;
|
||||
},
|
||||
|
||||
async getPauseHoldFacts({ companyId, issueId, wakeAgentId, deferredContextSeed, requestedByActorType, requestedByActorId }) {
|
||||
const activePauseHold = await treeControlSvc.getActivePauseHoldGate(companyId, issueId);
|
||||
if (!activePauseHold) {
|
||||
return {
|
||||
activePauseHold: false,
|
||||
treeHoldInteractionWake: false,
|
||||
holdId: null,
|
||||
rootIssueId: null,
|
||||
mode: null,
|
||||
reason: null,
|
||||
releasePolicy: null,
|
||||
};
|
||||
}
|
||||
const treeHoldInteractionWake = await isVerifiedIssueTreeControlInteractionWake(tx, {
|
||||
companyId,
|
||||
issueId,
|
||||
agentId: wakeAgentId,
|
||||
contextSnapshot: deferredContextSeed,
|
||||
requestedByActorType,
|
||||
requestedByActorId,
|
||||
});
|
||||
return {
|
||||
activePauseHold: true,
|
||||
treeHoldInteractionWake,
|
||||
holdId: activePauseHold.holdId,
|
||||
rootIssueId: activePauseHold.rootIssueId,
|
||||
mode: activePauseHold.mode,
|
||||
reason: activePauseHold.reason,
|
||||
releasePolicy: activePauseHold.releasePolicy,
|
||||
};
|
||||
},
|
||||
|
||||
async getCommentSelfAuthorship({ companyId, issueId, finishingRunId, commentIds }) {
|
||||
const rows = await tx
|
||||
.select({ createdByRunId: issueComments.createdByRunId })
|
||||
.from(issueComments)
|
||||
.where(and(eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId), inArray(issueComments.id, commentIds)));
|
||||
return { allSelfAuthored: rows.length > 0 && rows.every((row) => row.createdByRunId === finishingRunId) };
|
||||
},
|
||||
|
||||
async reopenIssue({ companyId, issueId }) {
|
||||
const updated = await issuesSvc.updateForCompany(issueId, companyId, { status: "todo", executionState: null }, tx);
|
||||
return updated ? toIssueSnapshot(updated as unknown as IssueRow) : null;
|
||||
},
|
||||
|
||||
async claimDeferredWakeForPromotion({ companyId, wakeId, now }) {
|
||||
const claimed = await tx
|
||||
.update(agentWakeupRequests)
|
||||
.set({
|
||||
status: "queued",
|
||||
reason: "issue_execution_promoted",
|
||||
claimedAt: null,
|
||||
finishedAt: null,
|
||||
error: null,
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(
|
||||
and(
|
||||
eq(agentWakeupRequests.id, wakeId),
|
||||
eq(agentWakeupRequests.companyId, companyId),
|
||||
eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS),
|
||||
),
|
||||
)
|
||||
.returning({ id: agentWakeupRequests.id });
|
||||
return claimed.length > 0;
|
||||
},
|
||||
|
||||
async finalizePromotedWake(input) {
|
||||
const newRun = await tx
|
||||
.insert(heartbeatRuns)
|
||||
.values({
|
||||
companyId: input.deferredAgent.companyId,
|
||||
agentId: input.deferredAgent.id,
|
||||
invocationSource: input.source,
|
||||
triggerDetail: input.triggerDetail,
|
||||
status: "queued",
|
||||
wakeupRequestId: input.wakeId,
|
||||
contextSnapshot: input.contextSnapshot,
|
||||
responsibleUserId: input.responsibleUserId,
|
||||
sessionIdBefore: input.sessionBefore,
|
||||
continuationAttempt: readContinuationAttempt(input.contextSnapshot.livenessContinuationAttempt),
|
||||
})
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
|
||||
// `claimDeferredWakeForPromotion` already moved this row off
|
||||
// `deferred_issue_execution` inside this same transaction, so no
|
||||
// concurrent claimer can still match that guard; this extra `runId is
|
||||
// null` guard only protects against writing the link twice.
|
||||
await tx
|
||||
.update(agentWakeupRequests)
|
||||
.set({ runId: newRun.id, updatedAt: input.now })
|
||||
.where(
|
||||
and(
|
||||
eq(agentWakeupRequests.id, input.wakeId),
|
||||
eq(agentWakeupRequests.companyId, input.companyId),
|
||||
isNull(agentWakeupRequests.runId),
|
||||
),
|
||||
);
|
||||
|
||||
// Promoted mention wakes are issue-scoped, not issue ownership
|
||||
// transfers. The lock-clearing step earlier in this transaction
|
||||
// already set `executionRunId` to null for this issue, so the `is
|
||||
// null` guard only protects against taking the lock twice.
|
||||
await tx
|
||||
.update(issues)
|
||||
.set({
|
||||
executionRunId: newRun.id,
|
||||
executionAgentNameKey: normalizeAgentNameKey(input.deferredAgent.name),
|
||||
executionLockedAt: input.now,
|
||||
updatedAt: input.now,
|
||||
})
|
||||
.where(
|
||||
and(
|
||||
eq(issues.id, input.issue.id),
|
||||
eq(issues.companyId, input.companyId),
|
||||
eq(issues.assigneeAgentId, input.deferredAgent.id),
|
||||
isNull(issues.executionRunId),
|
||||
),
|
||||
);
|
||||
|
||||
return toRunSummary(newRun);
|
||||
},
|
||||
|
||||
async hasExistingExecutionPath({ companyId, issueId, excludeRunId, agentId }) {
|
||||
const row = await tx
|
||||
.select({ id: heartbeatRuns.id })
|
||||
.from(heartbeatRuns)
|
||||
.where(
|
||||
and(
|
||||
eq(heartbeatRuns.companyId, companyId),
|
||||
inArray(heartbeatRuns.status, [...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES]),
|
||||
sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}`,
|
||||
sql`${heartbeatRuns.id} <> ${excludeRunId}`,
|
||||
agentId ? eq(heartbeatRuns.agentId, agentId) : sql`true`,
|
||||
),
|
||||
)
|
||||
.limit(1)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return row !== null;
|
||||
},
|
||||
|
||||
async hasExplicitBlockerPath({ companyId, issueId }) {
|
||||
const row = await tx
|
||||
.select({ issueId: issueRelations.issueId })
|
||||
.from(issueRelations)
|
||||
.innerJoin(issues, eq(issueRelations.issueId, issues.id))
|
||||
.where(
|
||||
and(
|
||||
eq(issueRelations.companyId, companyId),
|
||||
eq(issueRelations.relatedIssueId, issueId),
|
||||
eq(issueRelations.type, "blocks"),
|
||||
eq(issues.companyId, companyId),
|
||||
notInArray(issues.status, ["done", "cancelled"]),
|
||||
isNull(issues.hiddenAt),
|
||||
),
|
||||
)
|
||||
.limit(1)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return row !== null;
|
||||
},
|
||||
|
||||
async isAutomaticRecoverySuppressedByPauseHold({ companyId, issueId }) {
|
||||
return isAutomaticRecoverySuppressedByPauseHold(tx, companyId, issueId, treeControlSvc);
|
||||
},
|
||||
|
||||
async buildBlockedRecoveryNotice({ noticeKind, issueStatus, finishingRun }) {
|
||||
if (noticeKind === "workspace_validation") {
|
||||
return { notice: buildWorkspaceValidationRecoveryNoticeSeed(), recoveryCause: WORKSPACE_VALIDATION_RECOVERY_CAUSE };
|
||||
}
|
||||
if (noticeKind === "configuration_incomplete") {
|
||||
return {
|
||||
notice: buildConfigurationIncompleteRecoveryNoticeSeed(finishingRun.configurationIncompletePayload),
|
||||
recoveryCause: CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE,
|
||||
};
|
||||
}
|
||||
if (noticeKind === "execution_review_participant") {
|
||||
return {
|
||||
notice: buildExecutionReviewParticipantRecoveryNoticeSeed(),
|
||||
recoveryCause: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE,
|
||||
};
|
||||
}
|
||||
return { notice: buildImmediateExecutionPathRecoveryNoticeSeed({ status: issueStatus }), recoveryCause: null };
|
||||
},
|
||||
|
||||
async queueReviewParticipantRecoveryRun({ companyId, issue, finishingRun, recoveryAgent, sessionBefore, now }) {
|
||||
const executionState = parseIssueExecutionState(issue.executionState);
|
||||
const wakeupRequest = await tx
|
||||
.insert(agentWakeupRequests)
|
||||
.values({
|
||||
companyId,
|
||||
agentId: recoveryAgent.id,
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
reason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON,
|
||||
payload: withRecoveryContext(
|
||||
{
|
||||
issueId: issue.id,
|
||||
retryOfRunId: finishingRun.id,
|
||||
retryReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON,
|
||||
currentStageId: executionState?.currentStageId ?? null,
|
||||
currentStageType: executionState?.currentStageType ?? null,
|
||||
},
|
||||
"normal_model",
|
||||
),
|
||||
status: "queued",
|
||||
requestedByActorType: "system",
|
||||
requestedByActorId: null,
|
||||
updatedAt: now,
|
||||
})
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
|
||||
const queuedRun = await tx
|
||||
.insert(heartbeatRuns)
|
||||
.values({
|
||||
companyId,
|
||||
agentId: recoveryAgent.id,
|
||||
invocationSource: "automation",
|
||||
triggerDetail: "system",
|
||||
status: "queued",
|
||||
wakeupRequestId: wakeupRequest.id,
|
||||
contextSnapshot: withRecoveryContext(
|
||||
{
|
||||
issueId: issue.id,
|
||||
taskId: issue.id,
|
||||
wakeReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON,
|
||||
retryReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON,
|
||||
source: "issue.execution_review_recovery",
|
||||
retryOfRunId: finishingRun.id,
|
||||
currentStageId: executionState?.currentStageId ?? null,
|
||||
currentStageType: executionState?.currentStageType ?? null,
|
||||
reviewRecoveryInstruction:
|
||||
"The previous reviewer run ended while this execution-review stage was still pending. Submit the review decision now, or mark the issue blocked with the exact unblock action.",
|
||||
},
|
||||
"normal_model",
|
||||
),
|
||||
sessionIdBefore: sessionBefore,
|
||||
retryOfRunId: finishingRun.id,
|
||||
updatedAt: now,
|
||||
})
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
|
||||
await tx
|
||||
.update(agentWakeupRequests)
|
||||
.set({ runId: queuedRun.id, updatedAt: now })
|
||||
.where(and(eq(agentWakeupRequests.id, wakeupRequest.id), eq(agentWakeupRequests.companyId, companyId)));
|
||||
|
||||
await tx
|
||||
.update(issues)
|
||||
.set({
|
||||
executionRunId: queuedRun.id,
|
||||
executionAgentNameKey: normalizeAgentNameKey(recoveryAgent.name),
|
||||
executionLockedAt: now,
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(and(eq(issues.id, issue.id), eq(issues.companyId, companyId)));
|
||||
|
||||
return toRunSummary(queuedRun);
|
||||
},
|
||||
|
||||
async queueImmediateRecoveryRun({ companyId, issue, finishingRun, recoveryAgent, sessionBefore, now }) {
|
||||
const retryReason = issue.status === "todo" ? "assignment_recovery" : "issue_continuation_needed";
|
||||
const recoveryReason = issue.status === "todo" ? "issue_assignment_recovery" : "issue_continuation_needed";
|
||||
const recoverySource = issue.status === "todo" ? "issue.assignment_recovery" : "issue.continuation_recovery";
|
||||
const recoveryContextSnapshot = withRecoveryContext(
|
||||
{
|
||||
issueId: issue.id,
|
||||
taskId: issue.id,
|
||||
wakeReason: recoveryReason,
|
||||
retryReason,
|
||||
source: recoverySource,
|
||||
retryOfRunId: finishingRun.id,
|
||||
},
|
||||
"normal_model",
|
||||
);
|
||||
|
||||
const routineEnvContext = await deps.getRoutineEnv({ companyId, issue });
|
||||
const responsibleUserId = await deps.resolveResponsibleUserId({
|
||||
companyId,
|
||||
contextSnapshot: recoveryContextSnapshot,
|
||||
issue,
|
||||
routineEnvContext,
|
||||
requestedByActorType: "system",
|
||||
requestedByActorId: null,
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
existingRunResponsibleUserId: finishingRun.responsibleUserId,
|
||||
});
|
||||
if (!responsibleUserId) {
|
||||
throw new WakeQueueApplicationError(
|
||||
"responsible_user_unresolved",
|
||||
"Unable to resolve responsible user for recovery heartbeat run",
|
||||
{
|
||||
runId: finishingRun.id,
|
||||
agentId: recoveryAgent.id,
|
||||
companyId,
|
||||
issueId: issue.id,
|
||||
wakeReason: recoveryReason,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
const wakeupRequest = await tx
|
||||
.insert(agentWakeupRequests)
|
||||
.values({
|
||||
companyId,
|
||||
agentId: recoveryAgent.id,
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
reason: recoveryReason,
|
||||
payload: withRecoveryContext({ issueId: issue.id, retryOfRunId: finishingRun.id }, "normal_model"),
|
||||
status: "queued",
|
||||
requestedByActorType: "system",
|
||||
requestedByActorId: null,
|
||||
updatedAt: now,
|
||||
})
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
|
||||
const queuedRun = await tx
|
||||
.insert(heartbeatRuns)
|
||||
.values({
|
||||
companyId,
|
||||
agentId: recoveryAgent.id,
|
||||
invocationSource: "automation",
|
||||
triggerDetail: "system",
|
||||
status: "queued",
|
||||
wakeupRequestId: wakeupRequest.id,
|
||||
contextSnapshot: recoveryContextSnapshot,
|
||||
responsibleUserId,
|
||||
sessionIdBefore: sessionBefore,
|
||||
retryOfRunId: finishingRun.id,
|
||||
updatedAt: now,
|
||||
})
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
|
||||
await tx
|
||||
.update(agentWakeupRequests)
|
||||
.set({ runId: queuedRun.id, updatedAt: now })
|
||||
.where(and(eq(agentWakeupRequests.id, wakeupRequest.id), eq(agentWakeupRequests.companyId, companyId)));
|
||||
|
||||
await tx
|
||||
.update(issues)
|
||||
.set({
|
||||
executionRunId: queuedRun.id,
|
||||
executionAgentNameKey: normalizeAgentNameKey(recoveryAgent.name),
|
||||
executionLockedAt: now,
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(and(eq(issues.id, issue.id), eq(issues.companyId, companyId)));
|
||||
|
||||
return toRunSummary(queuedRun);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function recordNativeTerminalRecoveryIfNeeded(tx: Db, run: HeartbeatRunRow, issue: IssueRow, now: Date): Promise<boolean> {
|
||||
const applies =
|
||||
run.runtimeMode === "native" &&
|
||||
["failed", "timed_out", "interrupted", "cancelled"].includes(run.status) &&
|
||||
issue.assigneeAgentId === run.agentId &&
|
||||
!["done", "cancelled"].includes(issue.status);
|
||||
if (!applies) return false;
|
||||
|
||||
const existing = await tx
|
||||
.select({ id: issueRecoveryActions.id })
|
||||
.from(issueRecoveryActions)
|
||||
.where(
|
||||
and(
|
||||
eq(issueRecoveryActions.companyId, issue.companyId),
|
||||
eq(issueRecoveryActions.sourceIssueId, issue.id),
|
||||
or(
|
||||
inArray(issueRecoveryActions.status, ["active", "escalated"]),
|
||||
sql`${issueRecoveryActions.evidence}->'automaticRecovery'->>'runId' = ${run.id}`,
|
||||
),
|
||||
),
|
||||
)
|
||||
.limit(1);
|
||||
if (!existing.length) {
|
||||
await tx
|
||||
.update(nativeRunFinalizations)
|
||||
.set({
|
||||
phase: "terminal_failure",
|
||||
leaseOwner: null,
|
||||
leaseExpiresAt: null,
|
||||
nextAttemptAt: null,
|
||||
recoveryState: "blocked",
|
||||
failureCode: "native_continuation_requires_reconciliation",
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(
|
||||
and(
|
||||
eq(nativeRunFinalizations.companyId, issue.companyId),
|
||||
eq(nativeRunFinalizations.runId, run.id),
|
||||
isNull(nativeRunFinalizations.resultId),
|
||||
),
|
||||
);
|
||||
await issueRecoveryActionService(tx).upsertSourceScoped({
|
||||
companyId: issue.companyId,
|
||||
sourceIssueId: issue.id,
|
||||
kind: "active_run_watchdog",
|
||||
ownerType: "board",
|
||||
returnOwnerAgentId: run.agentId,
|
||||
cause: "native_continuation_requires_reconciliation",
|
||||
fingerprint: `native-continuation:${run.id}`,
|
||||
evidence: { runId: run.id, originalFailureCode: run.errorCode },
|
||||
nextAction:
|
||||
"Inspect the original failure and reconcile the previous execution before continuing. Automatic recovery cannot start another incident.",
|
||||
maxAttempts: 3,
|
||||
wakePolicy: null,
|
||||
supersedeOnIdentityChange: true,
|
||||
});
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAdapterDeps): IssueLockWriter {
|
||||
return {
|
||||
async withIssueExecutionLock(input, fn): Promise<ReleaseTransactionResult & { run: RunSnapshot }> {
|
||||
return db.transaction(async (rawTx) => {
|
||||
const tx = rawTx as unknown as Db;
|
||||
const run = await tx
|
||||
.select()
|
||||
.from(heartbeatRuns)
|
||||
.where(and(eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.companyId, input.companyId)))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (!run) {
|
||||
throw new Error(`wake-queue: run ${input.runId} was not found while releasing issue execution`);
|
||||
}
|
||||
const runSnapshot = toRunSnapshot(run);
|
||||
const contextIssueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId);
|
||||
|
||||
// Lock the context issue (if any) and every issue that still references this
|
||||
// run, in id order, so two concurrent finalizations can never deadlock on
|
||||
// each other's row-lock acquisition order.
|
||||
await tx.execute(
|
||||
contextIssueId
|
||||
? sql`
|
||||
select id from issues
|
||||
where company_id = ${input.companyId}
|
||||
and (
|
||||
id = ${contextIssueId}
|
||||
or execution_run_id = ${run.id}
|
||||
or checkout_run_id = ${run.id}
|
||||
)
|
||||
order by id
|
||||
for update
|
||||
`
|
||||
: sql`
|
||||
select id from issues
|
||||
where company_id = ${input.companyId}
|
||||
and (execution_run_id = ${run.id} or checkout_run_id = ${run.id})
|
||||
order by id
|
||||
for update
|
||||
`,
|
||||
);
|
||||
|
||||
const candidateIssues = await tx
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(
|
||||
and(
|
||||
eq(issues.companyId, input.companyId),
|
||||
contextIssueId
|
||||
? or(eq(issues.id, contextIssueId), eq(issues.executionRunId, run.id), eq(issues.checkoutRunId, run.id))
|
||||
: or(eq(issues.executionRunId, run.id), eq(issues.checkoutRunId, run.id)),
|
||||
),
|
||||
)
|
||||
.orderBy(asc(issues.id));
|
||||
|
||||
// Two separate updates: a retry can move `executionRunId` to a new run
|
||||
// while `checkoutRunId` still points at this one finishing.
|
||||
await tx
|
||||
.update(issues)
|
||||
.set({ executionRunId: null, executionAgentNameKey: null, executionLockedAt: null, updatedAt: input.now })
|
||||
.where(and(eq(issues.companyId, input.companyId), eq(issues.executionRunId, run.id)));
|
||||
await tx
|
||||
.update(issues)
|
||||
.set({ checkoutRunId: null, updatedAt: input.now })
|
||||
.where(and(eq(issues.companyId, input.companyId), eq(issues.checkoutRunId, run.id)));
|
||||
|
||||
const issueRow =
|
||||
(contextIssueId ? candidateIssues.find((candidate) => candidate.id === contextIssueId) : candidateIssues[0]) ?? null;
|
||||
|
||||
if (!issueRow || (issueRow.executionRunId && issueRow.executionRunId !== run.id)) {
|
||||
return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot };
|
||||
}
|
||||
|
||||
if (
|
||||
(isWorkspaceValidationFailedRun(run) || isConfigurationIncompleteFailedRun(run)) &&
|
||||
(issueRow.status === "todo" || issueRow.status === "in_progress") &&
|
||||
!issueRow.assigneeUserId &&
|
||||
issueRow.assigneeAgentId === run.agentId
|
||||
) {
|
||||
const configurationIncomplete = isConfigurationIncompleteFailedRun(run);
|
||||
const notice = configurationIncomplete
|
||||
? buildConfigurationIncompleteRecoveryNoticeSeed(runSnapshot.configurationIncompletePayload)
|
||||
: buildWorkspaceValidationRecoveryNoticeSeed();
|
||||
return {
|
||||
outcome: {
|
||||
kind: "blocked",
|
||||
issue: toIssueSnapshot(issueRow),
|
||||
previousStatus: issueRow.status as "todo" | "in_progress",
|
||||
notice,
|
||||
recoveryCause: configurationIncomplete ? CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE : WORKSPACE_VALIDATION_RECOVERY_CAUSE,
|
||||
},
|
||||
postCommitEffects: [],
|
||||
run: runSnapshot,
|
||||
};
|
||||
}
|
||||
|
||||
if (legacyExecutionNeedsReconciliation(run)) {
|
||||
return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot };
|
||||
}
|
||||
|
||||
// An operator stop never promotes old queued work by itself. The next
|
||||
// explicit wake adopts those messages atomically when it queues a run.
|
||||
if (run.status === "cancelled" && parseObject(run.resultJson?.executionCancellation).state === "acknowledged") {
|
||||
return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot };
|
||||
}
|
||||
|
||||
if (await recordNativeTerminalRecoveryIfNeeded(tx, run, issueRow, input.now)) {
|
||||
return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot };
|
||||
}
|
||||
|
||||
const locked: LockedIssueExecution = { primaryIssue: toIssueSnapshot(issueRow), run: runSnapshot };
|
||||
const result = await fn(locked, { reader: buildReader(tx, deps), writer: buildWriter(tx, deps) });
|
||||
return { ...result, run: runSnapshot };
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,244 @@
|
|||
import type { ReleaseRecoveryBlockedNoticeKind } from "../domain/policy.js";
|
||||
import type {
|
||||
InvokableAgentSnapshot,
|
||||
IssueSnapshot,
|
||||
PostCommitEffect,
|
||||
ReleaseOutcome,
|
||||
RunSnapshot,
|
||||
RunSummary,
|
||||
} from "./types.js";
|
||||
|
||||
export type { InvokableAgentSnapshot, IssueSnapshot, RunSnapshot, RunSummary };
|
||||
|
||||
/** The primary issue a locked release resolves to, plus the finishing run the lock step already loaded. */
|
||||
export type LockedIssueExecution = {
|
||||
primaryIssue: IssueSnapshot;
|
||||
run: RunSnapshot;
|
||||
};
|
||||
|
||||
export type ReleaseTransactionResult = {
|
||||
outcome: ReleaseOutcome;
|
||||
postCommitEffects: PostCommitEffect[];
|
||||
};
|
||||
|
||||
/** Read-only lookups the release use case needs, each scoped to a company. */
|
||||
export interface WakeQueueReader {
|
||||
findInvokableAgent(input: { companyId: string; agentId: string }): Promise<InvokableAgentSnapshot | null>;
|
||||
/**
|
||||
* Takes the transaction-scoped issue snapshot, not an issue id, so this
|
||||
* port never re-reads the issue on a separate connection while the
|
||||
* module's own transaction is open.
|
||||
*/
|
||||
resolveResponsibleUserId(input: {
|
||||
companyId: string;
|
||||
contextSnapshot: Record<string, unknown>;
|
||||
issue: IssueSnapshot;
|
||||
/** From a prior `getRoutineEnv` call against the same issue; pass `{ routineId: null, env: null, responsibleUserId: null }` when the issue is not a routine execution. */
|
||||
routineEnvContext: { routineId: string | null; env: unknown; responsibleUserId: string | null };
|
||||
requestedByActorType: "user" | "agent" | "system" | null;
|
||||
requestedByActorId: string | null;
|
||||
source: string;
|
||||
triggerDetail: string | null;
|
||||
existingRunResponsibleUserId: string | null;
|
||||
}): Promise<string | null>;
|
||||
/**
|
||||
* Takes the transaction-scoped issue snapshot, not an issue id, so this
|
||||
* port never re-reads the issue on a separate connection while the
|
||||
* module's own transaction is open.
|
||||
*/
|
||||
getRoutineEnv(input: {
|
||||
companyId: string;
|
||||
issue: IssueSnapshot;
|
||||
}): Promise<{ routineId: string | null; env: unknown; responsibleUserId: string | null }>;
|
||||
resolveSessionBeforeForWakeup(input: {
|
||||
companyId: string;
|
||||
agentId: string;
|
||||
taskKey: string | null;
|
||||
}): Promise<string | null>;
|
||||
}
|
||||
|
||||
export type DeferredWakeCandidate = {
|
||||
id: string;
|
||||
companyId: string;
|
||||
agentId: string;
|
||||
reason: string | null;
|
||||
source: string | null;
|
||||
triggerDetail: string | null;
|
||||
requestedByActorType: string | null;
|
||||
requestedByActorId: string | null;
|
||||
payload: Record<string, unknown>;
|
||||
/** The queued comment ids the wake's queued-comment context carries, already extracted from the payload. */
|
||||
queuedCommentIds: string[];
|
||||
/** True when the wake carries an independent reason to continue even with no live queued comments. */
|
||||
preservesIndependentContinuation: boolean;
|
||||
/** `payload._paperclipWakeContext`, already parsed to a plain object. */
|
||||
deferredContextSeed: Record<string, unknown>;
|
||||
/** The comment ids the wake's context snapshot carries (a separate set from queuedCommentIds), used for the reopen check. */
|
||||
deferredCommentIds: string[];
|
||||
wakeReason: string | null;
|
||||
};
|
||||
|
||||
export type PromoteDeferredWakeInput = {
|
||||
companyId: string;
|
||||
wakeId: string;
|
||||
deferredAgent: InvokableAgentSnapshot;
|
||||
issue: IssueSnapshot;
|
||||
finishingRun: RunSnapshot;
|
||||
contextSnapshot: Record<string, unknown>;
|
||||
reason: string;
|
||||
source: string;
|
||||
triggerDetail: string | null;
|
||||
payload: Record<string, unknown>;
|
||||
responsibleUserId: string;
|
||||
sessionBefore: string | null;
|
||||
now: Date;
|
||||
};
|
||||
|
||||
/** The transaction-scoped write operations that drain and resolve the deferred-wake queue. */
|
||||
export interface WakeQueueWriter {
|
||||
claimNextDeferredWake(input: { companyId: string; issueId: string }): Promise<DeferredWakeCandidate | null>;
|
||||
getQueuedCommentLiveness(input: {
|
||||
companyId: string;
|
||||
issueId: string;
|
||||
wakeAgentId: string;
|
||||
finishingRunId: string;
|
||||
finishingRunAgentId: string;
|
||||
queuedCommentIds: string[];
|
||||
}): Promise<{ liveNonSelfCommentIds: string[]; containedSelfAuthoredComment: boolean }>;
|
||||
/** Cancels the wake with `status = 'deferred_issue_execution'` as an atomic compare-and-set guard. */
|
||||
cancelDeferredWake(input: {
|
||||
companyId: string;
|
||||
wakeId: string;
|
||||
reason: string;
|
||||
now: Date;
|
||||
}): Promise<boolean>;
|
||||
normalizeDeferredWakeCommentIds(input: {
|
||||
companyId: string;
|
||||
wakeId: string;
|
||||
/** The wake's current payload, as already read by `claimNextDeferredWake`, used as the rewrite base. */
|
||||
payload: Record<string, unknown>;
|
||||
liveCommentIds: string[];
|
||||
now: Date;
|
||||
}): Promise<DeferredWakeCandidate | null>;
|
||||
/** Sets `status = 'failed'` guarded by the current `deferred_issue_execution` status. */
|
||||
failDeferredWake(input: { companyId: string; wakeId: string; now: Date }): Promise<boolean>;
|
||||
getPauseHoldFacts(input: {
|
||||
companyId: string;
|
||||
issueId: string;
|
||||
wakeAgentId: string;
|
||||
deferredContextSeed: Record<string, unknown>;
|
||||
requestedByActorType: string | null;
|
||||
requestedByActorId: string | null;
|
||||
}): Promise<{
|
||||
activePauseHold: boolean;
|
||||
treeHoldInteractionWake: boolean;
|
||||
holdId: string | null;
|
||||
rootIssueId: string | null;
|
||||
mode: string | null;
|
||||
reason: string | null;
|
||||
releasePolicy: unknown;
|
||||
}>;
|
||||
getCommentSelfAuthorship(input: {
|
||||
companyId: string;
|
||||
issueId: string;
|
||||
finishingRunId: string;
|
||||
commentIds: string[];
|
||||
}): Promise<{ allSelfAuthored: boolean }>;
|
||||
reopenIssue(input: { companyId: string; issueId: string; runId: string }): Promise<IssueSnapshot | null>;
|
||||
/**
|
||||
* Atomically claims the wake for promotion, guarded on its current
|
||||
* `deferred_issue_execution` status. Call this before any other write in
|
||||
* the promotion path (including a reopen), so a lost race here can never
|
||||
* leave another write committed underneath it. Returns `false` when a
|
||||
* concurrent writer already changed the wake's status.
|
||||
*/
|
||||
claimDeferredWakeForPromotion(input: { companyId: string; wakeId: string; now: Date }): Promise<boolean>;
|
||||
/**
|
||||
* Finalizes a wake that `claimDeferredWakeForPromotion` already claimed:
|
||||
* inserts the queued run, links it back onto the wake row, and takes the
|
||||
* issue's execution lock. Call only after that claim returns `true`.
|
||||
*/
|
||||
finalizePromotedWake(input: PromoteDeferredWakeInput): Promise<RunSummary>;
|
||||
/** An open run already on this issue (optionally scoped to one agent) that would race a new recovery run. */
|
||||
hasExistingExecutionPath(input: {
|
||||
companyId: string;
|
||||
issueId: string;
|
||||
excludeRunId: string;
|
||||
agentId: string | null;
|
||||
}): Promise<boolean>;
|
||||
/** An open, non-hidden issue that still lists this issue as a `blocks` predecessor. */
|
||||
hasExplicitBlockerPath(input: { companyId: string; issueId: string }): Promise<boolean>;
|
||||
isAutomaticRecoverySuppressedByPauseHold(input: { companyId: string; issueId: string }): Promise<boolean>;
|
||||
/** Builds the stranded-recovery notice content for a `blocked` outcome; pure formatting, kept behind the writer so `services/recovery/stranded-notice` stays out of the application layer. */
|
||||
buildBlockedRecoveryNotice(input: {
|
||||
noticeKind: ReleaseRecoveryBlockedNoticeKind;
|
||||
issueStatus: "todo" | "in_progress";
|
||||
finishingRun: RunSnapshot;
|
||||
}): Promise<{ notice: Record<string, unknown>; recoveryCause: string | null }>;
|
||||
queueReviewParticipantRecoveryRun(input: {
|
||||
companyId: string;
|
||||
issue: IssueSnapshot;
|
||||
finishingRun: RunSnapshot;
|
||||
recoveryAgent: InvokableAgentSnapshot;
|
||||
sessionBefore: string | null;
|
||||
now: Date;
|
||||
}): Promise<RunSummary>;
|
||||
/**
|
||||
* Builds the recovery context snapshot, resolves the responsible user
|
||||
* from it, and queues the run. Throws `WakeQueueApplicationError` with
|
||||
* code `responsible_user_unresolved` when no responsible user resolves,
|
||||
* without queuing anything.
|
||||
*/
|
||||
queueImmediateRecoveryRun(input: {
|
||||
companyId: string;
|
||||
issue: IssueSnapshot;
|
||||
finishingRun: RunSnapshot;
|
||||
recoveryAgent: InvokableAgentSnapshot;
|
||||
sessionBefore: string | null;
|
||||
now: Date;
|
||||
}): Promise<RunSummary>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Owns the module's own transaction: loads the finishing run, locks the
|
||||
* context issue and every sibling issue in id order, clears the two
|
||||
* release-lock columns, and picks the primary issue. When the primary
|
||||
* issue is missing, already reclaimed, or resolved by an early exit
|
||||
* (workspace-validation block, legacy reconciliation, a native-runtime
|
||||
* terminal failure), the adapter returns that outcome directly without
|
||||
* calling `fn`. Otherwise it calls `fn` with the locked issue and run, and
|
||||
* with `reader`/`writer` ports bound to the same transaction, so every
|
||||
* call `fn` makes through them participates in the one transaction this
|
||||
* method owns.
|
||||
*/
|
||||
export interface IssueLockWriter {
|
||||
withIssueExecutionLock(
|
||||
input: { companyId: string; runId: string; now: Date },
|
||||
fn: (
|
||||
locked: LockedIssueExecution,
|
||||
ports: { reader: WakeQueueReader; writer: WakeQueueWriter },
|
||||
) => Promise<ReleaseTransactionResult>,
|
||||
): Promise<ReleaseTransactionResult & { run: RunSnapshot }>;
|
||||
}
|
||||
|
||||
export type StrandedAssignedIssueEscalationInput = {
|
||||
issue: IssueSnapshot;
|
||||
previousStatus: "todo" | "in_progress" | "in_review";
|
||||
latestRun: RunSnapshot;
|
||||
notice: Record<string, unknown>;
|
||||
recoveryCause: string | null;
|
||||
};
|
||||
|
||||
export type StrandedRecoveryInPlaceEscalationInput = {
|
||||
issue: IssueSnapshot;
|
||||
previousStatus: "todo" | "in_progress" | "in_review";
|
||||
latestRun: RunSnapshot;
|
||||
};
|
||||
|
||||
/** Wraps `services/recovery`'s stranded-issue escalation, called only after the release transaction commits. */
|
||||
export interface RecoveryEscalationPort {
|
||||
escalateStrandedAssignedIssue(input: StrandedAssignedIssueEscalationInput): Promise<void>;
|
||||
escalateStrandedRecoveryIssueInPlace(input: StrandedRecoveryInPlaceEscalationInput): Promise<void>;
|
||||
}
|
||||
|
||||
export type { PostCommitEffect, ReleaseOutcome };
|
||||
|
|
@ -0,0 +1,99 @@
|
|||
export type RunSummary = {
|
||||
id: string;
|
||||
companyId: string;
|
||||
agentId: string;
|
||||
invocationSource: string;
|
||||
triggerDetail: string | null;
|
||||
wakeupRequestId: string | null;
|
||||
};
|
||||
|
||||
export type RunSnapshot = {
|
||||
id: string;
|
||||
companyId: string;
|
||||
agentId: string;
|
||||
status: string;
|
||||
runtimeMode: string | null;
|
||||
errorCode: string | null;
|
||||
responsibleUserId: string | null;
|
||||
/** The run's own context snapshot, kept as plain JSON so ports carry no drizzle types. */
|
||||
contextSnapshot: Record<string, unknown>;
|
||||
/** `resultJson.configurationIncomplete`, already parsed; non-null only on a configuration-incomplete failed run. */
|
||||
configurationIncompletePayload: Record<string, unknown> | null;
|
||||
};
|
||||
|
||||
export type IssueSnapshot = {
|
||||
id: string;
|
||||
companyId: string;
|
||||
identifier: string;
|
||||
status: string;
|
||||
assigneeAgentId: string | null;
|
||||
assigneeUserId: string | null;
|
||||
hiddenAt: Date | null;
|
||||
originKind: string | null;
|
||||
monitorNextCheckAt: Date | null;
|
||||
executionState: Record<string, unknown> | null;
|
||||
/** Carried so the routine-env and responsible-user reader ports can use this
|
||||
* transaction-scoped snapshot instead of reading the issue again. */
|
||||
responsibleUserId: string | null;
|
||||
parentId: string | null;
|
||||
originId: string | null;
|
||||
originRunId: string | null;
|
||||
};
|
||||
|
||||
export type InvokableAgentSnapshot = {
|
||||
id: string;
|
||||
companyId: string;
|
||||
name: string | null;
|
||||
invokable: boolean;
|
||||
};
|
||||
|
||||
/** A new heartbeat run reached the queued state and should be published and dispatched. */
|
||||
export type RunQueuedEffect = {
|
||||
kind: "run_queued";
|
||||
run: RunSummary;
|
||||
};
|
||||
|
||||
/** An issue reopened from done/cancelled because a live deferred comment wake promoted on it. */
|
||||
export type IssueReopenedEffect = {
|
||||
kind: "issue_reopened";
|
||||
companyId: string;
|
||||
agentId: string;
|
||||
runId: string;
|
||||
issueId: string;
|
||||
identifier: string;
|
||||
reopenedFrom: string;
|
||||
};
|
||||
|
||||
/** Explicit post-commit work a caller applies only after the release transaction commits. */
|
||||
export type PostCommitEffect = RunQueuedEffect | IssueReopenedEffect;
|
||||
|
||||
export type ReleaseOutcome =
|
||||
| { kind: "released" }
|
||||
| { kind: "promoted"; run: RunSummary }
|
||||
| { kind: "queued_review_participant_recovery"; run: RunSummary }
|
||||
| { kind: "queued_recovery"; run: RunSummary }
|
||||
| {
|
||||
kind: "blocked";
|
||||
issue: IssueSnapshot;
|
||||
previousStatus: "todo" | "in_progress" | "in_review";
|
||||
notice: Record<string, unknown>;
|
||||
recoveryCause: string | null;
|
||||
}
|
||||
| {
|
||||
kind: "blocked_recovery_in_place";
|
||||
issue: IssueSnapshot;
|
||||
previousStatus: "todo" | "in_progress" | "in_review";
|
||||
};
|
||||
|
||||
export type WakeQueueApplicationErrorCode = "responsible_user_unresolved";
|
||||
|
||||
export class WakeQueueApplicationError extends Error {
|
||||
constructor(
|
||||
readonly code: WakeQueueApplicationErrorCode,
|
||||
message: string,
|
||||
readonly details: Record<string, unknown> = {},
|
||||
) {
|
||||
super(message);
|
||||
this.name = "WakeQueueApplicationError";
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,331 @@
|
|||
import { describe, expect, it, vi } from "vitest";
|
||||
import { createReleaseIssueExecution } from "./use-cases.js";
|
||||
import { WakeQueueApplicationError } from "./types.js";
|
||||
import type {
|
||||
DeferredWakeCandidate,
|
||||
InvokableAgentSnapshot,
|
||||
IssueLockWriter,
|
||||
IssueSnapshot,
|
||||
PromoteDeferredWakeInput,
|
||||
RecoveryEscalationPort,
|
||||
RunSnapshot,
|
||||
RunSummary,
|
||||
WakeQueueReader,
|
||||
WakeQueueWriter,
|
||||
} from "./ports.js";
|
||||
|
||||
const RUN: RunSnapshot = {
|
||||
id: "run-1",
|
||||
companyId: "company-1",
|
||||
agentId: "finishing-agent",
|
||||
status: "failed",
|
||||
runtimeMode: "process",
|
||||
errorCode: null,
|
||||
responsibleUserId: "user-1",
|
||||
contextSnapshot: {},
|
||||
configurationIncompletePayload: null,
|
||||
};
|
||||
|
||||
const ISSUE: IssueSnapshot = {
|
||||
id: "issue-1",
|
||||
companyId: "company-1",
|
||||
identifier: "PAP-1",
|
||||
status: "in_progress",
|
||||
assigneeAgentId: "finishing-agent",
|
||||
assigneeUserId: null,
|
||||
hiddenAt: null,
|
||||
originKind: null,
|
||||
monitorNextCheckAt: null,
|
||||
executionState: null,
|
||||
responsibleUserId: null,
|
||||
parentId: null,
|
||||
originId: null,
|
||||
originRunId: null,
|
||||
};
|
||||
|
||||
const AGENT: InvokableAgentSnapshot = {
|
||||
id: "deferred-agent",
|
||||
companyId: "company-1",
|
||||
name: "Deferred Agent",
|
||||
invokable: true,
|
||||
};
|
||||
|
||||
function wakeCandidate(overrides: Partial<DeferredWakeCandidate> = {}): DeferredWakeCandidate {
|
||||
return {
|
||||
id: overrides.id ?? "wake-1",
|
||||
companyId: "company-1",
|
||||
agentId: "deferred-agent",
|
||||
reason: "issue_commented",
|
||||
source: "automation",
|
||||
triggerDetail: null,
|
||||
requestedByActorType: "user",
|
||||
requestedByActorId: "actor-1",
|
||||
payload: {},
|
||||
queuedCommentIds: [],
|
||||
preservesIndependentContinuation: false,
|
||||
deferredContextSeed: {},
|
||||
deferredCommentIds: [],
|
||||
wakeReason: "issue_commented",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function runSummary(id: string): RunSummary {
|
||||
return {
|
||||
id,
|
||||
companyId: "company-1",
|
||||
agentId: "deferred-agent",
|
||||
invocationSource: "automation",
|
||||
triggerDetail: null,
|
||||
wakeupRequestId: `wakeup-${id}`,
|
||||
};
|
||||
}
|
||||
|
||||
function createFakeReader(overrides: Partial<WakeQueueReader> = {}): WakeQueueReader {
|
||||
return {
|
||||
findInvokableAgent: vi.fn(async () => AGENT),
|
||||
resolveResponsibleUserId: vi.fn(async () => "user-1"),
|
||||
getRoutineEnv: vi.fn(async () => ({ routineId: null, env: null, responsibleUserId: null })),
|
||||
resolveSessionBeforeForWakeup: vi.fn(async () => null),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function createFakeWriter(overrides: Partial<WakeQueueWriter> = {}): WakeQueueWriter {
|
||||
return {
|
||||
claimNextDeferredWake: vi.fn(async () => null),
|
||||
getQueuedCommentLiveness: vi.fn(async () => ({ liveNonSelfCommentIds: [], containedSelfAuthoredComment: false })),
|
||||
cancelDeferredWake: vi.fn(async () => true),
|
||||
normalizeDeferredWakeCommentIds: vi.fn(async (input) => wakeCandidate({ id: input.wakeId, queuedCommentIds: input.liveCommentIds })),
|
||||
failDeferredWake: vi.fn(async () => true),
|
||||
getPauseHoldFacts: vi.fn(async () => ({
|
||||
activePauseHold: false,
|
||||
treeHoldInteractionWake: false,
|
||||
holdId: null,
|
||||
rootIssueId: null,
|
||||
mode: null,
|
||||
reason: null,
|
||||
releasePolicy: null,
|
||||
})),
|
||||
getCommentSelfAuthorship: vi.fn(async () => ({ allSelfAuthored: false })),
|
||||
reopenIssue: vi.fn(async () => null),
|
||||
claimDeferredWakeForPromotion: vi.fn(async () => true),
|
||||
finalizePromotedWake: vi.fn(async (input) => runSummary(input.wakeId)),
|
||||
hasExistingExecutionPath: vi.fn(async () => false),
|
||||
hasExplicitBlockerPath: vi.fn(async () => false),
|
||||
isAutomaticRecoverySuppressedByPauseHold: vi.fn(async () => false),
|
||||
buildBlockedRecoveryNotice: vi.fn(async () => ({ notice: {}, recoveryCause: null })),
|
||||
queueReviewParticipantRecoveryRun: vi.fn(async () => runSummary("review-recovery")),
|
||||
queueImmediateRecoveryRun: vi.fn(async () => runSummary("immediate-recovery")),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function createFakeIssueLock(reader: WakeQueueReader, writer: WakeQueueWriter): IssueLockWriter {
|
||||
return {
|
||||
withIssueExecutionLock: vi.fn(async (_input, fn) => {
|
||||
const result = await fn({ primaryIssue: ISSUE, run: RUN }, { reader, writer });
|
||||
return { ...result, run: RUN };
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function createFakeRecovery(): RecoveryEscalationPort {
|
||||
return {
|
||||
escalateStrandedAssignedIssue: vi.fn(async () => {}),
|
||||
escalateStrandedRecoveryIssueInPlace: vi.fn(async () => {}),
|
||||
};
|
||||
}
|
||||
|
||||
describe("releaseIssueExecution", () => {
|
||||
it("processes the deferred wakes in requestedAt order", async () => {
|
||||
const claimOrder: string[] = [];
|
||||
const queue = [wakeCandidate({ id: "wake-earliest" }), wakeCandidate({ id: "wake-latest" })];
|
||||
const writer = createFakeWriter({
|
||||
claimNextDeferredWake: vi.fn(async () => {
|
||||
const next = queue.shift() ?? null;
|
||||
if (next) claimOrder.push(next.id);
|
||||
return next;
|
||||
}),
|
||||
});
|
||||
// Every wake fails invokability so the loop keeps draining without promoting.
|
||||
const reader = createFakeReader({ findInvokableAgent: vi.fn(async () => null) });
|
||||
const issueLock = createFakeIssueLock(reader, writer);
|
||||
const releaseIssueExecution = createReleaseIssueExecution({ issueLock, recovery: createFakeRecovery() });
|
||||
|
||||
await releaseIssueExecution({ companyId: "company-1", runId: "run-1", now: new Date() });
|
||||
|
||||
expect(claimOrder).toEqual(["wake-earliest", "wake-latest"]);
|
||||
expect(writer.failDeferredWake).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("stops the loop after the first promotion", async () => {
|
||||
const claimNextDeferredWake = vi.fn(async () => wakeCandidate({ id: "wake-promotes" }));
|
||||
const writer = createFakeWriter({ claimNextDeferredWake });
|
||||
const reader = createFakeReader();
|
||||
const issueLock = createFakeIssueLock(reader, writer);
|
||||
const releaseIssueExecution = createReleaseIssueExecution({ issueLock, recovery: createFakeRecovery() });
|
||||
|
||||
const result = await releaseIssueExecution({ companyId: "company-1", runId: "run-1", now: new Date() });
|
||||
|
||||
expect(result.outcome.kind).toBe("promoted");
|
||||
expect(claimNextDeferredWake).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("continues the loop after a cancel outcome, a fail outcome, and a normalize outcome, then promotes", async () => {
|
||||
const queue = [
|
||||
// cancel_empty: queued comments, none live, no independent continuation.
|
||||
wakeCandidate({ id: "wake-cancel", queuedCommentIds: ["c1"] }),
|
||||
// fail_not_invokable: agent lookup misses for this one wake only.
|
||||
wakeCandidate({ id: "wake-fail", agentId: "uninvokable-agent" }),
|
||||
// normalize: queued comments differ from the live set, then promotes.
|
||||
wakeCandidate({ id: "wake-normalize", queuedCommentIds: ["c1", "c2"] }),
|
||||
];
|
||||
const claimNextDeferredWake = vi.fn(async () => queue.shift() ?? null);
|
||||
const findInvokableAgent = vi.fn(async (input: { agentId: string }) =>
|
||||
input.agentId === "deferred-agent" ? AGENT : null,
|
||||
);
|
||||
const getQueuedCommentLiveness = vi.fn(async (input: { queuedCommentIds: string[] }) =>
|
||||
input.queuedCommentIds.length === 1
|
||||
? { liveNonSelfCommentIds: [], containedSelfAuthoredComment: false }
|
||||
: { liveNonSelfCommentIds: ["c2"], containedSelfAuthoredComment: false },
|
||||
);
|
||||
const writer = createFakeWriter({ claimNextDeferredWake, getQueuedCommentLiveness });
|
||||
const reader = createFakeReader({ findInvokableAgent });
|
||||
const issueLock = createFakeIssueLock(reader, writer);
|
||||
const releaseIssueExecution = createReleaseIssueExecution({ issueLock, recovery: createFakeRecovery() });
|
||||
|
||||
const result = await releaseIssueExecution({ companyId: "company-1", runId: "run-1", now: new Date() });
|
||||
|
||||
expect(writer.cancelDeferredWake).toHaveBeenCalledTimes(1);
|
||||
expect(writer.failDeferredWake).toHaveBeenCalledTimes(1);
|
||||
expect(writer.normalizeDeferredWakeCommentIds).toHaveBeenCalledTimes(1);
|
||||
expect(claimNextDeferredWake).toHaveBeenCalledTimes(3);
|
||||
expect(result.outcome.kind).toBe("promoted");
|
||||
});
|
||||
|
||||
it("returns the post-commit effects as data without running them", async () => {
|
||||
const writer = createFakeWriter({ claimNextDeferredWake: vi.fn(async () => wakeCandidate()) });
|
||||
const reader = createFakeReader();
|
||||
const issueLock = createFakeIssueLock(reader, writer);
|
||||
const recovery = createFakeRecovery();
|
||||
const releaseIssueExecution = createReleaseIssueExecution({ issueLock, recovery });
|
||||
|
||||
const result = await releaseIssueExecution({ companyId: "company-1", runId: "run-1", now: new Date() });
|
||||
|
||||
expect(result.postCommitEffects).toEqual([{ kind: "run_queued", run: runSummary("wake-1") }]);
|
||||
expect(recovery.escalateStrandedAssignedIssue).not.toHaveBeenCalled();
|
||||
expect(recovery.escalateStrandedRecoveryIssueInPlace).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("carries the deferred wake's raw issue, interaction, execution-stage, and accepted-plan context onto the promoted run, and clears only the rendered text projections", async () => {
|
||||
const finalizePromotedWake = vi.fn(async (input: PromoteDeferredWakeInput) => runSummary(input.wakeId));
|
||||
const writer = createFakeWriter({
|
||||
claimNextDeferredWake: vi.fn(async () =>
|
||||
wakeCandidate({
|
||||
deferredContextSeed: {
|
||||
issueId: ISSUE.id,
|
||||
wakeCommentIds: ["comment-1"],
|
||||
// A queue-time render from a prior coalesced run. Promotion must
|
||||
// not persist this alongside the current (unrelated) comment id.
|
||||
paperclipTaskMarkdown: "queue-time markdown",
|
||||
paperclipTaskMarkdownCompact: "queue-time compact markdown",
|
||||
paperclipWake: { commentId: "comment-1" },
|
||||
executionStage: { stage: "review" },
|
||||
planReviewInteraction: { acceptedTargetRevision: { revisionId: "revision-1" } },
|
||||
acceptedPlanWakeRouting: { targetAgentId: "agent-1" },
|
||||
},
|
||||
}),
|
||||
),
|
||||
finalizePromotedWake,
|
||||
});
|
||||
const reader = createFakeReader();
|
||||
const issueLock = createFakeIssueLock(reader, writer);
|
||||
const releaseIssueExecution = createReleaseIssueExecution({ issueLock, recovery: createFakeRecovery() });
|
||||
|
||||
const result = await releaseIssueExecution({ companyId: "company-1", runId: "run-1", now: new Date() });
|
||||
|
||||
expect(result.outcome.kind).toBe("promoted");
|
||||
expect(finalizePromotedWake).toHaveBeenCalledTimes(1);
|
||||
const promotedContextSnapshot = finalizePromotedWake.mock.calls[0]![0].contextSnapshot;
|
||||
// The rendered text is cleared; `executeRun` rebuilds it, with proper
|
||||
// trust-based redaction, from the current issue and comment rows before
|
||||
// the run dispatches.
|
||||
expect(promotedContextSnapshot.paperclipTaskMarkdown).toBeUndefined();
|
||||
expect(promotedContextSnapshot.paperclipTaskMarkdownCompact).toBeUndefined();
|
||||
expect(promotedContextSnapshot.paperclipWake).toBeUndefined();
|
||||
// The raw fields that render depends on are not dropped.
|
||||
expect(promotedContextSnapshot.issueId).toBe(ISSUE.id);
|
||||
expect(promotedContextSnapshot.executionStage).toEqual({ stage: "review" });
|
||||
expect(promotedContextSnapshot.planReviewInteraction).toEqual({
|
||||
acceptedTargetRevision: { revisionId: "revision-1" },
|
||||
});
|
||||
expect(promotedContextSnapshot.acceptedPlanWakeRouting).toEqual({ targetAgentId: "agent-1" });
|
||||
});
|
||||
|
||||
it("never reopens the issue when the promotion claim loses the race, and moves on to the next wake", async () => {
|
||||
const doneIssue: IssueSnapshot = { ...ISSUE, status: "done" };
|
||||
const queue = [
|
||||
// Carries a comment that would reopen the done issue, but the
|
||||
// promotion claim below loses the race before that reopen can run.
|
||||
wakeCandidate({
|
||||
id: "wake-lost-race",
|
||||
deferredCommentIds: ["c1"],
|
||||
requestedByActorType: "user",
|
||||
}),
|
||||
wakeCandidate({ id: "wake-promotes" }),
|
||||
];
|
||||
const claimNextDeferredWake = vi.fn(async () => queue.shift() ?? null);
|
||||
const claimDeferredWakeForPromotion = vi.fn(async ({ wakeId }: { wakeId: string }) => wakeId !== "wake-lost-race");
|
||||
const reopenIssue = vi.fn(async () => null);
|
||||
const writer = createFakeWriter({ claimNextDeferredWake, claimDeferredWakeForPromotion, reopenIssue });
|
||||
const reader = createFakeReader();
|
||||
const issueLock: IssueLockWriter = {
|
||||
withIssueExecutionLock: vi.fn(async (_input, fn) => {
|
||||
const result = await fn({ primaryIssue: doneIssue, run: RUN }, { reader, writer });
|
||||
return { ...result, run: RUN };
|
||||
}),
|
||||
};
|
||||
const releaseIssueExecution = createReleaseIssueExecution({ issueLock, recovery: createFakeRecovery() });
|
||||
|
||||
const result = await releaseIssueExecution({ companyId: "company-1", runId: "run-1", now: new Date() });
|
||||
|
||||
expect(reopenIssue).not.toHaveBeenCalled();
|
||||
expect(claimDeferredWakeForPromotion).toHaveBeenCalledTimes(2);
|
||||
expect(result.outcome.kind).toBe("promoted");
|
||||
expect(result.postCommitEffects).toEqual([{ kind: "run_queued", run: runSummary("wake-promotes") }]);
|
||||
});
|
||||
|
||||
it("throws WakeQueueApplicationError with code responsible_user_unresolved when the responsible user cannot resolve", async () => {
|
||||
const writer = createFakeWriter({ claimNextDeferredWake: vi.fn(async () => wakeCandidate()) });
|
||||
const reader = createFakeReader({ resolveResponsibleUserId: vi.fn(async () => null) });
|
||||
const issueLock = createFakeIssueLock(reader, writer);
|
||||
const releaseIssueExecution = createReleaseIssueExecution({ issueLock, recovery: createFakeRecovery() });
|
||||
|
||||
await expect(
|
||||
releaseIssueExecution({ companyId: "company-1", runId: "run-1", now: new Date() }),
|
||||
).rejects.toMatchObject({
|
||||
constructor: WakeQueueApplicationError,
|
||||
code: "responsible_user_unresolved",
|
||||
});
|
||||
});
|
||||
|
||||
it("escalates through the recovery port for a blocked outcome, after the transaction resolves", async () => {
|
||||
const writer = createFakeWriter({
|
||||
claimNextDeferredWake: vi.fn(async () => null),
|
||||
hasExistingExecutionPath: vi.fn(async () => false),
|
||||
isAutomaticRecoverySuppressedByPauseHold: vi.fn(async () => false),
|
||||
buildBlockedRecoveryNotice: vi.fn(async () => ({ notice: { kind: "immediate_execution_path" }, recoveryCause: "immediate_execution_path" })),
|
||||
});
|
||||
// The recovery agent (the finishing run's own agent) is not invokable, which forces "blocked".
|
||||
const reader = createFakeReader({ findInvokableAgent: vi.fn(async () => null) });
|
||||
const issueLock = createFakeIssueLock(reader, writer);
|
||||
const recovery = createFakeRecovery();
|
||||
const releaseIssueExecution = createReleaseIssueExecution({ issueLock, recovery });
|
||||
|
||||
const result = await releaseIssueExecution({ companyId: "company-1", runId: "run-1", now: new Date() });
|
||||
|
||||
expect(result.outcome.kind).toBe("blocked");
|
||||
expect(recovery.escalateStrandedAssignedIssue).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,503 @@
|
|||
import { enrichPromotedWakeContext } from "../domain/context.js";
|
||||
import { decideDeferredWake, decideReleaseRecovery } from "../domain/policy.js";
|
||||
import type {
|
||||
IssueLockWriter,
|
||||
IssueSnapshot,
|
||||
LockedIssueExecution,
|
||||
RecoveryEscalationPort,
|
||||
ReleaseTransactionResult,
|
||||
RunSnapshot,
|
||||
WakeQueueReader,
|
||||
WakeQueueWriter,
|
||||
} from "./ports.js";
|
||||
import type { PostCommitEffect, ReleaseOutcome } from "./types.js";
|
||||
import { WakeQueueApplicationError } from "./types.js";
|
||||
|
||||
const ISSUE_DISPOSITION_REPAIR_RETRY_REASON = "issue_disposition_repair";
|
||||
const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON = "execution_review_participant_recovery";
|
||||
const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASONS = new Set([
|
||||
"execution_review_requested",
|
||||
"execution_approval_requested",
|
||||
]);
|
||||
const HEARTBEAT_RUN_TERMINAL_STATUSES = new Set([
|
||||
"succeeded",
|
||||
"failed",
|
||||
"timed_out",
|
||||
"cancelled",
|
||||
]);
|
||||
const UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES = new Set([
|
||||
"failed",
|
||||
"timed_out",
|
||||
"cancelled",
|
||||
]);
|
||||
const STRANDED_ISSUE_RECOVERY_ORIGIN_KIND = "stranded_issue_recovery";
|
||||
const WORKSPACE_VALIDATION_FAILURE_CODE = "workspace_validation_failed";
|
||||
const CONFIGURATION_INCOMPLETE_FAILURE_CODE = "configuration_incomplete";
|
||||
|
||||
function readNonEmptyString(value: unknown): string | null {
|
||||
return typeof value === "string" && value.trim().length > 0 ? value : null;
|
||||
}
|
||||
|
||||
function isWorkspaceValidationFailedRun(run: Pick<RunSnapshot, "errorCode">): boolean {
|
||||
return run.errorCode === WORKSPACE_VALIDATION_FAILURE_CODE;
|
||||
}
|
||||
|
||||
function isConfigurationIncompleteFailedRun(run: Pick<RunSnapshot, "errorCode">): boolean {
|
||||
return run.errorCode === CONFIGURATION_INCOMPLETE_FAILURE_CODE || run.errorCode === "model_not_found";
|
||||
}
|
||||
|
||||
function isExecutionReviewParticipantRecoveryRun(run: Pick<RunSnapshot, "contextSnapshot">): boolean {
|
||||
return readNonEmptyString(run.contextSnapshot.retryReason) === EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON;
|
||||
}
|
||||
|
||||
function isExecutionReviewParticipantRecoveryEligibleRun(run: Pick<RunSnapshot, "contextSnapshot">): boolean {
|
||||
const wakeReason = readNonEmptyString(run.contextSnapshot.wakeReason);
|
||||
return (wakeReason !== null && EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASONS.has(wakeReason))
|
||||
|| isExecutionReviewParticipantRecoveryRun(run);
|
||||
}
|
||||
|
||||
function didAutomaticRecoveryFail(
|
||||
run: Pick<RunSnapshot, "status" | "contextSnapshot">,
|
||||
expectedRetryReason: "assignment_recovery" | "issue_continuation_needed",
|
||||
): boolean {
|
||||
const latestRetryReason = readNonEmptyString(run.contextSnapshot.retryReason);
|
||||
return latestRetryReason === expectedRetryReason && UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES.has(run.status);
|
||||
}
|
||||
|
||||
function currentAgentParticipant(issue: IssueSnapshot): { agentId: string } | null {
|
||||
const executionState = issue.executionState;
|
||||
if (!executionState || executionState.status !== "pending") return null;
|
||||
const participant = executionState.currentParticipant as Record<string, unknown> | null | undefined;
|
||||
if (!participant || participant.type !== "agent") return null;
|
||||
const agentId = readNonEmptyString(participant.agentId);
|
||||
return agentId ? { agentId } : null;
|
||||
}
|
||||
|
||||
export type ReleaseIssueExecutionInput = {
|
||||
companyId: string;
|
||||
runId: string;
|
||||
now: Date;
|
||||
suppressImmediateRecovery?: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
* Drains the deferred-wake queue for the issue a run just released, in
|
||||
* `requestedAt` order, promoting at most one wake. When the queue empties
|
||||
* without a promotion, decides the release-recovery outcome. Every read and
|
||||
* write happens through `ports`, already bound to the module's own
|
||||
* transaction by the caller.
|
||||
*/
|
||||
async function runReleaseDrain(
|
||||
locked: LockedIssueExecution,
|
||||
ports: { reader: WakeQueueReader; writer: WakeQueueWriter },
|
||||
input: ReleaseIssueExecutionInput,
|
||||
): Promise<ReleaseTransactionResult> {
|
||||
const { run } = locked;
|
||||
let issue = locked.primaryIssue;
|
||||
const postCommitEffects: PostCommitEffect[] = [];
|
||||
|
||||
while (true) {
|
||||
const candidate = await ports.writer.claimNextDeferredWake({ companyId: run.companyId, issueId: issue.id });
|
||||
if (!candidate) break;
|
||||
|
||||
let liveness = { liveNonSelfCommentIds: candidate.queuedCommentIds, containedSelfAuthoredComment: false };
|
||||
if (candidate.queuedCommentIds.length > 0) {
|
||||
liveness = await ports.writer.getQueuedCommentLiveness({
|
||||
companyId: run.companyId,
|
||||
issueId: issue.id,
|
||||
wakeAgentId: candidate.agentId,
|
||||
finishingRunId: run.id,
|
||||
finishingRunAgentId: run.agentId,
|
||||
queuedCommentIds: candidate.queuedCommentIds,
|
||||
});
|
||||
}
|
||||
const liveCommentIdsChanged =
|
||||
liveness.liveNonSelfCommentIds.length !== candidate.queuedCommentIds.length ||
|
||||
liveness.liveNonSelfCommentIds.some((id, index) => id !== candidate.queuedCommentIds[index]);
|
||||
|
||||
const deferredAgent = await ports.reader.findInvokableAgent({ companyId: run.companyId, agentId: candidate.agentId });
|
||||
const pauseHold = await ports.writer.getPauseHoldFacts({
|
||||
companyId: run.companyId,
|
||||
issueId: issue.id,
|
||||
wakeAgentId: candidate.agentId,
|
||||
deferredContextSeed: candidate.deferredContextSeed,
|
||||
requestedByActorType: candidate.requestedByActorType,
|
||||
requestedByActorId: candidate.requestedByActorId,
|
||||
});
|
||||
|
||||
let decision = decideDeferredWake({
|
||||
queuedComment: {
|
||||
hasQueuedCommentIds: candidate.queuedCommentIds.length > 0,
|
||||
liveNonSelfCommentIdsLength: liveness.liveNonSelfCommentIds.length,
|
||||
queuedCommentIdsLength: candidate.queuedCommentIds.length,
|
||||
liveCommentIdsChanged,
|
||||
containedSelfAuthoredComment: liveness.containedSelfAuthoredComment,
|
||||
preservesIndependentContinuation: candidate.preservesIndependentContinuation,
|
||||
},
|
||||
agent: { agentFound: deferredAgent !== null, invokable: deferredAgent?.invokable ?? false },
|
||||
pauseHold: { activePauseHold: pauseHold.activePauseHold, treeHoldInteractionWake: pauseHold.treeHoldInteractionWake },
|
||||
});
|
||||
|
||||
if (decision.kind === "cancel_empty") {
|
||||
await ports.writer.cancelDeferredWake({
|
||||
companyId: run.companyId,
|
||||
wakeId: candidate.id,
|
||||
reason: decision.selfAuthored
|
||||
? "Deferred wake contained only comments authored by the finishing run"
|
||||
: "Queued messages were discarded before promotion",
|
||||
now: input.now,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
let workingCandidate = candidate;
|
||||
if (decision.kind === "normalize") {
|
||||
const normalized = await ports.writer.normalizeDeferredWakeCommentIds({
|
||||
companyId: run.companyId,
|
||||
wakeId: candidate.id,
|
||||
payload: candidate.payload,
|
||||
liveCommentIds: liveness.liveNonSelfCommentIds,
|
||||
now: input.now,
|
||||
});
|
||||
if (!normalized) continue;
|
||||
workingCandidate = normalized;
|
||||
// Re-decide with the same agent/pause-hold facts already fetched above; the
|
||||
// comment-id set now matches, so only fail/cancel-pause-hold/promote can result.
|
||||
decision = decideDeferredWake({
|
||||
queuedComment: {
|
||||
hasQueuedCommentIds: workingCandidate.queuedCommentIds.length > 0,
|
||||
liveNonSelfCommentIdsLength: liveness.liveNonSelfCommentIds.length,
|
||||
queuedCommentIdsLength: liveness.liveNonSelfCommentIds.length,
|
||||
liveCommentIdsChanged: false,
|
||||
containedSelfAuthoredComment: liveness.containedSelfAuthoredComment,
|
||||
preservesIndependentContinuation: workingCandidate.preservesIndependentContinuation,
|
||||
},
|
||||
agent: { agentFound: deferredAgent !== null, invokable: deferredAgent?.invokable ?? false },
|
||||
pauseHold: { activePauseHold: pauseHold.activePauseHold, treeHoldInteractionWake: pauseHold.treeHoldInteractionWake },
|
||||
});
|
||||
}
|
||||
|
||||
if (decision.kind === "fail_not_invokable") {
|
||||
await ports.writer.failDeferredWake({ companyId: run.companyId, wakeId: workingCandidate.id, now: input.now });
|
||||
continue;
|
||||
}
|
||||
|
||||
if (decision.kind === "cancel_pause_hold") {
|
||||
await ports.writer.cancelDeferredWake({
|
||||
companyId: run.companyId,
|
||||
wakeId: workingCandidate.id,
|
||||
reason: "Deferred wake suppressed by active subtree pause hold",
|
||||
now: input.now,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// decision.kind === "promote"
|
||||
const invokableAgent = deferredAgent!;
|
||||
|
||||
// Claim the wake for promotion before any other write in this branch
|
||||
// (design choice: claim first, then reopen). A reopen write, or its
|
||||
// `issue_reopened` post-commit effect, must never survive a lost race on
|
||||
// this compare-and-set. When the claim fails, a concurrent writer already
|
||||
// changed the wake's status, so this candidate is gone; move on to the
|
||||
// next one instead of ending the drain.
|
||||
const claimedForPromotion = await ports.writer.claimDeferredWakeForPromotion({
|
||||
companyId: run.companyId,
|
||||
wakeId: workingCandidate.id,
|
||||
now: input.now,
|
||||
});
|
||||
if (!claimedForPromotion) continue;
|
||||
|
||||
let currentIssue = issue;
|
||||
|
||||
if (workingCandidate.deferredCommentIds.length > 0 && (currentIssue.status === "done" || currentIssue.status === "cancelled")) {
|
||||
const selfAuthorship = await ports.writer.getCommentSelfAuthorship({
|
||||
companyId: run.companyId,
|
||||
issueId: currentIssue.id,
|
||||
finishingRunId: run.id,
|
||||
commentIds: workingCandidate.deferredCommentIds,
|
||||
});
|
||||
const shouldReopen =
|
||||
!selfAuthorship.allSelfAuthored &&
|
||||
(workingCandidate.requestedByActorType === "user" || workingCandidate.wakeReason === "issue_reopened_via_comment");
|
||||
if (shouldReopen) {
|
||||
const reopened = await ports.writer.reopenIssue({ companyId: run.companyId, issueId: currentIssue.id, runId: run.id });
|
||||
if (reopened) {
|
||||
postCommitEffects.push({
|
||||
kind: "issue_reopened",
|
||||
companyId: reopened.companyId,
|
||||
agentId: invokableAgent.id,
|
||||
runId: run.id,
|
||||
issueId: reopened.id,
|
||||
identifier: reopened.identifier,
|
||||
reopenedFrom: currentIssue.status,
|
||||
});
|
||||
currentIssue = reopened;
|
||||
issue = reopened;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const promotedReason = workingCandidate.reason ?? "issue_execution_promoted";
|
||||
const promotedSource = workingCandidate.source ?? "automation";
|
||||
const promotedTriggerDetail = workingCandidate.triggerDetail ?? null;
|
||||
const promotedPayload = { ...workingCandidate.payload };
|
||||
delete promotedPayload["_paperclipWakeContext"];
|
||||
|
||||
const promotedContextSeed: Record<string, unknown> = { ...workingCandidate.deferredContextSeed };
|
||||
if (pauseHold.activePauseHold) {
|
||||
promotedContextSeed.treeHoldInteraction = true;
|
||||
promotedContextSeed.activeTreeHold = {
|
||||
holdId: pauseHold.holdId,
|
||||
rootIssueId: pauseHold.rootIssueId,
|
||||
mode: pauseHold.mode,
|
||||
reason: pauseHold.reason,
|
||||
releasePolicy: pauseHold.releasePolicy,
|
||||
interaction: true,
|
||||
};
|
||||
}
|
||||
|
||||
const { contextSnapshot: promotedContextSnapshot, taskKey: promotedTaskKey } = enrichPromotedWakeContext({
|
||||
contextSnapshot: promotedContextSeed,
|
||||
reason: promotedReason,
|
||||
source: promotedSource,
|
||||
triggerDetail: promotedTriggerDetail,
|
||||
payload: promotedPayload,
|
||||
});
|
||||
|
||||
const sessionBefore =
|
||||
readNonEmptyString(promotedContextSnapshot.resumeSessionDisplayId) ??
|
||||
(await ports.reader.resolveSessionBeforeForWakeup({
|
||||
companyId: run.companyId,
|
||||
agentId: invokableAgent.id,
|
||||
taskKey: promotedTaskKey,
|
||||
}));
|
||||
|
||||
const promotedRoutineEnvContext = await ports.reader.getRoutineEnv({
|
||||
companyId: invokableAgent.companyId,
|
||||
issue: currentIssue,
|
||||
});
|
||||
const responsibleUserId = await ports.reader.resolveResponsibleUserId({
|
||||
companyId: invokableAgent.companyId,
|
||||
contextSnapshot: promotedContextSnapshot,
|
||||
issue: currentIssue,
|
||||
routineEnvContext: promotedRoutineEnvContext,
|
||||
requestedByActorType: workingCandidate.requestedByActorType as "user" | "agent" | "system" | null,
|
||||
requestedByActorId: workingCandidate.requestedByActorId,
|
||||
source: promotedSource,
|
||||
triggerDetail: promotedTriggerDetail,
|
||||
existingRunResponsibleUserId: run.responsibleUserId,
|
||||
});
|
||||
if (!responsibleUserId) {
|
||||
throw new WakeQueueApplicationError(
|
||||
"responsible_user_unresolved",
|
||||
"Unable to resolve responsible user for promoted heartbeat run",
|
||||
{
|
||||
runId: run.id,
|
||||
agentId: invokableAgent.id,
|
||||
companyId: invokableAgent.companyId,
|
||||
issueId: currentIssue.id,
|
||||
wakeReason: readNonEmptyString(promotedContextSnapshot.wakeReason),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
const promotedRun = await ports.writer.finalizePromotedWake({
|
||||
companyId: run.companyId,
|
||||
wakeId: workingCandidate.id,
|
||||
deferredAgent: invokableAgent,
|
||||
issue: currentIssue,
|
||||
finishingRun: run,
|
||||
contextSnapshot: promotedContextSnapshot,
|
||||
reason: promotedReason,
|
||||
source: promotedSource,
|
||||
triggerDetail: promotedTriggerDetail,
|
||||
payload: promotedPayload,
|
||||
responsibleUserId,
|
||||
sessionBefore,
|
||||
now: input.now,
|
||||
});
|
||||
|
||||
postCommitEffects.push({ kind: "run_queued", run: promotedRun });
|
||||
return { outcome: { kind: "promoted", run: promotedRun }, postCommitEffects };
|
||||
}
|
||||
|
||||
return runReleaseRecoveryTail(issue, run, ports.reader, ports.writer, input, postCommitEffects);
|
||||
}
|
||||
|
||||
async function runReleaseRecoveryTail(
|
||||
issue: IssueSnapshot,
|
||||
run: RunSnapshot,
|
||||
reader: WakeQueueReader,
|
||||
writer: WakeQueueWriter,
|
||||
input: ReleaseIssueExecutionInput,
|
||||
postCommitEffects: PostCommitEffect[],
|
||||
): Promise<ReleaseTransactionResult> {
|
||||
const suppressImmediateRecovery = input.suppressImmediateRecovery ?? false;
|
||||
const isStrandedRecoveryOrigin = issue.originKind === STRANDED_ISSUE_RECOVERY_ORIGIN_KIND;
|
||||
const recoveryAgent = await reader.findInvokableAgent({ companyId: issue.companyId, agentId: run.agentId });
|
||||
|
||||
const currentParticipant = currentAgentParticipant(issue);
|
||||
const reviewParticipantApplies =
|
||||
issue.status === "in_review" &&
|
||||
!issue.assigneeUserId &&
|
||||
currentParticipant !== null &&
|
||||
currentParticipant.agentId === run.agentId &&
|
||||
isExecutionReviewParticipantRecoveryEligibleRun(run) &&
|
||||
HEARTBEAT_RUN_TERMINAL_STATUSES.has(run.status);
|
||||
|
||||
const immediateApplies =
|
||||
(issue.status === "todo" || issue.status === "in_progress") &&
|
||||
!issue.assigneeUserId &&
|
||||
!issue.hiddenAt &&
|
||||
issue.assigneeAgentId === run.agentId &&
|
||||
(run.status === "failed" || run.status === "timed_out" || run.status === "cancelled");
|
||||
|
||||
const suppressedByPauseHold = (reviewParticipantApplies || immediateApplies)
|
||||
? await writer.isAutomaticRecoverySuppressedByPauseHold({ companyId: issue.companyId, issueId: issue.id })
|
||||
: false;
|
||||
|
||||
const hasExistingExecutionPath = reviewParticipantApplies
|
||||
? await writer.hasExistingExecutionPath({
|
||||
companyId: issue.companyId,
|
||||
issueId: issue.id,
|
||||
excludeRunId: run.id,
|
||||
agentId: currentParticipant?.agentId ?? null,
|
||||
})
|
||||
: immediateApplies
|
||||
? await writer.hasExistingExecutionPath({ companyId: issue.companyId, issueId: issue.id, excludeRunId: run.id, agentId: null })
|
||||
: false;
|
||||
|
||||
const hasExplicitBlockerPath = immediateApplies && !reviewParticipantApplies
|
||||
? await writer.hasExplicitBlockerPath({ companyId: issue.companyId, issueId: issue.id })
|
||||
: false;
|
||||
|
||||
const expectedRetryReason: "assignment_recovery" | "issue_continuation_needed" =
|
||||
issue.status === "todo" ? "assignment_recovery" : "issue_continuation_needed";
|
||||
|
||||
const decision = decideReleaseRecovery({
|
||||
suppressImmediateRecovery,
|
||||
reviewParticipant: {
|
||||
applies: reviewParticipantApplies,
|
||||
hasExistingExecutionPath,
|
||||
hasPersistedMonitor: Boolean(issue.monitorNextCheckAt),
|
||||
suppressedByPauseHold,
|
||||
isStrandedRecoveryOrigin,
|
||||
recoveryAgentPresent: recoveryAgent !== null,
|
||||
recoveryAgentInvokable: recoveryAgent?.invokable ?? false,
|
||||
isExecutionReviewParticipantRecoveryRun: isExecutionReviewParticipantRecoveryRun(run),
|
||||
},
|
||||
immediate: {
|
||||
applies: immediateApplies,
|
||||
isDispositionRepairRetry: readNonEmptyString(run.contextSnapshot.retryReason) === ISSUE_DISPOSITION_REPAIR_RETRY_REASON,
|
||||
hasExistingExecutionPath,
|
||||
hasPersistedMonitor: Boolean(issue.monitorNextCheckAt),
|
||||
hasExplicitBlockerPath,
|
||||
suppressedByPauseHold,
|
||||
isStrandedRecoveryOrigin,
|
||||
recoveryAgentPresent: recoveryAgent !== null,
|
||||
recoveryAgentInvokable: recoveryAgent?.invokable ?? false,
|
||||
isWorkspaceValidationFailedRun: isWorkspaceValidationFailedRun(run),
|
||||
isConfigurationIncompleteFailedRun: isConfigurationIncompleteFailedRun(run),
|
||||
automaticRecoveryAlreadyFailed: didAutomaticRecoveryFail(run, expectedRetryReason),
|
||||
},
|
||||
});
|
||||
|
||||
if (decision.kind === "released") {
|
||||
return { outcome: { kind: "released" }, postCommitEffects };
|
||||
}
|
||||
|
||||
if (decision.kind === "blocked_recovery_in_place") {
|
||||
return {
|
||||
outcome: { kind: "blocked_recovery_in_place", issue, previousStatus: statusForBlock(issue) },
|
||||
postCommitEffects,
|
||||
};
|
||||
}
|
||||
|
||||
if (decision.kind === "blocked") {
|
||||
const { notice, recoveryCause } = await writer.buildBlockedRecoveryNotice({
|
||||
noticeKind: decision.notice,
|
||||
issueStatus: issue.status === "todo" ? "todo" : "in_progress",
|
||||
finishingRun: run,
|
||||
});
|
||||
return {
|
||||
outcome: {
|
||||
kind: "blocked",
|
||||
issue,
|
||||
previousStatus: statusForBlock(issue),
|
||||
notice,
|
||||
recoveryCause,
|
||||
},
|
||||
postCommitEffects,
|
||||
};
|
||||
}
|
||||
|
||||
const sessionBefore = await reader.resolveSessionBeforeForWakeup({
|
||||
companyId: issue.companyId,
|
||||
agentId: recoveryAgent!.id,
|
||||
taskKey: readNonEmptyString(run.contextSnapshot.taskKey) ?? readNonEmptyString(run.contextSnapshot.issueId),
|
||||
});
|
||||
|
||||
if (decision.kind === "queue_review_participant_recovery") {
|
||||
const queuedRun = await writer.queueReviewParticipantRecoveryRun({
|
||||
companyId: issue.companyId,
|
||||
issue,
|
||||
finishingRun: run,
|
||||
recoveryAgent: recoveryAgent!,
|
||||
sessionBefore,
|
||||
now: input.now,
|
||||
});
|
||||
postCommitEffects.push({ kind: "run_queued", run: queuedRun });
|
||||
return { outcome: { kind: "queued_review_participant_recovery", run: queuedRun }, postCommitEffects };
|
||||
}
|
||||
|
||||
// decision.kind === "queue_recovery"; the adapter builds the recovery
|
||||
// context snapshot and resolves the responsible user from it, throwing
|
||||
// WakeQueueApplicationError when no responsible user resolves.
|
||||
const queuedRun = await writer.queueImmediateRecoveryRun({
|
||||
companyId: issue.companyId,
|
||||
issue,
|
||||
finishingRun: run,
|
||||
recoveryAgent: recoveryAgent!,
|
||||
sessionBefore,
|
||||
now: input.now,
|
||||
});
|
||||
postCommitEffects.push({ kind: "run_queued", run: queuedRun });
|
||||
return { outcome: { kind: "queued_recovery", run: queuedRun }, postCommitEffects };
|
||||
}
|
||||
|
||||
function statusForBlock(issue: IssueSnapshot): "todo" | "in_progress" | "in_review" {
|
||||
return issue.status === "todo" || issue.status === "in_review" ? issue.status : "in_progress";
|
||||
}
|
||||
|
||||
export function createReleaseIssueExecution(deps: {
|
||||
issueLock: IssueLockWriter;
|
||||
recovery: RecoveryEscalationPort;
|
||||
}) {
|
||||
return async function releaseIssueExecution(
|
||||
input: ReleaseIssueExecutionInput,
|
||||
): Promise<{ outcome: ReleaseOutcome; postCommitEffects: PostCommitEffect[] }> {
|
||||
const result = await deps.issueLock.withIssueExecutionLock(
|
||||
{ companyId: input.companyId, runId: input.runId, now: input.now },
|
||||
(locked, ports) => runReleaseDrain(locked, ports, input),
|
||||
);
|
||||
|
||||
if (result.outcome.kind === "blocked") {
|
||||
await deps.recovery.escalateStrandedAssignedIssue({
|
||||
issue: result.outcome.issue,
|
||||
previousStatus: result.outcome.previousStatus,
|
||||
latestRun: result.run,
|
||||
notice: result.outcome.notice,
|
||||
recoveryCause: result.outcome.recoveryCause,
|
||||
});
|
||||
} else if (result.outcome.kind === "blocked_recovery_in_place") {
|
||||
await deps.recovery.escalateStrandedRecoveryIssueInPlace({
|
||||
issue: result.outcome.issue,
|
||||
previousStatus: result.outcome.previousStatus,
|
||||
latestRun: result.run,
|
||||
});
|
||||
}
|
||||
|
||||
return { outcome: result.outcome, postCommitEffects: result.postCommitEffects };
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,124 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { enrichPromotedWakeContext } from "./context.js";
|
||||
|
||||
describe("enrichPromotedWakeContext", () => {
|
||||
it("clears stale derived comment projections when the canonical id list is empty, but keeps an independent interaction continuation", () => {
|
||||
const contextSnapshot: Record<string, unknown> = {
|
||||
// Normalization already dropped the canonical wake comment ids upstream
|
||||
// of this function; no wakeCommentIds field remains on the snapshot.
|
||||
paperclipWake: { commentId: "comment-2" },
|
||||
paperclipWakeComment: { id: "comment-2", body: "stale body" },
|
||||
paperclipTaskMarkdown: "stale markdown",
|
||||
paperclipTaskMarkdownCompact: "stale compact markdown",
|
||||
// Independent interaction continuation the promoted run must keep.
|
||||
interactionId: "interaction-1",
|
||||
interactionKind: "request_confirmation",
|
||||
interactionStatus: "accepted",
|
||||
};
|
||||
|
||||
const result = enrichPromotedWakeContext({
|
||||
contextSnapshot,
|
||||
reason: "issue_execution_promoted",
|
||||
source: "automation",
|
||||
triggerDetail: null,
|
||||
payload: { mutation: "interaction" },
|
||||
});
|
||||
|
||||
expect(result.contextSnapshot.wakeCommentIds).toBeUndefined();
|
||||
expect(result.contextSnapshot.commentId).toBeUndefined();
|
||||
expect(result.contextSnapshot.wakeCommentId).toBeUndefined();
|
||||
expect(result.contextSnapshot.paperclipWake).toBeUndefined();
|
||||
expect(result.contextSnapshot.paperclipWakeComment).toBeUndefined();
|
||||
expect(result.contextSnapshot.paperclipTaskMarkdown).toBeUndefined();
|
||||
expect(result.contextSnapshot.paperclipTaskMarkdownCompact).toBeUndefined();
|
||||
// The independent interaction continuation survives the clear.
|
||||
expect(result.contextSnapshot.interactionId).toBe("interaction-1");
|
||||
expect(result.contextSnapshot.interactionStatus).toBe("accepted");
|
||||
});
|
||||
|
||||
it("sets the canonical id and latest-id fields when the recomputed list has entries, and still clears the stale projections", () => {
|
||||
const contextSnapshot: Record<string, unknown> = {
|
||||
wakeCommentIds: ["comment-1"],
|
||||
paperclipWake: { commentId: "comment-1" },
|
||||
paperclipWakeComment: { id: "comment-1", body: "will be rebuilt" },
|
||||
paperclipTaskMarkdown: "will be rebuilt",
|
||||
paperclipTaskMarkdownCompact: "will be rebuilt",
|
||||
};
|
||||
|
||||
const result = enrichPromotedWakeContext({
|
||||
contextSnapshot,
|
||||
reason: "issue_execution_promoted",
|
||||
source: "automation",
|
||||
triggerDetail: null,
|
||||
payload: {},
|
||||
});
|
||||
|
||||
expect(result.contextSnapshot.wakeCommentIds).toEqual(["comment-1"]);
|
||||
expect(result.contextSnapshot.commentId).toBe("comment-1");
|
||||
expect(result.contextSnapshot.wakeCommentId).toBe("comment-1");
|
||||
// The derived projections are cleared so the run rebuilds them from the
|
||||
// canonical ids instead of carrying forward a queue-time snapshot.
|
||||
expect(result.contextSnapshot.paperclipWake).toBeUndefined();
|
||||
expect(result.contextSnapshot.paperclipWakeComment).toBeUndefined();
|
||||
expect(result.contextSnapshot.paperclipTaskMarkdown).toBeUndefined();
|
||||
expect(result.contextSnapshot.paperclipTaskMarkdownCompact).toBeUndefined();
|
||||
});
|
||||
|
||||
it("adds a new id from the payload to the canonical list and clears the stale projections", () => {
|
||||
const contextSnapshot: Record<string, unknown> = {
|
||||
wakeCommentIds: ["comment-1"],
|
||||
paperclipWakeComment: { id: "comment-1", body: "stale body" },
|
||||
};
|
||||
|
||||
const result = enrichPromotedWakeContext({
|
||||
contextSnapshot,
|
||||
reason: "issue_execution_promoted",
|
||||
source: "automation",
|
||||
triggerDetail: null,
|
||||
payload: { commentId: "comment-2" },
|
||||
});
|
||||
|
||||
expect(result.contextSnapshot.wakeCommentIds).toEqual(["comment-1", "comment-2"]);
|
||||
expect(result.contextSnapshot.commentId).toBe("comment-2");
|
||||
expect(result.contextSnapshot.wakeCommentId).toBe("comment-2");
|
||||
expect(result.contextSnapshot.paperclipWakeComment).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps the raw issue, execution-stage, and accepted-plan fields a dispatch rebuild needs, alongside the cleared text projections", () => {
|
||||
const contextSnapshot: Record<string, unknown> = {
|
||||
wakeCommentIds: ["comment-1"],
|
||||
paperclipWake: { commentId: "comment-1" },
|
||||
paperclipTaskMarkdown: "stale markdown",
|
||||
paperclipTaskMarkdownCompact: "stale compact markdown",
|
||||
// Raw context a dispatch-time rebuild reads directly off the promoted
|
||||
// run; none of it is derived from the comment ids above, so none of it
|
||||
// should be cleared alongside the stale text projections.
|
||||
issueId: "issue-1",
|
||||
executionStage: { stage: "review" },
|
||||
planReviewInteraction: { acceptedTargetRevision: { revisionId: "revision-1" } },
|
||||
acceptedPlanWakeRouting: { targetAgentId: "agent-1" },
|
||||
workspaceRefreshReason: "accepted_plan_confirmation",
|
||||
};
|
||||
|
||||
const result = enrichPromotedWakeContext({
|
||||
contextSnapshot,
|
||||
reason: "issue_execution_promoted",
|
||||
source: "automation",
|
||||
triggerDetail: null,
|
||||
payload: {},
|
||||
});
|
||||
|
||||
// The rendered text is cleared; dispatch rebuilds it from the raw fields.
|
||||
expect(result.contextSnapshot.paperclipWake).toBeUndefined();
|
||||
expect(result.contextSnapshot.paperclipTaskMarkdown).toBeUndefined();
|
||||
expect(result.contextSnapshot.paperclipTaskMarkdownCompact).toBeUndefined();
|
||||
// The raw fields the rebuild needs are untouched.
|
||||
expect(result.contextSnapshot.issueId).toBe("issue-1");
|
||||
expect(result.contextSnapshot.executionStage).toEqual({ stage: "review" });
|
||||
expect(result.contextSnapshot.planReviewInteraction).toEqual({
|
||||
acceptedTargetRevision: { revisionId: "revision-1" },
|
||||
});
|
||||
expect(result.contextSnapshot.acceptedPlanWakeRouting).toEqual({ targetAgentId: "agent-1" });
|
||||
expect(result.contextSnapshot.workspaceRefreshReason).toBe("accepted_plan_confirmation");
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,165 @@
|
|||
// Pure helpers that build the context snapshot for a promoted deferred wake.
|
||||
// `extractWakeCommentIds`, `deriveCommentId`, and `WAKE_COMMENT_IDS_KEY` come
|
||||
// from the run-dispatch module so the two modules read one wake context
|
||||
// shape; this file holds only the small pieces that are specific to
|
||||
// building the release half's promoted-run snapshot.
|
||||
|
||||
import { extractWakeCommentIds, WAKE_COMMENT_IDS_KEY } from "../../run-dispatch/index.js";
|
||||
|
||||
const PAPERCLIP_WAKE_PAYLOAD_KEY = "paperclipWake";
|
||||
const PAPERCLIP_WAKE_COMMENT_KEY = "paperclipWakeComment";
|
||||
const PAPERCLIP_TASK_MARKDOWN_KEY = "paperclipTaskMarkdown";
|
||||
const PAPERCLIP_TASK_MARKDOWN_COMPACT_KEY = "paperclipTaskMarkdownCompact";
|
||||
|
||||
const INTERACTION_CONTINUATION_CONTEXT_KEYS = [
|
||||
"interactionId",
|
||||
"interactionKind",
|
||||
"interactionStatus",
|
||||
"continuationPolicy",
|
||||
"checkboxSelection",
|
||||
"itemVerdicts",
|
||||
"newlyResolvedItemIds",
|
||||
] as const;
|
||||
|
||||
function readNonEmptyString(value: unknown): string | null {
|
||||
return typeof value === "string" && value.trim().length > 0 ? value : null;
|
||||
}
|
||||
|
||||
function deriveTaskKey(
|
||||
contextSnapshot: Record<string, unknown> | null | undefined,
|
||||
payload: Record<string, unknown> | null | undefined,
|
||||
): string | null {
|
||||
return (
|
||||
readNonEmptyString(contextSnapshot?.taskKey) ??
|
||||
readNonEmptyString(contextSnapshot?.taskId) ??
|
||||
readNonEmptyString(contextSnapshot?.issueId) ??
|
||||
readNonEmptyString(payload?.taskKey) ??
|
||||
readNonEmptyString(payload?.taskId) ??
|
||||
readNonEmptyString(payload?.issueId) ??
|
||||
null
|
||||
);
|
||||
}
|
||||
|
||||
function mergeWakeCommentIds(...values: Array<unknown>): string[] {
|
||||
const merged: string[] = [];
|
||||
const append = (value: unknown) => {
|
||||
const normalized = readNonEmptyString(value);
|
||||
if (!normalized || merged.includes(normalized)) return;
|
||||
merged.push(normalized);
|
||||
};
|
||||
for (const value of values) {
|
||||
if (Array.isArray(value)) {
|
||||
for (const entry of value) append(entry);
|
||||
continue;
|
||||
}
|
||||
if (typeof value === "object" && value !== null) {
|
||||
const candidate = value as Record<string, unknown>;
|
||||
const batched = extractWakeCommentIds(candidate);
|
||||
if (batched.length > 0) {
|
||||
for (const entry of batched) append(entry);
|
||||
continue;
|
||||
}
|
||||
append(candidate.wakeCommentId);
|
||||
append(candidate.commentId);
|
||||
continue;
|
||||
}
|
||||
append(value);
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
export function hasInteractionContinuationWakeContext(contextSnapshot: Record<string, unknown>): boolean {
|
||||
return INTERACTION_CONTINUATION_CONTEXT_KEYS.some((key) => readNonEmptyString(contextSnapshot[key]));
|
||||
}
|
||||
|
||||
function isInteractionResolutionWakePayload(payload: Record<string, unknown> | null | undefined): boolean {
|
||||
return readNonEmptyString(payload?.mutation) === "interaction";
|
||||
}
|
||||
|
||||
function normalizeInteractionContinuationWakeContext(
|
||||
contextSnapshot: Record<string, unknown>,
|
||||
payload: Record<string, unknown> | null | undefined,
|
||||
): void {
|
||||
if (isInteractionResolutionWakePayload(payload)) return;
|
||||
for (const key of INTERACTION_CONTINUATION_CONTEXT_KEYS) {
|
||||
delete contextSnapshot[key];
|
||||
}
|
||||
}
|
||||
|
||||
export type EnrichPromotedWakeContextInput = {
|
||||
contextSnapshot: Record<string, unknown>;
|
||||
reason: string | null;
|
||||
source: string | null;
|
||||
triggerDetail: string | null;
|
||||
payload: Record<string, unknown> | null;
|
||||
};
|
||||
|
||||
export type EnrichPromotedWakeContextResult = {
|
||||
contextSnapshot: Record<string, unknown>;
|
||||
taskKey: string | null;
|
||||
};
|
||||
|
||||
/**
|
||||
* Fills the promoted run's context snapshot with the fields the original
|
||||
* wake enrichment always fills, without overwriting a field the deferred
|
||||
* wake already carried. The deferred wake's own context snapshot was
|
||||
* already enriched once when it was first queued, so most calls only
|
||||
* normalize the interaction-continuation keys.
|
||||
*/
|
||||
export function enrichPromotedWakeContext(
|
||||
input: EnrichPromotedWakeContextInput,
|
||||
): EnrichPromotedWakeContextResult {
|
||||
const contextSnapshot = { ...input.contextSnapshot };
|
||||
const { reason, source, triggerDetail, payload } = input;
|
||||
const commentIdFromPayload = readNonEmptyString(payload?.["commentId"]);
|
||||
const taskKey = deriveTaskKey(contextSnapshot, payload);
|
||||
const wakeCommentIds = mergeWakeCommentIds(contextSnapshot, commentIdFromPayload);
|
||||
|
||||
if (!readNonEmptyString(contextSnapshot["wakeReason"]) && reason) {
|
||||
contextSnapshot.wakeReason = reason;
|
||||
}
|
||||
if (!readNonEmptyString(contextSnapshot["taskKey"]) && taskKey) {
|
||||
contextSnapshot.taskKey = taskKey;
|
||||
}
|
||||
if (!readNonEmptyString(contextSnapshot["commentId"]) && commentIdFromPayload) {
|
||||
contextSnapshot.commentId = commentIdFromPayload;
|
||||
}
|
||||
// The wake payload, resolved comment, and task-markdown snapshots below are
|
||||
// rendered text built from the canonical comment ids and the issue state at
|
||||
// queue time. This function recomputes the canonical ids on every call
|
||||
// (`wakeCommentIds`), so it must not let a rendered snapshot from a stale
|
||||
// id list carry forward: clear all four, then restore only the canonical
|
||||
// id and latest-id fields, and only when the recomputed list actually has
|
||||
// entries.
|
||||
//
|
||||
// This function does not render a replacement in its place. The
|
||||
// promoted run still carries every raw field the render depends on
|
||||
// (`issueId`, `wakeReason`, interaction and accepted-plan fields, and the
|
||||
// canonical comment ids set below): `heartbeatService`'s `executeRun`
|
||||
// rebuilds the rendered text from those raw fields and the current issue
|
||||
// and comment rows, with the trust-based redaction that rendering needs,
|
||||
// before it persists the run and dispatches it to the agent. Rendering a
|
||||
// second, simplified copy here would either skip that redaction (an unsafe
|
||||
// shortcut for quarantined comment content) or duplicate it outside the
|
||||
// one place it is proven correct, so this function leaves the render to
|
||||
// dispatch and only guarantees the raw inputs survive.
|
||||
delete contextSnapshot[PAPERCLIP_WAKE_PAYLOAD_KEY];
|
||||
delete contextSnapshot[PAPERCLIP_WAKE_COMMENT_KEY];
|
||||
delete contextSnapshot[PAPERCLIP_TASK_MARKDOWN_KEY];
|
||||
delete contextSnapshot[PAPERCLIP_TASK_MARKDOWN_COMPACT_KEY];
|
||||
if (wakeCommentIds.length > 0) {
|
||||
const latestCommentId = wakeCommentIds[wakeCommentIds.length - 1];
|
||||
contextSnapshot[WAKE_COMMENT_IDS_KEY] = wakeCommentIds;
|
||||
contextSnapshot.commentId = latestCommentId;
|
||||
contextSnapshot.wakeCommentId = latestCommentId;
|
||||
}
|
||||
if (!readNonEmptyString(contextSnapshot["wakeSource"]) && source) {
|
||||
contextSnapshot.wakeSource = source;
|
||||
}
|
||||
if (!readNonEmptyString(contextSnapshot["wakeTriggerDetail"]) && triggerDetail) {
|
||||
contextSnapshot.wakeTriggerDetail = triggerDetail;
|
||||
}
|
||||
normalizeInteractionContinuationWakeContext(contextSnapshot, payload);
|
||||
|
||||
return { contextSnapshot, taskKey };
|
||||
}
|
||||
|
|
@ -0,0 +1,353 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
decideDeferredWake,
|
||||
decideReleaseRecovery,
|
||||
type DeferredWakeFacts,
|
||||
type ReleaseRecoveryFacts,
|
||||
} from "./policy.js";
|
||||
|
||||
const baseDeferredWakeFacts: DeferredWakeFacts = {
|
||||
queuedComment: {
|
||||
hasQueuedCommentIds: false,
|
||||
liveNonSelfCommentIdsLength: 0,
|
||||
queuedCommentIdsLength: 0,
|
||||
liveCommentIdsChanged: false,
|
||||
containedSelfAuthoredComment: false,
|
||||
preservesIndependentContinuation: false,
|
||||
},
|
||||
agent: { agentFound: true, invokable: true },
|
||||
pauseHold: { activePauseHold: false, treeHoldInteractionWake: false },
|
||||
};
|
||||
|
||||
describe("decideDeferredWake", () => {
|
||||
const cases: Array<{
|
||||
name: string;
|
||||
facts: DeferredWakeFacts;
|
||||
expected: ReturnType<typeof decideDeferredWake>;
|
||||
}> = [
|
||||
{
|
||||
name: "cancel_empty: all queued comments discarded and no independent continuation",
|
||||
facts: {
|
||||
...baseDeferredWakeFacts,
|
||||
queuedComment: {
|
||||
hasQueuedCommentIds: true,
|
||||
liveNonSelfCommentIdsLength: 0,
|
||||
queuedCommentIdsLength: 2,
|
||||
liveCommentIdsChanged: true,
|
||||
containedSelfAuthoredComment: false,
|
||||
preservesIndependentContinuation: false,
|
||||
},
|
||||
},
|
||||
expected: { kind: "cancel_empty", selfAuthored: false },
|
||||
},
|
||||
{
|
||||
name: "cancel_empty: self-authored comments discarded, error text reflects self-authorship",
|
||||
facts: {
|
||||
...baseDeferredWakeFacts,
|
||||
queuedComment: {
|
||||
hasQueuedCommentIds: true,
|
||||
liveNonSelfCommentIdsLength: 0,
|
||||
queuedCommentIdsLength: 1,
|
||||
liveCommentIdsChanged: true,
|
||||
containedSelfAuthoredComment: true,
|
||||
preservesIndependentContinuation: false,
|
||||
},
|
||||
},
|
||||
expected: { kind: "cancel_empty", selfAuthored: true },
|
||||
},
|
||||
{
|
||||
name: "normalize: no live comments, but an independent continuation reason still rewrites the queued id list",
|
||||
facts: {
|
||||
...baseDeferredWakeFacts,
|
||||
queuedComment: {
|
||||
hasQueuedCommentIds: true,
|
||||
liveNonSelfCommentIdsLength: 0,
|
||||
queuedCommentIdsLength: 1,
|
||||
liveCommentIdsChanged: true,
|
||||
containedSelfAuthoredComment: false,
|
||||
preservesIndependentContinuation: true,
|
||||
},
|
||||
},
|
||||
expected: { kind: "normalize" },
|
||||
},
|
||||
{
|
||||
name: "promote: an independent continuation reason keeps the wake alive with the live id set already matching",
|
||||
facts: {
|
||||
...baseDeferredWakeFacts,
|
||||
queuedComment: {
|
||||
hasQueuedCommentIds: true,
|
||||
liveNonSelfCommentIdsLength: 0,
|
||||
queuedCommentIdsLength: 0,
|
||||
liveCommentIdsChanged: false,
|
||||
containedSelfAuthoredComment: false,
|
||||
preservesIndependentContinuation: true,
|
||||
},
|
||||
},
|
||||
expected: { kind: "promote" },
|
||||
},
|
||||
{
|
||||
name: "normalize: the live comment id set differs from the queued set",
|
||||
facts: {
|
||||
...baseDeferredWakeFacts,
|
||||
queuedComment: {
|
||||
hasQueuedCommentIds: true,
|
||||
liveNonSelfCommentIdsLength: 1,
|
||||
queuedCommentIdsLength: 2,
|
||||
liveCommentIdsChanged: true,
|
||||
containedSelfAuthoredComment: false,
|
||||
preservesIndependentContinuation: false,
|
||||
},
|
||||
},
|
||||
expected: { kind: "normalize" },
|
||||
},
|
||||
{
|
||||
name: "fail_not_invokable: the agent lookup returns not-found",
|
||||
facts: {
|
||||
...baseDeferredWakeFacts,
|
||||
agent: { agentFound: false, invokable: false },
|
||||
},
|
||||
expected: { kind: "fail_not_invokable" },
|
||||
},
|
||||
{
|
||||
name: "fail_not_invokable: the agent is found but not invokable",
|
||||
facts: {
|
||||
...baseDeferredWakeFacts,
|
||||
agent: { agentFound: true, invokable: false },
|
||||
},
|
||||
expected: { kind: "fail_not_invokable" },
|
||||
},
|
||||
{
|
||||
name: "cancel_pause_hold: an active pause hold with no verified tree-hold interaction",
|
||||
facts: {
|
||||
...baseDeferredWakeFacts,
|
||||
pauseHold: { activePauseHold: true, treeHoldInteractionWake: false },
|
||||
},
|
||||
expected: { kind: "cancel_pause_hold" },
|
||||
},
|
||||
{
|
||||
name: "promote: an active pause hold but a verified tree-hold interaction wake survives it",
|
||||
facts: {
|
||||
...baseDeferredWakeFacts,
|
||||
pauseHold: { activePauseHold: true, treeHoldInteractionWake: true },
|
||||
},
|
||||
expected: { kind: "promote" },
|
||||
},
|
||||
{
|
||||
name: "promote: no queued comments, an invokable agent, and no pause hold",
|
||||
facts: baseDeferredWakeFacts,
|
||||
expected: { kind: "promote" },
|
||||
},
|
||||
];
|
||||
|
||||
for (const testCase of cases) {
|
||||
it(testCase.name, () => {
|
||||
expect(decideDeferredWake(testCase.facts)).toEqual(testCase.expected);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
const baseReleaseRecoveryFacts: ReleaseRecoveryFacts = {
|
||||
suppressImmediateRecovery: false,
|
||||
reviewParticipant: {
|
||||
applies: false,
|
||||
hasExistingExecutionPath: false,
|
||||
hasPersistedMonitor: false,
|
||||
suppressedByPauseHold: false,
|
||||
isStrandedRecoveryOrigin: false,
|
||||
recoveryAgentPresent: true,
|
||||
recoveryAgentInvokable: true,
|
||||
isExecutionReviewParticipantRecoveryRun: false,
|
||||
},
|
||||
immediate: {
|
||||
applies: false,
|
||||
isDispositionRepairRetry: false,
|
||||
hasExistingExecutionPath: false,
|
||||
hasPersistedMonitor: false,
|
||||
hasExplicitBlockerPath: false,
|
||||
suppressedByPauseHold: false,
|
||||
isStrandedRecoveryOrigin: false,
|
||||
recoveryAgentPresent: true,
|
||||
recoveryAgentInvokable: true,
|
||||
isWorkspaceValidationFailedRun: false,
|
||||
isConfigurationIncompleteFailedRun: false,
|
||||
automaticRecoveryAlreadyFailed: false,
|
||||
},
|
||||
};
|
||||
|
||||
describe("decideReleaseRecovery", () => {
|
||||
const cases: Array<{
|
||||
name: string;
|
||||
facts: ReleaseRecoveryFacts;
|
||||
expected: ReturnType<typeof decideReleaseRecovery>;
|
||||
}> = [
|
||||
{
|
||||
name: "released: neither the review-participant nor the immediate-recovery branch applies",
|
||||
facts: baseReleaseRecoveryFacts,
|
||||
expected: { kind: "released" },
|
||||
},
|
||||
{
|
||||
name: "released: immediate recovery applies but the caller asked to suppress it",
|
||||
facts: {
|
||||
...baseReleaseRecoveryFacts,
|
||||
suppressImmediateRecovery: true,
|
||||
immediate: { ...baseReleaseRecoveryFacts.immediate, applies: true },
|
||||
},
|
||||
expected: { kind: "released" },
|
||||
},
|
||||
{
|
||||
name: "released: immediate recovery applies but an existing execution path already covers it",
|
||||
facts: {
|
||||
...baseReleaseRecoveryFacts,
|
||||
immediate: {
|
||||
...baseReleaseRecoveryFacts.immediate,
|
||||
applies: true,
|
||||
hasExistingExecutionPath: true,
|
||||
},
|
||||
},
|
||||
expected: { kind: "released" },
|
||||
},
|
||||
{
|
||||
name: "released: immediate recovery applies but the finishing run carried the disposition-repair retry reason",
|
||||
facts: {
|
||||
...baseReleaseRecoveryFacts,
|
||||
immediate: {
|
||||
...baseReleaseRecoveryFacts.immediate,
|
||||
applies: true,
|
||||
isDispositionRepairRetry: true,
|
||||
},
|
||||
},
|
||||
expected: { kind: "released" },
|
||||
},
|
||||
{
|
||||
name: "released: immediate recovery applies but an explicit blocker path exists",
|
||||
facts: {
|
||||
...baseReleaseRecoveryFacts,
|
||||
immediate: {
|
||||
...baseReleaseRecoveryFacts.immediate,
|
||||
applies: true,
|
||||
hasExplicitBlockerPath: true,
|
||||
},
|
||||
},
|
||||
expected: { kind: "released" },
|
||||
},
|
||||
{
|
||||
name: "blocked_recovery_in_place: immediate recovery applies on a stranded-issue-recovery origin",
|
||||
facts: {
|
||||
...baseReleaseRecoveryFacts,
|
||||
immediate: {
|
||||
...baseReleaseRecoveryFacts.immediate,
|
||||
applies: true,
|
||||
isStrandedRecoveryOrigin: true,
|
||||
},
|
||||
},
|
||||
expected: { kind: "blocked_recovery_in_place" },
|
||||
},
|
||||
{
|
||||
name: "blocked: immediate recovery applies but the recovery agent is not invokable",
|
||||
facts: {
|
||||
...baseReleaseRecoveryFacts,
|
||||
immediate: {
|
||||
...baseReleaseRecoveryFacts.immediate,
|
||||
applies: true,
|
||||
recoveryAgentInvokable: false,
|
||||
},
|
||||
},
|
||||
expected: { kind: "blocked", notice: "immediate_execution_path" },
|
||||
},
|
||||
{
|
||||
name: "blocked: immediate recovery applies and the run failed workspace validation",
|
||||
facts: {
|
||||
...baseReleaseRecoveryFacts,
|
||||
immediate: {
|
||||
...baseReleaseRecoveryFacts.immediate,
|
||||
applies: true,
|
||||
isWorkspaceValidationFailedRun: true,
|
||||
},
|
||||
},
|
||||
expected: { kind: "blocked", notice: "workspace_validation" },
|
||||
},
|
||||
{
|
||||
name: "blocked: immediate recovery applies and the run failed on incomplete configuration",
|
||||
facts: {
|
||||
...baseReleaseRecoveryFacts,
|
||||
immediate: {
|
||||
...baseReleaseRecoveryFacts.immediate,
|
||||
applies: true,
|
||||
isConfigurationIncompleteFailedRun: true,
|
||||
},
|
||||
},
|
||||
expected: { kind: "blocked", notice: "configuration_incomplete" },
|
||||
},
|
||||
{
|
||||
name: "queue_recovery: immediate recovery applies and no suppression or block condition fires",
|
||||
facts: {
|
||||
...baseReleaseRecoveryFacts,
|
||||
immediate: { ...baseReleaseRecoveryFacts.immediate, applies: true },
|
||||
},
|
||||
expected: { kind: "queue_recovery" },
|
||||
},
|
||||
{
|
||||
name: "released: review-participant recovery applies but a persisted monitor already covers it",
|
||||
facts: {
|
||||
...baseReleaseRecoveryFacts,
|
||||
reviewParticipant: {
|
||||
...baseReleaseRecoveryFacts.reviewParticipant,
|
||||
applies: true,
|
||||
hasPersistedMonitor: true,
|
||||
},
|
||||
},
|
||||
expected: { kind: "released" },
|
||||
},
|
||||
{
|
||||
name: "blocked_recovery_in_place: review-participant recovery applies on a stranded-issue-recovery origin",
|
||||
facts: {
|
||||
...baseReleaseRecoveryFacts,
|
||||
reviewParticipant: {
|
||||
...baseReleaseRecoveryFacts.reviewParticipant,
|
||||
applies: true,
|
||||
isStrandedRecoveryOrigin: true,
|
||||
},
|
||||
},
|
||||
expected: { kind: "blocked_recovery_in_place" },
|
||||
},
|
||||
{
|
||||
name: "blocked: review-participant recovery applies but the finishing run was itself that recovery retry",
|
||||
facts: {
|
||||
...baseReleaseRecoveryFacts,
|
||||
reviewParticipant: {
|
||||
...baseReleaseRecoveryFacts.reviewParticipant,
|
||||
applies: true,
|
||||
isExecutionReviewParticipantRecoveryRun: true,
|
||||
},
|
||||
},
|
||||
expected: { kind: "blocked", notice: "execution_review_participant" },
|
||||
},
|
||||
{
|
||||
name: "queue_review_participant_recovery: review-participant recovery applies and no suppression or block condition fires",
|
||||
facts: {
|
||||
...baseReleaseRecoveryFacts,
|
||||
reviewParticipant: { ...baseReleaseRecoveryFacts.reviewParticipant, applies: true },
|
||||
},
|
||||
expected: { kind: "queue_review_participant_recovery" },
|
||||
},
|
||||
{
|
||||
name: "review-participant recovery is evaluated before immediate recovery when both apply",
|
||||
facts: {
|
||||
...baseReleaseRecoveryFacts,
|
||||
reviewParticipant: { ...baseReleaseRecoveryFacts.reviewParticipant, applies: true },
|
||||
immediate: {
|
||||
...baseReleaseRecoveryFacts.immediate,
|
||||
applies: true,
|
||||
isStrandedRecoveryOrigin: true,
|
||||
},
|
||||
},
|
||||
expected: { kind: "queue_review_participant_recovery" },
|
||||
},
|
||||
];
|
||||
|
||||
for (const testCase of cases) {
|
||||
it(testCase.name, () => {
|
||||
expect(decideReleaseRecovery(testCase.facts)).toEqual(testCase.expected);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
|
@ -0,0 +1,200 @@
|
|||
// Pure decision rules for the release half of the deferred issue-execution
|
||||
// wake state machine:
|
||||
// - the per-wake decision (decideDeferredWake), applied to the earliest
|
||||
// deferred wake queued against the issue a run just released
|
||||
// - the release-recovery decision (decideReleaseRecovery), applied once
|
||||
// the deferred-wake queue is empty and no wake was promoted
|
||||
// The caller reads the database and packs the result into a facts object.
|
||||
// This file only branches on that facts object; it never queries a
|
||||
// database, reads the clock, or reads the wake context payload directly.
|
||||
|
||||
export type DeferredWakeQueuedCommentFacts = {
|
||||
/** True when the wake carries one or more queued comment ids to check. */
|
||||
hasQueuedCommentIds: boolean;
|
||||
/** Count of queued comment ids that are still live and not self-authored by the finishing run. */
|
||||
liveNonSelfCommentIdsLength: number;
|
||||
/** Count of queued comment ids the wake originally carried. */
|
||||
queuedCommentIdsLength: number;
|
||||
/** True when the live, non-self comment id list differs from the queued list. */
|
||||
liveCommentIdsChanged: boolean;
|
||||
/** True when every discarded comment id was authored by the finishing run. */
|
||||
containedSelfAuthoredComment: boolean;
|
||||
/** True when the wake carries an independent reason to continue even with no live comments. */
|
||||
preservesIndependentContinuation: boolean;
|
||||
};
|
||||
|
||||
export type DeferredWakeAgentFacts = {
|
||||
/** True when the wake's agent exists in the issue's own company. */
|
||||
agentFound: boolean;
|
||||
/** True when the agent is invokable (status, org chain). Meaningless when agentFound is false. */
|
||||
invokable: boolean;
|
||||
};
|
||||
|
||||
export type DeferredWakePauseHoldFacts = {
|
||||
/** True when an active subtree pause hold covers the issue. */
|
||||
activePauseHold: boolean;
|
||||
/** True when the wake is a verified issue-tree-control interaction wake that survives a pause hold. */
|
||||
treeHoldInteractionWake: boolean;
|
||||
};
|
||||
|
||||
export type DeferredWakeFacts = {
|
||||
queuedComment: DeferredWakeQueuedCommentFacts;
|
||||
agent: DeferredWakeAgentFacts;
|
||||
pauseHold: DeferredWakePauseHoldFacts;
|
||||
};
|
||||
|
||||
export type DeferredWakeDecision =
|
||||
| { kind: "cancel_empty"; selfAuthored: boolean }
|
||||
| { kind: "normalize" }
|
||||
| { kind: "fail_not_invokable" }
|
||||
| { kind: "cancel_pause_hold" }
|
||||
| { kind: "promote" };
|
||||
|
||||
/**
|
||||
* Decides what to do with the earliest deferred wake queued against the
|
||||
* issue a run just released. The caller applies a "normalize" decision (a
|
||||
* queued-comment-id rewrite) and calls this function again with facts that
|
||||
* reflect the rewrite, so a single wake can normalize and then also fail,
|
||||
* cancel, or promote in the same drain step — matching the order the
|
||||
* original state machine always evaluated them in.
|
||||
*/
|
||||
export function decideDeferredWake(facts: DeferredWakeFacts): DeferredWakeDecision {
|
||||
const { queuedComment, agent, pauseHold } = facts;
|
||||
|
||||
if (
|
||||
queuedComment.hasQueuedCommentIds &&
|
||||
queuedComment.liveNonSelfCommentIdsLength === 0 &&
|
||||
!queuedComment.preservesIndependentContinuation
|
||||
) {
|
||||
return { kind: "cancel_empty", selfAuthored: queuedComment.containedSelfAuthoredComment };
|
||||
}
|
||||
|
||||
if (queuedComment.hasQueuedCommentIds && queuedComment.liveCommentIdsChanged) {
|
||||
return { kind: "normalize" };
|
||||
}
|
||||
|
||||
if (!agent.agentFound || !agent.invokable) {
|
||||
return { kind: "fail_not_invokable" };
|
||||
}
|
||||
|
||||
if (pauseHold.activePauseHold && !pauseHold.treeHoldInteractionWake) {
|
||||
return { kind: "cancel_pause_hold" };
|
||||
}
|
||||
|
||||
return { kind: "promote" };
|
||||
}
|
||||
|
||||
export type ReleaseRecoveryReviewParticipantFacts = {
|
||||
/** True when the issue is in_review, unassigned to a user, and waiting on the finishing run as the current agent participant. */
|
||||
applies: boolean;
|
||||
hasExistingExecutionPath: boolean;
|
||||
hasPersistedMonitor: boolean;
|
||||
suppressedByPauseHold: boolean;
|
||||
isStrandedRecoveryOrigin: boolean;
|
||||
recoveryAgentPresent: boolean;
|
||||
recoveryAgentInvokable: boolean;
|
||||
/** True when the finishing run was itself a review-participant-recovery retry. */
|
||||
isExecutionReviewParticipantRecoveryRun: boolean;
|
||||
};
|
||||
|
||||
export type ReleaseRecoveryImmediateFacts = {
|
||||
/** True when the issue is todo/in_progress, unassigned to a user, not hidden, still assigned to the finishing run's agent, and the run ended failed/timed_out/cancelled. */
|
||||
applies: boolean;
|
||||
/** True when the finishing run itself carried the disposition-repair retry reason. */
|
||||
isDispositionRepairRetry: boolean;
|
||||
hasExistingExecutionPath: boolean;
|
||||
hasPersistedMonitor: boolean;
|
||||
hasExplicitBlockerPath: boolean;
|
||||
suppressedByPauseHold: boolean;
|
||||
isStrandedRecoveryOrigin: boolean;
|
||||
recoveryAgentPresent: boolean;
|
||||
recoveryAgentInvokable: boolean;
|
||||
isWorkspaceValidationFailedRun: boolean;
|
||||
isConfigurationIncompleteFailedRun: boolean;
|
||||
/** didAutomaticRecoveryFail(run, expectedRetryReason) for the issue's own status branch. */
|
||||
automaticRecoveryAlreadyFailed: boolean;
|
||||
};
|
||||
|
||||
export type ReleaseRecoveryFacts = {
|
||||
/** options.suppressImmediateRecovery on the caller's release request. */
|
||||
suppressImmediateRecovery: boolean;
|
||||
reviewParticipant: ReleaseRecoveryReviewParticipantFacts;
|
||||
immediate: ReleaseRecoveryImmediateFacts;
|
||||
};
|
||||
|
||||
export type ReleaseRecoveryBlockedNoticeKind =
|
||||
| "workspace_validation"
|
||||
| "configuration_incomplete"
|
||||
| "execution_review_participant"
|
||||
| "immediate_execution_path";
|
||||
|
||||
export type ReleaseRecoveryDecision =
|
||||
| { kind: "released" }
|
||||
| { kind: "blocked_recovery_in_place" }
|
||||
| { kind: "blocked"; notice: ReleaseRecoveryBlockedNoticeKind }
|
||||
| { kind: "queue_review_participant_recovery" }
|
||||
| { kind: "queue_recovery" };
|
||||
|
||||
/**
|
||||
* Decides the release-recovery outcome once the deferred-wake queue is
|
||||
* empty and no wake was promoted. The review-participant branch and the
|
||||
* assignment-and-continuation branch are the two ways a released issue can
|
||||
* still need automatic recovery; this function evaluates both as one
|
||||
* decision, review-participant first, matching the original evaluation
|
||||
* order.
|
||||
*/
|
||||
export function decideReleaseRecovery(facts: ReleaseRecoveryFacts): ReleaseRecoveryDecision {
|
||||
const { reviewParticipant, immediate } = facts;
|
||||
|
||||
if (reviewParticipant.applies) {
|
||||
if (
|
||||
facts.suppressImmediateRecovery ||
|
||||
reviewParticipant.hasExistingExecutionPath ||
|
||||
reviewParticipant.hasPersistedMonitor ||
|
||||
reviewParticipant.suppressedByPauseHold
|
||||
) {
|
||||
return { kind: "released" };
|
||||
}
|
||||
if (reviewParticipant.isStrandedRecoveryOrigin) {
|
||||
return { kind: "blocked_recovery_in_place" };
|
||||
}
|
||||
const shouldBlock =
|
||||
!reviewParticipant.recoveryAgentInvokable ||
|
||||
!reviewParticipant.recoveryAgentPresent ||
|
||||
reviewParticipant.isExecutionReviewParticipantRecoveryRun;
|
||||
if (shouldBlock) {
|
||||
return { kind: "blocked", notice: "execution_review_participant" };
|
||||
}
|
||||
return { kind: "queue_review_participant_recovery" };
|
||||
}
|
||||
|
||||
if (immediate.isDispositionRepairRetry) return { kind: "released" };
|
||||
if (!immediate.applies) return { kind: "released" };
|
||||
if (facts.suppressImmediateRecovery) return { kind: "released" };
|
||||
if (
|
||||
immediate.hasExistingExecutionPath ||
|
||||
immediate.hasPersistedMonitor ||
|
||||
immediate.hasExplicitBlockerPath
|
||||
) {
|
||||
return { kind: "released" };
|
||||
}
|
||||
if (immediate.suppressedByPauseHold) return { kind: "released" };
|
||||
if (immediate.isStrandedRecoveryOrigin) return { kind: "blocked_recovery_in_place" };
|
||||
|
||||
const shouldBlockImmediately =
|
||||
!immediate.recoveryAgentInvokable ||
|
||||
!immediate.recoveryAgentPresent ||
|
||||
immediate.isWorkspaceValidationFailedRun ||
|
||||
immediate.isConfigurationIncompleteFailedRun ||
|
||||
immediate.automaticRecoveryAlreadyFailed;
|
||||
if (shouldBlockImmediately) {
|
||||
const notice: ReleaseRecoveryBlockedNoticeKind = immediate.isWorkspaceValidationFailedRun
|
||||
? "workspace_validation"
|
||||
: immediate.isConfigurationIncompleteFailedRun
|
||||
? "configuration_incomplete"
|
||||
: "immediate_execution_path";
|
||||
return { kind: "blocked", notice };
|
||||
}
|
||||
|
||||
return { kind: "queue_recovery" };
|
||||
}
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
import type { Db } from "@paperclipai/db";
|
||||
import { createPostgresWakeQueueAdapter } from "./adapters/postgres.js";
|
||||
import { createReleaseIssueExecution } from "./application/use-cases.js";
|
||||
import type {
|
||||
IssueSnapshot,
|
||||
RecoveryEscalationPort,
|
||||
RunSnapshot,
|
||||
WakeQueueReader,
|
||||
} from "./application/ports.js";
|
||||
|
||||
export type {
|
||||
PostCommitEffect,
|
||||
ReleaseOutcome,
|
||||
RunSummary,
|
||||
} from "./application/types.js";
|
||||
export { WakeQueueApplicationError } from "./application/types.js";
|
||||
export type { IssueSnapshot, RunSnapshot, RecoveryEscalationPort } from "./application/ports.js";
|
||||
export type { ReleaseIssueExecutionInput } from "./application/use-cases.js";
|
||||
|
||||
export type WakeQueueDeps = {
|
||||
/** Stays in `heartbeat.ts`; resolves the responsible user for a promoted or recovery run seed. */
|
||||
resolveResponsibleUserId: WakeQueueReader["resolveResponsibleUserId"];
|
||||
/** Stays in `heartbeat.ts`; reads the routine environment context for an execution issue. */
|
||||
getRoutineEnv: WakeQueueReader["getRoutineEnv"];
|
||||
/** Stays in `heartbeat.ts`; resolves the session-before display id for a wakeup. */
|
||||
resolveSessionBeforeForWakeup: WakeQueueReader["resolveSessionBeforeForWakeup"];
|
||||
/** `services/recovery`'s stranded-issue escalation, called only after the release transaction commits. */
|
||||
recovery: RecoveryEscalationPort;
|
||||
};
|
||||
|
||||
/**
|
||||
* Composes the wake-queue module: the Postgres adapter (which owns the
|
||||
* release transaction) and the release use case. `heartbeat.ts` holds the
|
||||
* only caller: it builds one instance per process next to
|
||||
* `createRunDispatch(db)` and delegates `releaseIssueExecutionAndPromote`'s
|
||||
* body to `releaseIssueExecution`.
|
||||
*/
|
||||
export function createWakeQueue(db: Db, deps: WakeQueueDeps) {
|
||||
const issueLock = createPostgresWakeQueueAdapter(db, {
|
||||
resolveResponsibleUserId: deps.resolveResponsibleUserId,
|
||||
getRoutineEnv: deps.getRoutineEnv,
|
||||
resolveSessionBeforeForWakeup: deps.resolveSessionBeforeForWakeup,
|
||||
});
|
||||
|
||||
return {
|
||||
releaseIssueExecution: createReleaseIssueExecution({ issueLock, recovery: deps.recovery }),
|
||||
};
|
||||
}
|
||||
|
||||
export type WakeQueue = ReturnType<typeof createWakeQueue>;
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -5638,7 +5638,7 @@ export function issueService(db: Db) {
|
|||
return row;
|
||||
}
|
||||
|
||||
return {
|
||||
const service = {
|
||||
clearExecutionRunIfTerminal,
|
||||
clearCheckoutRunIfTerminal,
|
||||
addStopRelayCommentIfNeeded,
|
||||
|
|
@ -7783,6 +7783,7 @@ export function issueService(db: Db) {
|
|||
blockedByIssueIds?: string[];
|
||||
actorAgentId?: string | null;
|
||||
actorUserId?: string | null;
|
||||
companyGuard?: string;
|
||||
},
|
||||
dbOrTx: any = db,
|
||||
postCommitActivityPublications?: ActivityPublication[],
|
||||
|
|
@ -7792,10 +7793,18 @@ export function issueService(db: Db) {
|
|||
const activityPublications = postCommitActivityPublications ?? ownedActivityPublications;
|
||||
const ownedPostCommitActions: IssuePostCommitAction[] = [];
|
||||
const queuedPostCommitActions = postCommitActions ?? ownedPostCommitActions;
|
||||
// A caller that supplies `companyGuard` gets the company added to
|
||||
// every read, lock, and write predicate below. A check before this
|
||||
// call is not a boundary: `issues.company_id` can change between
|
||||
// that check and this write, so the predicate must carry the
|
||||
// company itself.
|
||||
const idPredicate = data.companyGuard !== undefined
|
||||
? and(eq(issues.id, id), eq(issues.companyId, data.companyGuard))
|
||||
: eq(issues.id, id);
|
||||
const existing = await dbOrTx
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(eq(issues.id, id))
|
||||
.where(idPredicate)
|
||||
.then((rows: Array<typeof issues.$inferSelect>) => rows[0] ?? null);
|
||||
if (!existing) return null;
|
||||
|
||||
|
|
@ -7804,6 +7813,7 @@ export function issueService(db: Db) {
|
|||
blockedByIssueIds,
|
||||
actorAgentId,
|
||||
actorUserId,
|
||||
companyGuard,
|
||||
...issueData
|
||||
} = data;
|
||||
const isolatedWorkspacesEnabled = (await instanceSettings.getExperimental()).enableIsolatedWorkspaces;
|
||||
|
|
@ -7955,7 +7965,7 @@ export function issueService(db: Db) {
|
|||
const receiptExisting = await tx
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(eq(issues.id, id))
|
||||
.where(idPredicate)
|
||||
.for("update")
|
||||
.then((rows: Array<typeof issues.$inferSelect>) => rows[0] ?? null);
|
||||
if (!receiptExisting) return null;
|
||||
|
|
@ -7994,7 +8004,7 @@ export function issueService(db: Db) {
|
|||
const updated = await tx
|
||||
.update(issues)
|
||||
.set(patch)
|
||||
.where(eq(issues.id, id))
|
||||
.where(idPredicate)
|
||||
.returning()
|
||||
.then((rows: Array<typeof issues.$inferSelect>) => rows[0] ?? null);
|
||||
if (!updated) return null;
|
||||
|
|
@ -9535,4 +9545,38 @@ export function issueService(db: Db) {
|
|||
}));
|
||||
},
|
||||
};
|
||||
|
||||
type IssueServiceApi = typeof service & {
|
||||
updateForCompany: (
|
||||
id: string,
|
||||
companyId: string,
|
||||
data: Parameters<typeof service.update>[1],
|
||||
dbOrTx?: any,
|
||||
postCommitActivityPublications?: ActivityPublication[],
|
||||
postCommitActions?: IssuePostCommitAction[],
|
||||
) => ReturnType<typeof service.update>;
|
||||
};
|
||||
const serviceApi = service as IssueServiceApi;
|
||||
|
||||
// A company-scoped wrapper around `update`. It passes the company as a
|
||||
// guard on every read, lock, and write predicate, so a caller with only
|
||||
// a company id and an issue id cannot update an issue in another company.
|
||||
serviceApi.updateForCompany = async (
|
||||
id,
|
||||
companyId,
|
||||
data,
|
||||
dbOrTx = db,
|
||||
postCommitActivityPublications,
|
||||
postCommitActions,
|
||||
) => {
|
||||
return service.update(
|
||||
id,
|
||||
{ ...data, companyGuard: companyId },
|
||||
dbOrTx,
|
||||
postCommitActivityPublications,
|
||||
postCommitActions,
|
||||
);
|
||||
};
|
||||
|
||||
return serviceApi;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue