diff --git a/server/src/__tests__/heartbeat-comment-wake-batching.test.ts b/server/src/__tests__/heartbeat-comment-wake-batching.test.ts index 9bc0e48011..a054564853 100644 --- a/server/src/__tests__/heartbeat-comment-wake-batching.test.ts +++ b/server/src/__tests__/heartbeat-comment-wake-batching.test.ts @@ -619,152 +619,6 @@ describeEmbeddedPostgres("heartbeat comment wake batching", () => { } }, 120_000); - it("cancels an empty deferred comment wake instead of promoting deleted input", async () => { - const gateway = await createControlledGatewayServer(); - const companyId = randomUUID(); - const agentId = randomUUID(); - const issueId = randomUUID(); - const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`; - const heartbeat = heartbeatService(db); - - try { - await db.insert(companies).values({ - id: companyId, - name: "Paperclip", - issuePrefix, - requireBoardApprovalForNewAgents: false, - defaultResponsibleUserId: "responsible-user", - }); - await db.insert(agents).values({ - id: agentId, - companyId, - name: "Gateway Agent", - role: "engineer", - status: "idle", - adapterType: "openclaw_gateway", - adapterConfig: { - url: gateway.url, - headers: { "x-openclaw-token": "gateway-token" }, - payloadTemplate: { message: "wake now" }, - waitTimeoutMs: 2_000, - }, - runtimeConfig: {}, - permissions: {}, - }); - await db.insert(issues).values({ - id: issueId, - companyId, - title: "Discard deferred follow-up", - status: "todo", - priority: "medium", - responsibleUserId: "responsible-user", - assigneeAgentId: agentId, - issueNumber: 1, - identifier: `${issuePrefix}-1`, - }); - const firstComment = await db - .insert(issueComments) - .values({ - companyId, - issueId, - authorUserId: "user-1", - body: "First comment", - }) - .returning() - .then((rows) => rows[0]); - const firstRun = await heartbeat.wakeup(agentId, { - source: "automation", - triggerDetail: "system", - reason: "issue_commented", - payload: { issueId, commentId: firstComment.id }, - contextSnapshot: { - issueId, - taskId: issueId, - commentId: firstComment.id, - wakeReason: "issue_commented", - }, - requestedByActorType: "user", - requestedByActorId: "user-1", - }); - expect(firstRun).not.toBeNull(); - await waitFor(async () => { - const current = await db - .select({ status: heartbeatRuns.status }) - .from(heartbeatRuns) - .where(eq(heartbeatRuns.id, firstRun!.id)) - .then((rows) => rows[0] ?? null); - return current?.status === "running"; - }); - - const discardedComment = await db - .insert(issueComments) - .values({ - companyId, - issueId, - authorUserId: "user-1", - body: "Delete this before the current turn finishes", - }) - .returning() - .then((rows) => rows[0]); - expect(await heartbeat.wakeup(agentId, { - source: "automation", - triggerDetail: "system", - reason: "issue_commented", - payload: { issueId, commentId: discardedComment.id }, - contextSnapshot: { - issueId, - taskId: issueId, - commentId: discardedComment.id, - wakeReason: "issue_commented", - }, - requestedByActorType: "user", - requestedByActorId: "user-1", - })).toBeNull(); - await waitFor(async () => db - .select({ id: agentWakeupRequests.id }) - .from(agentWakeupRequests) - .where(and( - eq(agentWakeupRequests.companyId, companyId), - eq(agentWakeupRequests.agentId, agentId), - eq(agentWakeupRequests.status, "deferred_issue_execution"), - )) - .then((rows) => Boolean(rows[0]))); - const deferredWake = await db - .select() - .from(agentWakeupRequests) - .where(and( - eq(agentWakeupRequests.companyId, companyId), - eq(agentWakeupRequests.agentId, agentId), - eq(agentWakeupRequests.status, "deferred_issue_execution"), - )) - .then((rows) => rows[0]); - if (!deferredWake) throw new Error("Expected a deferred comment wake"); - await db.delete(issueComments).where(eq(issueComments.id, discardedComment.id)); - - gateway.releaseFirstWait(); - await waitFor(async () => { - const wake = await db - .select({ status: agentWakeupRequests.status }) - .from(agentWakeupRequests) - .where(eq(agentWakeupRequests.id, deferredWake.id)) - .then((rows) => rows[0] ?? null); - return wake?.status === "cancelled"; - }, 90_000); - await heartbeat.drainActiveRunExecutions(); - - expect(gateway.getAgentPayloads()).toHaveLength(1); - const runs = await db - .select({ id: heartbeatRuns.id }) - .from(heartbeatRuns) - .where(eq(heartbeatRuns.agentId, agentId)); - expect(runs.map((run) => run.id)).toEqual([firstRun!.id]); - } finally { - gateway.releaseFirstWait(); - await heartbeat.drainActiveRunExecutions(); - await gateway.close(); - } - }, 120_000); - it("retains deferred comments for reconciliation after cancelling an unknown provider outcome", async () => { const gateway = await createControlledGatewayServer(); const companyId = randomUUID(); @@ -1312,363 +1166,6 @@ describeEmbeddedPostgres("heartbeat comment wake batching", () => { } }, 120_000); - it("cancels a deferred wake containing only a comment authored by the closing run", async () => { - const gateway = await createControlledGatewayServer(); - const companyId = randomUUID(); - const agentId = randomUUID(); - const issueId = randomUUID(); - const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`; - const heartbeat = heartbeatService(db); - - try { - await db.insert(companies).values({ - id: companyId, - name: "Paperclip", - issuePrefix, - requireBoardApprovalForNewAgents: false, - defaultResponsibleUserId: "responsible-user", - }); - - await db.insert(agents).values({ - id: agentId, - companyId, - name: "Local CLI Agent", - role: "engineer", - status: "idle", - adapterType: "openclaw_gateway", - adapterConfig: { - url: gateway.url, - headers: { - "x-openclaw-token": "gateway-token", - }, - payloadTemplate: { - message: "wake now", - }, - waitTimeoutMs: 2_000, - }, - runtimeConfig: {}, - permissions: {}, - }); - - await db.insert(issues).values({ - id: issueId, - companyId, - title: "Self-comment must not reopen", - status: "todo", - priority: "medium", - responsibleUserId: "responsible-user", - assigneeAgentId: agentId, - issueNumber: 1, - identifier: `${issuePrefix}-1`, - }); - - const firstRun = await heartbeat.wakeup(agentId, { - source: "assignment", - triggerDetail: "system", - reason: "issue_assigned", - payload: { issueId }, - contextSnapshot: { - issueId, - taskId: issueId, - wakeReason: "issue_assigned", - }, - requestedByActorType: "system", - requestedByActorId: null, - }); - - expect(firstRun).not.toBeNull(); - await waitFor(async () => { - const run = await db - .select({ status: heartbeatRuns.status }) - .from(heartbeatRuns) - .where(eq(heartbeatRuns.id, firstRun!.id)) - .then((rows) => rows[0] ?? null); - return run?.status === "running"; - }); - await waitFor(() => gateway.getAgentPayloads().length === 1); - - // Local-CLI agents post comments under user auth, but stamp the heartbeat - // run id on each comment via createdByRunId. Simulate that here: a "user" - // comment that was actually authored by the run that is about to close - // the issue. Without the Path A guard this would trigger a reopen. - const selfComment = await db - .insert(issueComments) - .values({ - companyId, - issueId, - authorUserId: "local-cli-user", - createdByRunId: firstRun?.id ?? null, - body: "Closing comment from the same run", - }) - .returning() - .then((rows) => rows[0]); - - const deferredRun = await heartbeat.wakeup(agentId, { - source: "automation", - triggerDetail: "system", - reason: "issue_commented", - payload: { issueId, commentId: selfComment.id }, - contextSnapshot: { - issueId, - taskId: issueId, - commentId: selfComment.id, - wakeCommentId: selfComment.id, - wakeReason: "issue_commented", - }, - requestedByActorType: "user", - requestedByActorId: "local-cli-user", - }); - - expect(deferredRun).toBeNull(); - - await waitFor(async () => { - const deferred = await db - .select() - .from(agentWakeupRequests) - .where( - and( - eq(agentWakeupRequests.companyId, companyId), - eq(agentWakeupRequests.agentId, agentId), - eq(agentWakeupRequests.status, "deferred_issue_execution"), - ), - ) - .then((rows) => rows[0] ?? null); - return Boolean(deferred); - }); - - // Running records admission. Wait for provider acceptance before - // simulating completion by that provider, or startup correctly rejects - // the already-closed task before this scenario reaches its follow-up. - await waitFor(() => gateway.getAgentPayloads().length >= 1); - await db - .update(issues) - .set({ - status: "done", - completedAt: new Date(), - executionRunId: null, - executionAgentNameKey: null, - executionLockedAt: null, - updatedAt: new Date(), - }) - .where(eq(issues.id, issueId)); - - gateway.releaseFirstWait(); - - await waitFor(async () => { - const run = await db - .select() - .from(heartbeatRuns) - .where(eq(heartbeatRuns.id, firstRun!.id)) - .then((rows) => rows[0] ?? null); - const deferred = await db - .select() - .from(agentWakeupRequests) - .where( - and( - eq(agentWakeupRequests.companyId, companyId), - eq(agentWakeupRequests.agentId, agentId), - ), - ) - .then((rows) => rows.find((request) => request.status === "cancelled") ?? null); - return ( - run?.status === "succeeded" && - deferred?.error === - "Deferred wake contained only comments authored by the finishing run" - ); - }, 90_000); - - expect(gateway.getAgentPayloads()).toHaveLength(1); - const runs = await db - .select() - .from(heartbeatRuns) - .where(eq(heartbeatRuns.agentId, agentId)); - expect(runs).toHaveLength(1); - - const issueAfterPromotion = await db - .select({ - status: issues.status, - completedAt: issues.completedAt, - }) - .from(issues) - .where(eq(issues.id, issueId)) - .then((rows) => rows[0] ?? null); - - expect(issueAfterPromotion).toMatchObject({ - status: "done", - }); - expect(issueAfterPromotion?.completedAt).not.toBeNull(); - } finally { - gateway.releaseFirstWait(); - await gateway.close(); - } - }, 120_000); - - it("promotes an interaction continuation after removing a coalesced self-authored comment", async () => { - const gateway = await createControlledGatewayServer(); - const companyId = randomUUID(); - const agentId = randomUUID(); - const issueId = randomUUID(); - const interactionId = randomUUID(); - const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`; - const heartbeat = heartbeatService(db); - - try { - await db.insert(companies).values({ - id: companyId, - name: "Paperclip", - issuePrefix, - requireBoardApprovalForNewAgents: false, - defaultResponsibleUserId: "responsible-user", - }); - - await db.insert(agents).values({ - id: agentId, - companyId, - name: "Local CLI Agent", - role: "engineer", - status: "idle", - adapterType: "openclaw_gateway", - adapterConfig: { - url: gateway.url, - headers: { - "x-openclaw-token": "gateway-token", - }, - payloadTemplate: { - message: "wake now", - }, - waitTimeoutMs: 2_000, - }, - runtimeConfig: {}, - permissions: {}, - }); - - await db.insert(issues).values({ - id: issueId, - companyId, - title: "Interaction continuation survives self-comment filtering", - status: "todo", - priority: "medium", - responsibleUserId: "responsible-user", - assigneeAgentId: agentId, - issueNumber: 1, - identifier: `${issuePrefix}-1`, - }); - - const firstRun = await heartbeat.wakeup(agentId, { - source: "assignment", - triggerDetail: "system", - reason: "issue_assigned", - payload: { issueId }, - contextSnapshot: { - issueId, - taskId: issueId, - wakeReason: "issue_assigned", - }, - requestedByActorType: "system", - requestedByActorId: null, - }); - - expect(firstRun).not.toBeNull(); - await waitFor(async () => { - const run = await db - .select({ status: heartbeatRuns.status }) - .from(heartbeatRuns) - .where(eq(heartbeatRuns.id, firstRun!.id)) - .then((rows) => rows[0] ?? null); - return run?.status === "running"; - }); - - const selfComment = await db - .insert(issueComments) - .values({ - companyId, - issueId, - authorUserId: "local-cli-user", - createdByRunId: firstRun!.id, - body: "Completion note from the source run", - }) - .returning() - .then((rows) => rows[0]); - - expect(await heartbeat.wakeup(agentId, { - source: "automation", - triggerDetail: "system", - reason: "issue_commented", - payload: { issueId, commentId: selfComment.id }, - contextSnapshot: { - issueId, - taskId: issueId, - commentId: selfComment.id, - wakeCommentId: selfComment.id, - wakeReason: "issue_commented", - }, - requestedByActorType: "user", - requestedByActorId: "local-cli-user", - })).toBeNull(); - - expect(await heartbeat.wakeup(agentId, { - source: "automation", - triggerDetail: "system", - reason: "issue_commented", - payload: { - issueId, - interactionId, - interactionKind: "request_confirmation", - interactionStatus: "accepted", - mutation: "interaction", - }, - contextSnapshot: { - issueId, - taskId: issueId, - interactionId, - interactionKind: "request_confirmation", - interactionStatus: "accepted", - wakeReason: "issue_commented", - source: "issue.interaction.respond", - }, - requestedByActorType: "user", - requestedByActorId: "user-1", - })).toBeNull(); - - gateway.releaseFirstWait(); - - await waitFor(async () => { - const runs = await db - .select() - .from(heartbeatRuns) - .where(eq(heartbeatRuns.agentId, agentId)) - .orderBy(asc(heartbeatRuns.createdAt)); - return ( - runs.length === 2 && - runs[0]?.status === "succeeded" && - runs[1]?.status === "succeeded" - ); - }, 90_000); - - const promotedRun = await db - .select() - .from(heartbeatRuns) - .where(eq(heartbeatRuns.agentId, agentId)) - .orderBy(asc(heartbeatRuns.createdAt)) - .then((runs) => runs[1] ?? null); - expect(promotedRun?.contextSnapshot).toMatchObject({ - interactionId, - interactionKind: "request_confirmation", - interactionStatus: "accepted", - }); - expect(promotedRun?.contextSnapshot).not.toMatchObject({ - wakeCommentIds: expect.anything(), - }); - expect(promotedRun?.contextSnapshot).not.toMatchObject({ - commentId: selfComment.id, - }); - expect(gateway.getAgentPayloads()).toHaveLength(2); - } finally { - gateway.releaseFirstWait(); - await gateway.close(); - } - }, 120_000); - it("still reopens a finished issue when a deferred batch mixes self-authored and human comments", async () => { const gateway = await createControlledGatewayServer(); const companyId = randomUUID(); @@ -1886,6 +1383,377 @@ describeEmbeddedPostgres("heartbeat comment wake batching", () => { } }, 120_000); + it("cancels a deferred comment wake when its only queued comment is deleted before promotion", async () => { + const gateway = await createControlledGatewayServer(); + const companyId = randomUUID(); + const agentId = randomUUID(); + const issueId = randomUUID(); + const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`; + const heartbeat = heartbeatService(db); + + try { + await db.insert(companies).values({ + id: companyId, + name: "Paperclip", + issuePrefix, + requireBoardApprovalForNewAgents: false, + defaultResponsibleUserId: "responsible-user", + }); + + await db.insert(agents).values({ + id: agentId, + companyId, + name: "Gateway Agent", + role: "engineer", + status: "idle", + adapterType: "openclaw_gateway", + adapterConfig: { + url: gateway.url, + headers: { + "x-openclaw-token": "gateway-token", + }, + payloadTemplate: { + message: "wake now", + }, + waitTimeoutMs: 2_000, + }, + runtimeConfig: {}, + permissions: {}, + }); + + await db.insert(issues).values({ + id: issueId, + companyId, + title: "Deleted follow-up must not reopen", + status: "todo", + priority: "medium", + responsibleUserId: "responsible-user", + assigneeAgentId: agentId, + issueNumber: 1, + identifier: `${issuePrefix}-1`, + }); + + const firstRun = await heartbeat.wakeup(agentId, { + source: "assignment", + triggerDetail: "system", + reason: "issue_assigned", + payload: { issueId }, + contextSnapshot: { + issueId, + taskId: issueId, + wakeReason: "issue_assigned", + }, + requestedByActorType: "system", + requestedByActorId: null, + }); + + expect(firstRun).not.toBeNull(); + await waitFor(async () => { + const run = await db + .select({ status: heartbeatRuns.status }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, firstRun!.id)) + .then((rows) => rows[0] ?? null); + return run?.status === "running"; + }); + await waitFor(() => gateway.getAgentPayloads().length === 1); + + const queuedComment = await db + .insert(issueComments) + .values({ + companyId, + issueId, + authorUserId: "user-1", + body: "Please look at this once you finish", + }) + .returning() + .then((rows) => rows[0]); + + const deferredRun = await heartbeat.wakeup(agentId, { + source: "automation", + triggerDetail: "system", + reason: "issue_commented", + payload: { issueId, commentId: queuedComment.id }, + contextSnapshot: { + issueId, + taskId: issueId, + commentId: queuedComment.id, + wakeCommentId: queuedComment.id, + wakeReason: "issue_commented", + }, + requestedByActorType: "user", + requestedByActorId: "user-1", + }); + + expect(deferredRun).toBeNull(); + + await waitFor(async () => { + const deferred = await db + .select() + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.agentId, agentId), + eq(agentWakeupRequests.status, "deferred_issue_execution"), + ), + ) + .then((rows) => rows[0] ?? null); + return Boolean(deferred); + }); + + const deferredWake = await db + .select({ id: agentWakeupRequests.id }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.agentId, agentId), + eq(agentWakeupRequests.status, "deferred_issue_execution"), + ), + ) + .then((rows) => rows[0] ?? null); + const deferredWakeId = deferredWake!.id; + + // The author retracts the comment before the first run finishes, so the + // real comment-liveness query must find zero live comments left in the + // queued batch. + await db.update(issueComments).set({ deletedAt: new Date() }).where(eq(issueComments.id, queuedComment.id)); + + // Running records admission. Wait for provider acceptance before + // simulating completion by that provider, or startup correctly rejects + // the already-closed task before this scenario reaches its follow-up. + await waitFor(() => gateway.getAgentPayloads().length >= 1); + await db + .update(issues) + .set({ + status: "done", + completedAt: new Date(), + executionRunId: null, + executionAgentNameKey: null, + executionLockedAt: null, + updatedAt: new Date(), + }) + .where(eq(issues.id, issueId)); + + gateway.releaseFirstWait(); + + await waitFor(async () => { + const [wake] = await db + .select({ status: agentWakeupRequests.status }) + .from(agentWakeupRequests) + .where(eq(agentWakeupRequests.id, deferredWakeId)); + return wake?.status === "cancelled"; + }, 90_000); + + const [cancelledWake] = await db + .select({ status: agentWakeupRequests.status, error: agentWakeupRequests.error }) + .from(agentWakeupRequests) + .where(eq(agentWakeupRequests.id, deferredWakeId)); + expect(cancelledWake).toMatchObject({ + status: "cancelled", + error: "Queued messages were discarded before promotion", + }); + + // No live comment remained, so the queue must not promote a second run + // and the issue must stay in the state the first run's completion left it in. + expect(gateway.getAgentPayloads()).toHaveLength(1); + const issueAfterCompletion = await db + .select({ status: issues.status, completedAt: issues.completedAt }) + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => rows[0] ?? null); + expect(issueAfterCompletion).toMatchObject({ status: "done" }); + expect(issueAfterCompletion?.completedAt).not.toBeNull(); + } finally { + gateway.releaseFirstWait(); + await gateway.close(); + } + }, 120_000); + + it("cancels a deferred comment wake when its only queued comment was authored by the finishing run", async () => { + const gateway = await createControlledGatewayServer(); + const companyId = randomUUID(); + const agentId = randomUUID(); + const issueId = randomUUID(); + const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`; + const heartbeat = heartbeatService(db); + + try { + await db.insert(companies).values({ + id: companyId, + name: "Paperclip", + issuePrefix, + requireBoardApprovalForNewAgents: false, + defaultResponsibleUserId: "responsible-user", + }); + + await db.insert(agents).values({ + id: agentId, + companyId, + name: "Local CLI Agent", + role: "engineer", + status: "idle", + adapterType: "openclaw_gateway", + adapterConfig: { + url: gateway.url, + headers: { + "x-openclaw-token": "gateway-token", + }, + payloadTemplate: { + message: "wake now", + }, + waitTimeoutMs: 2_000, + }, + runtimeConfig: {}, + permissions: {}, + }); + + await db.insert(issues).values({ + id: issueId, + companyId, + title: "Self-authored note must not reopen", + status: "todo", + priority: "medium", + responsibleUserId: "responsible-user", + assigneeAgentId: agentId, + issueNumber: 1, + identifier: `${issuePrefix}-1`, + }); + + const firstRun = await heartbeat.wakeup(agentId, { + source: "assignment", + triggerDetail: "system", + reason: "issue_assigned", + payload: { issueId }, + contextSnapshot: { + issueId, + taskId: issueId, + wakeReason: "issue_assigned", + }, + requestedByActorType: "system", + requestedByActorId: null, + }); + + expect(firstRun).not.toBeNull(); + await waitFor(async () => { + const run = await db + .select({ status: heartbeatRuns.status }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, firstRun!.id)) + .then((rows) => rows[0] ?? null); + return run?.status === "running"; + }); + + const selfComment = await db + .insert(issueComments) + .values({ + companyId, + issueId, + authorUserId: "local-cli-user", + createdByRunId: firstRun?.id ?? null, + body: "Closing note from the same run", + }) + .returning() + .then((rows) => rows[0]); + + const deferredRun = await heartbeat.wakeup(agentId, { + source: "automation", + triggerDetail: "system", + reason: "issue_commented", + payload: { issueId, commentId: selfComment.id }, + contextSnapshot: { + issueId, + taskId: issueId, + commentId: selfComment.id, + wakeCommentId: selfComment.id, + wakeReason: "issue_commented", + }, + requestedByActorType: "user", + requestedByActorId: "local-cli-user", + }); + + expect(deferredRun).toBeNull(); + + await waitFor(async () => { + const deferred = await db + .select() + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.agentId, agentId), + eq(agentWakeupRequests.status, "deferred_issue_execution"), + ), + ) + .then((rows) => rows[0] ?? null); + return Boolean(deferred); + }); + + const queuedWake = await db + .select({ id: agentWakeupRequests.id }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.agentId, agentId), + eq(agentWakeupRequests.status, "deferred_issue_execution"), + ), + ) + .then((rows) => rows[0] ?? null); + + // Running records admission. Wait for provider acceptance before + // simulating completion by that provider, or startup correctly rejects + // the already-closed task before this scenario reaches its follow-up. + await waitFor(() => gateway.getAgentPayloads().length >= 1); + await db + .update(issues) + .set({ + status: "done", + completedAt: new Date(), + executionRunId: null, + executionAgentNameKey: null, + executionLockedAt: null, + updatedAt: new Date(), + }) + .where(eq(issues.id, issueId)); + + gateway.releaseFirstWait(); + + await waitFor(async () => { + const [wake] = await db + .select({ status: agentWakeupRequests.status }) + .from(agentWakeupRequests) + .where(eq(agentWakeupRequests.id, queuedWake!.id)); + return wake?.status === "cancelled"; + }, 90_000); + + const [cancelledWake] = await db + .select({ status: agentWakeupRequests.status, error: agentWakeupRequests.error }) + .from(agentWakeupRequests) + .where(eq(agentWakeupRequests.id, queuedWake!.id)); + expect(cancelledWake).toMatchObject({ + status: "cancelled", + error: "Deferred wake contained only comments authored by the finishing run", + }); + + // The only queued comment came from the run that just finished, so the + // queue must not promote a second run for the same agent to re-read its + // own note, and the issue must stay in the state the run's completion left it in. + expect(gateway.getAgentPayloads()).toHaveLength(1); + const issueAfterCompletion = await db + .select({ status: issues.status, completedAt: issues.completedAt }) + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => rows[0] ?? null); + expect(issueAfterCompletion).toMatchObject({ status: "done" }); + expect(issueAfterCompletion?.completedAt).not.toBeNull(); + } finally { + gateway.releaseFirstWait(); + await gateway.close(); + } + }, 120_000); + it("queues exactly one follow-up run when an issue-bound run exits without a comment", async () => { const gateway = await createControlledGatewayServer(); const companyId = randomUUID(); diff --git a/server/src/modules/wake-queue/adapters/postgres.test.ts b/server/src/modules/wake-queue/adapters/postgres.test.ts new file mode 100644 index 0000000000..f49ec7cdb4 --- /dev/null +++ b/server/src/modules/wake-queue/adapters/postgres.test.ts @@ -0,0 +1,439 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { + agentWakeupRequests, + agents, + companies, + createDb, + heartbeatRuns, + issueComments, + issues, +} from "@paperclipai/db"; +import { + getEmbeddedPostgresTestSupport, + startEmbeddedPostgresTestDatabase, +} from "../../../__tests__/helpers/embedded-postgres.js"; +import { createPostgresWakeQueueAdapter } from "./postgres.js"; +import type { WakeQueuePostgresAdapterDeps } from "./postgres.js"; + +// Proves the atomicity and company-scope properties the security review +// requires: every mutation names `companyId` in its own SQL `WHERE` clause, +// a foreign-company row is invisible to a read, and a deferred-status +// compare-and-set that affects no row leaves no other trace. The decision +// branching itself is proven against plain facts in `domain/policy.test.ts`. +const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; + +if (!embeddedPostgresSupport.supported) { + console.warn( + `Skipping embedded Postgres wake-queue adapter tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`, + ); +} + +describeEmbeddedPostgres("wake-queue postgres adapter", () => { + let db!: ReturnType; + let tempDb: Awaited> | null = null; + + const stubDeps: WakeQueuePostgresAdapterDeps = { + resolveResponsibleUserId: async () => "responsible-user", + getRoutineEnv: async () => ({ routineId: null, env: null, responsibleUserId: null }), + resolveSessionBeforeForWakeup: async () => null, + }; + + beforeAll(async () => { + tempDb = await startEmbeddedPostgresTestDatabase("paperclip-wake-queue-postgres-adapter-"); + db = createDb(tempDb.connectionString); + }, 20_000); + + afterEach(async () => { + await db.delete(issueComments); + // `heartbeat_runs.wakeup_request_id` references `agent_wakeup_requests.id`, + // so the run row must go first. + await db.delete(heartbeatRuns); + await db.delete(agentWakeupRequests); + await db.delete(issues); + await db.delete(agents); + await db.delete(companies); + }); + + afterAll(async () => { + await tempDb?.cleanup(); + }); + + async function seedCompany(): Promise { + const companyId = randomUUID(); + await db.insert(companies).values({ + id: companyId, + name: "Paperclip", + issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`, + requireBoardApprovalForNewAgents: false, + defaultResponsibleUserId: "responsible-user", + }); + return companyId; + } + + async function seedAgent(input: { companyId: string; name?: string }): Promise { + const agentId = randomUUID(); + await db.insert(agents).values({ + id: agentId, + companyId: input.companyId, + name: input.name ?? "CodexCoder", + role: "engineer", + status: "active", + adapterType: "codex_local", + adapterConfig: {}, + runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } }, + permissions: {}, + }); + return agentId; + } + + async function seedIssue(input: { + companyId: string; + issueId?: string; + status?: string; + assigneeAgentId?: string | null; + executionRunId?: string | null; + checkoutRunId?: string | null; + }): Promise { + const issueId = input.issueId ?? randomUUID(); + await db.insert(issues).values({ + id: issueId, + companyId: input.companyId, + title: "Wake-queue adapter fixture issue", + status: input.status ?? "in_progress", + priority: "medium", + assigneeAgentId: input.assigneeAgentId ?? null, + executionRunId: input.executionRunId ?? null, + checkoutRunId: input.checkoutRunId ?? null, + }); + return issueId; + } + + async function seedRun(input: { + companyId: string; + agentId: string; + status?: string; + contextSnapshot?: Record; + errorCode?: string | null; + runtimeMode?: string; + }): Promise { + const runId = randomUUID(); + await db.insert(heartbeatRuns).values({ + id: runId, + companyId: input.companyId, + agentId: input.agentId, + invocationSource: "on_demand", + status: input.status ?? "failed", + contextSnapshot: input.contextSnapshot ?? {}, + errorCode: input.errorCode ?? null, + }); + return runId; + } + + async function seedDeferredWake(input: { + companyId: string; + agentId: string; + issueId: string; + requestedByActorType?: string; + requestedByActorId?: string | null; + payload?: Record; + }): Promise { + const id = randomUUID(); + await db.insert(agentWakeupRequests).values({ + id, + companyId: input.companyId, + agentId: input.agentId, + source: "automation", + reason: "issue_commented", + status: "deferred_issue_execution", + requestedByActorType: input.requestedByActorType ?? "user", + requestedByActorId: input.requestedByActorId ?? null, + payload: { issueId: input.issueId, ...(input.payload ?? {}) }, + }); + return id; + } + + // Review test (a): a foreign-company agent id produces the current failed + // wake status and the current error text, and creates no run. + it("fails a deferred wake whose agent belongs to a different company, without creating a run", async () => { + const companyId = await seedCompany(); + const otherCompanyId = await seedCompany(); + const foreignAgentId = await seedAgent({ companyId: otherCompanyId }); + const finishingAgentId = await seedAgent({ companyId }); + const issueId = await seedIssue({ companyId, assigneeAgentId: finishingAgentId, status: "in_progress" }); + // A finishing run status other than the legacy-reconciliation set (failed, + // timed_out, interrupted, cancelled) reaches the module's own drain logic. + const runId = await seedRun({ companyId, agentId: finishingAgentId, contextSnapshot: { issueId }, status: "succeeded" }); + await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId)); + const wakeId = await seedDeferredWake({ companyId, agentId: foreignAgentId, issueId }); + + const adapter = createPostgresWakeQueueAdapter(db, stubDeps); + const result = await adapter.withIssueExecutionLock({ companyId, runId, now: new Date() }, async (locked, ports) => { + const candidate = await ports.writer.claimNextDeferredWake({ companyId, issueId: locked.primaryIssue.id }); + expect(candidate?.id).toBe(wakeId); + const agent = await ports.reader.findInvokableAgent({ companyId, agentId: foreignAgentId }); + expect(agent).toBeNull(); + const failed = await ports.writer.failDeferredWake({ companyId, wakeId: candidate!.id, now: new Date() }); + expect(failed).toBe(true); + return { outcome: { kind: "released" as const }, postCommitEffects: [] }; + }); + expect(result.outcome.kind).toBe("released"); + + const wakeRow = (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeId)))[0]; + expect(wakeRow?.status).toBe("failed"); + expect(wakeRow?.error).toBe("Deferred wake could not be promoted: agent is not invokable"); + expect(wakeRow?.runId).toBeNull(); + const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId)); + expect(runs).toHaveLength(1); + expect(runs[0]!.id).toBe(runId); + }); + + // Review test (b): each release adapter mutation with a foreign company + // affects no row. + it("scopes every release mutation to its own company and affects no row across a company boundary", async () => { + const companyId = await seedCompany(); + const otherCompanyId = await seedCompany(); + const agentId = await seedAgent({ companyId }); + const issueId = await seedIssue({ companyId, assigneeAgentId: agentId, status: "blocked" }); + await db.update(issues).set({ executionState: { phase: "running" } }).where(eq(issues.id, issueId)); + const wakeId = await seedDeferredWake({ companyId, agentId, issueId }); + const runId = await seedRun({ companyId, agentId, contextSnapshot: { issueId }, status: "succeeded" }); + + const adapter = createPostgresWakeQueueAdapter(db, stubDeps); + await adapter.withIssueExecutionLock( + { companyId, runId, now: new Date() }, + async (_locked, ports) => { + const cancelledUnderWrongCompany = await ports.writer.cancelDeferredWake({ + companyId: otherCompanyId, + wakeId, + reason: "cross-company cancel attempt", + now: new Date(), + }); + expect(cancelledUnderWrongCompany).toBe(false); + + const failedUnderWrongCompany = await ports.writer.failDeferredWake({ + companyId: otherCompanyId, + wakeId, + now: new Date(), + }); + expect(failedUnderWrongCompany).toBe(false); + + const normalizedUnderWrongCompany = await ports.writer.normalizeDeferredWakeCommentIds({ + companyId: otherCompanyId, + wakeId, + payload: { issueId }, + liveCommentIds: ["c1"], + now: new Date(), + }); + expect(normalizedUnderWrongCompany).toBeNull(); + + const reopenedUnderWrongCompany = await ports.writer.reopenIssue({ + companyId: otherCompanyId, + issueId, + runId, + }); + expect(reopenedUnderWrongCompany).toBeNull(); + + return { outcome: { kind: "released" as const }, postCommitEffects: [] }; + }, + ); + + const wakeRow = (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeId)))[0]; + expect(wakeRow?.status).toBe("deferred_issue_execution"); + expect(wakeRow?.error).toBeNull(); + const issueRow = (await db.select().from(issues).where(eq(issues.id, issueId)))[0]; + expect(issueRow?.status).toBe("blocked"); + expect(issueRow?.executionState).toEqual({ phase: "running" }); + }); + + // Review defect: the reopen path must carry the company into every read, + // lock, and write. A check before the write is not a boundary, because + // `issues.company_id` can change between that check and the write. + it("refuses to reopen an issue for a company that does not own it, and leaves the issue untouched", async () => { + const companyId = await seedCompany(); + const otherCompanyId = await seedCompany(); + const agentId = await seedAgent({ companyId }); + const issueId = await seedIssue({ companyId, assigneeAgentId: agentId, status: "blocked" }); + await db.update(issues).set({ executionState: { phase: "running" } }).where(eq(issues.id, issueId)); + const runId = await seedRun({ companyId, agentId, contextSnapshot: { issueId }, status: "succeeded" }); + + const adapter = createPostgresWakeQueueAdapter(db, stubDeps); + const captured: { reopened: { status: string; executionState: Record | null } | null } = { + reopened: null, + }; + await adapter.withIssueExecutionLock({ companyId, runId, now: new Date() }, async (_locked, ports) => { + captured.reopened = await ports.writer.reopenIssue({ companyId: otherCompanyId, issueId, runId }); + return { outcome: { kind: "released" as const }, postCommitEffects: [] }; + }); + expect(captured.reopened).toBeNull(); + + const issueRow = (await db.select().from(issues).where(eq(issues.id, issueId)))[0]; + expect(issueRow?.status).toBe("blocked"); + expect(issueRow?.executionState).toEqual({ phase: "running" }); + }); + + it("reopens an issue for the company that owns it, and clears the execution state", async () => { + const companyId = await seedCompany(); + const agentId = await seedAgent({ companyId }); + const issueId = await seedIssue({ companyId, assigneeAgentId: agentId, status: "blocked" }); + await db.update(issues).set({ executionState: { phase: "running" } }).where(eq(issues.id, issueId)); + const runId = await seedRun({ companyId, agentId, contextSnapshot: { issueId }, status: "succeeded" }); + + const adapter = createPostgresWakeQueueAdapter(db, stubDeps); + const captured: { reopened: { status: string; executionState: Record | null } | null } = { + reopened: null, + }; + await adapter.withIssueExecutionLock({ companyId, runId, now: new Date() }, async (_locked, ports) => { + captured.reopened = await ports.writer.reopenIssue({ companyId, issueId, runId }); + return { outcome: { kind: "released" as const }, postCommitEffects: [] }; + }); + expect(captured.reopened?.status).toBe("todo"); + expect(captured.reopened?.executionState).toBeNull(); + + const issueRow = (await db.select().from(issues).where(eq(issues.id, issueId)))[0]; + expect(issueRow?.status).toBe("todo"); + expect(issueRow?.executionState).toBeNull(); + }); + + // Review test (c): a deferred-status compare-and-set that affects no row + // claims nothing, and no other write in the promotion path ever runs. + it("fails the promotion claim when the deferred-status compare-and-set loses the race, before any other write", async () => { + const companyId = await seedCompany(); + const agentId = await seedAgent({ companyId }); + const issueId = await seedIssue({ companyId, assigneeAgentId: agentId }); + const wakeId = await seedDeferredWake({ companyId, agentId, issueId }); + // A concurrent finalization already claimed this wake before the promotion claim runs. + await db.update(agentWakeupRequests).set({ status: "cancelled" }).where(eq(agentWakeupRequests.id, wakeId)); + + const adapter = createPostgresWakeQueueAdapter(db, stubDeps); + const runId = await seedRun({ companyId, agentId, contextSnapshot: { issueId }, status: "succeeded" }); + const result = await adapter.withIssueExecutionLock({ companyId, runId, now: new Date() }, async (_locked, ports) => { + const claimed = await ports.writer.claimDeferredWakeForPromotion({ companyId, wakeId, now: new Date() }); + expect(claimed).toBe(false); + return { outcome: { kind: "released" as const }, postCommitEffects: [] }; + }); + expect(result.outcome.kind).toBe("released"); + + const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId)); + expect(runs).toHaveLength(1); + expect(runs[0]!.id).toBe(runId); + const issueRow = (await db.select().from(issues).where(eq(issues.id, issueId)))[0]; + expect(issueRow?.executionRunId).toBeNull(); + const wakeRow = (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeId)))[0]; + expect(wakeRow?.status).toBe("cancelled"); + }); + + // `finalizePromotedWake`'s own writes guard against clobbering state a + // concurrent write already changed: it never takes the issue's execution + // lock away from a run that already holds it, and it never overwrites a + // `runId` a wake row already carries. Both guards only matter as defense + // in depth today (the release drain calls this at most once per + // transaction), so this drives the port directly to prove the SQL itself, + // independent of that call pattern. + it("guards finalizePromotedWake's own writes against clobbering an already-set execution lock or runId", async () => { + const companyId = await seedCompany(); + const agentId = await seedAgent({ companyId }); + const issueId = await seedIssue({ companyId, assigneeAgentId: agentId }); + const wakeIdA = await seedDeferredWake({ companyId, agentId, issueId }); + const wakeIdB = await seedDeferredWake({ companyId, agentId, issueId }); + const runId = await seedRun({ companyId, agentId, contextSnapshot: { issueId }, status: "succeeded" }); + const deferredAgent = { id: agentId, companyId, name: "CodexCoder", invokable: true }; + + const finalizedRunIds: string[] = []; + const adapter = createPostgresWakeQueueAdapter(db, stubDeps); + await adapter.withIssueExecutionLock({ companyId, runId, now: new Date() }, async (locked, ports) => { + const finalize = async (wakeId: string) => { + const promoted = await ports.writer.finalizePromotedWake({ + companyId, + wakeId, + deferredAgent, + issue: locked.primaryIssue, + finishingRun: locked.run, + contextSnapshot: { issueId }, + reason: "issue_execution_promoted", + source: "automation", + triggerDetail: null, + payload: {}, + responsibleUserId: "responsible-user", + sessionBefore: null, + now: new Date(), + }); + finalizedRunIds.push(promoted.id); + }; + // The issue's execution lock is free; this call takes it. + await finalize(wakeIdA); + // The issue's execution lock is already held by the first call's run, + // so this call's issue-lock write must no-op even though a run is + // still created. + await finalize(wakeIdB); + // Repeating the same wake must not overwrite its now-set `runId`. + await finalize(wakeIdA); + return { outcome: { kind: "released" as const }, postCommitEffects: [] }; + }); + const [runA, runB, runC] = finalizedRunIds; + + const wakeRowA = (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeIdA)))[0]; + const wakeRowB = (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeIdB)))[0]; + expect(wakeRowA?.runId).toBe(runA); + expect(wakeRowB?.runId).toBe(runB); + const issueRow = (await db.select().from(issues).where(eq(issues.id, issueId)))[0]; + expect(issueRow?.executionRunId).toBe(runA); + expect(issueRow?.executionRunId).not.toBe(runB); + expect(issueRow?.executionRunId).not.toBe(runC); + const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId)); + // All three finalize calls each still insert their own run row. + expect(runs.map((run) => run.id).sort()).toEqual([runId, runA, runB, runC].sort()); + }); + + it("locks the context issue and every sibling issue in id order, and two concurrent releases do not deadlock", async () => { + const companyId = await seedCompany(); + const agentId = await seedAgent({ companyId }); + const issueA = await seedIssue({ companyId, assigneeAgentId: agentId }); + const issueB = await seedIssue({ companyId, assigneeAgentId: agentId }); + const runA = await seedRun({ companyId, agentId, contextSnapshot: { issueId: issueA } }); + const runB = await seedRun({ companyId, agentId, contextSnapshot: { issueId: issueB } }); + await db.update(issues).set({ executionRunId: runA, checkoutRunId: runB }).where(eq(issues.id, issueA)); + await db.update(issues).set({ executionRunId: runB, checkoutRunId: runA }).where(eq(issues.id, issueB)); + + const adapterA = createPostgresWakeQueueAdapter(db, stubDeps); + const adapterB = createPostgresWakeQueueAdapter(db, stubDeps); + const releaseA = adapterA.withIssueExecutionLock({ companyId, runId: runA, now: new Date() }, async (_locked, _ports) => { + await new Promise((resolve) => setTimeout(resolve, 25)); + return { outcome: { kind: "released" as const }, postCommitEffects: [] }; + }); + const releaseB = adapterB.withIssueExecutionLock({ companyId, runId: runB, now: new Date() }, async (_locked, _ports) => { + await new Promise((resolve) => setTimeout(resolve, 25)); + return { outcome: { kind: "released" as const }, postCommitEffects: [] }; + }); + + await expect(Promise.all([releaseA, releaseB])).resolves.toBeDefined(); + + const rows = await db.select().from(issues).where(eq(issues.companyId, companyId)); + for (const row of rows) { + expect(row.executionRunId).toBeNull(); + expect(row.checkoutRunId).toBeNull(); + } + }); + + it("clears both lock columns on every sibling and keeps a transferred executionRunId", async () => { + const companyId = await seedCompany(); + const agentId = await seedAgent({ companyId }); + const finishingRunId = await seedRun({ companyId, agentId, contextSnapshot: {} }); + const retryRunId = await seedRun({ companyId, agentId, contextSnapshot: {}, status: "queued" }); + const issueId = await seedIssue({ companyId, assigneeAgentId: agentId, executionRunId: retryRunId, checkoutRunId: finishingRunId }); + + const adapter = createPostgresWakeQueueAdapter(db, stubDeps); + const result = await adapter.withIssueExecutionLock({ companyId, runId: finishingRunId, now: new Date() }, async () => ({ + outcome: { kind: "released" as const }, + postCommitEffects: [], + })); + expect(result.outcome.kind).toBe("released"); + + const issueRow = (await db.select().from(issues).where(eq(issues.id, issueId)))[0]; + // executionRunId already pointed at the retry, not the finishing run, so it must survive. + expect(issueRow?.executionRunId).toBe(retryRunId); + expect(issueRow?.checkoutRunId).toBeNull(); + }); +}); diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts new file mode 100644 index 0000000000..f3fd525818 --- /dev/null +++ b/server/src/modules/wake-queue/adapters/postgres.ts @@ -0,0 +1,817 @@ +import { and, asc, eq, inArray, isNull, notInArray, or, sql } from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { + agentWakeupRequests, + agents, + heartbeatRuns, + issueComments, + issueRecoveryActions, + issueRelations, + issues, + nativeRunFinalizations, +} from "@paperclipai/db"; +import { legacyExecutionNeedsReconciliation } from "../../../services/legacy-execution-recovery.js"; +import { evaluateAgentInvokability } from "../../../services/agent-invokability.js"; +import { issueTreeControlService, isVerifiedIssueTreeControlInteractionWake } from "../../../services/issue-tree-control.js"; +import { isAutomaticRecoverySuppressedByPauseHold } from "../../../services/recovery/pause-hold-guard.js"; +import { issueService } from "../../../services/issues.js"; +import { issueRecoveryActionService } from "../../../services/issue-recovery-actions.js"; +import { readContinuationAttempt } from "../../../services/recovery/run-liveness-continuations.js"; +import { withRecoveryContext } from "../../../services/recovery/status-only-context.js"; +import { parseIssueExecutionState } from "../../../services/issue-execution-policy.js"; +import { + buildConfigurationIncompleteRecoveryNoticeSeed, + buildExecutionReviewParticipantRecoveryNoticeSeed, + buildImmediateExecutionPathRecoveryNoticeSeed, + buildWorkspaceValidationRecoveryNoticeSeed, +} from "../../../services/recovery/stranded-notice.js"; +import { + queuedCommentIdsFromWakePayload, + withQueuedCommentIdsInWakePayload, +} from "../../../services/issue-queued-comment-queue.js"; +import { extractWakeCommentIds } from "../../run-dispatch/index.js"; +import { hasInteractionContinuationWakeContext } from "../domain/context.js"; +import type { + DeferredWakeCandidate, + InvokableAgentSnapshot, + IssueLockWriter, + IssueSnapshot, + LockedIssueExecution, + ReleaseTransactionResult, + RunSnapshot, + WakeQueueReader, + WakeQueueWriter, +} from "../application/ports.js"; +import type { RunSummary } from "../application/types.js"; +import { WakeQueueApplicationError } from "../application/types.js"; + +const DEFERRED_WAKE_STATUS = "deferred_issue_execution"; +const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext"; +const WORKSPACE_VALIDATION_FAILURE_CODE = "workspace_validation_failed"; +const CONFIGURATION_INCOMPLETE_FAILURE_CODE = "configuration_incomplete"; +const WORKSPACE_VALIDATION_RECOVERY_CAUSE = "workspace_validation_failed"; +const CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE = "configuration_incomplete"; +const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE = "execution_review_participant_recovery"; +const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON = "execution_review_participant_recovery"; +const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON = "execution_review_participant_recovery"; +const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const; + +type HeartbeatRunRow = typeof heartbeatRuns.$inferSelect; +type IssueRow = typeof issues.$inferSelect; + +function parseObject(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record) + : {}; +} + +function readNonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value : null; +} + +function normalizeAgentNameKey(value: string | null | undefined): string | null { + if (typeof value !== "string") return null; + const normalized = value.trim().toLowerCase(); + return normalized.length > 0 ? normalized : null; +} + +function isWorkspaceValidationFailedRun(run: Pick): boolean { + return run.errorCode === WORKSPACE_VALIDATION_FAILURE_CODE; +} + +function isConfigurationIncompleteFailedRun(run: Pick): boolean { + return run.errorCode === CONFIGURATION_INCOMPLETE_FAILURE_CODE || run.errorCode === "model_not_found"; +} + +function toRunSnapshot(row: HeartbeatRunRow): RunSnapshot { + const configurationIncompletePayload = parseObject(parseObject(row.resultJson).configurationIncomplete); + return { + id: row.id, + companyId: row.companyId, + agentId: row.agentId, + status: row.status, + runtimeMode: row.runtimeMode, + errorCode: row.errorCode, + responsibleUserId: row.responsibleUserId, + contextSnapshot: parseObject(row.contextSnapshot), + configurationIncompletePayload: Object.keys(configurationIncompletePayload).length > 0 ? configurationIncompletePayload : null, + }; +} + +function toIssueSnapshot(row: IssueRow): IssueSnapshot { + return { + id: row.id, + companyId: row.companyId, + identifier: row.identifier ?? "", + status: row.status, + assigneeAgentId: row.assigneeAgentId, + assigneeUserId: row.assigneeUserId, + hiddenAt: row.hiddenAt, + originKind: row.originKind, + monitorNextCheckAt: row.monitorNextCheckAt, + executionState: (row.executionState as Record | null) ?? null, + responsibleUserId: row.responsibleUserId, + parentId: row.parentId, + originId: row.originId, + originRunId: row.originRunId, + }; +} + +function toRunSummary(row: HeartbeatRunRow): RunSummary { + return { + id: row.id, + companyId: row.companyId, + agentId: row.agentId, + invocationSource: row.invocationSource, + triggerDetail: row.triggerDetail, + wakeupRequestId: row.wakeupRequestId, + }; +} + +function toDeferredWakeCandidate(row: typeof agentWakeupRequests.$inferSelect): DeferredWakeCandidate { + const payload = parseObject(row.payload); + const queuedCommentIds = queuedCommentIdsFromWakePayload(payload); + const deferredContextSeed = parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]); + const deferredCommentIds = extractWakeCommentIds(deferredContextSeed); + const wakeReason = readNonEmptyString(deferredContextSeed.wakeReason); + const queuedReason = wakeReason ?? readNonEmptyString(row.reason); + const queuedWakeIsCommentOnly = + !queuedReason || + queuedReason === "issue_commented" || + queuedReason === "issue_reopened_via_comment" || + queuedReason === "issue_comment_mentioned"; + const preservesIndependentContinuation = + hasInteractionContinuationWakeContext(deferredContextSeed) || + deferredContextSeed.resumeIntent === true || + !queuedWakeIsCommentOnly; + + return { + id: row.id, + companyId: row.companyId, + agentId: row.agentId, + reason: row.reason, + source: row.source, + triggerDetail: row.triggerDetail, + requestedByActorType: row.requestedByActorType, + requestedByActorId: row.requestedByActorId, + payload, + queuedCommentIds, + preservesIndependentContinuation, + deferredContextSeed, + deferredCommentIds, + wakeReason, + }; +} + +export type WakeQueuePostgresAdapterDeps = { + resolveResponsibleUserId: WakeQueueReader["resolveResponsibleUserId"]; + getRoutineEnv: WakeQueueReader["getRoutineEnv"]; + resolveSessionBeforeForWakeup: WakeQueueReader["resolveSessionBeforeForWakeup"]; +}; + +function buildReader(tx: Db, deps: WakeQueuePostgresAdapterDeps): WakeQueueReader { + return { + async findInvokableAgent({ companyId, agentId }): Promise { + const agent = await tx + .select() + .from(agents) + .where(and(eq(agents.id, agentId), eq(agents.companyId, companyId))) + .then((rows) => rows[0] ?? null); + if (!agent) return null; + const companyAgents = await tx + .select({ id: agents.id, companyId: agents.companyId, name: agents.name, reportsTo: agents.reportsTo, status: agents.status }) + .from(agents) + .where(eq(agents.companyId, companyId)); + const invokability = evaluateAgentInvokability(agent, companyAgents); + return { id: agent.id, companyId: agent.companyId, name: agent.name, invokable: invokability.invokable }; + }, + resolveResponsibleUserId: deps.resolveResponsibleUserId, + getRoutineEnv: deps.getRoutineEnv, + resolveSessionBeforeForWakeup: deps.resolveSessionBeforeForWakeup, + }; +} + +function buildWriter(tx: Db, deps: WakeQueuePostgresAdapterDeps): WakeQueueWriter { + const treeControlSvc = issueTreeControlService(tx); + const issuesSvc = issueService(tx); + + return { + async claimNextDeferredWake({ companyId, issueId }) { + const row = await tx + .select() + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS), + sql`${agentWakeupRequests.payload} ->> 'issueId' = ${issueId}`, + ), + ) + .orderBy(asc(agentWakeupRequests.requestedAt)) + .limit(1) + .then((rows) => rows[0] ?? null); + return row ? toDeferredWakeCandidate(row) : null; + }, + + async getQueuedCommentLiveness({ companyId, issueId, wakeAgentId, finishingRunId, finishingRunAgentId, queuedCommentIds }) { + const rows = await tx + .select({ id: issueComments.id, deletedAt: issueComments.deletedAt, createdByRunId: issueComments.createdByRunId }) + .from(issueComments) + .where(and(eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId), inArray(issueComments.id, queuedCommentIds))); + const targetsFinishingRunAgent = wakeAgentId === finishingRunAgentId; + const liveNonSelfCommentIds = queuedCommentIds.filter((commentId) => { + const row = rows.find((candidate) => candidate.id === commentId); + return Boolean(row && !row.deletedAt && (!targetsFinishingRunAgent || row.createdByRunId !== finishingRunId)); + }); + const containedSelfAuthoredComment = rows.some( + (row) => targetsFinishingRunAgent && !row.deletedAt && row.createdByRunId === finishingRunId, + ); + return { liveNonSelfCommentIds, containedSelfAuthoredComment }; + }, + + async cancelDeferredWake({ companyId, wakeId, reason, now }) { + const rows = await tx + .update(agentWakeupRequests) + .set({ status: "cancelled", finishedAt: now, error: reason, updatedAt: now }) + .where( + and( + eq(agentWakeupRequests.id, wakeId), + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS), + ), + ) + .returning({ id: agentWakeupRequests.id }); + return rows.length > 0; + }, + + async normalizeDeferredWakeCommentIds({ companyId, wakeId, payload, liveCommentIds, now }) { + const rows = await tx + .update(agentWakeupRequests) + .set({ payload: withQueuedCommentIdsInWakePayload(payload, liveCommentIds), updatedAt: now }) + .where( + and( + eq(agentWakeupRequests.id, wakeId), + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS), + ), + ) + .returning(); + const row = rows[0]; + return row ? toDeferredWakeCandidate(row) : null; + }, + + async failDeferredWake({ companyId, wakeId, now }) { + const rows = await tx + .update(agentWakeupRequests) + .set({ + status: "failed", + finishedAt: now, + error: "Deferred wake could not be promoted: agent is not invokable", + updatedAt: now, + }) + .where( + and( + eq(agentWakeupRequests.id, wakeId), + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS), + ), + ) + .returning({ id: agentWakeupRequests.id }); + return rows.length > 0; + }, + + async getPauseHoldFacts({ companyId, issueId, wakeAgentId, deferredContextSeed, requestedByActorType, requestedByActorId }) { + const activePauseHold = await treeControlSvc.getActivePauseHoldGate(companyId, issueId); + if (!activePauseHold) { + return { + activePauseHold: false, + treeHoldInteractionWake: false, + holdId: null, + rootIssueId: null, + mode: null, + reason: null, + releasePolicy: null, + }; + } + const treeHoldInteractionWake = await isVerifiedIssueTreeControlInteractionWake(tx, { + companyId, + issueId, + agentId: wakeAgentId, + contextSnapshot: deferredContextSeed, + requestedByActorType, + requestedByActorId, + }); + return { + activePauseHold: true, + treeHoldInteractionWake, + holdId: activePauseHold.holdId, + rootIssueId: activePauseHold.rootIssueId, + mode: activePauseHold.mode, + reason: activePauseHold.reason, + releasePolicy: activePauseHold.releasePolicy, + }; + }, + + async getCommentSelfAuthorship({ companyId, issueId, finishingRunId, commentIds }) { + const rows = await tx + .select({ createdByRunId: issueComments.createdByRunId }) + .from(issueComments) + .where(and(eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId), inArray(issueComments.id, commentIds))); + return { allSelfAuthored: rows.length > 0 && rows.every((row) => row.createdByRunId === finishingRunId) }; + }, + + async reopenIssue({ companyId, issueId }) { + const updated = await issuesSvc.updateForCompany(issueId, companyId, { status: "todo", executionState: null }, tx); + return updated ? toIssueSnapshot(updated as unknown as IssueRow) : null; + }, + + async claimDeferredWakeForPromotion({ companyId, wakeId, now }) { + const claimed = await tx + .update(agentWakeupRequests) + .set({ + status: "queued", + reason: "issue_execution_promoted", + claimedAt: null, + finishedAt: null, + error: null, + updatedAt: now, + }) + .where( + and( + eq(agentWakeupRequests.id, wakeId), + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS), + ), + ) + .returning({ id: agentWakeupRequests.id }); + return claimed.length > 0; + }, + + async finalizePromotedWake(input) { + const newRun = await tx + .insert(heartbeatRuns) + .values({ + companyId: input.deferredAgent.companyId, + agentId: input.deferredAgent.id, + invocationSource: input.source, + triggerDetail: input.triggerDetail, + status: "queued", + wakeupRequestId: input.wakeId, + contextSnapshot: input.contextSnapshot, + responsibleUserId: input.responsibleUserId, + sessionIdBefore: input.sessionBefore, + continuationAttempt: readContinuationAttempt(input.contextSnapshot.livenessContinuationAttempt), + }) + .returning() + .then((rows) => rows[0]); + + // `claimDeferredWakeForPromotion` already moved this row off + // `deferred_issue_execution` inside this same transaction, so no + // concurrent claimer can still match that guard; this extra `runId is + // null` guard only protects against writing the link twice. + await tx + .update(agentWakeupRequests) + .set({ runId: newRun.id, updatedAt: input.now }) + .where( + and( + eq(agentWakeupRequests.id, input.wakeId), + eq(agentWakeupRequests.companyId, input.companyId), + isNull(agentWakeupRequests.runId), + ), + ); + + // Promoted mention wakes are issue-scoped, not issue ownership + // transfers. The lock-clearing step earlier in this transaction + // already set `executionRunId` to null for this issue, so the `is + // null` guard only protects against taking the lock twice. + await tx + .update(issues) + .set({ + executionRunId: newRun.id, + executionAgentNameKey: normalizeAgentNameKey(input.deferredAgent.name), + executionLockedAt: input.now, + updatedAt: input.now, + }) + .where( + and( + eq(issues.id, input.issue.id), + eq(issues.companyId, input.companyId), + eq(issues.assigneeAgentId, input.deferredAgent.id), + isNull(issues.executionRunId), + ), + ); + + return toRunSummary(newRun); + }, + + async hasExistingExecutionPath({ companyId, issueId, excludeRunId, agentId }) { + const row = await tx + .select({ id: heartbeatRuns.id }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, companyId), + inArray(heartbeatRuns.status, [...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES]), + sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}`, + sql`${heartbeatRuns.id} <> ${excludeRunId}`, + agentId ? eq(heartbeatRuns.agentId, agentId) : sql`true`, + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null); + return row !== null; + }, + + async hasExplicitBlockerPath({ companyId, issueId }) { + const row = await tx + .select({ issueId: issueRelations.issueId }) + .from(issueRelations) + .innerJoin(issues, eq(issueRelations.issueId, issues.id)) + .where( + and( + eq(issueRelations.companyId, companyId), + eq(issueRelations.relatedIssueId, issueId), + eq(issueRelations.type, "blocks"), + eq(issues.companyId, companyId), + notInArray(issues.status, ["done", "cancelled"]), + isNull(issues.hiddenAt), + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null); + return row !== null; + }, + + async isAutomaticRecoverySuppressedByPauseHold({ companyId, issueId }) { + return isAutomaticRecoverySuppressedByPauseHold(tx, companyId, issueId, treeControlSvc); + }, + + async buildBlockedRecoveryNotice({ noticeKind, issueStatus, finishingRun }) { + if (noticeKind === "workspace_validation") { + return { notice: buildWorkspaceValidationRecoveryNoticeSeed(), recoveryCause: WORKSPACE_VALIDATION_RECOVERY_CAUSE }; + } + if (noticeKind === "configuration_incomplete") { + return { + notice: buildConfigurationIncompleteRecoveryNoticeSeed(finishingRun.configurationIncompletePayload), + recoveryCause: CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE, + }; + } + if (noticeKind === "execution_review_participant") { + return { + notice: buildExecutionReviewParticipantRecoveryNoticeSeed(), + recoveryCause: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE, + }; + } + return { notice: buildImmediateExecutionPathRecoveryNoticeSeed({ status: issueStatus }), recoveryCause: null }; + }, + + async queueReviewParticipantRecoveryRun({ companyId, issue, finishingRun, recoveryAgent, sessionBefore, now }) { + const executionState = parseIssueExecutionState(issue.executionState); + const wakeupRequest = await tx + .insert(agentWakeupRequests) + .values({ + companyId, + agentId: recoveryAgent.id, + source: "automation", + triggerDetail: "system", + reason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON, + payload: withRecoveryContext( + { + issueId: issue.id, + retryOfRunId: finishingRun.id, + retryReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON, + currentStageId: executionState?.currentStageId ?? null, + currentStageType: executionState?.currentStageType ?? null, + }, + "normal_model", + ), + status: "queued", + requestedByActorType: "system", + requestedByActorId: null, + updatedAt: now, + }) + .returning() + .then((rows) => rows[0]); + + const queuedRun = await tx + .insert(heartbeatRuns) + .values({ + companyId, + agentId: recoveryAgent.id, + invocationSource: "automation", + triggerDetail: "system", + status: "queued", + wakeupRequestId: wakeupRequest.id, + contextSnapshot: withRecoveryContext( + { + issueId: issue.id, + taskId: issue.id, + wakeReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON, + retryReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON, + source: "issue.execution_review_recovery", + retryOfRunId: finishingRun.id, + currentStageId: executionState?.currentStageId ?? null, + currentStageType: executionState?.currentStageType ?? null, + reviewRecoveryInstruction: + "The previous reviewer run ended while this execution-review stage was still pending. Submit the review decision now, or mark the issue blocked with the exact unblock action.", + }, + "normal_model", + ), + sessionIdBefore: sessionBefore, + retryOfRunId: finishingRun.id, + updatedAt: now, + }) + .returning() + .then((rows) => rows[0]); + + await tx + .update(agentWakeupRequests) + .set({ runId: queuedRun.id, updatedAt: now }) + .where(and(eq(agentWakeupRequests.id, wakeupRequest.id), eq(agentWakeupRequests.companyId, companyId))); + + await tx + .update(issues) + .set({ + executionRunId: queuedRun.id, + executionAgentNameKey: normalizeAgentNameKey(recoveryAgent.name), + executionLockedAt: now, + updatedAt: now, + }) + .where(and(eq(issues.id, issue.id), eq(issues.companyId, companyId))); + + return toRunSummary(queuedRun); + }, + + async queueImmediateRecoveryRun({ companyId, issue, finishingRun, recoveryAgent, sessionBefore, now }) { + const retryReason = issue.status === "todo" ? "assignment_recovery" : "issue_continuation_needed"; + const recoveryReason = issue.status === "todo" ? "issue_assignment_recovery" : "issue_continuation_needed"; + const recoverySource = issue.status === "todo" ? "issue.assignment_recovery" : "issue.continuation_recovery"; + const recoveryContextSnapshot = withRecoveryContext( + { + issueId: issue.id, + taskId: issue.id, + wakeReason: recoveryReason, + retryReason, + source: recoverySource, + retryOfRunId: finishingRun.id, + }, + "normal_model", + ); + + const routineEnvContext = await deps.getRoutineEnv({ companyId, issue }); + const responsibleUserId = await deps.resolveResponsibleUserId({ + companyId, + contextSnapshot: recoveryContextSnapshot, + issue, + routineEnvContext, + requestedByActorType: "system", + requestedByActorId: null, + source: "automation", + triggerDetail: "system", + existingRunResponsibleUserId: finishingRun.responsibleUserId, + }); + if (!responsibleUserId) { + throw new WakeQueueApplicationError( + "responsible_user_unresolved", + "Unable to resolve responsible user for recovery heartbeat run", + { + runId: finishingRun.id, + agentId: recoveryAgent.id, + companyId, + issueId: issue.id, + wakeReason: recoveryReason, + }, + ); + } + + const wakeupRequest = await tx + .insert(agentWakeupRequests) + .values({ + companyId, + agentId: recoveryAgent.id, + source: "automation", + triggerDetail: "system", + reason: recoveryReason, + payload: withRecoveryContext({ issueId: issue.id, retryOfRunId: finishingRun.id }, "normal_model"), + status: "queued", + requestedByActorType: "system", + requestedByActorId: null, + updatedAt: now, + }) + .returning() + .then((rows) => rows[0]); + + const queuedRun = await tx + .insert(heartbeatRuns) + .values({ + companyId, + agentId: recoveryAgent.id, + invocationSource: "automation", + triggerDetail: "system", + status: "queued", + wakeupRequestId: wakeupRequest.id, + contextSnapshot: recoveryContextSnapshot, + responsibleUserId, + sessionIdBefore: sessionBefore, + retryOfRunId: finishingRun.id, + updatedAt: now, + }) + .returning() + .then((rows) => rows[0]); + + await tx + .update(agentWakeupRequests) + .set({ runId: queuedRun.id, updatedAt: now }) + .where(and(eq(agentWakeupRequests.id, wakeupRequest.id), eq(agentWakeupRequests.companyId, companyId))); + + await tx + .update(issues) + .set({ + executionRunId: queuedRun.id, + executionAgentNameKey: normalizeAgentNameKey(recoveryAgent.name), + executionLockedAt: now, + updatedAt: now, + }) + .where(and(eq(issues.id, issue.id), eq(issues.companyId, companyId))); + + return toRunSummary(queuedRun); + }, + }; +} + +async function recordNativeTerminalRecoveryIfNeeded(tx: Db, run: HeartbeatRunRow, issue: IssueRow, now: Date): Promise { + const applies = + run.runtimeMode === "native" && + ["failed", "timed_out", "interrupted", "cancelled"].includes(run.status) && + issue.assigneeAgentId === run.agentId && + !["done", "cancelled"].includes(issue.status); + if (!applies) return false; + + const existing = await tx + .select({ id: issueRecoveryActions.id }) + .from(issueRecoveryActions) + .where( + and( + eq(issueRecoveryActions.companyId, issue.companyId), + eq(issueRecoveryActions.sourceIssueId, issue.id), + or( + inArray(issueRecoveryActions.status, ["active", "escalated"]), + sql`${issueRecoveryActions.evidence}->'automaticRecovery'->>'runId' = ${run.id}`, + ), + ), + ) + .limit(1); + if (!existing.length) { + await tx + .update(nativeRunFinalizations) + .set({ + phase: "terminal_failure", + leaseOwner: null, + leaseExpiresAt: null, + nextAttemptAt: null, + recoveryState: "blocked", + failureCode: "native_continuation_requires_reconciliation", + updatedAt: now, + }) + .where( + and( + eq(nativeRunFinalizations.companyId, issue.companyId), + eq(nativeRunFinalizations.runId, run.id), + isNull(nativeRunFinalizations.resultId), + ), + ); + await issueRecoveryActionService(tx).upsertSourceScoped({ + companyId: issue.companyId, + sourceIssueId: issue.id, + kind: "active_run_watchdog", + ownerType: "board", + returnOwnerAgentId: run.agentId, + cause: "native_continuation_requires_reconciliation", + fingerprint: `native-continuation:${run.id}`, + evidence: { runId: run.id, originalFailureCode: run.errorCode }, + nextAction: + "Inspect the original failure and reconcile the previous execution before continuing. Automatic recovery cannot start another incident.", + maxAttempts: 3, + wakePolicy: null, + supersedeOnIdentityChange: true, + }); + } + return true; +} + +export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAdapterDeps): IssueLockWriter { + return { + async withIssueExecutionLock(input, fn): Promise { + return db.transaction(async (rawTx) => { + const tx = rawTx as unknown as Db; + const run = await tx + .select() + .from(heartbeatRuns) + .where(and(eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.companyId, input.companyId))) + .then((rows) => rows[0] ?? null); + if (!run) { + throw new Error(`wake-queue: run ${input.runId} was not found while releasing issue execution`); + } + const runSnapshot = toRunSnapshot(run); + const contextIssueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId); + + // Lock the context issue (if any) and every issue that still references this + // run, in id order, so two concurrent finalizations can never deadlock on + // each other's row-lock acquisition order. + await tx.execute( + contextIssueId + ? sql` + select id from issues + where company_id = ${input.companyId} + and ( + id = ${contextIssueId} + or execution_run_id = ${run.id} + or checkout_run_id = ${run.id} + ) + order by id + for update + ` + : sql` + select id from issues + where company_id = ${input.companyId} + and (execution_run_id = ${run.id} or checkout_run_id = ${run.id}) + order by id + for update + `, + ); + + const candidateIssues = await tx + .select() + .from(issues) + .where( + and( + eq(issues.companyId, input.companyId), + contextIssueId + ? or(eq(issues.id, contextIssueId), eq(issues.executionRunId, run.id), eq(issues.checkoutRunId, run.id)) + : or(eq(issues.executionRunId, run.id), eq(issues.checkoutRunId, run.id)), + ), + ) + .orderBy(asc(issues.id)); + + // Two separate updates: a retry can move `executionRunId` to a new run + // while `checkoutRunId` still points at this one finishing. + await tx + .update(issues) + .set({ executionRunId: null, executionAgentNameKey: null, executionLockedAt: null, updatedAt: input.now }) + .where(and(eq(issues.companyId, input.companyId), eq(issues.executionRunId, run.id))); + await tx + .update(issues) + .set({ checkoutRunId: null, updatedAt: input.now }) + .where(and(eq(issues.companyId, input.companyId), eq(issues.checkoutRunId, run.id))); + + const issueRow = + (contextIssueId ? candidateIssues.find((candidate) => candidate.id === contextIssueId) : candidateIssues[0]) ?? null; + + if (!issueRow || (issueRow.executionRunId && issueRow.executionRunId !== run.id)) { + return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot }; + } + + if ( + (isWorkspaceValidationFailedRun(run) || isConfigurationIncompleteFailedRun(run)) && + (issueRow.status === "todo" || issueRow.status === "in_progress") && + !issueRow.assigneeUserId && + issueRow.assigneeAgentId === run.agentId + ) { + const configurationIncomplete = isConfigurationIncompleteFailedRun(run); + const notice = configurationIncomplete + ? buildConfigurationIncompleteRecoveryNoticeSeed(runSnapshot.configurationIncompletePayload) + : buildWorkspaceValidationRecoveryNoticeSeed(); + return { + outcome: { + kind: "blocked", + issue: toIssueSnapshot(issueRow), + previousStatus: issueRow.status as "todo" | "in_progress", + notice, + recoveryCause: configurationIncomplete ? CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE : WORKSPACE_VALIDATION_RECOVERY_CAUSE, + }, + postCommitEffects: [], + run: runSnapshot, + }; + } + + if (legacyExecutionNeedsReconciliation(run)) { + return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot }; + } + + // An operator stop never promotes old queued work by itself. The next + // explicit wake adopts those messages atomically when it queues a run. + if (run.status === "cancelled" && parseObject(run.resultJson?.executionCancellation).state === "acknowledged") { + return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot }; + } + + if (await recordNativeTerminalRecoveryIfNeeded(tx, run, issueRow, input.now)) { + return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot }; + } + + const locked: LockedIssueExecution = { primaryIssue: toIssueSnapshot(issueRow), run: runSnapshot }; + const result = await fn(locked, { reader: buildReader(tx, deps), writer: buildWriter(tx, deps) }); + return { ...result, run: runSnapshot }; + }); + }, + }; +} diff --git a/server/src/modules/wake-queue/application/ports.ts b/server/src/modules/wake-queue/application/ports.ts new file mode 100644 index 0000000000..bce32babfd --- /dev/null +++ b/server/src/modules/wake-queue/application/ports.ts @@ -0,0 +1,244 @@ +import type { ReleaseRecoveryBlockedNoticeKind } from "../domain/policy.js"; +import type { + InvokableAgentSnapshot, + IssueSnapshot, + PostCommitEffect, + ReleaseOutcome, + RunSnapshot, + RunSummary, +} from "./types.js"; + +export type { InvokableAgentSnapshot, IssueSnapshot, RunSnapshot, RunSummary }; + +/** The primary issue a locked release resolves to, plus the finishing run the lock step already loaded. */ +export type LockedIssueExecution = { + primaryIssue: IssueSnapshot; + run: RunSnapshot; +}; + +export type ReleaseTransactionResult = { + outcome: ReleaseOutcome; + postCommitEffects: PostCommitEffect[]; +}; + +/** Read-only lookups the release use case needs, each scoped to a company. */ +export interface WakeQueueReader { + findInvokableAgent(input: { companyId: string; agentId: string }): Promise; + /** + * Takes the transaction-scoped issue snapshot, not an issue id, so this + * port never re-reads the issue on a separate connection while the + * module's own transaction is open. + */ + resolveResponsibleUserId(input: { + companyId: string; + contextSnapshot: Record; + issue: IssueSnapshot; + /** From a prior `getRoutineEnv` call against the same issue; pass `{ routineId: null, env: null, responsibleUserId: null }` when the issue is not a routine execution. */ + routineEnvContext: { routineId: string | null; env: unknown; responsibleUserId: string | null }; + requestedByActorType: "user" | "agent" | "system" | null; + requestedByActorId: string | null; + source: string; + triggerDetail: string | null; + existingRunResponsibleUserId: string | null; + }): Promise; + /** + * Takes the transaction-scoped issue snapshot, not an issue id, so this + * port never re-reads the issue on a separate connection while the + * module's own transaction is open. + */ + getRoutineEnv(input: { + companyId: string; + issue: IssueSnapshot; + }): Promise<{ routineId: string | null; env: unknown; responsibleUserId: string | null }>; + resolveSessionBeforeForWakeup(input: { + companyId: string; + agentId: string; + taskKey: string | null; + }): Promise; +} + +export type DeferredWakeCandidate = { + id: string; + companyId: string; + agentId: string; + reason: string | null; + source: string | null; + triggerDetail: string | null; + requestedByActorType: string | null; + requestedByActorId: string | null; + payload: Record; + /** The queued comment ids the wake's queued-comment context carries, already extracted from the payload. */ + queuedCommentIds: string[]; + /** True when the wake carries an independent reason to continue even with no live queued comments. */ + preservesIndependentContinuation: boolean; + /** `payload._paperclipWakeContext`, already parsed to a plain object. */ + deferredContextSeed: Record; + /** The comment ids the wake's context snapshot carries (a separate set from queuedCommentIds), used for the reopen check. */ + deferredCommentIds: string[]; + wakeReason: string | null; +}; + +export type PromoteDeferredWakeInput = { + companyId: string; + wakeId: string; + deferredAgent: InvokableAgentSnapshot; + issue: IssueSnapshot; + finishingRun: RunSnapshot; + contextSnapshot: Record; + reason: string; + source: string; + triggerDetail: string | null; + payload: Record; + responsibleUserId: string; + sessionBefore: string | null; + now: Date; +}; + +/** The transaction-scoped write operations that drain and resolve the deferred-wake queue. */ +export interface WakeQueueWriter { + claimNextDeferredWake(input: { companyId: string; issueId: string }): Promise; + getQueuedCommentLiveness(input: { + companyId: string; + issueId: string; + wakeAgentId: string; + finishingRunId: string; + finishingRunAgentId: string; + queuedCommentIds: string[]; + }): Promise<{ liveNonSelfCommentIds: string[]; containedSelfAuthoredComment: boolean }>; + /** Cancels the wake with `status = 'deferred_issue_execution'` as an atomic compare-and-set guard. */ + cancelDeferredWake(input: { + companyId: string; + wakeId: string; + reason: string; + now: Date; + }): Promise; + normalizeDeferredWakeCommentIds(input: { + companyId: string; + wakeId: string; + /** The wake's current payload, as already read by `claimNextDeferredWake`, used as the rewrite base. */ + payload: Record; + liveCommentIds: string[]; + now: Date; + }): Promise; + /** Sets `status = 'failed'` guarded by the current `deferred_issue_execution` status. */ + failDeferredWake(input: { companyId: string; wakeId: string; now: Date }): Promise; + getPauseHoldFacts(input: { + companyId: string; + issueId: string; + wakeAgentId: string; + deferredContextSeed: Record; + requestedByActorType: string | null; + requestedByActorId: string | null; + }): Promise<{ + activePauseHold: boolean; + treeHoldInteractionWake: boolean; + holdId: string | null; + rootIssueId: string | null; + mode: string | null; + reason: string | null; + releasePolicy: unknown; + }>; + getCommentSelfAuthorship(input: { + companyId: string; + issueId: string; + finishingRunId: string; + commentIds: string[]; + }): Promise<{ allSelfAuthored: boolean }>; + reopenIssue(input: { companyId: string; issueId: string; runId: string }): Promise; + /** + * Atomically claims the wake for promotion, guarded on its current + * `deferred_issue_execution` status. Call this before any other write in + * the promotion path (including a reopen), so a lost race here can never + * leave another write committed underneath it. Returns `false` when a + * concurrent writer already changed the wake's status. + */ + claimDeferredWakeForPromotion(input: { companyId: string; wakeId: string; now: Date }): Promise; + /** + * Finalizes a wake that `claimDeferredWakeForPromotion` already claimed: + * inserts the queued run, links it back onto the wake row, and takes the + * issue's execution lock. Call only after that claim returns `true`. + */ + finalizePromotedWake(input: PromoteDeferredWakeInput): Promise; + /** An open run already on this issue (optionally scoped to one agent) that would race a new recovery run. */ + hasExistingExecutionPath(input: { + companyId: string; + issueId: string; + excludeRunId: string; + agentId: string | null; + }): Promise; + /** An open, non-hidden issue that still lists this issue as a `blocks` predecessor. */ + hasExplicitBlockerPath(input: { companyId: string; issueId: string }): Promise; + isAutomaticRecoverySuppressedByPauseHold(input: { companyId: string; issueId: string }): Promise; + /** Builds the stranded-recovery notice content for a `blocked` outcome; pure formatting, kept behind the writer so `services/recovery/stranded-notice` stays out of the application layer. */ + buildBlockedRecoveryNotice(input: { + noticeKind: ReleaseRecoveryBlockedNoticeKind; + issueStatus: "todo" | "in_progress"; + finishingRun: RunSnapshot; + }): Promise<{ notice: Record; recoveryCause: string | null }>; + queueReviewParticipantRecoveryRun(input: { + companyId: string; + issue: IssueSnapshot; + finishingRun: RunSnapshot; + recoveryAgent: InvokableAgentSnapshot; + sessionBefore: string | null; + now: Date; + }): Promise; + /** + * Builds the recovery context snapshot, resolves the responsible user + * from it, and queues the run. Throws `WakeQueueApplicationError` with + * code `responsible_user_unresolved` when no responsible user resolves, + * without queuing anything. + */ + queueImmediateRecoveryRun(input: { + companyId: string; + issue: IssueSnapshot; + finishingRun: RunSnapshot; + recoveryAgent: InvokableAgentSnapshot; + sessionBefore: string | null; + now: Date; + }): Promise; +} + +/** + * Owns the module's own transaction: loads the finishing run, locks the + * context issue and every sibling issue in id order, clears the two + * release-lock columns, and picks the primary issue. When the primary + * issue is missing, already reclaimed, or resolved by an early exit + * (workspace-validation block, legacy reconciliation, a native-runtime + * terminal failure), the adapter returns that outcome directly without + * calling `fn`. Otherwise it calls `fn` with the locked issue and run, and + * with `reader`/`writer` ports bound to the same transaction, so every + * call `fn` makes through them participates in the one transaction this + * method owns. + */ +export interface IssueLockWriter { + withIssueExecutionLock( + input: { companyId: string; runId: string; now: Date }, + fn: ( + locked: LockedIssueExecution, + ports: { reader: WakeQueueReader; writer: WakeQueueWriter }, + ) => Promise, + ): Promise; +} + +export type StrandedAssignedIssueEscalationInput = { + issue: IssueSnapshot; + previousStatus: "todo" | "in_progress" | "in_review"; + latestRun: RunSnapshot; + notice: Record; + recoveryCause: string | null; +}; + +export type StrandedRecoveryInPlaceEscalationInput = { + issue: IssueSnapshot; + previousStatus: "todo" | "in_progress" | "in_review"; + latestRun: RunSnapshot; +}; + +/** Wraps `services/recovery`'s stranded-issue escalation, called only after the release transaction commits. */ +export interface RecoveryEscalationPort { + escalateStrandedAssignedIssue(input: StrandedAssignedIssueEscalationInput): Promise; + escalateStrandedRecoveryIssueInPlace(input: StrandedRecoveryInPlaceEscalationInput): Promise; +} + +export type { PostCommitEffect, ReleaseOutcome }; diff --git a/server/src/modules/wake-queue/application/types.ts b/server/src/modules/wake-queue/application/types.ts new file mode 100644 index 0000000000..cb126d644d --- /dev/null +++ b/server/src/modules/wake-queue/application/types.ts @@ -0,0 +1,99 @@ +export type RunSummary = { + id: string; + companyId: string; + agentId: string; + invocationSource: string; + triggerDetail: string | null; + wakeupRequestId: string | null; +}; + +export type RunSnapshot = { + id: string; + companyId: string; + agentId: string; + status: string; + runtimeMode: string | null; + errorCode: string | null; + responsibleUserId: string | null; + /** The run's own context snapshot, kept as plain JSON so ports carry no drizzle types. */ + contextSnapshot: Record; + /** `resultJson.configurationIncomplete`, already parsed; non-null only on a configuration-incomplete failed run. */ + configurationIncompletePayload: Record | null; +}; + +export type IssueSnapshot = { + id: string; + companyId: string; + identifier: string; + status: string; + assigneeAgentId: string | null; + assigneeUserId: string | null; + hiddenAt: Date | null; + originKind: string | null; + monitorNextCheckAt: Date | null; + executionState: Record | null; + /** Carried so the routine-env and responsible-user reader ports can use this + * transaction-scoped snapshot instead of reading the issue again. */ + responsibleUserId: string | null; + parentId: string | null; + originId: string | null; + originRunId: string | null; +}; + +export type InvokableAgentSnapshot = { + id: string; + companyId: string; + name: string | null; + invokable: boolean; +}; + +/** A new heartbeat run reached the queued state and should be published and dispatched. */ +export type RunQueuedEffect = { + kind: "run_queued"; + run: RunSummary; +}; + +/** An issue reopened from done/cancelled because a live deferred comment wake promoted on it. */ +export type IssueReopenedEffect = { + kind: "issue_reopened"; + companyId: string; + agentId: string; + runId: string; + issueId: string; + identifier: string; + reopenedFrom: string; +}; + +/** Explicit post-commit work a caller applies only after the release transaction commits. */ +export type PostCommitEffect = RunQueuedEffect | IssueReopenedEffect; + +export type ReleaseOutcome = + | { kind: "released" } + | { kind: "promoted"; run: RunSummary } + | { kind: "queued_review_participant_recovery"; run: RunSummary } + | { kind: "queued_recovery"; run: RunSummary } + | { + kind: "blocked"; + issue: IssueSnapshot; + previousStatus: "todo" | "in_progress" | "in_review"; + notice: Record; + recoveryCause: string | null; + } + | { + kind: "blocked_recovery_in_place"; + issue: IssueSnapshot; + previousStatus: "todo" | "in_progress" | "in_review"; + }; + +export type WakeQueueApplicationErrorCode = "responsible_user_unresolved"; + +export class WakeQueueApplicationError extends Error { + constructor( + readonly code: WakeQueueApplicationErrorCode, + message: string, + readonly details: Record = {}, + ) { + super(message); + this.name = "WakeQueueApplicationError"; + } +} diff --git a/server/src/modules/wake-queue/application/use-cases.test.ts b/server/src/modules/wake-queue/application/use-cases.test.ts new file mode 100644 index 0000000000..81f0719e00 --- /dev/null +++ b/server/src/modules/wake-queue/application/use-cases.test.ts @@ -0,0 +1,331 @@ +import { describe, expect, it, vi } from "vitest"; +import { createReleaseIssueExecution } from "./use-cases.js"; +import { WakeQueueApplicationError } from "./types.js"; +import type { + DeferredWakeCandidate, + InvokableAgentSnapshot, + IssueLockWriter, + IssueSnapshot, + PromoteDeferredWakeInput, + RecoveryEscalationPort, + RunSnapshot, + RunSummary, + WakeQueueReader, + WakeQueueWriter, +} from "./ports.js"; + +const RUN: RunSnapshot = { + id: "run-1", + companyId: "company-1", + agentId: "finishing-agent", + status: "failed", + runtimeMode: "process", + errorCode: null, + responsibleUserId: "user-1", + contextSnapshot: {}, + configurationIncompletePayload: null, +}; + +const ISSUE: IssueSnapshot = { + id: "issue-1", + companyId: "company-1", + identifier: "PAP-1", + status: "in_progress", + assigneeAgentId: "finishing-agent", + assigneeUserId: null, + hiddenAt: null, + originKind: null, + monitorNextCheckAt: null, + executionState: null, + responsibleUserId: null, + parentId: null, + originId: null, + originRunId: null, +}; + +const AGENT: InvokableAgentSnapshot = { + id: "deferred-agent", + companyId: "company-1", + name: "Deferred Agent", + invokable: true, +}; + +function wakeCandidate(overrides: Partial = {}): DeferredWakeCandidate { + return { + id: overrides.id ?? "wake-1", + companyId: "company-1", + agentId: "deferred-agent", + reason: "issue_commented", + source: "automation", + triggerDetail: null, + requestedByActorType: "user", + requestedByActorId: "actor-1", + payload: {}, + queuedCommentIds: [], + preservesIndependentContinuation: false, + deferredContextSeed: {}, + deferredCommentIds: [], + wakeReason: "issue_commented", + ...overrides, + }; +} + +function runSummary(id: string): RunSummary { + return { + id, + companyId: "company-1", + agentId: "deferred-agent", + invocationSource: "automation", + triggerDetail: null, + wakeupRequestId: `wakeup-${id}`, + }; +} + +function createFakeReader(overrides: Partial = {}): WakeQueueReader { + return { + findInvokableAgent: vi.fn(async () => AGENT), + resolveResponsibleUserId: vi.fn(async () => "user-1"), + getRoutineEnv: vi.fn(async () => ({ routineId: null, env: null, responsibleUserId: null })), + resolveSessionBeforeForWakeup: vi.fn(async () => null), + ...overrides, + }; +} + +function createFakeWriter(overrides: Partial = {}): WakeQueueWriter { + return { + claimNextDeferredWake: vi.fn(async () => null), + getQueuedCommentLiveness: vi.fn(async () => ({ liveNonSelfCommentIds: [], containedSelfAuthoredComment: false })), + cancelDeferredWake: vi.fn(async () => true), + normalizeDeferredWakeCommentIds: vi.fn(async (input) => wakeCandidate({ id: input.wakeId, queuedCommentIds: input.liveCommentIds })), + failDeferredWake: vi.fn(async () => true), + getPauseHoldFacts: vi.fn(async () => ({ + activePauseHold: false, + treeHoldInteractionWake: false, + holdId: null, + rootIssueId: null, + mode: null, + reason: null, + releasePolicy: null, + })), + getCommentSelfAuthorship: vi.fn(async () => ({ allSelfAuthored: false })), + reopenIssue: vi.fn(async () => null), + claimDeferredWakeForPromotion: vi.fn(async () => true), + finalizePromotedWake: vi.fn(async (input) => runSummary(input.wakeId)), + hasExistingExecutionPath: vi.fn(async () => false), + hasExplicitBlockerPath: vi.fn(async () => false), + isAutomaticRecoverySuppressedByPauseHold: vi.fn(async () => false), + buildBlockedRecoveryNotice: vi.fn(async () => ({ notice: {}, recoveryCause: null })), + queueReviewParticipantRecoveryRun: vi.fn(async () => runSummary("review-recovery")), + queueImmediateRecoveryRun: vi.fn(async () => runSummary("immediate-recovery")), + ...overrides, + }; +} + +function createFakeIssueLock(reader: WakeQueueReader, writer: WakeQueueWriter): IssueLockWriter { + return { + withIssueExecutionLock: vi.fn(async (_input, fn) => { + const result = await fn({ primaryIssue: ISSUE, run: RUN }, { reader, writer }); + return { ...result, run: RUN }; + }), + }; +} + +function createFakeRecovery(): RecoveryEscalationPort { + return { + escalateStrandedAssignedIssue: vi.fn(async () => {}), + escalateStrandedRecoveryIssueInPlace: vi.fn(async () => {}), + }; +} + +describe("releaseIssueExecution", () => { + it("processes the deferred wakes in requestedAt order", async () => { + const claimOrder: string[] = []; + const queue = [wakeCandidate({ id: "wake-earliest" }), wakeCandidate({ id: "wake-latest" })]; + const writer = createFakeWriter({ + claimNextDeferredWake: vi.fn(async () => { + const next = queue.shift() ?? null; + if (next) claimOrder.push(next.id); + return next; + }), + }); + // Every wake fails invokability so the loop keeps draining without promoting. + const reader = createFakeReader({ findInvokableAgent: vi.fn(async () => null) }); + const issueLock = createFakeIssueLock(reader, writer); + const releaseIssueExecution = createReleaseIssueExecution({ issueLock, recovery: createFakeRecovery() }); + + await releaseIssueExecution({ companyId: "company-1", runId: "run-1", now: new Date() }); + + expect(claimOrder).toEqual(["wake-earliest", "wake-latest"]); + expect(writer.failDeferredWake).toHaveBeenCalledTimes(2); + }); + + it("stops the loop after the first promotion", async () => { + const claimNextDeferredWake = vi.fn(async () => wakeCandidate({ id: "wake-promotes" })); + const writer = createFakeWriter({ claimNextDeferredWake }); + const reader = createFakeReader(); + const issueLock = createFakeIssueLock(reader, writer); + const releaseIssueExecution = createReleaseIssueExecution({ issueLock, recovery: createFakeRecovery() }); + + const result = await releaseIssueExecution({ companyId: "company-1", runId: "run-1", now: new Date() }); + + expect(result.outcome.kind).toBe("promoted"); + expect(claimNextDeferredWake).toHaveBeenCalledTimes(1); + }); + + it("continues the loop after a cancel outcome, a fail outcome, and a normalize outcome, then promotes", async () => { + const queue = [ + // cancel_empty: queued comments, none live, no independent continuation. + wakeCandidate({ id: "wake-cancel", queuedCommentIds: ["c1"] }), + // fail_not_invokable: agent lookup misses for this one wake only. + wakeCandidate({ id: "wake-fail", agentId: "uninvokable-agent" }), + // normalize: queued comments differ from the live set, then promotes. + wakeCandidate({ id: "wake-normalize", queuedCommentIds: ["c1", "c2"] }), + ]; + const claimNextDeferredWake = vi.fn(async () => queue.shift() ?? null); + const findInvokableAgent = vi.fn(async (input: { agentId: string }) => + input.agentId === "deferred-agent" ? AGENT : null, + ); + const getQueuedCommentLiveness = vi.fn(async (input: { queuedCommentIds: string[] }) => + input.queuedCommentIds.length === 1 + ? { liveNonSelfCommentIds: [], containedSelfAuthoredComment: false } + : { liveNonSelfCommentIds: ["c2"], containedSelfAuthoredComment: false }, + ); + const writer = createFakeWriter({ claimNextDeferredWake, getQueuedCommentLiveness }); + const reader = createFakeReader({ findInvokableAgent }); + const issueLock = createFakeIssueLock(reader, writer); + const releaseIssueExecution = createReleaseIssueExecution({ issueLock, recovery: createFakeRecovery() }); + + const result = await releaseIssueExecution({ companyId: "company-1", runId: "run-1", now: new Date() }); + + expect(writer.cancelDeferredWake).toHaveBeenCalledTimes(1); + expect(writer.failDeferredWake).toHaveBeenCalledTimes(1); + expect(writer.normalizeDeferredWakeCommentIds).toHaveBeenCalledTimes(1); + expect(claimNextDeferredWake).toHaveBeenCalledTimes(3); + expect(result.outcome.kind).toBe("promoted"); + }); + + it("returns the post-commit effects as data without running them", async () => { + const writer = createFakeWriter({ claimNextDeferredWake: vi.fn(async () => wakeCandidate()) }); + const reader = createFakeReader(); + const issueLock = createFakeIssueLock(reader, writer); + const recovery = createFakeRecovery(); + const releaseIssueExecution = createReleaseIssueExecution({ issueLock, recovery }); + + const result = await releaseIssueExecution({ companyId: "company-1", runId: "run-1", now: new Date() }); + + expect(result.postCommitEffects).toEqual([{ kind: "run_queued", run: runSummary("wake-1") }]); + expect(recovery.escalateStrandedAssignedIssue).not.toHaveBeenCalled(); + expect(recovery.escalateStrandedRecoveryIssueInPlace).not.toHaveBeenCalled(); + }); + + it("carries the deferred wake's raw issue, interaction, execution-stage, and accepted-plan context onto the promoted run, and clears only the rendered text projections", async () => { + const finalizePromotedWake = vi.fn(async (input: PromoteDeferredWakeInput) => runSummary(input.wakeId)); + const writer = createFakeWriter({ + claimNextDeferredWake: vi.fn(async () => + wakeCandidate({ + deferredContextSeed: { + issueId: ISSUE.id, + wakeCommentIds: ["comment-1"], + // A queue-time render from a prior coalesced run. Promotion must + // not persist this alongside the current (unrelated) comment id. + paperclipTaskMarkdown: "queue-time markdown", + paperclipTaskMarkdownCompact: "queue-time compact markdown", + paperclipWake: { commentId: "comment-1" }, + executionStage: { stage: "review" }, + planReviewInteraction: { acceptedTargetRevision: { revisionId: "revision-1" } }, + acceptedPlanWakeRouting: { targetAgentId: "agent-1" }, + }, + }), + ), + finalizePromotedWake, + }); + const reader = createFakeReader(); + const issueLock = createFakeIssueLock(reader, writer); + const releaseIssueExecution = createReleaseIssueExecution({ issueLock, recovery: createFakeRecovery() }); + + const result = await releaseIssueExecution({ companyId: "company-1", runId: "run-1", now: new Date() }); + + expect(result.outcome.kind).toBe("promoted"); + expect(finalizePromotedWake).toHaveBeenCalledTimes(1); + const promotedContextSnapshot = finalizePromotedWake.mock.calls[0]![0].contextSnapshot; + // The rendered text is cleared; `executeRun` rebuilds it, with proper + // trust-based redaction, from the current issue and comment rows before + // the run dispatches. + expect(promotedContextSnapshot.paperclipTaskMarkdown).toBeUndefined(); + expect(promotedContextSnapshot.paperclipTaskMarkdownCompact).toBeUndefined(); + expect(promotedContextSnapshot.paperclipWake).toBeUndefined(); + // The raw fields that render depends on are not dropped. + expect(promotedContextSnapshot.issueId).toBe(ISSUE.id); + expect(promotedContextSnapshot.executionStage).toEqual({ stage: "review" }); + expect(promotedContextSnapshot.planReviewInteraction).toEqual({ + acceptedTargetRevision: { revisionId: "revision-1" }, + }); + expect(promotedContextSnapshot.acceptedPlanWakeRouting).toEqual({ targetAgentId: "agent-1" }); + }); + + it("never reopens the issue when the promotion claim loses the race, and moves on to the next wake", async () => { + const doneIssue: IssueSnapshot = { ...ISSUE, status: "done" }; + const queue = [ + // Carries a comment that would reopen the done issue, but the + // promotion claim below loses the race before that reopen can run. + wakeCandidate({ + id: "wake-lost-race", + deferredCommentIds: ["c1"], + requestedByActorType: "user", + }), + wakeCandidate({ id: "wake-promotes" }), + ]; + const claimNextDeferredWake = vi.fn(async () => queue.shift() ?? null); + const claimDeferredWakeForPromotion = vi.fn(async ({ wakeId }: { wakeId: string }) => wakeId !== "wake-lost-race"); + const reopenIssue = vi.fn(async () => null); + const writer = createFakeWriter({ claimNextDeferredWake, claimDeferredWakeForPromotion, reopenIssue }); + const reader = createFakeReader(); + const issueLock: IssueLockWriter = { + withIssueExecutionLock: vi.fn(async (_input, fn) => { + const result = await fn({ primaryIssue: doneIssue, run: RUN }, { reader, writer }); + return { ...result, run: RUN }; + }), + }; + const releaseIssueExecution = createReleaseIssueExecution({ issueLock, recovery: createFakeRecovery() }); + + const result = await releaseIssueExecution({ companyId: "company-1", runId: "run-1", now: new Date() }); + + expect(reopenIssue).not.toHaveBeenCalled(); + expect(claimDeferredWakeForPromotion).toHaveBeenCalledTimes(2); + expect(result.outcome.kind).toBe("promoted"); + expect(result.postCommitEffects).toEqual([{ kind: "run_queued", run: runSummary("wake-promotes") }]); + }); + + it("throws WakeQueueApplicationError with code responsible_user_unresolved when the responsible user cannot resolve", async () => { + const writer = createFakeWriter({ claimNextDeferredWake: vi.fn(async () => wakeCandidate()) }); + const reader = createFakeReader({ resolveResponsibleUserId: vi.fn(async () => null) }); + const issueLock = createFakeIssueLock(reader, writer); + const releaseIssueExecution = createReleaseIssueExecution({ issueLock, recovery: createFakeRecovery() }); + + await expect( + releaseIssueExecution({ companyId: "company-1", runId: "run-1", now: new Date() }), + ).rejects.toMatchObject({ + constructor: WakeQueueApplicationError, + code: "responsible_user_unresolved", + }); + }); + + it("escalates through the recovery port for a blocked outcome, after the transaction resolves", async () => { + const writer = createFakeWriter({ + claimNextDeferredWake: vi.fn(async () => null), + hasExistingExecutionPath: vi.fn(async () => false), + isAutomaticRecoverySuppressedByPauseHold: vi.fn(async () => false), + buildBlockedRecoveryNotice: vi.fn(async () => ({ notice: { kind: "immediate_execution_path" }, recoveryCause: "immediate_execution_path" })), + }); + // The recovery agent (the finishing run's own agent) is not invokable, which forces "blocked". + const reader = createFakeReader({ findInvokableAgent: vi.fn(async () => null) }); + const issueLock = createFakeIssueLock(reader, writer); + const recovery = createFakeRecovery(); + const releaseIssueExecution = createReleaseIssueExecution({ issueLock, recovery }); + + const result = await releaseIssueExecution({ companyId: "company-1", runId: "run-1", now: new Date() }); + + expect(result.outcome.kind).toBe("blocked"); + expect(recovery.escalateStrandedAssignedIssue).toHaveBeenCalledTimes(1); + }); +}); diff --git a/server/src/modules/wake-queue/application/use-cases.ts b/server/src/modules/wake-queue/application/use-cases.ts new file mode 100644 index 0000000000..6a07e32498 --- /dev/null +++ b/server/src/modules/wake-queue/application/use-cases.ts @@ -0,0 +1,503 @@ +import { enrichPromotedWakeContext } from "../domain/context.js"; +import { decideDeferredWake, decideReleaseRecovery } from "../domain/policy.js"; +import type { + IssueLockWriter, + IssueSnapshot, + LockedIssueExecution, + RecoveryEscalationPort, + ReleaseTransactionResult, + RunSnapshot, + WakeQueueReader, + WakeQueueWriter, +} from "./ports.js"; +import type { PostCommitEffect, ReleaseOutcome } from "./types.js"; +import { WakeQueueApplicationError } from "./types.js"; + +const ISSUE_DISPOSITION_REPAIR_RETRY_REASON = "issue_disposition_repair"; +const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON = "execution_review_participant_recovery"; +const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASONS = new Set([ + "execution_review_requested", + "execution_approval_requested", +]); +const HEARTBEAT_RUN_TERMINAL_STATUSES = new Set([ + "succeeded", + "failed", + "timed_out", + "cancelled", +]); +const UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES = new Set([ + "failed", + "timed_out", + "cancelled", +]); +const STRANDED_ISSUE_RECOVERY_ORIGIN_KIND = "stranded_issue_recovery"; +const WORKSPACE_VALIDATION_FAILURE_CODE = "workspace_validation_failed"; +const CONFIGURATION_INCOMPLETE_FAILURE_CODE = "configuration_incomplete"; + +function readNonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value : null; +} + +function isWorkspaceValidationFailedRun(run: Pick): boolean { + return run.errorCode === WORKSPACE_VALIDATION_FAILURE_CODE; +} + +function isConfigurationIncompleteFailedRun(run: Pick): boolean { + return run.errorCode === CONFIGURATION_INCOMPLETE_FAILURE_CODE || run.errorCode === "model_not_found"; +} + +function isExecutionReviewParticipantRecoveryRun(run: Pick): boolean { + return readNonEmptyString(run.contextSnapshot.retryReason) === EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON; +} + +function isExecutionReviewParticipantRecoveryEligibleRun(run: Pick): boolean { + const wakeReason = readNonEmptyString(run.contextSnapshot.wakeReason); + return (wakeReason !== null && EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASONS.has(wakeReason)) + || isExecutionReviewParticipantRecoveryRun(run); +} + +function didAutomaticRecoveryFail( + run: Pick, + expectedRetryReason: "assignment_recovery" | "issue_continuation_needed", +): boolean { + const latestRetryReason = readNonEmptyString(run.contextSnapshot.retryReason); + return latestRetryReason === expectedRetryReason && UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES.has(run.status); +} + +function currentAgentParticipant(issue: IssueSnapshot): { agentId: string } | null { + const executionState = issue.executionState; + if (!executionState || executionState.status !== "pending") return null; + const participant = executionState.currentParticipant as Record | null | undefined; + if (!participant || participant.type !== "agent") return null; + const agentId = readNonEmptyString(participant.agentId); + return agentId ? { agentId } : null; +} + +export type ReleaseIssueExecutionInput = { + companyId: string; + runId: string; + now: Date; + suppressImmediateRecovery?: boolean; +}; + +/** + * Drains the deferred-wake queue for the issue a run just released, in + * `requestedAt` order, promoting at most one wake. When the queue empties + * without a promotion, decides the release-recovery outcome. Every read and + * write happens through `ports`, already bound to the module's own + * transaction by the caller. + */ +async function runReleaseDrain( + locked: LockedIssueExecution, + ports: { reader: WakeQueueReader; writer: WakeQueueWriter }, + input: ReleaseIssueExecutionInput, +): Promise { + const { run } = locked; + let issue = locked.primaryIssue; + const postCommitEffects: PostCommitEffect[] = []; + + while (true) { + const candidate = await ports.writer.claimNextDeferredWake({ companyId: run.companyId, issueId: issue.id }); + if (!candidate) break; + + let liveness = { liveNonSelfCommentIds: candidate.queuedCommentIds, containedSelfAuthoredComment: false }; + if (candidate.queuedCommentIds.length > 0) { + liveness = await ports.writer.getQueuedCommentLiveness({ + companyId: run.companyId, + issueId: issue.id, + wakeAgentId: candidate.agentId, + finishingRunId: run.id, + finishingRunAgentId: run.agentId, + queuedCommentIds: candidate.queuedCommentIds, + }); + } + const liveCommentIdsChanged = + liveness.liveNonSelfCommentIds.length !== candidate.queuedCommentIds.length || + liveness.liveNonSelfCommentIds.some((id, index) => id !== candidate.queuedCommentIds[index]); + + const deferredAgent = await ports.reader.findInvokableAgent({ companyId: run.companyId, agentId: candidate.agentId }); + const pauseHold = await ports.writer.getPauseHoldFacts({ + companyId: run.companyId, + issueId: issue.id, + wakeAgentId: candidate.agentId, + deferredContextSeed: candidate.deferredContextSeed, + requestedByActorType: candidate.requestedByActorType, + requestedByActorId: candidate.requestedByActorId, + }); + + let decision = decideDeferredWake({ + queuedComment: { + hasQueuedCommentIds: candidate.queuedCommentIds.length > 0, + liveNonSelfCommentIdsLength: liveness.liveNonSelfCommentIds.length, + queuedCommentIdsLength: candidate.queuedCommentIds.length, + liveCommentIdsChanged, + containedSelfAuthoredComment: liveness.containedSelfAuthoredComment, + preservesIndependentContinuation: candidate.preservesIndependentContinuation, + }, + agent: { agentFound: deferredAgent !== null, invokable: deferredAgent?.invokable ?? false }, + pauseHold: { activePauseHold: pauseHold.activePauseHold, treeHoldInteractionWake: pauseHold.treeHoldInteractionWake }, + }); + + if (decision.kind === "cancel_empty") { + await ports.writer.cancelDeferredWake({ + companyId: run.companyId, + wakeId: candidate.id, + reason: decision.selfAuthored + ? "Deferred wake contained only comments authored by the finishing run" + : "Queued messages were discarded before promotion", + now: input.now, + }); + continue; + } + + let workingCandidate = candidate; + if (decision.kind === "normalize") { + const normalized = await ports.writer.normalizeDeferredWakeCommentIds({ + companyId: run.companyId, + wakeId: candidate.id, + payload: candidate.payload, + liveCommentIds: liveness.liveNonSelfCommentIds, + now: input.now, + }); + if (!normalized) continue; + workingCandidate = normalized; + // Re-decide with the same agent/pause-hold facts already fetched above; the + // comment-id set now matches, so only fail/cancel-pause-hold/promote can result. + decision = decideDeferredWake({ + queuedComment: { + hasQueuedCommentIds: workingCandidate.queuedCommentIds.length > 0, + liveNonSelfCommentIdsLength: liveness.liveNonSelfCommentIds.length, + queuedCommentIdsLength: liveness.liveNonSelfCommentIds.length, + liveCommentIdsChanged: false, + containedSelfAuthoredComment: liveness.containedSelfAuthoredComment, + preservesIndependentContinuation: workingCandidate.preservesIndependentContinuation, + }, + agent: { agentFound: deferredAgent !== null, invokable: deferredAgent?.invokable ?? false }, + pauseHold: { activePauseHold: pauseHold.activePauseHold, treeHoldInteractionWake: pauseHold.treeHoldInteractionWake }, + }); + } + + if (decision.kind === "fail_not_invokable") { + await ports.writer.failDeferredWake({ companyId: run.companyId, wakeId: workingCandidate.id, now: input.now }); + continue; + } + + if (decision.kind === "cancel_pause_hold") { + await ports.writer.cancelDeferredWake({ + companyId: run.companyId, + wakeId: workingCandidate.id, + reason: "Deferred wake suppressed by active subtree pause hold", + now: input.now, + }); + continue; + } + + // decision.kind === "promote" + const invokableAgent = deferredAgent!; + + // Claim the wake for promotion before any other write in this branch + // (design choice: claim first, then reopen). A reopen write, or its + // `issue_reopened` post-commit effect, must never survive a lost race on + // this compare-and-set. When the claim fails, a concurrent writer already + // changed the wake's status, so this candidate is gone; move on to the + // next one instead of ending the drain. + const claimedForPromotion = await ports.writer.claimDeferredWakeForPromotion({ + companyId: run.companyId, + wakeId: workingCandidate.id, + now: input.now, + }); + if (!claimedForPromotion) continue; + + let currentIssue = issue; + + if (workingCandidate.deferredCommentIds.length > 0 && (currentIssue.status === "done" || currentIssue.status === "cancelled")) { + const selfAuthorship = await ports.writer.getCommentSelfAuthorship({ + companyId: run.companyId, + issueId: currentIssue.id, + finishingRunId: run.id, + commentIds: workingCandidate.deferredCommentIds, + }); + const shouldReopen = + !selfAuthorship.allSelfAuthored && + (workingCandidate.requestedByActorType === "user" || workingCandidate.wakeReason === "issue_reopened_via_comment"); + if (shouldReopen) { + const reopened = await ports.writer.reopenIssue({ companyId: run.companyId, issueId: currentIssue.id, runId: run.id }); + if (reopened) { + postCommitEffects.push({ + kind: "issue_reopened", + companyId: reopened.companyId, + agentId: invokableAgent.id, + runId: run.id, + issueId: reopened.id, + identifier: reopened.identifier, + reopenedFrom: currentIssue.status, + }); + currentIssue = reopened; + issue = reopened; + } + } + } + + const promotedReason = workingCandidate.reason ?? "issue_execution_promoted"; + const promotedSource = workingCandidate.source ?? "automation"; + const promotedTriggerDetail = workingCandidate.triggerDetail ?? null; + const promotedPayload = { ...workingCandidate.payload }; + delete promotedPayload["_paperclipWakeContext"]; + + const promotedContextSeed: Record = { ...workingCandidate.deferredContextSeed }; + if (pauseHold.activePauseHold) { + promotedContextSeed.treeHoldInteraction = true; + promotedContextSeed.activeTreeHold = { + holdId: pauseHold.holdId, + rootIssueId: pauseHold.rootIssueId, + mode: pauseHold.mode, + reason: pauseHold.reason, + releasePolicy: pauseHold.releasePolicy, + interaction: true, + }; + } + + const { contextSnapshot: promotedContextSnapshot, taskKey: promotedTaskKey } = enrichPromotedWakeContext({ + contextSnapshot: promotedContextSeed, + reason: promotedReason, + source: promotedSource, + triggerDetail: promotedTriggerDetail, + payload: promotedPayload, + }); + + const sessionBefore = + readNonEmptyString(promotedContextSnapshot.resumeSessionDisplayId) ?? + (await ports.reader.resolveSessionBeforeForWakeup({ + companyId: run.companyId, + agentId: invokableAgent.id, + taskKey: promotedTaskKey, + })); + + const promotedRoutineEnvContext = await ports.reader.getRoutineEnv({ + companyId: invokableAgent.companyId, + issue: currentIssue, + }); + const responsibleUserId = await ports.reader.resolveResponsibleUserId({ + companyId: invokableAgent.companyId, + contextSnapshot: promotedContextSnapshot, + issue: currentIssue, + routineEnvContext: promotedRoutineEnvContext, + requestedByActorType: workingCandidate.requestedByActorType as "user" | "agent" | "system" | null, + requestedByActorId: workingCandidate.requestedByActorId, + source: promotedSource, + triggerDetail: promotedTriggerDetail, + existingRunResponsibleUserId: run.responsibleUserId, + }); + if (!responsibleUserId) { + throw new WakeQueueApplicationError( + "responsible_user_unresolved", + "Unable to resolve responsible user for promoted heartbeat run", + { + runId: run.id, + agentId: invokableAgent.id, + companyId: invokableAgent.companyId, + issueId: currentIssue.id, + wakeReason: readNonEmptyString(promotedContextSnapshot.wakeReason), + }, + ); + } + + const promotedRun = await ports.writer.finalizePromotedWake({ + companyId: run.companyId, + wakeId: workingCandidate.id, + deferredAgent: invokableAgent, + issue: currentIssue, + finishingRun: run, + contextSnapshot: promotedContextSnapshot, + reason: promotedReason, + source: promotedSource, + triggerDetail: promotedTriggerDetail, + payload: promotedPayload, + responsibleUserId, + sessionBefore, + now: input.now, + }); + + postCommitEffects.push({ kind: "run_queued", run: promotedRun }); + return { outcome: { kind: "promoted", run: promotedRun }, postCommitEffects }; + } + + return runReleaseRecoveryTail(issue, run, ports.reader, ports.writer, input, postCommitEffects); +} + +async function runReleaseRecoveryTail( + issue: IssueSnapshot, + run: RunSnapshot, + reader: WakeQueueReader, + writer: WakeQueueWriter, + input: ReleaseIssueExecutionInput, + postCommitEffects: PostCommitEffect[], +): Promise { + const suppressImmediateRecovery = input.suppressImmediateRecovery ?? false; + const isStrandedRecoveryOrigin = issue.originKind === STRANDED_ISSUE_RECOVERY_ORIGIN_KIND; + const recoveryAgent = await reader.findInvokableAgent({ companyId: issue.companyId, agentId: run.agentId }); + + const currentParticipant = currentAgentParticipant(issue); + const reviewParticipantApplies = + issue.status === "in_review" && + !issue.assigneeUserId && + currentParticipant !== null && + currentParticipant.agentId === run.agentId && + isExecutionReviewParticipantRecoveryEligibleRun(run) && + HEARTBEAT_RUN_TERMINAL_STATUSES.has(run.status); + + const immediateApplies = + (issue.status === "todo" || issue.status === "in_progress") && + !issue.assigneeUserId && + !issue.hiddenAt && + issue.assigneeAgentId === run.agentId && + (run.status === "failed" || run.status === "timed_out" || run.status === "cancelled"); + + const suppressedByPauseHold = (reviewParticipantApplies || immediateApplies) + ? await writer.isAutomaticRecoverySuppressedByPauseHold({ companyId: issue.companyId, issueId: issue.id }) + : false; + + const hasExistingExecutionPath = reviewParticipantApplies + ? await writer.hasExistingExecutionPath({ + companyId: issue.companyId, + issueId: issue.id, + excludeRunId: run.id, + agentId: currentParticipant?.agentId ?? null, + }) + : immediateApplies + ? await writer.hasExistingExecutionPath({ companyId: issue.companyId, issueId: issue.id, excludeRunId: run.id, agentId: null }) + : false; + + const hasExplicitBlockerPath = immediateApplies && !reviewParticipantApplies + ? await writer.hasExplicitBlockerPath({ companyId: issue.companyId, issueId: issue.id }) + : false; + + const expectedRetryReason: "assignment_recovery" | "issue_continuation_needed" = + issue.status === "todo" ? "assignment_recovery" : "issue_continuation_needed"; + + const decision = decideReleaseRecovery({ + suppressImmediateRecovery, + reviewParticipant: { + applies: reviewParticipantApplies, + hasExistingExecutionPath, + hasPersistedMonitor: Boolean(issue.monitorNextCheckAt), + suppressedByPauseHold, + isStrandedRecoveryOrigin, + recoveryAgentPresent: recoveryAgent !== null, + recoveryAgentInvokable: recoveryAgent?.invokable ?? false, + isExecutionReviewParticipantRecoveryRun: isExecutionReviewParticipantRecoveryRun(run), + }, + immediate: { + applies: immediateApplies, + isDispositionRepairRetry: readNonEmptyString(run.contextSnapshot.retryReason) === ISSUE_DISPOSITION_REPAIR_RETRY_REASON, + hasExistingExecutionPath, + hasPersistedMonitor: Boolean(issue.monitorNextCheckAt), + hasExplicitBlockerPath, + suppressedByPauseHold, + isStrandedRecoveryOrigin, + recoveryAgentPresent: recoveryAgent !== null, + recoveryAgentInvokable: recoveryAgent?.invokable ?? false, + isWorkspaceValidationFailedRun: isWorkspaceValidationFailedRun(run), + isConfigurationIncompleteFailedRun: isConfigurationIncompleteFailedRun(run), + automaticRecoveryAlreadyFailed: didAutomaticRecoveryFail(run, expectedRetryReason), + }, + }); + + if (decision.kind === "released") { + return { outcome: { kind: "released" }, postCommitEffects }; + } + + if (decision.kind === "blocked_recovery_in_place") { + return { + outcome: { kind: "blocked_recovery_in_place", issue, previousStatus: statusForBlock(issue) }, + postCommitEffects, + }; + } + + if (decision.kind === "blocked") { + const { notice, recoveryCause } = await writer.buildBlockedRecoveryNotice({ + noticeKind: decision.notice, + issueStatus: issue.status === "todo" ? "todo" : "in_progress", + finishingRun: run, + }); + return { + outcome: { + kind: "blocked", + issue, + previousStatus: statusForBlock(issue), + notice, + recoveryCause, + }, + postCommitEffects, + }; + } + + const sessionBefore = await reader.resolveSessionBeforeForWakeup({ + companyId: issue.companyId, + agentId: recoveryAgent!.id, + taskKey: readNonEmptyString(run.contextSnapshot.taskKey) ?? readNonEmptyString(run.contextSnapshot.issueId), + }); + + if (decision.kind === "queue_review_participant_recovery") { + const queuedRun = await writer.queueReviewParticipantRecoveryRun({ + companyId: issue.companyId, + issue, + finishingRun: run, + recoveryAgent: recoveryAgent!, + sessionBefore, + now: input.now, + }); + postCommitEffects.push({ kind: "run_queued", run: queuedRun }); + return { outcome: { kind: "queued_review_participant_recovery", run: queuedRun }, postCommitEffects }; + } + + // decision.kind === "queue_recovery"; the adapter builds the recovery + // context snapshot and resolves the responsible user from it, throwing + // WakeQueueApplicationError when no responsible user resolves. + const queuedRun = await writer.queueImmediateRecoveryRun({ + companyId: issue.companyId, + issue, + finishingRun: run, + recoveryAgent: recoveryAgent!, + sessionBefore, + now: input.now, + }); + postCommitEffects.push({ kind: "run_queued", run: queuedRun }); + return { outcome: { kind: "queued_recovery", run: queuedRun }, postCommitEffects }; +} + +function statusForBlock(issue: IssueSnapshot): "todo" | "in_progress" | "in_review" { + return issue.status === "todo" || issue.status === "in_review" ? issue.status : "in_progress"; +} + +export function createReleaseIssueExecution(deps: { + issueLock: IssueLockWriter; + recovery: RecoveryEscalationPort; +}) { + return async function releaseIssueExecution( + input: ReleaseIssueExecutionInput, + ): Promise<{ outcome: ReleaseOutcome; postCommitEffects: PostCommitEffect[] }> { + const result = await deps.issueLock.withIssueExecutionLock( + { companyId: input.companyId, runId: input.runId, now: input.now }, + (locked, ports) => runReleaseDrain(locked, ports, input), + ); + + if (result.outcome.kind === "blocked") { + await deps.recovery.escalateStrandedAssignedIssue({ + issue: result.outcome.issue, + previousStatus: result.outcome.previousStatus, + latestRun: result.run, + notice: result.outcome.notice, + recoveryCause: result.outcome.recoveryCause, + }); + } else if (result.outcome.kind === "blocked_recovery_in_place") { + await deps.recovery.escalateStrandedRecoveryIssueInPlace({ + issue: result.outcome.issue, + previousStatus: result.outcome.previousStatus, + latestRun: result.run, + }); + } + + return { outcome: result.outcome, postCommitEffects: result.postCommitEffects }; + }; +} diff --git a/server/src/modules/wake-queue/domain/context.test.ts b/server/src/modules/wake-queue/domain/context.test.ts new file mode 100644 index 0000000000..d4fe457f71 --- /dev/null +++ b/server/src/modules/wake-queue/domain/context.test.ts @@ -0,0 +1,124 @@ +import { describe, expect, it } from "vitest"; +import { enrichPromotedWakeContext } from "./context.js"; + +describe("enrichPromotedWakeContext", () => { + it("clears stale derived comment projections when the canonical id list is empty, but keeps an independent interaction continuation", () => { + const contextSnapshot: Record = { + // Normalization already dropped the canonical wake comment ids upstream + // of this function; no wakeCommentIds field remains on the snapshot. + paperclipWake: { commentId: "comment-2" }, + paperclipWakeComment: { id: "comment-2", body: "stale body" }, + paperclipTaskMarkdown: "stale markdown", + paperclipTaskMarkdownCompact: "stale compact markdown", + // Independent interaction continuation the promoted run must keep. + interactionId: "interaction-1", + interactionKind: "request_confirmation", + interactionStatus: "accepted", + }; + + const result = enrichPromotedWakeContext({ + contextSnapshot, + reason: "issue_execution_promoted", + source: "automation", + triggerDetail: null, + payload: { mutation: "interaction" }, + }); + + expect(result.contextSnapshot.wakeCommentIds).toBeUndefined(); + expect(result.contextSnapshot.commentId).toBeUndefined(); + expect(result.contextSnapshot.wakeCommentId).toBeUndefined(); + expect(result.contextSnapshot.paperclipWake).toBeUndefined(); + expect(result.contextSnapshot.paperclipWakeComment).toBeUndefined(); + expect(result.contextSnapshot.paperclipTaskMarkdown).toBeUndefined(); + expect(result.contextSnapshot.paperclipTaskMarkdownCompact).toBeUndefined(); + // The independent interaction continuation survives the clear. + expect(result.contextSnapshot.interactionId).toBe("interaction-1"); + expect(result.contextSnapshot.interactionStatus).toBe("accepted"); + }); + + it("sets the canonical id and latest-id fields when the recomputed list has entries, and still clears the stale projections", () => { + const contextSnapshot: Record = { + wakeCommentIds: ["comment-1"], + paperclipWake: { commentId: "comment-1" }, + paperclipWakeComment: { id: "comment-1", body: "will be rebuilt" }, + paperclipTaskMarkdown: "will be rebuilt", + paperclipTaskMarkdownCompact: "will be rebuilt", + }; + + const result = enrichPromotedWakeContext({ + contextSnapshot, + reason: "issue_execution_promoted", + source: "automation", + triggerDetail: null, + payload: {}, + }); + + expect(result.contextSnapshot.wakeCommentIds).toEqual(["comment-1"]); + expect(result.contextSnapshot.commentId).toBe("comment-1"); + expect(result.contextSnapshot.wakeCommentId).toBe("comment-1"); + // The derived projections are cleared so the run rebuilds them from the + // canonical ids instead of carrying forward a queue-time snapshot. + expect(result.contextSnapshot.paperclipWake).toBeUndefined(); + expect(result.contextSnapshot.paperclipWakeComment).toBeUndefined(); + expect(result.contextSnapshot.paperclipTaskMarkdown).toBeUndefined(); + expect(result.contextSnapshot.paperclipTaskMarkdownCompact).toBeUndefined(); + }); + + it("adds a new id from the payload to the canonical list and clears the stale projections", () => { + const contextSnapshot: Record = { + wakeCommentIds: ["comment-1"], + paperclipWakeComment: { id: "comment-1", body: "stale body" }, + }; + + const result = enrichPromotedWakeContext({ + contextSnapshot, + reason: "issue_execution_promoted", + source: "automation", + triggerDetail: null, + payload: { commentId: "comment-2" }, + }); + + expect(result.contextSnapshot.wakeCommentIds).toEqual(["comment-1", "comment-2"]); + expect(result.contextSnapshot.commentId).toBe("comment-2"); + expect(result.contextSnapshot.wakeCommentId).toBe("comment-2"); + expect(result.contextSnapshot.paperclipWakeComment).toBeUndefined(); + }); + + it("keeps the raw issue, execution-stage, and accepted-plan fields a dispatch rebuild needs, alongside the cleared text projections", () => { + const contextSnapshot: Record = { + wakeCommentIds: ["comment-1"], + paperclipWake: { commentId: "comment-1" }, + paperclipTaskMarkdown: "stale markdown", + paperclipTaskMarkdownCompact: "stale compact markdown", + // Raw context a dispatch-time rebuild reads directly off the promoted + // run; none of it is derived from the comment ids above, so none of it + // should be cleared alongside the stale text projections. + issueId: "issue-1", + executionStage: { stage: "review" }, + planReviewInteraction: { acceptedTargetRevision: { revisionId: "revision-1" } }, + acceptedPlanWakeRouting: { targetAgentId: "agent-1" }, + workspaceRefreshReason: "accepted_plan_confirmation", + }; + + const result = enrichPromotedWakeContext({ + contextSnapshot, + reason: "issue_execution_promoted", + source: "automation", + triggerDetail: null, + payload: {}, + }); + + // The rendered text is cleared; dispatch rebuilds it from the raw fields. + expect(result.contextSnapshot.paperclipWake).toBeUndefined(); + expect(result.contextSnapshot.paperclipTaskMarkdown).toBeUndefined(); + expect(result.contextSnapshot.paperclipTaskMarkdownCompact).toBeUndefined(); + // The raw fields the rebuild needs are untouched. + expect(result.contextSnapshot.issueId).toBe("issue-1"); + expect(result.contextSnapshot.executionStage).toEqual({ stage: "review" }); + expect(result.contextSnapshot.planReviewInteraction).toEqual({ + acceptedTargetRevision: { revisionId: "revision-1" }, + }); + expect(result.contextSnapshot.acceptedPlanWakeRouting).toEqual({ targetAgentId: "agent-1" }); + expect(result.contextSnapshot.workspaceRefreshReason).toBe("accepted_plan_confirmation"); + }); +}); diff --git a/server/src/modules/wake-queue/domain/context.ts b/server/src/modules/wake-queue/domain/context.ts new file mode 100644 index 0000000000..d9f31ef6e3 --- /dev/null +++ b/server/src/modules/wake-queue/domain/context.ts @@ -0,0 +1,165 @@ +// Pure helpers that build the context snapshot for a promoted deferred wake. +// `extractWakeCommentIds`, `deriveCommentId`, and `WAKE_COMMENT_IDS_KEY` come +// from the run-dispatch module so the two modules read one wake context +// shape; this file holds only the small pieces that are specific to +// building the release half's promoted-run snapshot. + +import { extractWakeCommentIds, WAKE_COMMENT_IDS_KEY } from "../../run-dispatch/index.js"; + +const PAPERCLIP_WAKE_PAYLOAD_KEY = "paperclipWake"; +const PAPERCLIP_WAKE_COMMENT_KEY = "paperclipWakeComment"; +const PAPERCLIP_TASK_MARKDOWN_KEY = "paperclipTaskMarkdown"; +const PAPERCLIP_TASK_MARKDOWN_COMPACT_KEY = "paperclipTaskMarkdownCompact"; + +const INTERACTION_CONTINUATION_CONTEXT_KEYS = [ + "interactionId", + "interactionKind", + "interactionStatus", + "continuationPolicy", + "checkboxSelection", + "itemVerdicts", + "newlyResolvedItemIds", +] as const; + +function readNonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value : null; +} + +function deriveTaskKey( + contextSnapshot: Record | null | undefined, + payload: Record | null | undefined, +): string | null { + return ( + readNonEmptyString(contextSnapshot?.taskKey) ?? + readNonEmptyString(contextSnapshot?.taskId) ?? + readNonEmptyString(contextSnapshot?.issueId) ?? + readNonEmptyString(payload?.taskKey) ?? + readNonEmptyString(payload?.taskId) ?? + readNonEmptyString(payload?.issueId) ?? + null + ); +} + +function mergeWakeCommentIds(...values: Array): string[] { + const merged: string[] = []; + const append = (value: unknown) => { + const normalized = readNonEmptyString(value); + if (!normalized || merged.includes(normalized)) return; + merged.push(normalized); + }; + for (const value of values) { + if (Array.isArray(value)) { + for (const entry of value) append(entry); + continue; + } + if (typeof value === "object" && value !== null) { + const candidate = value as Record; + const batched = extractWakeCommentIds(candidate); + if (batched.length > 0) { + for (const entry of batched) append(entry); + continue; + } + append(candidate.wakeCommentId); + append(candidate.commentId); + continue; + } + append(value); + } + return merged; +} + +export function hasInteractionContinuationWakeContext(contextSnapshot: Record): boolean { + return INTERACTION_CONTINUATION_CONTEXT_KEYS.some((key) => readNonEmptyString(contextSnapshot[key])); +} + +function isInteractionResolutionWakePayload(payload: Record | null | undefined): boolean { + return readNonEmptyString(payload?.mutation) === "interaction"; +} + +function normalizeInteractionContinuationWakeContext( + contextSnapshot: Record, + payload: Record | null | undefined, +): void { + if (isInteractionResolutionWakePayload(payload)) return; + for (const key of INTERACTION_CONTINUATION_CONTEXT_KEYS) { + delete contextSnapshot[key]; + } +} + +export type EnrichPromotedWakeContextInput = { + contextSnapshot: Record; + reason: string | null; + source: string | null; + triggerDetail: string | null; + payload: Record | null; +}; + +export type EnrichPromotedWakeContextResult = { + contextSnapshot: Record; + taskKey: string | null; +}; + +/** + * Fills the promoted run's context snapshot with the fields the original + * wake enrichment always fills, without overwriting a field the deferred + * wake already carried. The deferred wake's own context snapshot was + * already enriched once when it was first queued, so most calls only + * normalize the interaction-continuation keys. + */ +export function enrichPromotedWakeContext( + input: EnrichPromotedWakeContextInput, +): EnrichPromotedWakeContextResult { + const contextSnapshot = { ...input.contextSnapshot }; + const { reason, source, triggerDetail, payload } = input; + const commentIdFromPayload = readNonEmptyString(payload?.["commentId"]); + const taskKey = deriveTaskKey(contextSnapshot, payload); + const wakeCommentIds = mergeWakeCommentIds(contextSnapshot, commentIdFromPayload); + + if (!readNonEmptyString(contextSnapshot["wakeReason"]) && reason) { + contextSnapshot.wakeReason = reason; + } + if (!readNonEmptyString(contextSnapshot["taskKey"]) && taskKey) { + contextSnapshot.taskKey = taskKey; + } + if (!readNonEmptyString(contextSnapshot["commentId"]) && commentIdFromPayload) { + contextSnapshot.commentId = commentIdFromPayload; + } + // The wake payload, resolved comment, and task-markdown snapshots below are + // rendered text built from the canonical comment ids and the issue state at + // queue time. This function recomputes the canonical ids on every call + // (`wakeCommentIds`), so it must not let a rendered snapshot from a stale + // id list carry forward: clear all four, then restore only the canonical + // id and latest-id fields, and only when the recomputed list actually has + // entries. + // + // This function does not render a replacement in its place. The + // promoted run still carries every raw field the render depends on + // (`issueId`, `wakeReason`, interaction and accepted-plan fields, and the + // canonical comment ids set below): `heartbeatService`'s `executeRun` + // rebuilds the rendered text from those raw fields and the current issue + // and comment rows, with the trust-based redaction that rendering needs, + // before it persists the run and dispatches it to the agent. Rendering a + // second, simplified copy here would either skip that redaction (an unsafe + // shortcut for quarantined comment content) or duplicate it outside the + // one place it is proven correct, so this function leaves the render to + // dispatch and only guarantees the raw inputs survive. + delete contextSnapshot[PAPERCLIP_WAKE_PAYLOAD_KEY]; + delete contextSnapshot[PAPERCLIP_WAKE_COMMENT_KEY]; + delete contextSnapshot[PAPERCLIP_TASK_MARKDOWN_KEY]; + delete contextSnapshot[PAPERCLIP_TASK_MARKDOWN_COMPACT_KEY]; + if (wakeCommentIds.length > 0) { + const latestCommentId = wakeCommentIds[wakeCommentIds.length - 1]; + contextSnapshot[WAKE_COMMENT_IDS_KEY] = wakeCommentIds; + contextSnapshot.commentId = latestCommentId; + contextSnapshot.wakeCommentId = latestCommentId; + } + if (!readNonEmptyString(contextSnapshot["wakeSource"]) && source) { + contextSnapshot.wakeSource = source; + } + if (!readNonEmptyString(contextSnapshot["wakeTriggerDetail"]) && triggerDetail) { + contextSnapshot.wakeTriggerDetail = triggerDetail; + } + normalizeInteractionContinuationWakeContext(contextSnapshot, payload); + + return { contextSnapshot, taskKey }; +} diff --git a/server/src/modules/wake-queue/domain/policy.test.ts b/server/src/modules/wake-queue/domain/policy.test.ts new file mode 100644 index 0000000000..61eb44d0e6 --- /dev/null +++ b/server/src/modules/wake-queue/domain/policy.test.ts @@ -0,0 +1,353 @@ +import { describe, expect, it } from "vitest"; +import { + decideDeferredWake, + decideReleaseRecovery, + type DeferredWakeFacts, + type ReleaseRecoveryFacts, +} from "./policy.js"; + +const baseDeferredWakeFacts: DeferredWakeFacts = { + queuedComment: { + hasQueuedCommentIds: false, + liveNonSelfCommentIdsLength: 0, + queuedCommentIdsLength: 0, + liveCommentIdsChanged: false, + containedSelfAuthoredComment: false, + preservesIndependentContinuation: false, + }, + agent: { agentFound: true, invokable: true }, + pauseHold: { activePauseHold: false, treeHoldInteractionWake: false }, +}; + +describe("decideDeferredWake", () => { + const cases: Array<{ + name: string; + facts: DeferredWakeFacts; + expected: ReturnType; + }> = [ + { + name: "cancel_empty: all queued comments discarded and no independent continuation", + facts: { + ...baseDeferredWakeFacts, + queuedComment: { + hasQueuedCommentIds: true, + liveNonSelfCommentIdsLength: 0, + queuedCommentIdsLength: 2, + liveCommentIdsChanged: true, + containedSelfAuthoredComment: false, + preservesIndependentContinuation: false, + }, + }, + expected: { kind: "cancel_empty", selfAuthored: false }, + }, + { + name: "cancel_empty: self-authored comments discarded, error text reflects self-authorship", + facts: { + ...baseDeferredWakeFacts, + queuedComment: { + hasQueuedCommentIds: true, + liveNonSelfCommentIdsLength: 0, + queuedCommentIdsLength: 1, + liveCommentIdsChanged: true, + containedSelfAuthoredComment: true, + preservesIndependentContinuation: false, + }, + }, + expected: { kind: "cancel_empty", selfAuthored: true }, + }, + { + name: "normalize: no live comments, but an independent continuation reason still rewrites the queued id list", + facts: { + ...baseDeferredWakeFacts, + queuedComment: { + hasQueuedCommentIds: true, + liveNonSelfCommentIdsLength: 0, + queuedCommentIdsLength: 1, + liveCommentIdsChanged: true, + containedSelfAuthoredComment: false, + preservesIndependentContinuation: true, + }, + }, + expected: { kind: "normalize" }, + }, + { + name: "promote: an independent continuation reason keeps the wake alive with the live id set already matching", + facts: { + ...baseDeferredWakeFacts, + queuedComment: { + hasQueuedCommentIds: true, + liveNonSelfCommentIdsLength: 0, + queuedCommentIdsLength: 0, + liveCommentIdsChanged: false, + containedSelfAuthoredComment: false, + preservesIndependentContinuation: true, + }, + }, + expected: { kind: "promote" }, + }, + { + name: "normalize: the live comment id set differs from the queued set", + facts: { + ...baseDeferredWakeFacts, + queuedComment: { + hasQueuedCommentIds: true, + liveNonSelfCommentIdsLength: 1, + queuedCommentIdsLength: 2, + liveCommentIdsChanged: true, + containedSelfAuthoredComment: false, + preservesIndependentContinuation: false, + }, + }, + expected: { kind: "normalize" }, + }, + { + name: "fail_not_invokable: the agent lookup returns not-found", + facts: { + ...baseDeferredWakeFacts, + agent: { agentFound: false, invokable: false }, + }, + expected: { kind: "fail_not_invokable" }, + }, + { + name: "fail_not_invokable: the agent is found but not invokable", + facts: { + ...baseDeferredWakeFacts, + agent: { agentFound: true, invokable: false }, + }, + expected: { kind: "fail_not_invokable" }, + }, + { + name: "cancel_pause_hold: an active pause hold with no verified tree-hold interaction", + facts: { + ...baseDeferredWakeFacts, + pauseHold: { activePauseHold: true, treeHoldInteractionWake: false }, + }, + expected: { kind: "cancel_pause_hold" }, + }, + { + name: "promote: an active pause hold but a verified tree-hold interaction wake survives it", + facts: { + ...baseDeferredWakeFacts, + pauseHold: { activePauseHold: true, treeHoldInteractionWake: true }, + }, + expected: { kind: "promote" }, + }, + { + name: "promote: no queued comments, an invokable agent, and no pause hold", + facts: baseDeferredWakeFacts, + expected: { kind: "promote" }, + }, + ]; + + for (const testCase of cases) { + it(testCase.name, () => { + expect(decideDeferredWake(testCase.facts)).toEqual(testCase.expected); + }); + } +}); + +const baseReleaseRecoveryFacts: ReleaseRecoveryFacts = { + suppressImmediateRecovery: false, + reviewParticipant: { + applies: false, + hasExistingExecutionPath: false, + hasPersistedMonitor: false, + suppressedByPauseHold: false, + isStrandedRecoveryOrigin: false, + recoveryAgentPresent: true, + recoveryAgentInvokable: true, + isExecutionReviewParticipantRecoveryRun: false, + }, + immediate: { + applies: false, + isDispositionRepairRetry: false, + hasExistingExecutionPath: false, + hasPersistedMonitor: false, + hasExplicitBlockerPath: false, + suppressedByPauseHold: false, + isStrandedRecoveryOrigin: false, + recoveryAgentPresent: true, + recoveryAgentInvokable: true, + isWorkspaceValidationFailedRun: false, + isConfigurationIncompleteFailedRun: false, + automaticRecoveryAlreadyFailed: false, + }, +}; + +describe("decideReleaseRecovery", () => { + const cases: Array<{ + name: string; + facts: ReleaseRecoveryFacts; + expected: ReturnType; + }> = [ + { + name: "released: neither the review-participant nor the immediate-recovery branch applies", + facts: baseReleaseRecoveryFacts, + expected: { kind: "released" }, + }, + { + name: "released: immediate recovery applies but the caller asked to suppress it", + facts: { + ...baseReleaseRecoveryFacts, + suppressImmediateRecovery: true, + immediate: { ...baseReleaseRecoveryFacts.immediate, applies: true }, + }, + expected: { kind: "released" }, + }, + { + name: "released: immediate recovery applies but an existing execution path already covers it", + facts: { + ...baseReleaseRecoveryFacts, + immediate: { + ...baseReleaseRecoveryFacts.immediate, + applies: true, + hasExistingExecutionPath: true, + }, + }, + expected: { kind: "released" }, + }, + { + name: "released: immediate recovery applies but the finishing run carried the disposition-repair retry reason", + facts: { + ...baseReleaseRecoveryFacts, + immediate: { + ...baseReleaseRecoveryFacts.immediate, + applies: true, + isDispositionRepairRetry: true, + }, + }, + expected: { kind: "released" }, + }, + { + name: "released: immediate recovery applies but an explicit blocker path exists", + facts: { + ...baseReleaseRecoveryFacts, + immediate: { + ...baseReleaseRecoveryFacts.immediate, + applies: true, + hasExplicitBlockerPath: true, + }, + }, + expected: { kind: "released" }, + }, + { + name: "blocked_recovery_in_place: immediate recovery applies on a stranded-issue-recovery origin", + facts: { + ...baseReleaseRecoveryFacts, + immediate: { + ...baseReleaseRecoveryFacts.immediate, + applies: true, + isStrandedRecoveryOrigin: true, + }, + }, + expected: { kind: "blocked_recovery_in_place" }, + }, + { + name: "blocked: immediate recovery applies but the recovery agent is not invokable", + facts: { + ...baseReleaseRecoveryFacts, + immediate: { + ...baseReleaseRecoveryFacts.immediate, + applies: true, + recoveryAgentInvokable: false, + }, + }, + expected: { kind: "blocked", notice: "immediate_execution_path" }, + }, + { + name: "blocked: immediate recovery applies and the run failed workspace validation", + facts: { + ...baseReleaseRecoveryFacts, + immediate: { + ...baseReleaseRecoveryFacts.immediate, + applies: true, + isWorkspaceValidationFailedRun: true, + }, + }, + expected: { kind: "blocked", notice: "workspace_validation" }, + }, + { + name: "blocked: immediate recovery applies and the run failed on incomplete configuration", + facts: { + ...baseReleaseRecoveryFacts, + immediate: { + ...baseReleaseRecoveryFacts.immediate, + applies: true, + isConfigurationIncompleteFailedRun: true, + }, + }, + expected: { kind: "blocked", notice: "configuration_incomplete" }, + }, + { + name: "queue_recovery: immediate recovery applies and no suppression or block condition fires", + facts: { + ...baseReleaseRecoveryFacts, + immediate: { ...baseReleaseRecoveryFacts.immediate, applies: true }, + }, + expected: { kind: "queue_recovery" }, + }, + { + name: "released: review-participant recovery applies but a persisted monitor already covers it", + facts: { + ...baseReleaseRecoveryFacts, + reviewParticipant: { + ...baseReleaseRecoveryFacts.reviewParticipant, + applies: true, + hasPersistedMonitor: true, + }, + }, + expected: { kind: "released" }, + }, + { + name: "blocked_recovery_in_place: review-participant recovery applies on a stranded-issue-recovery origin", + facts: { + ...baseReleaseRecoveryFacts, + reviewParticipant: { + ...baseReleaseRecoveryFacts.reviewParticipant, + applies: true, + isStrandedRecoveryOrigin: true, + }, + }, + expected: { kind: "blocked_recovery_in_place" }, + }, + { + name: "blocked: review-participant recovery applies but the finishing run was itself that recovery retry", + facts: { + ...baseReleaseRecoveryFacts, + reviewParticipant: { + ...baseReleaseRecoveryFacts.reviewParticipant, + applies: true, + isExecutionReviewParticipantRecoveryRun: true, + }, + }, + expected: { kind: "blocked", notice: "execution_review_participant" }, + }, + { + name: "queue_review_participant_recovery: review-participant recovery applies and no suppression or block condition fires", + facts: { + ...baseReleaseRecoveryFacts, + reviewParticipant: { ...baseReleaseRecoveryFacts.reviewParticipant, applies: true }, + }, + expected: { kind: "queue_review_participant_recovery" }, + }, + { + name: "review-participant recovery is evaluated before immediate recovery when both apply", + facts: { + ...baseReleaseRecoveryFacts, + reviewParticipant: { ...baseReleaseRecoveryFacts.reviewParticipant, applies: true }, + immediate: { + ...baseReleaseRecoveryFacts.immediate, + applies: true, + isStrandedRecoveryOrigin: true, + }, + }, + expected: { kind: "queue_review_participant_recovery" }, + }, + ]; + + for (const testCase of cases) { + it(testCase.name, () => { + expect(decideReleaseRecovery(testCase.facts)).toEqual(testCase.expected); + }); + } +}); diff --git a/server/src/modules/wake-queue/domain/policy.ts b/server/src/modules/wake-queue/domain/policy.ts new file mode 100644 index 0000000000..7fd959a2d8 --- /dev/null +++ b/server/src/modules/wake-queue/domain/policy.ts @@ -0,0 +1,200 @@ +// Pure decision rules for the release half of the deferred issue-execution +// wake state machine: +// - the per-wake decision (decideDeferredWake), applied to the earliest +// deferred wake queued against the issue a run just released +// - the release-recovery decision (decideReleaseRecovery), applied once +// the deferred-wake queue is empty and no wake was promoted +// The caller reads the database and packs the result into a facts object. +// This file only branches on that facts object; it never queries a +// database, reads the clock, or reads the wake context payload directly. + +export type DeferredWakeQueuedCommentFacts = { + /** True when the wake carries one or more queued comment ids to check. */ + hasQueuedCommentIds: boolean; + /** Count of queued comment ids that are still live and not self-authored by the finishing run. */ + liveNonSelfCommentIdsLength: number; + /** Count of queued comment ids the wake originally carried. */ + queuedCommentIdsLength: number; + /** True when the live, non-self comment id list differs from the queued list. */ + liveCommentIdsChanged: boolean; + /** True when every discarded comment id was authored by the finishing run. */ + containedSelfAuthoredComment: boolean; + /** True when the wake carries an independent reason to continue even with no live comments. */ + preservesIndependentContinuation: boolean; +}; + +export type DeferredWakeAgentFacts = { + /** True when the wake's agent exists in the issue's own company. */ + agentFound: boolean; + /** True when the agent is invokable (status, org chain). Meaningless when agentFound is false. */ + invokable: boolean; +}; + +export type DeferredWakePauseHoldFacts = { + /** True when an active subtree pause hold covers the issue. */ + activePauseHold: boolean; + /** True when the wake is a verified issue-tree-control interaction wake that survives a pause hold. */ + treeHoldInteractionWake: boolean; +}; + +export type DeferredWakeFacts = { + queuedComment: DeferredWakeQueuedCommentFacts; + agent: DeferredWakeAgentFacts; + pauseHold: DeferredWakePauseHoldFacts; +}; + +export type DeferredWakeDecision = + | { kind: "cancel_empty"; selfAuthored: boolean } + | { kind: "normalize" } + | { kind: "fail_not_invokable" } + | { kind: "cancel_pause_hold" } + | { kind: "promote" }; + +/** + * Decides what to do with the earliest deferred wake queued against the + * issue a run just released. The caller applies a "normalize" decision (a + * queued-comment-id rewrite) and calls this function again with facts that + * reflect the rewrite, so a single wake can normalize and then also fail, + * cancel, or promote in the same drain step — matching the order the + * original state machine always evaluated them in. + */ +export function decideDeferredWake(facts: DeferredWakeFacts): DeferredWakeDecision { + const { queuedComment, agent, pauseHold } = facts; + + if ( + queuedComment.hasQueuedCommentIds && + queuedComment.liveNonSelfCommentIdsLength === 0 && + !queuedComment.preservesIndependentContinuation + ) { + return { kind: "cancel_empty", selfAuthored: queuedComment.containedSelfAuthoredComment }; + } + + if (queuedComment.hasQueuedCommentIds && queuedComment.liveCommentIdsChanged) { + return { kind: "normalize" }; + } + + if (!agent.agentFound || !agent.invokable) { + return { kind: "fail_not_invokable" }; + } + + if (pauseHold.activePauseHold && !pauseHold.treeHoldInteractionWake) { + return { kind: "cancel_pause_hold" }; + } + + return { kind: "promote" }; +} + +export type ReleaseRecoveryReviewParticipantFacts = { + /** True when the issue is in_review, unassigned to a user, and waiting on the finishing run as the current agent participant. */ + applies: boolean; + hasExistingExecutionPath: boolean; + hasPersistedMonitor: boolean; + suppressedByPauseHold: boolean; + isStrandedRecoveryOrigin: boolean; + recoveryAgentPresent: boolean; + recoveryAgentInvokable: boolean; + /** True when the finishing run was itself a review-participant-recovery retry. */ + isExecutionReviewParticipantRecoveryRun: boolean; +}; + +export type ReleaseRecoveryImmediateFacts = { + /** True when the issue is todo/in_progress, unassigned to a user, not hidden, still assigned to the finishing run's agent, and the run ended failed/timed_out/cancelled. */ + applies: boolean; + /** True when the finishing run itself carried the disposition-repair retry reason. */ + isDispositionRepairRetry: boolean; + hasExistingExecutionPath: boolean; + hasPersistedMonitor: boolean; + hasExplicitBlockerPath: boolean; + suppressedByPauseHold: boolean; + isStrandedRecoveryOrigin: boolean; + recoveryAgentPresent: boolean; + recoveryAgentInvokable: boolean; + isWorkspaceValidationFailedRun: boolean; + isConfigurationIncompleteFailedRun: boolean; + /** didAutomaticRecoveryFail(run, expectedRetryReason) for the issue's own status branch. */ + automaticRecoveryAlreadyFailed: boolean; +}; + +export type ReleaseRecoveryFacts = { + /** options.suppressImmediateRecovery on the caller's release request. */ + suppressImmediateRecovery: boolean; + reviewParticipant: ReleaseRecoveryReviewParticipantFacts; + immediate: ReleaseRecoveryImmediateFacts; +}; + +export type ReleaseRecoveryBlockedNoticeKind = + | "workspace_validation" + | "configuration_incomplete" + | "execution_review_participant" + | "immediate_execution_path"; + +export type ReleaseRecoveryDecision = + | { kind: "released" } + | { kind: "blocked_recovery_in_place" } + | { kind: "blocked"; notice: ReleaseRecoveryBlockedNoticeKind } + | { kind: "queue_review_participant_recovery" } + | { kind: "queue_recovery" }; + +/** + * Decides the release-recovery outcome once the deferred-wake queue is + * empty and no wake was promoted. The review-participant branch and the + * assignment-and-continuation branch are the two ways a released issue can + * still need automatic recovery; this function evaluates both as one + * decision, review-participant first, matching the original evaluation + * order. + */ +export function decideReleaseRecovery(facts: ReleaseRecoveryFacts): ReleaseRecoveryDecision { + const { reviewParticipant, immediate } = facts; + + if (reviewParticipant.applies) { + if ( + facts.suppressImmediateRecovery || + reviewParticipant.hasExistingExecutionPath || + reviewParticipant.hasPersistedMonitor || + reviewParticipant.suppressedByPauseHold + ) { + return { kind: "released" }; + } + if (reviewParticipant.isStrandedRecoveryOrigin) { + return { kind: "blocked_recovery_in_place" }; + } + const shouldBlock = + !reviewParticipant.recoveryAgentInvokable || + !reviewParticipant.recoveryAgentPresent || + reviewParticipant.isExecutionReviewParticipantRecoveryRun; + if (shouldBlock) { + return { kind: "blocked", notice: "execution_review_participant" }; + } + return { kind: "queue_review_participant_recovery" }; + } + + if (immediate.isDispositionRepairRetry) return { kind: "released" }; + if (!immediate.applies) return { kind: "released" }; + if (facts.suppressImmediateRecovery) return { kind: "released" }; + if ( + immediate.hasExistingExecutionPath || + immediate.hasPersistedMonitor || + immediate.hasExplicitBlockerPath + ) { + return { kind: "released" }; + } + if (immediate.suppressedByPauseHold) return { kind: "released" }; + if (immediate.isStrandedRecoveryOrigin) return { kind: "blocked_recovery_in_place" }; + + const shouldBlockImmediately = + !immediate.recoveryAgentInvokable || + !immediate.recoveryAgentPresent || + immediate.isWorkspaceValidationFailedRun || + immediate.isConfigurationIncompleteFailedRun || + immediate.automaticRecoveryAlreadyFailed; + if (shouldBlockImmediately) { + const notice: ReleaseRecoveryBlockedNoticeKind = immediate.isWorkspaceValidationFailedRun + ? "workspace_validation" + : immediate.isConfigurationIncompleteFailedRun + ? "configuration_incomplete" + : "immediate_execution_path"; + return { kind: "blocked", notice }; + } + + return { kind: "queue_recovery" }; +} diff --git a/server/src/modules/wake-queue/index.ts b/server/src/modules/wake-queue/index.ts new file mode 100644 index 0000000000..4c794e452e --- /dev/null +++ b/server/src/modules/wake-queue/index.ts @@ -0,0 +1,50 @@ +import type { Db } from "@paperclipai/db"; +import { createPostgresWakeQueueAdapter } from "./adapters/postgres.js"; +import { createReleaseIssueExecution } from "./application/use-cases.js"; +import type { + IssueSnapshot, + RecoveryEscalationPort, + RunSnapshot, + WakeQueueReader, +} from "./application/ports.js"; + +export type { + PostCommitEffect, + ReleaseOutcome, + RunSummary, +} from "./application/types.js"; +export { WakeQueueApplicationError } from "./application/types.js"; +export type { IssueSnapshot, RunSnapshot, RecoveryEscalationPort } from "./application/ports.js"; +export type { ReleaseIssueExecutionInput } from "./application/use-cases.js"; + +export type WakeQueueDeps = { + /** Stays in `heartbeat.ts`; resolves the responsible user for a promoted or recovery run seed. */ + resolveResponsibleUserId: WakeQueueReader["resolveResponsibleUserId"]; + /** Stays in `heartbeat.ts`; reads the routine environment context for an execution issue. */ + getRoutineEnv: WakeQueueReader["getRoutineEnv"]; + /** Stays in `heartbeat.ts`; resolves the session-before display id for a wakeup. */ + resolveSessionBeforeForWakeup: WakeQueueReader["resolveSessionBeforeForWakeup"]; + /** `services/recovery`'s stranded-issue escalation, called only after the release transaction commits. */ + recovery: RecoveryEscalationPort; +}; + +/** + * Composes the wake-queue module: the Postgres adapter (which owns the + * release transaction) and the release use case. `heartbeat.ts` holds the + * only caller: it builds one instance per process next to + * `createRunDispatch(db)` and delegates `releaseIssueExecutionAndPromote`'s + * body to `releaseIssueExecution`. + */ +export function createWakeQueue(db: Db, deps: WakeQueueDeps) { + const issueLock = createPostgresWakeQueueAdapter(db, { + resolveResponsibleUserId: deps.resolveResponsibleUserId, + getRoutineEnv: deps.getRoutineEnv, + resolveSessionBeforeForWakeup: deps.resolveSessionBeforeForWakeup, + }); + + return { + releaseIssueExecution: createReleaseIssueExecution({ issueLock, recovery: deps.recovery }), + }; +} + +export type WakeQueue = ReturnType; diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 78bfa70dcb..37a38c592f 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -3,7 +3,6 @@ import { legacyExecutionNeedsReconciliation, terminalizeLegacyExecution } from " import { adapterExecutionControls, createAdapterExecutionControl, waitForAdapterStop } from "./adapter-execution-control.js"; import { executionFailureRetryCount } from "./execution-recovery-attempt.js"; import { buildHeartbeatRunStatusLiveEventPayload } from "./heartbeat-run-status-payload.js"; -import { issueRecoveryActionService } from "./issue-recovery-actions.js"; export { buildHeartbeatRunStatusLiveEventPayload } from "./heartbeat-run-status-payload.js"; import { rebindContinuationContract } from "./native-runtime/continuation-contract.js"; import { buildExecutionContinuation } from "./execution-continuation.js"; @@ -376,13 +375,9 @@ import { SUCCESSFUL_RUN_HANDOFF_REQUIRED_NOTICE_BODY, readContinuationAttempt, } from "./recovery/index.js"; -import { isAutomaticRecoverySuppressedByPauseHold } from "./recovery/pause-hold-guard.js"; import { SANDBOX_PROVIDER_PLUGIN_NOT_READY_REASON, - buildConfigurationIncompleteRecoveryNoticeSeed, - buildExecutionReviewParticipantRecoveryNoticeSeed, - buildImmediateExecutionPathRecoveryNoticeSeed, - buildWorkspaceValidationRecoveryNoticeSeed, + type StrandedRecoveryNoticeSeed, } from "./recovery/stranded-notice.js"; import { withRecoveryContext } from "./recovery/status-only-context.js"; import { @@ -403,6 +398,11 @@ import { allowsIssueInteractionWake, isResolvedInteractionContinuationWakeContext, } from "../modules/run-dispatch/index.js"; +import { + createWakeQueue, + WakeQueueApplicationError, + type PostCommitEffect as WakeQueuePostCommitEffect, +} from "../modules/wake-queue/index.js"; import { buildIssueReviewPathLostIdempotencyKey, decideIssueReviewPathRecovery, @@ -795,13 +795,6 @@ function buildSandboxProviderPluginNotReadyResultJson( }; } -function readConfigurationIncompletePayload( - run: Pick | null | undefined, -): Record | null { - const payload = parseObject(parseObject(run?.resultJson).configurationIncomplete); - return Object.keys(payload).length > 0 ? payload : null; -} - function buildUnresolvedWorkspaceBaseRefResultJson( run: typeof heartbeatRuns.$inferSelect, error: UnresolvedWorkspaceBaseRefError, @@ -4830,52 +4823,6 @@ export function summarizeHeartbeatRunListResultJson(input: { return Object.keys(summary).length > 0 ? summary : null; } -function didAutomaticRecoveryFail( - latestRun: Pick< - typeof heartbeatRuns.$inferSelect, - "status" | "contextSnapshot" - > | null, - expectedRetryReason: - | "assignment_recovery" - | "issue_continuation_needed" - | typeof EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON, -) { - if (!latestRun) return false; - - const latestContext = parseObject(latestRun.contextSnapshot); - const latestRetryReason = readNonEmptyString(latestContext.retryReason); - return ( - latestRetryReason === expectedRetryReason && - UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES.includes( - latestRun.status as (typeof UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES)[number], - ) - ); -} - -function isExecutionReviewParticipantRecoveryRun( - run: Pick | null, -) { - if (!run) return false; - const context = parseObject(run.contextSnapshot); - return ( - readNonEmptyString(context.retryReason) === - EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON - ); -} - -function isExecutionReviewParticipantRecoveryEligibleRun( - run: Pick | null, -) { - if (!run) return false; - const context = parseObject(run.contextSnapshot); - const wakeReason = readNonEmptyString(context.wakeReason); - return ( - wakeReason === "execution_review_requested" || - wakeReason === "execution_approval_requested" || - isExecutionReviewParticipantRecoveryRun(run) - ); -} - function normalizeLedgerBillingType(value: unknown): BillingType { const raw = readNonEmptyString(value); switch (raw) { @@ -8499,6 +8446,134 @@ export function heartbeatService( } } + const wakeQueue = createWakeQueue(db, { + resolveResponsibleUserId: async (input) => { + // `input.issue` is the wake-queue module's own transaction-scoped + // snapshot; using it here, instead of re-reading the issue through + // `getIssueExecutionContext`, keeps this read off a second connection + // while the module's transaction is open, and keeps it seeing the + // in-transaction issue status rather than a stale one. + return resolveResponsibleUserIdForRunSeed({ + companyId: input.companyId, + contextSnapshot: input.contextSnapshot, + issueContext: input.issue, + // The wake-queue module's port type widens `env` to `unknown` so its + // application layer stays free of this file's routine env type; the + // value always comes from this file's own getRoutineEnvForExecutionIssue. + routineEnvContext: input.routineEnvContext as Awaited< + ReturnType + >, + requestedByActorType: input.requestedByActorType, + requestedByActorId: input.requestedByActorId, + source: input.source as WakeupOptions["source"], + triggerDetail: input.triggerDetail as WakeupOptions["triggerDetail"], + existingRunResponsibleUserId: input.existingRunResponsibleUserId, + }); + }, + getRoutineEnv: async (input) => { + // Same reason as `resolveResponsibleUserId` above: use the passed-in + // transaction-scoped issue snapshot instead of reading the issue again. + return getRoutineEnvForExecutionIssue(input.companyId, input.issue); + }, + resolveSessionBeforeForWakeup: async (input) => { + // Scoped to this port only, so a wake-queue agent id can never resolve + // a session against another company's agent row. The shared `getAgent` + // helper below has no company predicate, so this reads the agent + // directly with the company named in its own `WHERE` clause. + const agent = await db + .select() + .from(agents) + .where(and(eq(agents.id, input.agentId), eq(agents.companyId, input.companyId))) + .then((rows) => rows[0] ?? null); + if (!agent) return null; + return resolveSessionBeforeForWakeup(agent, input.taskKey); + }, + recovery: { + // The wake-queue module's plain snapshots hold only the fields the + // release decision needs; escalation needs the full row, so this + // re-reads both by id after the release transaction has committed. + escalateStrandedAssignedIssue: async (input) => { + const [issueRow] = await db + .select() + .from(issues) + .where(and(eq(issues.id, input.issue.id), eq(issues.companyId, input.issue.companyId))); + const [runRow] = await db + .select() + .from(heartbeatRuns) + .where(and(eq(heartbeatRuns.id, input.latestRun.id), eq(heartbeatRuns.companyId, input.latestRun.companyId))); + if (!issueRow || !runRow) return; + await recovery.escalateStrandedAssignedIssue({ + issue: issueRow, + previousStatus: input.previousStatus, + latestRun: runRow, + notice: input.notice as StrandedRecoveryNoticeSeed, + recoveryCause: (input.recoveryCause ?? undefined) as + | typeof WORKSPACE_VALIDATION_RECOVERY_CAUSE + | typeof CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE + | typeof EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE + | undefined, + }); + }, + escalateStrandedRecoveryIssueInPlace: async (input) => { + const [issueRow] = await db + .select() + .from(issues) + .where(and(eq(issues.id, input.issue.id), eq(issues.companyId, input.issue.companyId))); + const [runRow] = await db + .select() + .from(heartbeatRuns) + .where(and(eq(heartbeatRuns.id, input.latestRun.id), eq(heartbeatRuns.companyId, input.latestRun.companyId))); + if (!issueRow || !runRow) return; + await recovery.escalateStrandedRecoveryIssueInPlace({ + issue: issueRow, + previousStatus: input.previousStatus, + latestRun: runRow, + }); + }, + }, + }); + + // Applies the post-commit effects a wake-queue release returns, exactly as + // the original release function did before its writes moved into that + // module: publish + dispatch a promoted or recovery run, log a reopened + // issue's activity entry. + async function applyWakeQueuePostCommitEffects(effects: WakeQueuePostCommitEffect[]) { + for (const effect of effects) { + if (effect.kind === "run_queued") { + publishLiveEvent({ + companyId: effect.run.companyId, + type: "heartbeat.run.queued", + payload: { + runId: effect.run.id, + agentId: effect.run.agentId, + invocationSource: effect.run.invocationSource, + triggerDetail: effect.run.triggerDetail, + wakeupRequestId: effect.run.wakeupRequestId, + }, + }); + await startNextQueuedRunForAgent(effect.run.agentId); + } else { + await logActivity(db, { + companyId: effect.companyId, + actorType: "system", + actorId: "heartbeat", + agentId: effect.agentId, + runId: effect.runId, + action: "issue.updated", + entityType: "issue", + entityId: effect.issueId, + details: { + status: "todo", + reopened: true, + reopenedFrom: effect.reopenedFrom, + source: "deferred_comment_wake", + identifier: effect.identifier, + }, + }); + } + } + } + function isPlanApprovalConfirmationPayload(payload: unknown) { const target = parseObject(parseObject(payload).target); return ( @@ -9164,7 +9239,7 @@ export function heartbeatService( async function getRoutineEnvForExecutionIssue( companyId: string, - issueContext: Awaited> | null, + issueContext: { originKind: string | null; originId: string | null; originRunId: string | null } | null, ) { if ( !issueContext || @@ -9317,7 +9392,7 @@ export function heartbeatService( async function resolveResponsibleUserIdForRunSeed(input: { companyId: string; contextSnapshot: Record; - issueContext: Awaited> | null; + issueContext: { id: string; responsibleUserId: string | null; parentId: string | null } | null; routineEnvContext: Awaited< ReturnType >; @@ -22602,1015 +22677,25 @@ export function heartbeatService( run: typeof heartbeatRuns.$inferSelect, options: { suppressImmediateRecovery?: boolean } = {}, ) { - const runContext = parseObject(run.contextSnapshot); - const contextIssueId = readNonEmptyString(runContext.issueId); - const taskKey = deriveTaskKeyWithHeartbeatFallback(runContext, null); - const recoveryAgent = await getAgent(run.agentId); - const recoveryAgentInvokable = - recoveryAgent && - recoveryAgent.status !== "paused" && - recoveryAgent.status !== "terminated" && - recoveryAgent.status !== "pending_approval"; - const recoverySessionBefore = recoveryAgentInvokable - ? await resolveSessionBeforeForWakeup(recoveryAgent, taskKey) - : null; - const recoveryAgentNameKey = normalizeAgentNameKey(recoveryAgent?.name); - - const promotionResult = await db.transaction(async (tx) => { - // Lock the context issue (if any) AND every issue that still references this run. - // - // A single run can hold execution locks on multiple issues: the caller's context - // issue (set via svc.checkout) plus any additional issues stamped by - // enqueueWakeup's "legacy run" fallback when the run was the only queued/running - // run matching their contextSnapshot.issueId. Historically this function only - // resolved and cleared the lock on *one* issue (rows[0]), leaving the others - // with an executionRunId pointing at a finalized run. Subsequent checkouts from - // the assigned agent then failed with 409 and the issue stayed blocked forever. - // `order by id` makes row-lock acquisition deterministic across concurrent - // finalizations, which keeps deadlock risk independent of PostgreSQL's plan - // choice when multiple issues match. - await tx.execute( - contextIssueId - ? sql` - select id from issues - where company_id = ${run.companyId} - and ( - id = ${contextIssueId} - or execution_run_id = ${run.id} - or checkout_run_id = ${run.id} - ) - order by id - for update - ` - : sql` - select id from issues - where company_id = ${run.companyId} - and (execution_run_id = ${run.id} or checkout_run_id = ${run.id}) - order by id - for update - `, - ); - - const candidateIssues = await tx - .select() - .from(issues) - .where( - and( - eq(issues.companyId, run.companyId), - contextIssueId - ? or( - eq(issues.id, contextIssueId), - eq(issues.executionRunId, run.id), - eq(issues.checkoutRunId, run.id), - ) - : or( - eq(issues.executionRunId, run.id), - eq(issues.checkoutRunId, run.id), - ), - ), - ) - .orderBy(asc(issues.id)); - - // Clear orphaned execution-lock columns that still point at this finalizing - // run, across every sibling issue in one statement so it scales with N - // orphans without N round-trips. Rows are already held under FOR UPDATE from - // the lock query above. - // - // The two columns are cleared in separate UPDATEs so we never clobber a - // retry's executionRunId pointer: when a process-loss or codex-transient - // retry is scheduled mid-finalization, it moves `executionRunId` from this - // run to the retry run while leaving `checkoutRunId` pinned at this run. - // Only the checkout column should be released in that case; the execution - // column now belongs to the retry. - const promotionUpdateTimestamp = new Date(); - await tx - .update(issues) - .set({ - executionRunId: null, - executionAgentNameKey: null, - executionLockedAt: null, - updatedAt: promotionUpdateTimestamp, - }) - .where( - and( - eq(issues.companyId, run.companyId), - eq(issues.executionRunId, run.id), - ), - ); - // `checkoutRunId` clear is symmetric to #6008's per-issue self-heal, - // extended to all siblings: covers paths where the issue's assignee or - // status changed between checkout and termination, which - // adoptStaleCheckoutRun's narrow WHERE clause cannot reach. - await tx - .update(issues) - .set({ - checkoutRunId: null, - updatedAt: promotionUpdateTimestamp, - }) - .where( - and( - eq(issues.companyId, run.companyId), - eq(issues.checkoutRunId, run.id), - ), - ); - - // Deferred-wake promotion is bound to a single primary issue: the run's context - // issue when present, otherwise the first candidate we found (preserves the - // legacy rows[0] selection for runs that were not tied to a specific issue). - let issue = - (contextIssueId - ? candidateIssues.find((candidate) => candidate.id === contextIssueId) - : candidateIssues[0]) ?? null; - - if (!issue) return null; - if (issue.executionRunId && issue.executionRunId !== run.id) return null; - - // Workspace-validation recovery: if the finalizing run failed workspace - // validation, surface the primary issue for the blocked-recovery comment path. - // Sibling lock cleanup is already done above; only the primary issue carries - // the recovery surface because the comment is attached to a single issue. - if ( - (isWorkspaceValidationFailedRun(run) || - isConfigurationIncompleteFailedRun(run)) && - (issue.status === "todo" || issue.status === "in_progress") && - !issue.assigneeUserId && - issue.assigneeAgentId === run.agentId - ) { - const configurationIncomplete = isConfigurationIncompleteFailedRun(run); - return { - kind: "blocked" as const, - issue, - previousStatus: issue.status, - notice: configurationIncomplete - ? buildConfigurationIncompleteRecoveryNoticeSeed(readConfigurationIncompletePayload(run)) - : buildWorkspaceValidationRecoveryNoticeSeed(), - recoveryCause: configurationIncomplete - ? CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE - : WORKSPACE_VALIDATION_RECOVERY_CAUSE, - }; - } - - if (legacyExecutionNeedsReconciliation(run)) return { kind: "released" as const }; - // An operator stop never promotes old queued work by itself. The next - // explicit wake adopts those messages atomically when it queues a run. - if (run.status === "cancelled" && parseObject(run.resultJson?.executionCancellation).state === "acknowledged") { - return { kind: "released" as const }; - } - - // Native recovery owns its entire incident. The legacy stranded-work - // fallback must not create a fresh run and reset a failed native budget, - // including failures before the native executor acquired its lease. - if (run.runtimeMode === "native" && ["failed", "timed_out", "interrupted", "cancelled"].includes(run.status) - && issue.assigneeAgentId === run.agentId && !["done", "cancelled"].includes(issue.status)) { - const existing = await tx.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and( - eq(issueRecoveryActions.companyId, issue.companyId), eq(issueRecoveryActions.sourceIssueId, issue.id), - or(inArray(issueRecoveryActions.status, ["active", "escalated"]), - sql`${issueRecoveryActions.evidence}->'automaticRecovery'->>'runId' = ${run.id}`), - )).limit(1); - if (!existing.length) { - await tx.update(nativeRunFinalizations).set({ phase: "terminal_failure", leaseOwner: null, leaseExpiresAt: null, - nextAttemptAt: null, recoveryState: "blocked", failureCode: "native_continuation_requires_reconciliation", - updatedAt: promotionUpdateTimestamp, - }).where(and(eq(nativeRunFinalizations.runId, run.id), isNull(nativeRunFinalizations.resultId))); - await issueRecoveryActionService(tx as unknown as Db).upsertSourceScoped({ - companyId: issue.companyId, sourceIssueId: issue.id, kind: "active_run_watchdog", ownerType: "board", - returnOwnerAgentId: run.agentId, cause: "native_continuation_requires_reconciliation", - fingerprint: `native-continuation:${run.id}`, evidence: { runId: run.id, originalFailureCode: run.errorCode }, - nextAction: "Inspect the original failure and reconcile the previous execution before continuing. Automatic recovery cannot start another incident.", - maxAttempts: 3, wakePolicy: null, supersedeOnIdentityChange: true, - }); - } - return { kind: "released" as const }; - } - - while (true) { - let deferred = await tx - .select() - .from(agentWakeupRequests) - .where( - and( - eq(agentWakeupRequests.companyId, issue.companyId), - eq(agentWakeupRequests.status, "deferred_issue_execution"), - sql`${agentWakeupRequests.payload} ->> 'issueId' = ${issue.id}`, - ), - ) - .orderBy(asc(agentWakeupRequests.requestedAt)) - .limit(1) - .then((rows) => rows[0] ?? null); - - if (!deferred) break; - - const queuedCommentIds = queuedCommentIdsFromWakePayload( - deferred.payload, - ); - if (queuedCommentIds.length > 0) { - const queuedCommentRows = await tx - .select({ - id: issueComments.id, - deletedAt: issueComments.deletedAt, - createdByRunId: issueComments.createdByRunId, - }) - .from(issueComments) - .where( - and( - eq(issueComments.companyId, issue.companyId), - eq(issueComments.issueId, issue.id), - inArray(issueComments.id, queuedCommentIds), - ), - ); - const targetsFinishingRunAgent = deferred.agentId === run.agentId; - const liveNonSelfCommentIds = queuedCommentIds.filter((commentId) => { - const row = queuedCommentRows.find( - (candidate) => candidate.id === commentId, - ); - return Boolean( - row && - !row.deletedAt && - (!targetsFinishingRunAgent || row.createdByRunId !== run.id), - ); - }); - const now = new Date(); - const queuedContext = parseObject( - parseObject(deferred.payload)[DEFERRED_WAKE_CONTEXT_KEY], - ); - const queuedReason = - readNonEmptyString(queuedContext.wakeReason) ?? - readNonEmptyString(deferred.reason); - const queuedWakeIsCommentOnly = - !queuedReason || - queuedReason === "issue_commented" || - queuedReason === "issue_reopened_via_comment" || - queuedReason === "issue_comment_mentioned"; - const preservesIndependentContinuation = - hasInteractionContinuationWakeContext(queuedContext) || - queuedContext.resumeIntent === true || - !queuedWakeIsCommentOnly; - - if ( - liveNonSelfCommentIds.length === 0 && - !preservesIndependentContinuation - ) { - const containedSelfAuthoredComment = queuedCommentRows.some( - (row) => - targetsFinishingRunAgent && - !row.deletedAt && - row.createdByRunId === run.id, - ); - await tx - .update(agentWakeupRequests) - .set({ - status: "cancelled", - finishedAt: now, - error: containedSelfAuthoredComment - ? "Deferred wake contained only comments authored by the finishing run" - : "Queued messages were discarded before promotion", - updatedAt: now, - }) - .where( - and( - eq(agentWakeupRequests.id, deferred.id), - eq(agentWakeupRequests.status, "deferred_issue_execution"), - ), - ); - continue; - } - - if ( - liveNonSelfCommentIds.length !== queuedCommentIds.length || - liveNonSelfCommentIds.some( - (commentId, index) => commentId !== queuedCommentIds[index], - ) - ) { - const normalizedWake = await tx - .update(agentWakeupRequests) - .set({ - payload: withQueuedCommentIdsInWakePayload( - deferred.payload, - liveNonSelfCommentIds, - ), - updatedAt: now, - }) - .where( - and( - eq(agentWakeupRequests.id, deferred.id), - eq(agentWakeupRequests.status, "deferred_issue_execution"), - ), - ) - .returning() - .then((rows) => rows[0] ?? null); - if (!normalizedWake) continue; - deferred = normalizedWake; - } - } - - const deferredAgent = await tx - .select() - .from(agents) - .where(eq(agents.id, deferred.agentId)) - .then((rows) => rows[0] ?? null); - - const companyAgents = deferredAgent - ? await tx - .select({ - id: agents.id, - companyId: agents.companyId, - name: agents.name, - reportsTo: agents.reportsTo, - status: agents.status, - }) - .from(agents) - .where(eq(agents.companyId, issue.companyId)) - : []; - const deferredInvokability = - deferredAgent?.companyId === issue.companyId - ? evaluateAgentInvokability(deferredAgent, companyAgents) - : evaluateAgentInvokability(null, companyAgents); - - if ( - !deferredAgent || - deferredAgent.companyId !== issue.companyId || - !deferredInvokability.invokable - ) { - await tx - .update(agentWakeupRequests) - .set({ - status: "failed", - finishedAt: new Date(), - error: - "Deferred wake could not be promoted: agent is not invokable", - updatedAt: new Date(), - }) - .where(eq(agentWakeupRequests.id, deferred.id)); - continue; - } - - const deferredPayload = parseObject(deferred.payload); - const deferredContextSeed = parseObject( - deferredPayload[DEFERRED_WAKE_CONTEXT_KEY], - ); - const activePauseHold = await treeControlSvc.getActivePauseHoldGate( - issue.companyId, - issue.id, - ); - const treeHoldInteractionWake = - activePauseHold && - (await isVerifiedIssueTreeControlInteractionWake(tx, { - companyId: issue.companyId, - issueId: issue.id, - agentId: deferred.agentId, - contextSnapshot: deferredContextSeed, - requestedByActorType: deferred.requestedByActorType, - requestedByActorId: deferred.requestedByActorId, - })); - if (activePauseHold && !treeHoldInteractionWake) { - await tx - .update(agentWakeupRequests) - .set({ - status: "cancelled", - finishedAt: new Date(), - error: "Deferred wake suppressed by active subtree pause hold", - updatedAt: new Date(), - }) - .where(eq(agentWakeupRequests.id, deferred.id)); - continue; - } - - const promotedContextSeed: Record = { - ...deferredContextSeed, - }; - if (activePauseHold) { - promotedContextSeed.treeHoldInteraction = true; - promotedContextSeed.activeTreeHold = { - holdId: activePauseHold.holdId, - rootIssueId: activePauseHold.rootIssueId, - mode: activePauseHold.mode, - reason: activePauseHold.reason, - releasePolicy: activePauseHold.releasePolicy, - interaction: true, - }; - } - const deferredCommentIds = extractWakeCommentIds(deferredContextSeed); - const deferredWakeReason = readNonEmptyString( - deferredContextSeed.wakeReason, - ); - // Local-CLI agents post comments under user auth, so a self-comment from - // the run that is now ending would otherwise look like a real human - // comment and trigger a reopen on the very issue this run just closed. - // Suppress reopen only when every referenced comment came from this run; - // mixed batches must still reopen because they contain a real follow-up. - let deferredCommentWakeIsSelfAuthored = false; - if (deferredCommentIds.length > 0) { - const deferredComments = await tx - .select({ createdByRunId: issueComments.createdByRunId }) - .from(issueComments) - .where( - and( - eq(issueComments.companyId, issue.companyId), - eq(issueComments.issueId, issue.id), - inArray(issueComments.id, deferredCommentIds), - ), - ) - .then((rows) => rows); - deferredCommentWakeIsSelfAuthored = - deferredComments.length > 0 && - deferredComments.every( - (comment) => comment.createdByRunId === run.id, - ); - } - // Only human/comment-reopen interactions should revive completed issues; - // system follow-ups such as retry or cleanup wakes must not reopen closed work. - const shouldReopenDeferredCommentWake = - deferredCommentIds.length > 0 && - !deferredCommentWakeIsSelfAuthored && - (issue.status === "done" || issue.status === "cancelled") && - (deferred.requestedByActorType === "user" || - deferredWakeReason === "issue_reopened_via_comment"); - let reopenedActivity: LogActivityInput | null = null; - - if (shouldReopenDeferredCommentWake) { - const reopenedFromStatus = issue.status; - const reopenedIssue = await issuesSvc.update( - issue.id, - { - status: "todo", - executionState: null, - }, - tx, - ); - if (reopenedIssue) { - issue = { - ...issue, - identifier: reopenedIssue.identifier, - status: reopenedIssue.status, - executionRunId: reopenedIssue.executionRunId, - }; - if (!readNonEmptyString(promotedContextSeed.reopenedFrom)) { - promotedContextSeed.reopenedFrom = reopenedFromStatus; - } - reopenedActivity = { - companyId: issue.companyId, - actorType: "system", - actorId: "heartbeat", - agentId: deferred.agentId, - runId: run.id, - action: "issue.updated", - entityType: "issue", - entityId: issue.id, - details: { - status: "todo", - reopened: true, - reopenedFrom: reopenedFromStatus, - source: "deferred_comment_wake", - identifier: issue.identifier, - }, - }; - } - } - - const promotedReason = - readNonEmptyString(deferred.reason) ?? "issue_execution_promoted"; - const promotedSource = - (readNonEmptyString(deferred.source) as WakeupOptions["source"]) ?? - "automation"; - const promotedTriggerDetail = - (readNonEmptyString( - deferred.triggerDetail, - ) as WakeupOptions["triggerDetail"]) ?? null; - const promotedPayload = deferredPayload; - delete promotedPayload[DEFERRED_WAKE_CONTEXT_KEY]; - - const { - contextSnapshot: promotedContextSnapshot, - taskKey: promotedTaskKey, - } = enrichWakeContextSnapshot({ - contextSnapshot: promotedContextSeed, - reason: promotedReason, - source: promotedSource, - triggerDetail: promotedTriggerDetail, - payload: promotedPayload, - }); - - const sessionBefore = - readNonEmptyString(promotedContextSnapshot.resumeSessionDisplayId) ?? - (await resolveSessionBeforeForWakeup(deferredAgent, promotedTaskKey)); - const promotedContinuationAttempt = readContinuationAttempt( - promotedContextSnapshot.livenessContinuationAttempt, - ); - const promotedResponsibleUserId = - await resolveResponsibleUserIdForRunSeed({ - companyId: deferredAgent.companyId, - contextSnapshot: promotedContextSnapshot, - issueContext: issue, - routineEnvContext: await getRoutineEnvForExecutionIssue( - deferredAgent.companyId, - issue, - ), - requestedByActorType: deferred.requestedByActorType as - "user" | "agent" | "system" | null, - requestedByActorId: deferred.requestedByActorId, - source: promotedSource, - triggerDetail: promotedTriggerDetail, - existingRunResponsibleUserId: run.responsibleUserId, - }); - if (!promotedResponsibleUserId) { - throw new HttpError( - 422, - "Unable to resolve responsible user for promoted heartbeat run", - { - code: "responsible_user_unresolved", - runId: run.id, - agentId: deferredAgent.id, - companyId: deferredAgent.companyId, - issueId: issue.id, - wakeReason: readNonEmptyString( - promotedContextSnapshot.wakeReason, - ), - }, - ); - } - const now = new Date(); - const newRun = await tx - .insert(heartbeatRuns) - .values({ - companyId: deferredAgent.companyId, - agentId: deferredAgent.id, - invocationSource: promotedSource, - triggerDetail: promotedTriggerDetail, - status: "queued", - wakeupRequestId: deferred.id, - contextSnapshot: promotedContextSnapshot, - responsibleUserId: promotedResponsibleUserId, - sessionIdBefore: sessionBefore, - continuationAttempt: promotedContinuationAttempt, - }) - .returning() - .then((rows) => rows[0]); - - await tx - .update(agentWakeupRequests) - .set({ - status: "queued", - reason: "issue_execution_promoted", - runId: newRun.id, - claimedAt: null, - finishedAt: null, - error: null, - updatedAt: now, - }) - .where(eq(agentWakeupRequests.id, deferred.id)); - - await tx - .update(issues) - .set({ - executionRunId: newRun.id, - executionAgentNameKey: normalizeAgentNameKey(deferredAgent.name), - executionLockedAt: now, - updatedAt: now, - }) - // Promoted mention wakes are issue-scoped, not issue ownership transfers. - .where( - and( - eq(issues.id, issue.id), - eq(issues.assigneeAgentId, deferredAgent.id), - ), - ); - - return { - kind: "promoted" as const, - run: newRun, - reopenedActivity, - }; - } - - const findExistingExecutionPath = (agentId?: string | null) => - tx - .select({ id: heartbeatRuns.id }) - .from(heartbeatRuns) - .where( - and( - eq(heartbeatRuns.companyId, issue.companyId), - inArray(heartbeatRuns.status, [ - ...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES, - ]), - sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issue.id}`, - sql`${heartbeatRuns.id} <> ${run.id}`, - agentId ? eq(heartbeatRuns.agentId, agentId) : sql`true`, - ), - ) - .limit(1) - .then((rows) => rows[0] ?? null); - - const issueHasPersistedMonitor = Boolean(issue.monitorNextCheckAt); - const findExplicitBlockerPath = () => - tx - .select({ id: issueRelations.issueId }) - .from(issueRelations) - .innerJoin(issues, eq(issueRelations.issueId, issues.id)) - .where( - and( - eq(issueRelations.companyId, issue.companyId), - eq(issueRelations.relatedIssueId, issue.id), - eq(issueRelations.type, "blocks"), - eq(issues.companyId, issue.companyId), - notInArray(issues.status, ["done", "cancelled"]), - isNull(issues.hiddenAt), - ), - ) - .limit(1) - .then((rows) => rows[0] ?? null); - const executionState = parseIssueExecutionState(issue.executionState); - const currentParticipant = - executionState?.status === "pending" - ? executionState.currentParticipant - : null; - const issueNeedsReviewParticipantRecovery = - issue.status === "in_review" && - !issue.assigneeUserId && - currentParticipant?.type === "agent" && - currentParticipant.agentId === run.agentId && - isExecutionReviewParticipantRecoveryEligibleRun(run) && - HEARTBEAT_RUN_TERMINAL_STATUSES.includes( - run.status as (typeof HEARTBEAT_RUN_TERMINAL_STATUSES)[number], - ); - - if (issueNeedsReviewParticipantRecovery) { - const existingReviewParticipantExecutionPath = - await findExistingExecutionPath(currentParticipant.agentId); - if ( - options.suppressImmediateRecovery || - existingReviewParticipantExecutionPath || - issueHasPersistedMonitor || - (await isAutomaticRecoverySuppressedByPauseHold( - db, - issue.companyId, - issue.id, - treeControlSvc, - )) - ) { - return { kind: "released" as const }; - } - - if (issue.originKind === RECOVERY_ORIGIN_KINDS.strandedIssueRecovery) { - return { - kind: "blocked_recovery_in_place" as const, - issue, - previousStatus: issue.status, - }; - } - - const shouldBlockReviewRecovery = - !recoveryAgentInvokable || - !recoveryAgent || - isExecutionReviewParticipantRecoveryRun(run); - if (shouldBlockReviewRecovery) { - return { - kind: "blocked" as const, - issue, - previousStatus: issue.status, - notice: buildExecutionReviewParticipantRecoveryNoticeSeed(), - recoveryCause: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE, - }; - } - - const now = new Date(); - const wakeupRequest = await tx - .insert(agentWakeupRequests) - .values({ - companyId: issue.companyId, - agentId: recoveryAgent.id, - source: "automation", - triggerDetail: "system", - reason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON, - payload: withRecoveryContext( - { - issueId: issue.id, - retryOfRunId: run.id, - retryReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON, - currentStageId: executionState?.currentStageId ?? null, - currentStageType: executionState?.currentStageType ?? null, - }, - "normal_model", - ), - status: "queued", - requestedByActorType: "system", - requestedByActorId: null, - updatedAt: now, - }) - .returning() - .then((rows) => rows[0]); - - const queuedRun = await tx - .insert(heartbeatRuns) - .values({ - companyId: issue.companyId, - agentId: recoveryAgent.id, - invocationSource: "automation", - triggerDetail: "system", - status: "queued", - wakeupRequestId: wakeupRequest.id, - contextSnapshot: withRecoveryContext( - { - issueId: issue.id, - taskId: issue.id, - wakeReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON, - retryReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON, - source: "issue.execution_review_recovery", - retryOfRunId: run.id, - currentStageId: executionState?.currentStageId ?? null, - currentStageType: executionState?.currentStageType ?? null, - reviewRecoveryInstruction: - "The previous reviewer run ended while this execution-review stage was still pending. Submit the review decision now, or mark the issue blocked with the exact unblock action.", - }, - "normal_model", - ), - sessionIdBefore: recoverySessionBefore, - retryOfRunId: run.id, - updatedAt: now, - }) - .returning() - .then((rows) => rows[0]); - - await tx - .update(agentWakeupRequests) - .set({ - runId: queuedRun.id, - updatedAt: now, - }) - .where(eq(agentWakeupRequests.id, wakeupRequest.id)); - - await tx - .update(issues) - .set({ - executionRunId: queuedRun.id, - executionAgentNameKey: recoveryAgentNameKey, - executionLockedAt: now, - updatedAt: now, - }) - .where(eq(issues.id, issue.id)); - - return { - kind: "queued_recovery" as const, - run: queuedRun, - }; - } - - const issueNeedsImmediateRecovery = - (issue.status === "todo" || issue.status === "in_progress") && - !issue.assigneeUserId && - !issue.hiddenAt && - issue.assigneeAgentId === run.agentId && - (run.status === "failed" || - run.status === "timed_out" || - run.status === "cancelled"); - - if ( - readNonEmptyString(parseObject(run.contextSnapshot).retryReason) === - ISSUE_DISPOSITION_REPAIR_RETRY_REASON - ) { - return { kind: "released" as const }; - } - - if (!issueNeedsImmediateRecovery) { - return { kind: "released" as const }; - } - if (options.suppressImmediateRecovery) { - return { kind: "released" as const }; - } - - const existingExecutionPath = await findExistingExecutionPath(); - if ( - existingExecutionPath || - issueHasPersistedMonitor || - (await findExplicitBlockerPath()) - ) { - return { kind: "released" as const }; - } - - if ( - await isAutomaticRecoverySuppressedByPauseHold( - db, - issue.companyId, - issue.id, - treeControlSvc, - ) - ) { - return { kind: "released" as const }; - } - - if (issue.originKind === RECOVERY_ORIGIN_KINDS.strandedIssueRecovery) { - return { - kind: "blocked_recovery_in_place" as const, - issue, - previousStatus: issue.status, - }; - } - - const shouldBlockImmediately = - !recoveryAgentInvokable || - !recoveryAgent || - isWorkspaceValidationFailedRun(run) || - isConfigurationIncompleteFailedRun(run) || - didAutomaticRecoveryFail( - run, - issue.status === "todo" - ? "assignment_recovery" - : "issue_continuation_needed", - ); - if (shouldBlockImmediately) { - const workspaceValidationFailure = isWorkspaceValidationFailedRun(run); - const configurationIncompleteFailure = - isConfigurationIncompleteFailedRun(run); - const notice = workspaceValidationFailure - ? buildWorkspaceValidationRecoveryNoticeSeed() - : configurationIncompleteFailure - ? buildConfigurationIncompleteRecoveryNoticeSeed(readConfigurationIncompletePayload(run)) - : buildImmediateExecutionPathRecoveryNoticeSeed({ - status: issue.status as "todo" | "in_progress", - }); - return { - kind: "blocked" as const, - issue, - previousStatus: issue.status, - notice, - recoveryCause: workspaceValidationFailure - ? WORKSPACE_VALIDATION_RECOVERY_CAUSE - : configurationIncompleteFailure - ? CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE - : undefined, - }; - } - - const retryReason = - issue.status === "todo" - ? "assignment_recovery" - : "issue_continuation_needed"; - const recoveryReason = - issue.status === "todo" - ? "issue_assignment_recovery" - : "issue_continuation_needed"; - const recoverySource = - issue.status === "todo" - ? "issue.assignment_recovery" - : "issue.continuation_recovery"; - const now = new Date(); - const recoveryContextSnapshot = withRecoveryContext( - { - issueId: issue.id, - taskId: issue.id, - wakeReason: recoveryReason, - retryReason, - source: recoverySource, - retryOfRunId: run.id, - }, - "normal_model", - ); - const responsibleUserId = await resolveResponsibleUserIdForRunSeed({ - companyId: issue.companyId, - contextSnapshot: recoveryContextSnapshot, - issueContext: issue, - routineEnvContext: await getRoutineEnvForExecutionIssue( - issue.companyId, - issue, - ), - requestedByActorType: "system", - requestedByActorId: null, - source: "automation", - triggerDetail: "system", - existingRunResponsibleUserId: run.responsibleUserId, + try { + const { postCommitEffects } = await wakeQueue.releaseIssueExecution({ + companyId: run.companyId, + runId: run.id, + now: new Date(), + suppressImmediateRecovery: options.suppressImmediateRecovery, }); - if (!responsibleUserId) { - throw new HttpError( - 422, - "Unable to resolve responsible user for recovery heartbeat run", - { - code: "responsible_user_unresolved", - runId: run.id, - agentId: recoveryAgent.id, - companyId: issue.companyId, - issueId: issue.id, - wakeReason: recoveryReason, - }, - ); + await applyWakeQueuePostCommitEffects(postCommitEffects); + } catch (error) { + if ( + error instanceof WakeQueueApplicationError && + error.code === "responsible_user_unresolved" + ) { + // Every other `responsible_user_unresolved` HttpError in this file + // carries `code` inside its own `details`; match that shape here too. + throw new HttpError(422, error.message, { code: error.code, ...error.details }); } - const wakeupRequest = await tx - .insert(agentWakeupRequests) - .values({ - companyId: issue.companyId, - agentId: recoveryAgent.id, - source: "automation", - triggerDetail: "system", - reason: recoveryReason, - payload: withRecoveryContext( - { - issueId: issue.id, - retryOfRunId: run.id, - }, - "normal_model", - ), - status: "queued", - requestedByActorType: "system", - requestedByActorId: null, - updatedAt: now, - }) - .returning() - .then((rows) => rows[0]); - - const queuedRun = await tx - .insert(heartbeatRuns) - .values({ - companyId: issue.companyId, - agentId: recoveryAgent.id, - invocationSource: "automation", - triggerDetail: "system", - status: "queued", - wakeupRequestId: wakeupRequest.id, - contextSnapshot: recoveryContextSnapshot, - responsibleUserId, - sessionIdBefore: recoverySessionBefore, - retryOfRunId: run.id, - updatedAt: now, - }) - .returning() - .then((rows) => rows[0]); - - await tx - .update(agentWakeupRequests) - .set({ - runId: queuedRun.id, - updatedAt: now, - }) - .where(eq(agentWakeupRequests.id, wakeupRequest.id)); - - await tx - .update(issues) - .set({ - executionRunId: queuedRun.id, - executionAgentNameKey: recoveryAgentNameKey, - executionLockedAt: now, - updatedAt: now, - }) - .where(eq(issues.id, issue.id)); - - return { - kind: "queued_recovery" as const, - run: queuedRun, - }; - }); - - if (promotionResult?.kind === "blocked") { - await recovery.escalateStrandedAssignedIssue({ - issue: promotionResult.issue, - previousStatus: promotionResult.previousStatus as - "todo" | "in_progress" | "in_review", - latestRun: run, - notice: promotionResult.notice, - recoveryCause: - promotionResult.recoveryCause === WORKSPACE_VALIDATION_RECOVERY_CAUSE - ? WORKSPACE_VALIDATION_RECOVERY_CAUSE - : promotionResult.recoveryCause === - CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE - ? CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE - : promotionResult.recoveryCause === - EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE - ? EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE - : undefined, - }); - return; + throw error; } - - if (promotionResult?.kind === "blocked_recovery_in_place") { - await recovery.escalateStrandedRecoveryIssueInPlace({ - issue: promotionResult.issue, - previousStatus: promotionResult.previousStatus as - "todo" | "in_progress" | "in_review", - latestRun: run, - }); - return; - } - - const promotedRun = promotionResult?.run ?? null; - if (!promotedRun) return; - - if ( - promotionResult?.kind === "promoted" && - promotionResult.reopenedActivity - ) { - await logActivity(db, promotionResult.reopenedActivity); - } - - publishLiveEvent({ - companyId: promotedRun.companyId, - type: "heartbeat.run.queued", - payload: { - runId: promotedRun.id, - agentId: promotedRun.agentId, - invocationSource: promotedRun.invocationSource, - triggerDetail: promotedRun.triggerDetail, - wakeupRequestId: promotedRun.wakeupRequestId, - }, - }); - - await startNextQueuedRunForAgent(promotedRun.agentId); } async function enqueueWakeup(agentId: string, opts: WakeupOptions = {}) { diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 91684013af..709132b2bf 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -5638,7 +5638,7 @@ export function issueService(db: Db) { return row; } - return { + const service = { clearExecutionRunIfTerminal, clearCheckoutRunIfTerminal, addStopRelayCommentIfNeeded, @@ -7783,6 +7783,7 @@ export function issueService(db: Db) { blockedByIssueIds?: string[]; actorAgentId?: string | null; actorUserId?: string | null; + companyGuard?: string; }, dbOrTx: any = db, postCommitActivityPublications?: ActivityPublication[], @@ -7792,10 +7793,18 @@ export function issueService(db: Db) { const activityPublications = postCommitActivityPublications ?? ownedActivityPublications; const ownedPostCommitActions: IssuePostCommitAction[] = []; const queuedPostCommitActions = postCommitActions ?? ownedPostCommitActions; + // A caller that supplies `companyGuard` gets the company added to + // every read, lock, and write predicate below. A check before this + // call is not a boundary: `issues.company_id` can change between + // that check and this write, so the predicate must carry the + // company itself. + const idPredicate = data.companyGuard !== undefined + ? and(eq(issues.id, id), eq(issues.companyId, data.companyGuard)) + : eq(issues.id, id); const existing = await dbOrTx .select() .from(issues) - .where(eq(issues.id, id)) + .where(idPredicate) .then((rows: Array) => rows[0] ?? null); if (!existing) return null; @@ -7804,6 +7813,7 @@ export function issueService(db: Db) { blockedByIssueIds, actorAgentId, actorUserId, + companyGuard, ...issueData } = data; const isolatedWorkspacesEnabled = (await instanceSettings.getExperimental()).enableIsolatedWorkspaces; @@ -7955,7 +7965,7 @@ export function issueService(db: Db) { const receiptExisting = await tx .select() .from(issues) - .where(eq(issues.id, id)) + .where(idPredicate) .for("update") .then((rows: Array) => rows[0] ?? null); if (!receiptExisting) return null; @@ -7994,7 +8004,7 @@ export function issueService(db: Db) { const updated = await tx .update(issues) .set(patch) - .where(eq(issues.id, id)) + .where(idPredicate) .returning() .then((rows: Array) => rows[0] ?? null); if (!updated) return null; @@ -9535,4 +9545,38 @@ export function issueService(db: Db) { })); }, }; + + type IssueServiceApi = typeof service & { + updateForCompany: ( + id: string, + companyId: string, + data: Parameters[1], + dbOrTx?: any, + postCommitActivityPublications?: ActivityPublication[], + postCommitActions?: IssuePostCommitAction[], + ) => ReturnType; + }; + const serviceApi = service as IssueServiceApi; + + // A company-scoped wrapper around `update`. It passes the company as a + // guard on every read, lock, and write predicate, so a caller with only + // a company id and an issue id cannot update an issue in another company. + serviceApi.updateForCompany = async ( + id, + companyId, + data, + dbOrTx = db, + postCommitActivityPublications, + postCommitActions, + ) => { + return service.update( + id, + { ...data, companyGuard: companyId }, + dbOrTx, + postCommitActivityPublications, + postCommitActions, + ); + }; + + return serviceApi; }