diff --git a/server/src/__tests__/routines-service.test.ts b/server/src/__tests__/routines-service.test.ts index 772747a873..cbdf42ee70 100644 --- a/server/src/__tests__/routines-service.test.ts +++ b/server/src/__tests__/routines-service.test.ts @@ -239,6 +239,136 @@ describeEmbeddedPostgres("routine service live-execution coalescing", () => { .then((rows) => rows[0]!); } + it("clears transient routine run failures when execution issues resume", async () => { + const { companyId, issueSvc, routine, svc } = await seedFixture(); + const runId = randomUUID(); + const executionIssue = await issueSvc.create(companyId, { + projectId: routine.projectId, + title: routine.title, + description: routine.description, + status: "blocked", + priority: routine.priority, + assigneeAgentId: routine.assigneeAgentId, + originKind: "routine_execution", + originId: routine.id, + originRunId: runId, + }); + + await db.insert(routineRuns).values({ + id: runId, + companyId, + routineId: routine.id, + source: "manual", + status: "issue_created", + triggeredAt: new Date("2026-07-16T12:00:00.000Z"), + linkedIssueId: executionIssue.id, + }); + + await svc.syncRunStatusForIssue(executionIssue.id); + const [failedRun] = await db.select().from(routineRuns).where(eq(routineRuns.id, runId)); + expect(failedRun).toMatchObject({ + status: "failed", + failureReason: "Execution issue moved to blocked", + triggerPayload: { + transientFailure: { + code: "execution_issue_status", + status: "blocked", + }, + }, + }); + await db.update(issues).set({ status: "in_progress" }).where(eq(issues.id, executionIssue.id)); + await svc.syncRunStatusForIssue(executionIssue.id); + + const [run] = await db.select().from(routineRuns).where(eq(routineRuns.id, runId)); + expect(run).toMatchObject({ + status: "issue_created", + failureReason: null, + completedAt: null, + triggerPayload: { + transientFailure: { + code: "execution_issue_status", + status: "blocked", + clearedAt: expect.any(String), + }, + }, + }); + + const clearedAt = (run?.triggerPayload as { transientFailure?: { clearedAt?: string } } | null) + ?.transientFailure?.clearedAt; + expect(clearedAt).toEqual(expect.any(String)); + + await db.update(issues).set({ status: "done" }).where(eq(issues.id, executionIssue.id)); + await svc.syncRunStatusForIssue(executionIssue.id); + + const [completedRun] = await db.select().from(routineRuns).where(eq(routineRuns.id, runId)); + expect(completedRun).toMatchObject({ + status: "completed", + failureReason: null, + triggerPayload: { + transientFailure: { + code: "execution_issue_status", + status: "blocked", + clearedAt, + }, + }, + }); + expect(completedRun?.completedAt).toBeInstanceOf(Date); + }); + + it("moves transient routine run failures into completion context", async () => { + const { companyId, issueSvc, routine, svc } = await seedFixture(); + const runId = randomUUID(); + const executionIssue = await issueSvc.create(companyId, { + projectId: routine.projectId, + title: routine.title, + description: routine.description, + status: "blocked", + priority: routine.priority, + assigneeAgentId: routine.assigneeAgentId, + originKind: "routine_execution", + originId: routine.id, + originRunId: runId, + }); + + await db.insert(routineRuns).values({ + id: runId, + companyId, + routineId: routine.id, + source: "manual", + status: "issue_created", + triggeredAt: new Date("2026-07-16T12:00:00.000Z"), + linkedIssueId: executionIssue.id, + triggerPayload: { input: "preserved" }, + }); + + await svc.syncRunStatusForIssue(executionIssue.id); + const [failedRun] = await db.select().from(routineRuns).where(eq(routineRuns.id, runId)); + expect(failedRun).toMatchObject({ + status: "failed", + failureReason: "Execution issue moved to blocked", + }); + await db.update(issues).set({ status: "done" }).where(eq(issues.id, executionIssue.id)); + await svc.syncRunStatusForIssue(executionIssue.id); + + const [run] = await db.select().from(routineRuns).where(eq(routineRuns.id, runId)); + expect(run).toMatchObject({ + status: "completed", + failureReason: null, + triggerPayload: { + input: "preserved", + transientFailure: { + code: "execution_issue_status", + status: "blocked", + reason: "Execution issue moved to blocked", + }, + }, + }); + expect(run?.completedAt).toBeInstanceOf(Date); + expect(run?.triggerPayload).toMatchObject({ + transientFailure: { clearedAt: expect.any(String) }, + }); + }); + it("filters listed routines by project", async () => { const { companyId, agentId, projectId, routine, svc } = await seedFixture(); const otherProjectId = randomUUID(); diff --git a/server/src/services/routines.ts b/server/src/services/routines.ts index 95f7a3e06b..f09ac75821 100644 --- a/server/src/services/routines.ts +++ b/server/src/services/routines.ts @@ -83,6 +83,8 @@ const LIVE_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retry"]; const TERMINAL_ISSUE_STATUSES = new Set(["done", "cancelled"]); const MAX_CATCH_UP_RUNS = 25; const MAX_ROUTINE_REVISIONS = 100; +const EXECUTION_ISSUE_TRANSIENT_FAILURE_CODE = "execution_issue_status"; +const EXECUTION_ISSUE_TRANSIENT_FAILURE_STATUSES = ["blocked", "cancelled"] as const; const ACTIVITY_GATE_IGNORED_ACTIONS = [ "issue.read_marked", "issue.read_unmarked", @@ -100,6 +102,37 @@ const WEEKDAY_INDEX: Record = { Sat: 6, }; +type ExecutionIssueTransientFailureStatus = (typeof EXECUTION_ISSUE_TRANSIENT_FAILURE_STATUSES)[number]; + +function executionIssueTransientFailureReason(status: ExecutionIssueTransientFailureStatus) { + return `Execution issue moved to ${status}`; +} + +function executionIssueTransientFailureStatusFromPayload(payload: unknown): ExecutionIssueTransientFailureStatus | null { + if (!payload || typeof payload !== "object" || Array.isArray(payload)) return null; + const transientFailure = (payload as Record).transientFailure; + if (!transientFailure || typeof transientFailure !== "object" || Array.isArray(transientFailure)) return null; + const record = transientFailure as Record; + if (record.code !== EXECUTION_ISSUE_TRANSIENT_FAILURE_CODE) return null; + return EXECUTION_ISSUE_TRANSIENT_FAILURE_STATUSES.find((status) => record.status === status) ?? null; +} + +function executionIssueTransientFailureClearedAtFromPayload(payload: unknown): string | null { + if (!payload || typeof payload !== "object" || Array.isArray(payload)) return null; + const transientFailure = (payload as Record).transientFailure; + if (!transientFailure || typeof transientFailure !== "object" || Array.isArray(transientFailure)) return null; + const clearedAt = (transientFailure as Record).clearedAt; + return typeof clearedAt === "string" ? clearedAt : null; +} + +function legacyExecutionIssueTransientFailureStatus( + failureReason: string | null, +): ExecutionIssueTransientFailureStatus | null { + return EXECUTION_ISSUE_TRANSIENT_FAILURE_STATUSES.find( + (status) => failureReason === executionIssueTransientFailureReason(status), + ) ?? null; +} + async function resolveCompanyDefaultResponsibleUserId(db: Db, companyId: string) { const company = await db .select({ defaultResponsibleUserId: companies.defaultResponsibleUserId }) @@ -3144,17 +3177,73 @@ export function routineService( .where(eq(issues.id, issueId)) .then((rows) => rows[0] ?? null); if (!issue || issue.originKind !== "routine_execution" || !issue.originRunId) return null; + const run = await db + .select({ + id: routineRuns.id, + status: routineRuns.status, + failureReason: routineRuns.failureReason, + triggerPayload: routineRuns.triggerPayload, + }) + .from(routineRuns) + .where(eq(routineRuns.id, issue.originRunId)) + .then((rows) => rows[0] ?? null); + if (!run) return null; if (issue.status === "done") { + const transientFailureStatus = executionIssueTransientFailureStatusFromPayload(run.triggerPayload) + ?? legacyExecutionIssueTransientFailureStatus(run.failureReason); + const transientFailureClearedAt = executionIssueTransientFailureClearedAtFromPayload(run.triggerPayload); return finalizeRun(issue.originRunId, { status: "completed", + failureReason: null, completedAt: new Date(), + ...(transientFailureStatus + ? { + triggerPayload: { + ...(run.triggerPayload ?? {}), + transientFailure: { + code: EXECUTION_ISSUE_TRANSIENT_FAILURE_CODE, + status: transientFailureStatus, + reason: executionIssueTransientFailureReason(transientFailureStatus), + clearedAt: transientFailureClearedAt ?? new Date().toISOString(), + }, + }, + } + : {}), }); } if (issue.status === "blocked" || issue.status === "cancelled") { + const failureReason = executionIssueTransientFailureReason(issue.status); return finalizeRun(issue.originRunId, { status: "failed", - failureReason: `Execution issue moved to ${issue.status}`, + failureReason, completedAt: new Date(), + triggerPayload: { + ...(run.triggerPayload ?? {}), + transientFailure: { + code: EXECUTION_ISSUE_TRANSIENT_FAILURE_CODE, + status: issue.status, + reason: failureReason, + recordedAt: new Date().toISOString(), + }, + }, + }); + } + const transientFailureStatus = executionIssueTransientFailureStatusFromPayload(run.triggerPayload) + ?? legacyExecutionIssueTransientFailureStatus(run.failureReason); + if (run.status === "failed" && transientFailureStatus) { + return finalizeRun(issue.originRunId, { + status: "issue_created", + failureReason: null, + completedAt: null, + triggerPayload: { + ...(run.triggerPayload ?? {}), + transientFailure: { + code: EXECUTION_ISSUE_TRANSIENT_FAILURE_CODE, + status: transientFailureStatus, + reason: executionIssueTransientFailureReason(transientFailureStatus), + clearedAt: new Date().toISOString(), + }, + }, }); } return null;