From 3b115f8f060264b6b8baff37418d40e7744b14e8 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 11 Sep 2026 14:07:17 -0500 Subject: [PATCH] test: strengthen chat handoff and repository evidence --- tests/e2e/agent-chat.spec.ts | 9 ++++ tests/runner-e2e/chat-flow.test.ts | 25 +++++++++ tests/runner-e2e/chat-flow.ts | 84 ++++++++++++++++++++++++------ 3 files changed, 102 insertions(+), 16 deletions(-) diff --git a/tests/e2e/agent-chat.spec.ts b/tests/e2e/agent-chat.spec.ts index 6ec8b34c75..7d6ad1d2c5 100644 --- a/tests/e2e/agent-chat.spec.ts +++ b/tests/e2e/agent-chat.spec.ts @@ -408,6 +408,14 @@ for (const direct of [false, true]) ); expect(projects).toHaveLength(1); expect(projects[0].workspaces).toHaveLength(2); + if (direct) { + await json(await request.post(`/api/projects/${projects[0].id}/workspaces`, { + data: { name: "Additional repository", repoUrl: "https://github.com/octocat/git-consortium" }, + })); + await expect(card.getByRole("link", { name: "Additional repository" })) + .toHaveAttribute("href", "https://github.com/octocat/git-consortium"); + await expect(card).toHaveCount(1); + } await send(page, "/new"); await expect .poll( @@ -418,6 +426,7 @@ for (const direct of [false, true]) .toBe(1); await page.reload(); await expect(card).toHaveCount(1); + if (direct) await expect(card.getByRole("link", { name: "Additional repository" })).toBeVisible(); await card .getByRole("link", { name: "Browser repositories", exact: true }) .click(); diff --git a/tests/runner-e2e/chat-flow.test.ts b/tests/runner-e2e/chat-flow.test.ts index 40f72a4e41..8f290cac4a 100644 --- a/tests/runner-e2e/chat-flow.test.ts +++ b/tests/runner-e2e/chat-flow.test.ts @@ -5,6 +5,7 @@ import { assertChatTaskHandoff, chatQuestionPresentation, chatRunFailure, + chatTaskCompletionFailure, collectChatRunEvidence, readRunningChatLog, isResetRun, @@ -183,6 +184,30 @@ describe("chat acceptance contracts", () => { "log returned 500", ); }); + it("fails terminal execution errors without preempting active retries or recovery", () => { + const failed = { + ...run, + status: "failed", + error: "provider rejected request", + }; + expect(chatTaskCompletionFailure(task, [failed])).toContain( + "provider rejected request", + ); + expect( + chatTaskCompletionFailure(task, [failed, { ...run, status: "queued" }]), + ).toBeUndefined(); + expect( + chatTaskCompletionFailure({ ...task, scheduledRetry: { id: "retry" } }, [ + failed, + ]), + ).toBeUndefined(); + expect( + chatTaskCompletionFailure( + { ...task, activeRecoveryAction: { id: "recovery" } }, + [failed], + ), + ).toBeUndefined(); + }); it("fails promptly on terminal provider failures while permitting only expected cancellations", () => { expect(chatRunFailure([run])).toBeUndefined(); expect(chatRunFailure([{ ...run, status: "running" }])).toBeUndefined(); diff --git a/tests/runner-e2e/chat-flow.ts b/tests/runner-e2e/chat-flow.ts index a044b70c3a..7d90593d6e 100644 --- a/tests/runner-e2e/chat-flow.ts +++ b/tests/runner-e2e/chat-flow.ts @@ -21,6 +21,8 @@ export interface ChatIssue { parentId?: string | null; projectId?: string | null; assigneeAgentId?: string | null; + scheduledRetry?: unknown; + activeRecoveryAction?: unknown; } export interface ChatRun { id: string; @@ -61,6 +63,19 @@ export function chatRunFailure( : undefined; } +export function chatTaskCompletionFailure( + task: ChatIssue, + runs: ChatRun[], +): string | undefined { + if ( + task.scheduledRetry || + task.activeRecoveryAction || + runs.some((run) => ["queued", "running"].includes(run.status)) + ) + return undefined; + return chatRunFailure(runs); +} + /** Match the shared question form's durable/native presentation, including custom labels. */ export function chatQuestionPresentation( payload: AskUserQuestionsPayload, @@ -360,6 +375,12 @@ export async function runChatFlow(input: { } else { let existingProject: { id: string; name: string } | undefined; let acceptedPlan: Plan | undefined; + let repositoryCatalogBefore: + { repositories: Array<{ id: string; url: string }> } | undefined; + const expectedRepositoryUrls = [ + "https://github.com/octocat/Hello-World", + "https://github.com/octocat/Spoon-Knife", + ]; if (caseId === "clarify-reuse") { existingProject = await api.post( `/api/companies/${f.company.id}/projects`, @@ -540,6 +561,14 @@ export async function runChatFlow(input: { source: await api.get(`/api/issues/${issue!.id}/documents/plan`), }); } else { + repositoryCatalogBefore = await api.get( + `/api/companies/${f.company.id}/project-repositories`, + ); + expect( + repositoryCatalogBefore!.repositories.filter((repository) => + expectedRepositoryUrls.includes(repository.url), + ), + ).toHaveLength(0); await turn( `Create a project called Repository Discussion ${nonce} for work spanning https://github.com/octocat/Hello-World and https://github.com/octocat/Spoon-Knife. These existing public repositories are not in our catalog; register both URLs. Then make one assigned task for yourself to write a two-sentence description of the intended project in an output document, containing ${marker}, and complete that task. No code changes or remote repository creation are needed.`, 2, @@ -548,13 +577,17 @@ export async function runChatFlow(input: { const children = await tasks(); expect(children).toHaveLength(1); const child = children[0]!; - await expect - .poll( - async () => - (await api.get(`/api/issues/${child.id}`)).status, - { timeout: 240_000 }, - ) - .toBe("done"); + await pollUntil({ + label: `execution task ${child.id} completes`, + deadlineAt: Date.now() + 240_000, + intervalMs: 1000, + load: async () => ({ + task: await api.get(`/api/issues/${child.id}`), + runs: await api.get(`/api/issues/${child.id}/runs`), + }), + accept: (state) => state.task.status === "done", + reject: (state) => chatTaskCompletionFailure(state.task, state.runs), + }); runs = await allRuns(); input.observe(issue!, runs); const plan = @@ -590,7 +623,7 @@ export async function runChatFlow(input: { Array<{ id: string; name: string; - workspaces: Array<{ repoUrl?: string }>; + workspaces: Array<{ id: string; name: string; repoUrl?: string }>; }> >(`/api/companies/${f.company.id}/projects`); expect(projects).toHaveLength(1); @@ -605,19 +638,38 @@ export async function runChatFlow(input: { ).toHaveCount(1); if (caseId === "multi-repository") { expect(projects[0]!.workspaces.map((w) => w.repoUrl).sort()).toEqual( - [ - "https://github.com/octocat/Hello-World", - "https://github.com/octocat/Spoon-Knife", - ].sort(), + [...expectedRepositoryUrls].sort(), ); + // URL registration is persisted as project workspaces; the discovery + // catalog continues to reflect authorized GitHub connections only. + expect( + projects[0]!.workspaces.every((workspace) => Boolean(workspace.id)), + ).toBe(true); + expect( + new Set(projects[0]!.workspaces.map((workspace) => workspace.id)) + .size, + ).toBe(2); + const persistedProject = await api.get<{ + workspaces: Array<{ id: string; repoUrl?: string }>; + }>(`/api/projects/${projects[0]!.id}`); + expect( + persistedProject.workspaces.map(({ id, repoUrl }) => ({ + id, + repoUrl, + })), + ).toEqual( + projects[0]!.workspaces.map(({ id, repoUrl }) => ({ id, repoUrl })), + ); + await input.evidence("chat-repository-registration.json", { + catalogBefore: repositoryCatalogBefore, + projectId: projects[0]!.id, + registeredWorkspaces: persistedProject.workspaces, + }); const projectCard = page.getByRole("article", { name: /Project created:/, }); // Repository labels may be customized; verify the actual destinations. - for (const repositoryUrl of [ - "https://github.com/octocat/Hello-World", - "https://github.com/octocat/Spoon-Knife", - ]) { + for (const repositoryUrl of expectedRepositoryUrls) { await expect( projectCard.locator(`a[href="${repositoryUrl}"]`), ).toBeVisible();