791 lines
29 KiB
TypeScript
791 lines
29 KiB
TypeScript
import { expect, type Page } from "@playwright/test";
|
|
import { pollUntil, type RunnerApi } from "./api.js";
|
|
import type {
|
|
AskUserQuestionsPayload,
|
|
PaperclipQuestionSetPayload,
|
|
} from "../../packages/shared/src/types/issue.js";
|
|
import type { LiveFixtureValues } from "./live-fixtures.js";
|
|
import type { MatrixExecution } from "./types.js";
|
|
import { chatMarker } from "./chat-cases.js";
|
|
|
|
// Public API observations only: this driver never fabricates provider results or writes DB state.
|
|
export interface ChatIssue {
|
|
id: string;
|
|
companyId: string;
|
|
title: string;
|
|
status: string;
|
|
identifier?: string;
|
|
conversationState?: string;
|
|
conversationSessionGeneration?: number;
|
|
conversationBoundaryCommentId?: string;
|
|
parentId?: string | null;
|
|
projectId?: string | null;
|
|
assigneeAgentId?: string | null;
|
|
scheduledRetry?: unknown;
|
|
activeRecoveryAction?: unknown;
|
|
}
|
|
export interface ChatRun {
|
|
id: string;
|
|
companyId: string;
|
|
agentId: string;
|
|
status: string;
|
|
error?: string | null;
|
|
errorCode?: string | null;
|
|
runtimeMode?: string;
|
|
contextSnapshot?: Record<string, unknown>;
|
|
resultJson?: Record<string, unknown>;
|
|
sessionIdBefore?: string | null;
|
|
sessionIdAfter?: string | null;
|
|
startedAt?: string;
|
|
}
|
|
type Comment = {
|
|
id: string;
|
|
body: string;
|
|
authorAgentId?: string;
|
|
createdByRunId?: string;
|
|
conversationSessionGeneration?: number;
|
|
};
|
|
type Plan = { body: string; latestRevisionId: string; updatedAt: string };
|
|
type ChatOutputDocument = Plan & { id: string; issueId: string; key: string };
|
|
|
|
/** Clarification may request information imperatively rather than end in a question mark. */
|
|
export function isChatClarificationReply(body: string): boolean {
|
|
if (body.includes("?")) return true;
|
|
const request = body.match(
|
|
/\b(?:please\s+(?:share|provide|clarify|confirm)|tell me|let me know)\b([\s\S]*)/i,
|
|
);
|
|
return Boolean(request?.[1].replace(/[\s:*-]/g, ""));
|
|
}
|
|
|
|
/** A requested output document may have a descriptive key; a copied plan is not output. */
|
|
export async function readChatOutputDocument(
|
|
api: Pick<RunnerApi, "get">,
|
|
issueId: string,
|
|
marker: string,
|
|
): Promise<ChatOutputDocument> {
|
|
const summaries = await api.get<Array<{ key: string }>>(
|
|
`/api/issues/${issueId}/documents`,
|
|
);
|
|
const documents = await Promise.all(
|
|
summaries
|
|
.filter((document) => document.key !== "plan")
|
|
.map((document) =>
|
|
api.get<ChatOutputDocument>(
|
|
`/api/issues/${issueId}/documents/${encodeURIComponent(document.key)}`,
|
|
),
|
|
),
|
|
);
|
|
const output = documents.find(
|
|
(document) => document.issueId === issueId && document.body.includes(marker),
|
|
);
|
|
if (!output)
|
|
throw new Error(
|
|
`Execution task ${issueId} has no non-plan output document containing ${marker}; document keys: ${summaries.map((document) => document.key).join(", ") || "none"}`,
|
|
);
|
|
expect(output.id).toBeTruthy();
|
|
expect(output.latestRevisionId).toBeTruthy();
|
|
return output;
|
|
}
|
|
|
|
export const isResetRun = (run: ChatRun) =>
|
|
run.contextSnapshot?.conversationReset === true ||
|
|
run.resultJson?.conversationReset === true;
|
|
export function chatRunFailure(
|
|
runs: ChatRun[],
|
|
allowCancelled = false,
|
|
): string | undefined {
|
|
const failed = runs.find(
|
|
(run) =>
|
|
["failed", "timed_out"].includes(run.status) ||
|
|
(!allowCancelled && run.status === "cancelled"),
|
|
);
|
|
return failed
|
|
? `run ${failed.id} ${failed.status}${failed.errorCode ? ` (${failed.errorCode})` : ""}${failed.error ? `: ${failed.error}` : ""}`
|
|
: undefined;
|
|
}
|
|
|
|
export function chatTaskCompletionFailure(
|
|
task: ChatIssue,
|
|
runs: ChatRun[],
|
|
): string | undefined {
|
|
if (
|
|
task.scheduledRetry ||
|
|
task.activeRecoveryAction ||
|
|
runs.some((run) => ["queued", "running"].includes(run.status))
|
|
)
|
|
return undefined;
|
|
return chatRunFailure(runs);
|
|
}
|
|
|
|
/** Match the shared question form's durable/native presentation, including custom labels. */
|
|
export function chatQuestionPresentation(
|
|
payload: AskUserQuestionsPayload,
|
|
): PaperclipQuestionSetPayload {
|
|
if (payload.questionSet) return payload.questionSet;
|
|
return {
|
|
schema: "paperclip.question_set.v1",
|
|
...(payload.submitLabel ? { submitLabel: payload.submitLabel } : {}),
|
|
questions: payload.questions.map((question) => {
|
|
const freeText = question.options.find((option) => option.freeText);
|
|
return {
|
|
id: question.id,
|
|
prompt: question.prompt,
|
|
required: question.required === true,
|
|
answerMode:
|
|
question.selectionMode === "multi" ? "multi_select" : "single_select",
|
|
...(freeText
|
|
? { customAnswer: { enabled: true as const, label: freeText.label } }
|
|
: {}),
|
|
};
|
|
}),
|
|
};
|
|
}
|
|
|
|
export function assertChatTaskHandoff(
|
|
task: ChatIssue,
|
|
runs: ChatRun[],
|
|
source: ChatIssue,
|
|
) {
|
|
expect(task.parentId).toBeNull();
|
|
expect(task.projectId).toBeTruthy();
|
|
expect(task.assigneeAgentId).toBe(source.assigneeAgentId);
|
|
expect(runs.length).toBeGreaterThan(0);
|
|
}
|
|
|
|
/** A running row can precede creation of its log file. Only that expected 404 is retryable. */
|
|
export async function readRunningChatLog(
|
|
api: Pick<RunnerApi, "request">,
|
|
runId: string,
|
|
): Promise<string | undefined> {
|
|
const response = await api.request.get(
|
|
`/api/heartbeat-runs/${runId}/log?limitBytes=65536`,
|
|
);
|
|
if (response.status() === 404) return undefined;
|
|
if (!response.ok())
|
|
throw new Error(`Run ${runId} log returned ${response.status()}`);
|
|
return ((await response.json()) as { content?: string }).content;
|
|
}
|
|
|
|
/** Synthetic reset runs have durable events but never start a provider log. */
|
|
export async function collectChatRunEvidence(
|
|
api: Pick<RunnerApi, "get">,
|
|
run: ChatRun,
|
|
) {
|
|
return {
|
|
runId: run.id,
|
|
log: isResetRun(run)
|
|
? null
|
|
: await api.get(`/api/heartbeat-runs/${run.id}/log?limitBytes=1048576`),
|
|
events: await api.get(`/api/heartbeat-runs/${run.id}/events?limit=1000`),
|
|
};
|
|
}
|
|
|
|
export function assertChatHandoff(
|
|
task: ChatIssue,
|
|
plan: Plan,
|
|
runs: ChatRun[],
|
|
source: ChatIssue,
|
|
) {
|
|
assertChatTaskHandoff(task, runs, source);
|
|
expect(plan.body.trim()).not.toBe("");
|
|
for (const run of runs) {
|
|
expect(Date.parse(plan.updatedAt)).toBeLessThanOrEqual(
|
|
Date.parse(run.startedAt!),
|
|
);
|
|
}
|
|
}
|
|
export async function sendChatMessage(page: Page, message: string) {
|
|
const composer = page.getByTestId("task-chat-composer-input").last();
|
|
await composer
|
|
.locator('[contenteditable="true"], textarea')
|
|
.first()
|
|
.fill(message);
|
|
await page.getByTestId("task-chat-composer-send").last().click();
|
|
}
|
|
|
|
export async function runChatFlow(input: {
|
|
page: Page;
|
|
api: RunnerApi;
|
|
fixtures: LiveFixtureValues;
|
|
execution: MatrixExecution;
|
|
nonce: string;
|
|
restart: () => Promise<void>;
|
|
observe: (issue: ChatIssue, runs: ChatRun[]) => void;
|
|
capture: (id: string, label: string, file: string) => Promise<void>;
|
|
evidence: (name: string, data: unknown) => Promise<void>;
|
|
}) {
|
|
const { page, api, fixtures: f, execution, nonce } = input;
|
|
const chatPath = `/api/companies/${f.company.id}/chats/${f.agent.id}`;
|
|
const route = `/${f.company.issuePrefix}/chats/${f.agent.id}`;
|
|
const marker = execution.task.buildVisibleMarker(nonce);
|
|
const draftMarker = chatMarker("DRAFT", nonce);
|
|
const caseId = execution.task.id;
|
|
let issue: ChatIssue;
|
|
let runs: ChatRun[] = [];
|
|
const settings = await api.get<Record<string, unknown>>(
|
|
"/api/instance/settings/experimental",
|
|
);
|
|
const allRuns = async () => {
|
|
const rows = await api.get<ChatRun[]>(
|
|
`/api/companies/${f.company.id}/heartbeat-runs?limit=100`,
|
|
);
|
|
return Promise.all(
|
|
rows.map((row) => api.get<ChatRun>(`/api/heartbeat-runs/${row.id}`)),
|
|
);
|
|
};
|
|
const tasks = () =>
|
|
api.get<ChatIssue[]>(`/api/companies/${f.company.id}/issues`);
|
|
const comments = async () =>
|
|
(
|
|
await api.get<Array<Comment & { createdAt: string }>>(
|
|
`/api/issues/${issue.id}/comments?order=asc`,
|
|
)
|
|
).sort(
|
|
(a, b) =>
|
|
a.createdAt.localeCompare(b.createdAt) || a.id.localeCompare(b.id),
|
|
);
|
|
const idle = async (minimumProviderRuns: number) => {
|
|
await pollUntil({
|
|
label: "chat turn settles to waiting",
|
|
deadlineAt: Date.now() + 240_000,
|
|
intervalMs: 1000,
|
|
load: async () => {
|
|
const resolved = await api.get<ChatIssue | null>(chatPath);
|
|
if (resolved) issue = resolved;
|
|
runs = await allRuns();
|
|
if (resolved) input.observe(resolved, runs);
|
|
return {
|
|
resolved: Boolean(resolved),
|
|
status: resolved?.status,
|
|
conversationState: resolved?.conversationState,
|
|
providerRunCount: runs.filter((run) => !isResetRun(run)).length,
|
|
activeRuns: runs
|
|
.filter((run) => ["queued", "running"].includes(run.status))
|
|
.map((run) => run.id),
|
|
failure: chatRunFailure(runs, caseId === "stop-new-resume"),
|
|
};
|
|
},
|
|
reject: (state) => state.failure,
|
|
accept: (state) =>
|
|
!state.failure &&
|
|
state.resolved &&
|
|
state.providerRunCount >= minimumProviderRuns &&
|
|
state.activeRuns.length === 0 &&
|
|
state.status === "in_review" &&
|
|
state.conversationState === "waiting",
|
|
});
|
|
};
|
|
const turn = async (text: string, count: number) => {
|
|
await sendChatMessage(page, text);
|
|
await idle(count);
|
|
};
|
|
const noTasks = async () => expect(await tasks()).toHaveLength(0);
|
|
try {
|
|
await api.patch("/api/instance/settings/experimental", {
|
|
enableAgentChat: true,
|
|
enableClassicTaskInterface: false,
|
|
});
|
|
expect(await api.get(chatPath)).toBeNull();
|
|
// Cold Vite startup can keep unrelated assets loading after the chat is
|
|
// interactive. The composer assertion below verifies actual UI readiness.
|
|
await page.goto(route, { waitUntil: "domcontentloaded", timeout: 60_000 });
|
|
await expect(page.getByTestId("task-chat-composer-input")).toBeVisible();
|
|
expect(await api.get(chatPath)).toBeNull();
|
|
expect(await allRuns()).toHaveLength(0);
|
|
|
|
if (
|
|
["continuity-restart", "new-session", "stop-new-resume"].includes(caseId)
|
|
) {
|
|
const secret = chatMarker("OLDCONTEXT", nonce);
|
|
await turn(
|
|
`For this conversation only, remember the phrase ${secret}. Just acknowledge briefly; no project or task is needed.`,
|
|
1,
|
|
);
|
|
const initialId = issue!.id;
|
|
const before = runs.filter((run) => !isResetRun(run))[0]!;
|
|
await noTasks();
|
|
await page.reload({ waitUntil: "domcontentloaded", timeout: 60_000 });
|
|
if (caseId === "continuity-restart") {
|
|
await turn(
|
|
"What phrase did I just ask you to remember? Reply with the phrase only.",
|
|
2,
|
|
);
|
|
expect(
|
|
(await comments()).filter((c) => c.authorAgentId).at(-1)?.body,
|
|
).toContain(secret);
|
|
const count = runs.length;
|
|
await input.restart();
|
|
await page.reload({ waitUntil: "domcontentloaded", timeout: 60_000 });
|
|
await idle(2);
|
|
expect(runs).toHaveLength(count);
|
|
await turn(
|
|
`We are done discussing it. Reply with ${marker} only; no further work.`,
|
|
3,
|
|
);
|
|
} else {
|
|
let cancelledId: string | undefined;
|
|
if (caseId === "stop-new-resume") {
|
|
await sendChatMessage(
|
|
page,
|
|
"Explain the history of gardening at length here, in 100 numbered paragraphs. This is discussion only; do not create work.",
|
|
);
|
|
await expect
|
|
.poll(
|
|
async () => {
|
|
runs = await allRuns();
|
|
const active = runs.find(
|
|
(run) => run.status === "running" && run.id !== before.id,
|
|
);
|
|
if (!active) return false;
|
|
const events = await api.get<Array<Record<string, unknown>>>(
|
|
`/api/heartbeat-runs/${active.id}/events?limit=1000`,
|
|
);
|
|
const log = await readRunningChatLog(api, active.id);
|
|
if (!(events.length || log?.length)) return false;
|
|
cancelledId = active.id;
|
|
return true;
|
|
},
|
|
{ timeout: 120_000 },
|
|
)
|
|
.toBe(true);
|
|
await page.getByTestId("task-chat-composer-stop").click();
|
|
await expect
|
|
.poll(
|
|
async () =>
|
|
(await api.get<ChatRun>(`/api/heartbeat-runs/${cancelledId}`))
|
|
.status,
|
|
)
|
|
.toBe("cancelled");
|
|
}
|
|
const oldComments = await comments();
|
|
await sendChatMessage(page, "/new");
|
|
await expect
|
|
.poll(
|
|
async () =>
|
|
(await api.get<ChatIssue>(chatPath))
|
|
.conversationSessionGeneration,
|
|
{ timeout: 30_000 },
|
|
)
|
|
.toBe(1);
|
|
await expect
|
|
.poll(async () => (await allRuns()).some(isResetRun))
|
|
.toBe(true);
|
|
await turn(
|
|
`Without reading older history or files, if you have a remembered phrase in your current context return it; otherwise reply exactly ${marker}. Do not look it up.`,
|
|
caseId === "new-session" ? 2 : 3,
|
|
);
|
|
const fresh = runs
|
|
.filter((run) => !isResetRun(run) && run.status === "succeeded")
|
|
.sort((a, b) => Date.parse(a.startedAt!) - Date.parse(b.startedAt!))
|
|
.at(-1)!;
|
|
expect(fresh.contextSnapshot?.conversationSessionGeneration).toBe(1);
|
|
expect(fresh.sessionIdBefore).toBeFalsy();
|
|
expect(
|
|
String(fresh.contextSnapshot?.paperclipTaskMarkdown ?? ""),
|
|
).not.toContain(secret);
|
|
if (before.sessionIdAfter && fresh.sessionIdAfter)
|
|
expect(fresh.sessionIdAfter).not.toBe(before.sessionIdAfter);
|
|
const replies = (await comments()).filter(
|
|
(c) => c.createdByRunId === fresh.id && c.authorAgentId,
|
|
);
|
|
expect(replies.map((c) => c.body).join("\n")).toContain(marker);
|
|
expect(replies.map((c) => c.body).join("\n")).not.toContain(secret);
|
|
const reset = runs.filter(isResetRun);
|
|
expect(reset).toHaveLength(1);
|
|
expect(
|
|
(await comments()).filter(
|
|
(c) => c.authorAgentId && c.createdByRunId === reset[0]!.id,
|
|
),
|
|
).toHaveLength(0);
|
|
expect(
|
|
(await comments()).filter((c) =>
|
|
oldComments.some((old) => old.id === c.id),
|
|
),
|
|
).toHaveLength(oldComments.length);
|
|
if (cancelledId)
|
|
expect(
|
|
(await comments()).filter((c) => c.createdByRunId === cancelledId),
|
|
).toEqual(
|
|
oldComments.filter((c) => c.createdByRunId === cancelledId),
|
|
);
|
|
await page.reload({ waitUntil: "domcontentloaded", timeout: 60_000 });
|
|
await expect(
|
|
page.getByText("New session", { exact: true }),
|
|
).toHaveCount(1);
|
|
}
|
|
expect(issue!.id).toBe(initialId);
|
|
await noTasks();
|
|
} else {
|
|
let existingProject: { id: string; name: string } | undefined;
|
|
let acceptedPlan: Plan | undefined;
|
|
let repositoryCatalogBefore:
|
|
{ repositories: Array<{ id: string; url: string }> } | undefined;
|
|
const expectedRepositoryUrls = [
|
|
"https://github.com/octocat/Hello-World",
|
|
"https://github.com/octocat/Spoon-Knife",
|
|
];
|
|
if (caseId === "clarify-reuse") {
|
|
existingProject = await api.post(
|
|
`/api/companies/${f.company.id}/projects`,
|
|
{
|
|
name: `Garden ${nonce}`,
|
|
description: "Garden club welcome notes and event announcements.",
|
|
},
|
|
);
|
|
await turn(
|
|
"I need a welcome note for our club. Help me clarify what information you need before assigning the work.",
|
|
1,
|
|
);
|
|
await noTasks();
|
|
const questions = await api.get<
|
|
Array<{
|
|
status: string;
|
|
kind: string;
|
|
payload: AskUserQuestionsPayload;
|
|
}>
|
|
>(`/api/issues/${issue!.id}/interactions`);
|
|
const pendingInteraction = questions.find(
|
|
(row) =>
|
|
row.status === "pending" && row.kind === "ask_user_questions",
|
|
);
|
|
const questionSet = pendingInteraction
|
|
? chatQuestionPresentation(pendingInteraction.payload)
|
|
: undefined;
|
|
const pendingQuestions = questionSet?.questions;
|
|
expect(
|
|
Boolean(pendingQuestions?.length) ||
|
|
(await comments()).some(
|
|
(c) => c.authorAgentId && isChatClarificationReply(c.body),
|
|
),
|
|
).toBe(true);
|
|
const clarification = `It is the garden club; use the existing Garden ${nonce} project. Make one assigned task for yourself to write a two-sentence welcome note. Include ${marker} in that note, save it as the output document, and finish that execution task. Please get it started now.`;
|
|
if (pendingQuestions?.length) {
|
|
for (const [index, question] of pendingQuestions.entries()) {
|
|
const textInput = page
|
|
.getByTestId("question-text-answer-composer")
|
|
.last();
|
|
if (await textInput.isVisible()) {
|
|
await textInput
|
|
.locator('[contenteditable="true"],textarea')
|
|
.first()
|
|
.fill(clarification);
|
|
} else {
|
|
await page
|
|
.getByRole(
|
|
question.answerMode === "multi_select" ? "checkbox" : "radio",
|
|
{
|
|
name: question.customAnswer?.label ?? "Other",
|
|
exact: true,
|
|
},
|
|
)
|
|
.last()
|
|
.click();
|
|
await page
|
|
.getByTestId("question-other-answer-composer")
|
|
.last()
|
|
.locator('[contenteditable="true"],textarea')
|
|
.first()
|
|
.fill(clarification);
|
|
}
|
|
await page
|
|
.getByRole("button", {
|
|
name:
|
|
index === pendingQuestions.length - 1
|
|
? (questionSet?.submitLabel ?? "Submit answers")
|
|
: "Next",
|
|
exact: true,
|
|
})
|
|
.last()
|
|
.click();
|
|
}
|
|
await idle(3);
|
|
} else await turn(clarification, 3);
|
|
} else if (caseId === "plan-handoff") {
|
|
await page.getByTestId("task-chat-composer-mode").click();
|
|
await page
|
|
.getByTestId("task-chat-composer-mode-menu")
|
|
.getByText("Plan mode", { exact: true })
|
|
.click();
|
|
await turn(
|
|
`Let's plan a two-sentence garden club welcome note. The finished welcome note itself must contain the exact phrase ${draftMarker}. Write a plan in the plan panel that includes this requirement, and present it for approval. When I approve the final revision, create a suitable repository-free project and an assigned task for yourself, copy the plan into that task, and have it save the note in its output document and finish. Do not create the project or task before approval.`,
|
|
1,
|
|
);
|
|
const draft = await api.get<Plan>(
|
|
`/api/issues/${issue!.id}/documents/plan`,
|
|
);
|
|
expect(draft.body).toContain(draftMarker);
|
|
await noTasks();
|
|
await input.capture(
|
|
"chat-plan-draft",
|
|
"Draft plan in the conversation",
|
|
"chat-plan-draft.png",
|
|
);
|
|
const initialInteractions = await api.get<
|
|
Array<{
|
|
status: string;
|
|
kind: string;
|
|
payload?: {
|
|
target?: { revisionId?: string };
|
|
rejectLabel?: string;
|
|
};
|
|
}>
|
|
>(`/api/issues/${issue!.id}/interactions`);
|
|
const initialApproval = initialInteractions.find(
|
|
(row) =>
|
|
row.status === "pending" &&
|
|
row.kind === "request_confirmation" &&
|
|
row.payload?.target?.revisionId === draft.latestRevisionId,
|
|
);
|
|
expect(
|
|
initialApproval,
|
|
"draft has a revision-bound approval",
|
|
).toBeTruthy();
|
|
const reviseButton = page
|
|
.getByRole("button", {
|
|
name: initialApproval!.payload?.rejectLabel ?? "Reject",
|
|
exact: true,
|
|
})
|
|
.last();
|
|
await reviseButton.click();
|
|
await page
|
|
.getByTestId("plan-revision-composer")
|
|
.last()
|
|
.locator('[contenteditable="true"],textarea')
|
|
.first()
|
|
.fill(
|
|
`Revise the plan: the finished welcome note itself must contain the exact phrase ${marker} instead of ${draftMarker}. Include that requirement in the revised plan. The execution task should save that welcome note in its output document. Present this revised plan for approval; wait for that approval before handing it off as agreed.`,
|
|
);
|
|
await reviseButton.click();
|
|
await idle(2);
|
|
const revised = await api.get<Plan>(
|
|
`/api/issues/${issue!.id}/documents/plan`,
|
|
);
|
|
expect(revised.body).toContain(marker);
|
|
expect(revised.body).not.toContain(draftMarker);
|
|
expect(revised.latestRevisionId).not.toBe(draft.latestRevisionId);
|
|
acceptedPlan = revised;
|
|
await noTasks();
|
|
const interactions = await api.get<
|
|
Array<{
|
|
id: string;
|
|
status: string;
|
|
kind: string;
|
|
payload?: {
|
|
target?: { revisionId?: string };
|
|
acceptLabel?: string;
|
|
};
|
|
}>
|
|
>(`/api/issues/${issue!.id}/interactions`);
|
|
const approval = interactions.find(
|
|
(row) =>
|
|
row.status === "pending" &&
|
|
row.kind === "request_confirmation" &&
|
|
row.payload?.target?.revisionId === revised.latestRevisionId,
|
|
);
|
|
expect(approval, "approval targets the revised plan").toBeTruthy();
|
|
await input.capture(
|
|
"chat-plan-revised",
|
|
"Revised plan before handoff",
|
|
"chat-plan-revised.png",
|
|
);
|
|
await page
|
|
.getByRole("button", {
|
|
name: approval!.payload?.acceptLabel ?? "Approve",
|
|
exact: true,
|
|
})
|
|
.last()
|
|
.click();
|
|
await idle(4);
|
|
await input.evidence("chat-plan-revisions.json", {
|
|
draft,
|
|
revised,
|
|
approval,
|
|
source: await api.get(`/api/issues/${issue!.id}/documents/plan`),
|
|
});
|
|
} else {
|
|
repositoryCatalogBefore = await api.get(
|
|
`/api/companies/${f.company.id}/project-repositories`,
|
|
);
|
|
expect(
|
|
repositoryCatalogBefore!.repositories.filter((repository) =>
|
|
expectedRepositoryUrls.includes(repository.url),
|
|
),
|
|
).toHaveLength(0);
|
|
await turn(
|
|
`Create a project called Repository Discussion ${nonce} for work spanning https://github.com/octocat/Hello-World and https://github.com/octocat/Spoon-Knife. These existing public repositories are not in our catalog; register both URLs. Then make one assigned task for yourself to write a two-sentence description of the intended project in an output document, containing ${marker}, and complete that task. No code changes or remote repository creation are needed.`,
|
|
2,
|
|
);
|
|
}
|
|
const children = await tasks();
|
|
expect(children).toHaveLength(1);
|
|
const child = children[0]!;
|
|
await pollUntil({
|
|
label: `execution task ${child.id} completes`,
|
|
deadlineAt: Date.now() + 240_000,
|
|
intervalMs: 1000,
|
|
load: async () => ({
|
|
task: await api.get<ChatIssue>(`/api/issues/${child.id}`),
|
|
runs: await api.get<ChatRun[]>(`/api/issues/${child.id}/runs`),
|
|
}),
|
|
accept: (state) => state.task.status === "done",
|
|
reject: (state) => chatTaskCompletionFailure(state.task, state.runs),
|
|
});
|
|
runs = await allRuns();
|
|
input.observe(issue!, runs);
|
|
const plan =
|
|
caseId === "plan-handoff"
|
|
? await api.get<Plan>(`/api/issues/${child.id}/documents/plan`)
|
|
: null;
|
|
const taskRuns = runs.filter(
|
|
(run) => run.contextSnapshot?.issueId === child.id,
|
|
);
|
|
if (plan) {
|
|
assertChatHandoff(child, plan, taskRuns, issue!);
|
|
expect(plan.body).toContain(marker);
|
|
expect(plan.body).not.toContain(draftMarker);
|
|
const sourcePlan = await api.get<Plan>(
|
|
`/api/issues/${issue!.id}/documents/plan`,
|
|
);
|
|
expect(sourcePlan.body).toBe(acceptedPlan!.body);
|
|
expect(sourcePlan.latestRevisionId).toBe(
|
|
acceptedPlan!.latestRevisionId,
|
|
);
|
|
} else assertChatTaskHandoff(child, taskRuns, issue!);
|
|
const output = await readChatOutputDocument(api, child.id, marker);
|
|
expect(output.body).toContain(marker);
|
|
await input.evidence("chat-execution-output.json", {
|
|
taskId: child.id,
|
|
document: output,
|
|
revisions: await api.get(
|
|
`/api/issues/${child.id}/documents/${encodeURIComponent(output.key)}/revisions`,
|
|
),
|
|
executionRunIds: taskRuns.map((run) => run.id),
|
|
});
|
|
expect(
|
|
(await comments())
|
|
.filter((c) => c.authorAgentId)
|
|
.map((c) => c.body)
|
|
.join("\n"),
|
|
).toMatch(new RegExp(`${child.id}|${child.identifier}`));
|
|
const projects = await api.get<
|
|
Array<{
|
|
id: string;
|
|
name: string;
|
|
workspaces: Array<{ id: string; name: string; repoUrl?: string }>;
|
|
}>
|
|
>(`/api/companies/${f.company.id}/projects`);
|
|
expect(projects).toHaveLength(1);
|
|
if (existingProject) {
|
|
expect(child.projectId).toBe(existingProject.id);
|
|
await expect(
|
|
page.getByRole("article", { name: /Project created:/ }),
|
|
).toHaveCount(0);
|
|
} else {
|
|
await expect(
|
|
page.getByRole("article", { name: /Project created:/ }),
|
|
).toHaveCount(1);
|
|
if (caseId === "multi-repository") {
|
|
expect(projects[0]!.workspaces.map((w) => w.repoUrl).sort()).toEqual(
|
|
[...expectedRepositoryUrls].sort(),
|
|
);
|
|
// URL registration is persisted as project workspaces; the discovery
|
|
// catalog continues to reflect authorized GitHub connections only.
|
|
expect(
|
|
projects[0]!.workspaces.every((workspace) => Boolean(workspace.id)),
|
|
).toBe(true);
|
|
expect(
|
|
new Set(projects[0]!.workspaces.map((workspace) => workspace.id))
|
|
.size,
|
|
).toBe(2);
|
|
const persistedProject = await api.get<{
|
|
workspaces: Array<{ id: string; repoUrl?: string }>;
|
|
}>(`/api/projects/${projects[0]!.id}`);
|
|
expect(
|
|
persistedProject.workspaces.map(({ id, repoUrl }) => ({
|
|
id,
|
|
repoUrl,
|
|
})),
|
|
).toEqual(
|
|
projects[0]!.workspaces.map(({ id, repoUrl }) => ({ id, repoUrl })),
|
|
);
|
|
await input.evidence("chat-repository-registration.json", {
|
|
catalogBefore: repositoryCatalogBefore,
|
|
projectId: projects[0]!.id,
|
|
registeredWorkspaces: persistedProject.workspaces,
|
|
});
|
|
const projectCard = page.getByRole("article", {
|
|
name: /Project created:/,
|
|
});
|
|
// Repository labels may be customized; verify the actual destinations.
|
|
for (const repositoryUrl of expectedRepositoryUrls) {
|
|
await expect(
|
|
projectCard.locator(`a[href="${repositoryUrl}"]`),
|
|
).toBeVisible();
|
|
}
|
|
} else
|
|
expect(projects[0]!.workspaces.filter((w) => w.repoUrl)).toHaveLength(
|
|
0,
|
|
);
|
|
await page.reload({ waitUntil: "domcontentloaded", timeout: 60_000 });
|
|
await expect(
|
|
page.getByRole("article", { name: /Project created:/ }),
|
|
).toHaveCount(1);
|
|
}
|
|
await input.evidence("chat-handoff.json", {
|
|
source: issue!,
|
|
task: child,
|
|
plan,
|
|
output,
|
|
projects,
|
|
});
|
|
}
|
|
await idle(execution.task.expectedRunCount);
|
|
expect(runs.filter((run) => !isResetRun(run))).toHaveLength(
|
|
execution.task.expectedRunCount,
|
|
);
|
|
for (const run of runs.filter((run) => !isResetRun(run))) {
|
|
expect(run.runtimeMode).toBe(execution.profile.expectedRuntimeMode);
|
|
expect(run.status).toBe(
|
|
caseId === "stop-new-resume" && run.status === "cancelled"
|
|
? "cancelled"
|
|
: "succeeded",
|
|
);
|
|
}
|
|
await input.evidence("api-state.json", {
|
|
issue: issue!,
|
|
runs,
|
|
runGroups: {
|
|
resets: runs.filter(isResetRun).map((run) => run.id),
|
|
cancelled: runs
|
|
.filter((run) => run.status === "cancelled")
|
|
.map((run) => run.id),
|
|
conversation: runs
|
|
.filter(
|
|
(run) =>
|
|
!isResetRun(run) && run.contextSnapshot?.issueId === issue!.id,
|
|
)
|
|
.map((run) => run.id),
|
|
handoff: runs
|
|
.filter((run) => run.contextSnapshot?.issueId !== issue!.id)
|
|
.map((run) => run.id),
|
|
},
|
|
comments: await comments(),
|
|
activity: await api.get(`/api/issues/${issue!.id}/activity`),
|
|
runEvidence: await Promise.all(
|
|
runs.map((run) => collectChatRunEvidence(api, run)),
|
|
),
|
|
});
|
|
await input.capture(
|
|
"final-state",
|
|
"Chat waiting after its verified workflow",
|
|
"final-state.png",
|
|
);
|
|
return { issue: issue!, runs };
|
|
} finally {
|
|
await api.patch("/api/instance/settings/experimental", {
|
|
enableAgentChat: settings.enableAgentChat,
|
|
enableClassicTaskInterface: settings.enableClassicTaskInterface,
|
|
});
|
|
}
|
|
}
|