838 lines
31 KiB
TypeScript
838 lines
31 KiB
TypeScript
import { expect, type Page } from "@playwright/test";
|
|
import { pollUntil, type RunnerApi } from "./api.js";
|
|
import type {
|
|
AskUserQuestionsPayload,
|
|
PaperclipQuestionSetPayload,
|
|
} from "../../packages/shared/src/types/issue.js";
|
|
import type { LiveFixtureValues } from "./live-fixtures.js";
|
|
import type { MatrixExecution } from "./types.js";
|
|
import { chatMarker } from "./chat-cases.js";
|
|
|
|
// Public API observations only: this driver never fabricates provider results or writes DB state.
|
|
export interface ChatIssue {
|
|
id: string;
|
|
companyId: string;
|
|
title: string;
|
|
status: string;
|
|
identifier?: string;
|
|
conversationState?: string;
|
|
conversationSessionGeneration?: number;
|
|
conversationBoundaryCommentId?: string;
|
|
parentId?: string | null;
|
|
projectId?: string | null;
|
|
assigneeAgentId?: string | null;
|
|
scheduledRetry?: unknown;
|
|
activeRecoveryAction?: unknown;
|
|
}
|
|
export interface ChatRun {
|
|
id: string;
|
|
companyId: string;
|
|
agentId: string;
|
|
status: string;
|
|
error?: string | null;
|
|
errorCode?: string | null;
|
|
runtimeMode?: string;
|
|
contextSnapshot?: Record<string, unknown>;
|
|
resultJson?: Record<string, unknown>;
|
|
sessionIdBefore?: string | null;
|
|
sessionIdAfter?: string | null;
|
|
startedAt?: string;
|
|
}
|
|
type Comment = {
|
|
id: string;
|
|
body: string;
|
|
authorAgentId?: string;
|
|
createdByRunId?: string;
|
|
conversationSessionGeneration?: number;
|
|
};
|
|
type Plan = { body: string; latestRevisionId: string; updatedAt: string };
|
|
type ChatOutputDocument = Plan & { id: string; issueId: string; key: string };
|
|
|
|
/** Clarification may request information imperatively rather than end in a question mark. */
|
|
export function isChatClarificationReply(body: string): boolean {
|
|
if (body.includes("?")) return true;
|
|
const request = body.match(
|
|
/\b(?:please\s+(?:share|provide|clarify|confirm)|tell me|let me know)\b([\s\S]*)/i,
|
|
) ?? body.match(
|
|
/\b(?:I|we)(?:'ll|\s+will)?\s+need\s+(?:(?:a|some|the|your|more|following|compact|short|few|additional)\s+){0,4}(?:brief|details|information|context|clarification)\b([\s\S]*)/i,
|
|
);
|
|
return Boolean(request && /[\p{L}\p{N}]/u.test(request[1]));
|
|
}
|
|
|
|
export function assertChatExecutionOutput(
|
|
body: string,
|
|
marker: string,
|
|
supersededMarker?: string,
|
|
): void {
|
|
expect(body).toContain(marker);
|
|
if (supersededMarker) expect(body).not.toContain(supersededMarker);
|
|
}
|
|
|
|
/** A requested output document may have a descriptive key; a copied plan is not output. */
|
|
export async function readChatOutputDocument(
|
|
api: Pick<RunnerApi, "get">,
|
|
issueId: string,
|
|
marker: string,
|
|
): Promise<ChatOutputDocument> {
|
|
const summaries = await api.get<Array<{ key: string }>>(
|
|
`/api/issues/${issueId}/documents`,
|
|
);
|
|
const documents = await Promise.all(
|
|
summaries
|
|
.filter((document) => document.key !== "plan")
|
|
.map((document) =>
|
|
api.get<ChatOutputDocument>(
|
|
`/api/issues/${issueId}/documents/${encodeURIComponent(document.key)}`,
|
|
),
|
|
),
|
|
);
|
|
const output = documents.find(
|
|
(document) => document.issueId === issueId && document.body.includes(marker),
|
|
);
|
|
if (!output)
|
|
throw new Error(
|
|
`Execution task ${issueId} has no non-plan output document containing ${marker}; document keys: ${summaries.map((document) => document.key).join(", ") || "none"}`,
|
|
);
|
|
expect(output.id).toBeTruthy();
|
|
expect(output.latestRevisionId).toBeTruthy();
|
|
return output;
|
|
}
|
|
|
|
export const isResetRun = (run: ChatRun) =>
|
|
run.contextSnapshot?.conversationReset === true ||
|
|
run.resultJson?.conversationReset === true;
|
|
export function chatRunFailure(
|
|
runs: ChatRun[],
|
|
allowCancelled = false,
|
|
): string | undefined {
|
|
const failed = runs.find(
|
|
(run) =>
|
|
["failed", "timed_out"].includes(run.status) ||
|
|
(!allowCancelled && run.status === "cancelled"),
|
|
);
|
|
return failed
|
|
? `run ${failed.id} ${failed.status}${failed.errorCode ? ` (${failed.errorCode})` : ""}${failed.error ? `: ${failed.error}` : ""}`
|
|
: undefined;
|
|
}
|
|
|
|
export function chatTaskCompletionFailure(
|
|
task: ChatIssue,
|
|
runs: ChatRun[],
|
|
): string | undefined {
|
|
if (
|
|
task.scheduledRetry ||
|
|
task.activeRecoveryAction ||
|
|
runs.some((run) => ["queued", "running"].includes(run.status))
|
|
)
|
|
return undefined;
|
|
return chatRunFailure(runs);
|
|
}
|
|
|
|
/** Require two settled observations so an in-flight finalization is not a failure. */
|
|
export function createChatIdleFailureDetector(minimumProviderRuns: number) {
|
|
let priorInconsistentState: string | undefined;
|
|
return (state: {
|
|
resolved: boolean;
|
|
status?: string;
|
|
conversationState?: string;
|
|
providerRunCount: number;
|
|
activeRuns: string[];
|
|
}): string | undefined => {
|
|
const inconsistentState =
|
|
state.resolved &&
|
|
state.providerRunCount >= minimumProviderRuns &&
|
|
state.activeRuns.length === 0 &&
|
|
state.conversationState === "waiting" &&
|
|
["blocked", "done", "cancelled"].includes(state.status ?? "")
|
|
? `${state.status}:${state.providerRunCount}`
|
|
: undefined;
|
|
const stable =
|
|
inconsistentState !== undefined &&
|
|
inconsistentState === priorInconsistentState;
|
|
priorInconsistentState = inconsistentState;
|
|
return stable
|
|
? `chat_idle_state_invariant: conversation is ${state.status} while waiting after ${state.providerRunCount} settled provider runs`
|
|
: undefined;
|
|
};
|
|
}
|
|
|
|
/** Match the shared question form's durable/native presentation, including custom labels. */
|
|
export function chatQuestionPresentation(
|
|
payload: AskUserQuestionsPayload,
|
|
): PaperclipQuestionSetPayload {
|
|
if (payload.questionSet) return payload.questionSet;
|
|
return {
|
|
schema: "paperclip.question_set.v1",
|
|
...(payload.submitLabel ? { submitLabel: payload.submitLabel } : {}),
|
|
questions: payload.questions.map((question) => {
|
|
const freeText = question.options.find((option) => option.freeText);
|
|
return {
|
|
id: question.id,
|
|
prompt: question.prompt,
|
|
required: question.required === true,
|
|
answerMode:
|
|
question.selectionMode === "multi" ? "multi_select" : "single_select",
|
|
...(freeText
|
|
? { customAnswer: { enabled: true as const, label: freeText.label } }
|
|
: {}),
|
|
};
|
|
}),
|
|
};
|
|
}
|
|
|
|
export function assertChatTaskHandoff(
|
|
task: ChatIssue,
|
|
runs: ChatRun[],
|
|
source: ChatIssue,
|
|
) {
|
|
expect(task.parentId).toBeNull();
|
|
expect(task.projectId).toBeTruthy();
|
|
expect(task.assigneeAgentId).toBe(source.assigneeAgentId);
|
|
expect(runs.length).toBeGreaterThan(0);
|
|
}
|
|
|
|
/** A running row can precede creation of its log file. Only that expected 404 is retryable. */
|
|
export async function readRunningChatLog(
|
|
api: Pick<RunnerApi, "request">,
|
|
runId: string,
|
|
): Promise<string | undefined> {
|
|
const response = await api.request.get(
|
|
`/api/heartbeat-runs/${runId}/log?limitBytes=65536`,
|
|
);
|
|
if (response.status() === 404) return undefined;
|
|
if (!response.ok())
|
|
throw new Error(`Run ${runId} log returned ${response.status()}`);
|
|
return ((await response.json()) as { content?: string }).content;
|
|
}
|
|
|
|
/** Synthetic reset runs have durable events but never start a provider log. */
|
|
export async function collectChatRunEvidence(
|
|
api: Pick<RunnerApi, "get">,
|
|
run: ChatRun,
|
|
) {
|
|
return {
|
|
runId: run.id,
|
|
log: isResetRun(run)
|
|
? null
|
|
: await api.get(`/api/heartbeat-runs/${run.id}/log?limitBytes=1048576`),
|
|
events: await api.get(`/api/heartbeat-runs/${run.id}/events?limit=1000`),
|
|
};
|
|
}
|
|
|
|
export function assertChatHandoff(
|
|
task: ChatIssue,
|
|
plan: Plan,
|
|
runs: ChatRun[],
|
|
source: ChatIssue,
|
|
) {
|
|
assertChatTaskHandoff(task, runs, source);
|
|
expect(plan.body.trim()).not.toBe("");
|
|
for (const run of runs) {
|
|
expect(Date.parse(plan.updatedAt)).toBeLessThanOrEqual(
|
|
Date.parse(run.startedAt!),
|
|
);
|
|
}
|
|
}
|
|
export async function sendChatMessage(page: Page, message: string) {
|
|
const composer = page.getByTestId("task-chat-composer-input").last();
|
|
await composer
|
|
.locator('[contenteditable="true"], textarea')
|
|
.first()
|
|
.fill(message);
|
|
await page.getByTestId("task-chat-composer-send").last().click();
|
|
}
|
|
|
|
export async function runChatFlow(input: {
|
|
page: Page;
|
|
api: RunnerApi;
|
|
fixtures: LiveFixtureValues;
|
|
execution: MatrixExecution;
|
|
nonce: string;
|
|
restart: () => Promise<void>;
|
|
observe: (issue: ChatIssue, runs: ChatRun[]) => void;
|
|
capture: (id: string, label: string, file: string) => Promise<void>;
|
|
evidence: (name: string, data: unknown) => Promise<void>;
|
|
}) {
|
|
const { page, api, fixtures: f, execution, nonce } = input;
|
|
const chatPath = `/api/companies/${f.company.id}/chats/${f.agent.id}`;
|
|
const route = `/${f.company.issuePrefix}/chats/${f.agent.id}`;
|
|
const marker = execution.task.buildVisibleMarker(nonce);
|
|
const draftMarker = chatMarker("DRAFT", nonce);
|
|
const caseId = execution.task.id;
|
|
let issue: ChatIssue;
|
|
let runs: ChatRun[] = [];
|
|
const settings = await api.get<Record<string, unknown>>(
|
|
"/api/instance/settings/experimental",
|
|
);
|
|
const allRuns = async () => {
|
|
const rows = await api.get<ChatRun[]>(
|
|
`/api/companies/${f.company.id}/heartbeat-runs?limit=100`,
|
|
);
|
|
return Promise.all(
|
|
rows.map((row) => api.get<ChatRun>(`/api/heartbeat-runs/${row.id}`)),
|
|
);
|
|
};
|
|
const tasks = () =>
|
|
api.get<ChatIssue[]>(`/api/companies/${f.company.id}/issues`);
|
|
const comments = async () =>
|
|
(
|
|
await api.get<Array<Comment & { createdAt: string }>>(
|
|
`/api/issues/${issue.id}/comments?order=asc`,
|
|
)
|
|
).sort(
|
|
(a, b) =>
|
|
a.createdAt.localeCompare(b.createdAt) || a.id.localeCompare(b.id),
|
|
);
|
|
const idle = async (minimumProviderRuns: number) => {
|
|
const inconsistentIdle = createChatIdleFailureDetector(minimumProviderRuns);
|
|
await pollUntil({
|
|
label: "chat turn settles to waiting",
|
|
deadlineAt: Date.now() + 240_000,
|
|
intervalMs: 1000,
|
|
load: async () => {
|
|
const resolved = await api.get<ChatIssue | null>(chatPath);
|
|
if (resolved) issue = resolved;
|
|
runs = await allRuns();
|
|
if (resolved) input.observe(resolved, runs);
|
|
return {
|
|
resolved: Boolean(resolved),
|
|
status: resolved?.status,
|
|
conversationState: resolved?.conversationState,
|
|
providerRunCount: runs.filter((run) => !isResetRun(run)).length,
|
|
activeRuns: runs
|
|
.filter((run) => ["queued", "running"].includes(run.status))
|
|
.map((run) => run.id),
|
|
failure: chatRunFailure(runs, caseId === "stop-new-resume"),
|
|
};
|
|
},
|
|
reject: (state) => state.failure ?? inconsistentIdle(state),
|
|
accept: (state) =>
|
|
!state.failure &&
|
|
state.resolved &&
|
|
state.providerRunCount >= minimumProviderRuns &&
|
|
state.activeRuns.length === 0 &&
|
|
state.status === "in_review" &&
|
|
state.conversationState === "waiting",
|
|
});
|
|
};
|
|
const turn = async (text: string, count: number) => {
|
|
await sendChatMessage(page, text);
|
|
await idle(count);
|
|
};
|
|
const noTasks = async () => expect(await tasks()).toHaveLength(0);
|
|
try {
|
|
await api.patch("/api/instance/settings/experimental", {
|
|
enableAgentChat: true,
|
|
enableClassicTaskInterface: false,
|
|
});
|
|
expect(await api.get(chatPath)).toBeNull();
|
|
// Cold Vite startup can keep unrelated assets loading after the chat is
|
|
// interactive. The composer assertion below verifies actual UI readiness.
|
|
await page.goto(route, { waitUntil: "domcontentloaded", timeout: 60_000 });
|
|
await expect(page.getByTestId("task-chat-composer-input")).toBeVisible();
|
|
expect(await api.get(chatPath)).toBeNull();
|
|
expect(await allRuns()).toHaveLength(0);
|
|
|
|
if (
|
|
["continuity-restart", "new-session", "stop-new-resume"].includes(caseId)
|
|
) {
|
|
const secret = chatMarker("OLDCONTEXT", nonce);
|
|
await turn(
|
|
`For this conversation only, remember the phrase ${secret}. Just acknowledge briefly; no project or task is needed.`,
|
|
1,
|
|
);
|
|
const initialId = issue!.id;
|
|
const before = runs.filter((run) => !isResetRun(run))[0]!;
|
|
await noTasks();
|
|
await page.reload({ waitUntil: "domcontentloaded", timeout: 60_000 });
|
|
if (caseId === "continuity-restart") {
|
|
await turn(
|
|
"What phrase did I just ask you to remember? Reply with the phrase only.",
|
|
2,
|
|
);
|
|
expect(
|
|
(await comments()).filter((c) => c.authorAgentId).at(-1)?.body,
|
|
).toContain(secret);
|
|
const count = runs.length;
|
|
await input.restart();
|
|
// Re-enter the canonical route after the server replaces its browser
|
|
// transport; reloading the stale document can target a detached page.
|
|
await page.goto(route, { waitUntil: "domcontentloaded", timeout: 60_000 });
|
|
await expect(page.getByTestId("task-chat-composer-input")).toBeVisible();
|
|
await idle(2);
|
|
expect(runs).toHaveLength(count);
|
|
await turn(
|
|
`We are done discussing it. Reply with ${marker} only; no further work.`,
|
|
3,
|
|
);
|
|
} else {
|
|
let cancelledId: string | undefined;
|
|
if (caseId === "stop-new-resume") {
|
|
await sendChatMessage(
|
|
page,
|
|
"Explain the history of gardening at length here, in 100 numbered paragraphs. This is discussion only; do not create work.",
|
|
);
|
|
await expect
|
|
.poll(
|
|
async () => {
|
|
runs = await allRuns();
|
|
const active = runs.find(
|
|
(run) => run.status === "running" && run.id !== before.id,
|
|
);
|
|
if (!active) return false;
|
|
const events = await api.get<Array<Record<string, unknown>>>(
|
|
`/api/heartbeat-runs/${active.id}/events?limit=1000`,
|
|
);
|
|
const log = await readRunningChatLog(api, active.id);
|
|
if (!(events.length || log?.length)) return false;
|
|
cancelledId = active.id;
|
|
return true;
|
|
},
|
|
{ timeout: 120_000 },
|
|
)
|
|
.toBe(true);
|
|
await page.getByTestId("task-chat-composer-stop").click();
|
|
await expect
|
|
.poll(
|
|
async () =>
|
|
(await api.get<ChatRun>(`/api/heartbeat-runs/${cancelledId}`))
|
|
.status,
|
|
)
|
|
.toBe("cancelled");
|
|
}
|
|
const oldComments = await comments();
|
|
await sendChatMessage(page, "/new");
|
|
await expect
|
|
.poll(
|
|
async () =>
|
|
(await api.get<ChatIssue>(chatPath))
|
|
.conversationSessionGeneration,
|
|
{ timeout: 30_000 },
|
|
)
|
|
.toBe(1);
|
|
await expect
|
|
.poll(async () => (await allRuns()).some(isResetRun))
|
|
.toBe(true);
|
|
await turn(
|
|
`Without reading older history or files, if you have a remembered phrase in your current context return it; otherwise reply exactly ${marker}. Do not look it up.`,
|
|
caseId === "new-session" ? 2 : 3,
|
|
);
|
|
const fresh = runs
|
|
.filter((run) => !isResetRun(run) && run.status === "succeeded")
|
|
.sort((a, b) => Date.parse(a.startedAt!) - Date.parse(b.startedAt!))
|
|
.at(-1)!;
|
|
expect(fresh.contextSnapshot?.conversationSessionGeneration).toBe(1);
|
|
expect(fresh.sessionIdBefore).toBeFalsy();
|
|
expect(
|
|
String(fresh.contextSnapshot?.paperclipTaskMarkdown ?? ""),
|
|
).not.toContain(secret);
|
|
if (before.sessionIdAfter && fresh.sessionIdAfter)
|
|
expect(fresh.sessionIdAfter).not.toBe(before.sessionIdAfter);
|
|
const replies = (await comments()).filter(
|
|
(c) => c.createdByRunId === fresh.id && c.authorAgentId,
|
|
);
|
|
expect(replies.map((c) => c.body).join("\n")).toContain(marker);
|
|
expect(replies.map((c) => c.body).join("\n")).not.toContain(secret);
|
|
const reset = runs.filter(isResetRun);
|
|
expect(reset).toHaveLength(1);
|
|
expect(
|
|
(await comments()).filter(
|
|
(c) => c.authorAgentId && c.createdByRunId === reset[0]!.id,
|
|
),
|
|
).toHaveLength(0);
|
|
expect(
|
|
(await comments()).filter((c) =>
|
|
oldComments.some((old) => old.id === c.id),
|
|
),
|
|
).toHaveLength(oldComments.length);
|
|
if (cancelledId)
|
|
expect(
|
|
(await comments()).filter((c) => c.createdByRunId === cancelledId),
|
|
).toEqual(
|
|
oldComments.filter((c) => c.createdByRunId === cancelledId),
|
|
);
|
|
await page.reload({ waitUntil: "domcontentloaded", timeout: 60_000 });
|
|
await expect(
|
|
page.getByText("New session", { exact: true }),
|
|
).toHaveCount(1);
|
|
}
|
|
expect(issue!.id).toBe(initialId);
|
|
await noTasks();
|
|
} else {
|
|
let existingProject: { id: string; name: string } | undefined;
|
|
let acceptedPlan: Plan | undefined;
|
|
let repositoryCatalogBefore:
|
|
{ repositories: Array<{ id: string; url: string }> } | undefined;
|
|
const expectedRepositoryUrls = [
|
|
"https://github.com/octocat/Hello-World",
|
|
"https://github.com/octocat/Spoon-Knife",
|
|
];
|
|
if (caseId === "clarify-reuse") {
|
|
existingProject = await api.post(
|
|
`/api/companies/${f.company.id}/projects`,
|
|
{
|
|
name: `Garden ${nonce}`,
|
|
description: "Garden club welcome notes and event announcements.",
|
|
},
|
|
);
|
|
await turn(
|
|
"I need a welcome note for our club. Help me clarify what information you need before assigning the work.",
|
|
1,
|
|
);
|
|
await noTasks();
|
|
const questions = await api.get<
|
|
Array<{
|
|
status: string;
|
|
kind: string;
|
|
payload: AskUserQuestionsPayload;
|
|
}>
|
|
>(`/api/issues/${issue!.id}/interactions`);
|
|
const pendingInteraction = questions.find(
|
|
(row) =>
|
|
row.status === "pending" && row.kind === "ask_user_questions",
|
|
);
|
|
const questionSet = pendingInteraction
|
|
? chatQuestionPresentation(pendingInteraction.payload)
|
|
: undefined;
|
|
const pendingQuestions = questionSet?.questions;
|
|
expect(
|
|
Boolean(pendingQuestions?.length) ||
|
|
(await comments()).some(
|
|
(c) => c.authorAgentId && isChatClarificationReply(c.body),
|
|
),
|
|
).toBe(true);
|
|
const clarification = `It is the garden club; use the existing Garden ${nonce} project. Make one assigned task for yourself to write a two-sentence welcome note. Include ${marker} in that note, save it as the output document, and finish that execution task. Please get it started now.`;
|
|
if (pendingQuestions?.length) {
|
|
for (const [index, question] of pendingQuestions.entries()) {
|
|
const textInput = page
|
|
.getByTestId("question-text-answer-composer")
|
|
.last();
|
|
if (await textInput.isVisible()) {
|
|
await textInput
|
|
.locator('[contenteditable="true"],textarea')
|
|
.first()
|
|
.fill(clarification);
|
|
} else {
|
|
await page
|
|
.getByRole(
|
|
question.answerMode === "multi_select" ? "checkbox" : "radio",
|
|
{
|
|
name: question.customAnswer?.label ?? "Other",
|
|
exact: true,
|
|
},
|
|
)
|
|
.last()
|
|
.click();
|
|
await page
|
|
.getByTestId("question-other-answer-composer")
|
|
.last()
|
|
.locator('[contenteditable="true"],textarea')
|
|
.first()
|
|
.fill(clarification);
|
|
}
|
|
await page
|
|
.getByRole("button", {
|
|
name:
|
|
index === pendingQuestions.length - 1
|
|
? (questionSet?.submitLabel ?? "Submit answers")
|
|
: "Next",
|
|
exact: true,
|
|
})
|
|
.last()
|
|
.click();
|
|
}
|
|
await idle(3);
|
|
} else await turn(clarification, 3);
|
|
} else if (caseId === "plan-handoff") {
|
|
await page.getByTestId("task-chat-composer-mode").click();
|
|
await page
|
|
.getByTestId("task-chat-composer-mode-menu")
|
|
.getByText("Plan mode", { exact: true })
|
|
.click();
|
|
await turn(
|
|
`Let's plan a two-sentence garden club welcome note. The finished welcome note itself must contain the exact phrase ${draftMarker}. Write a plan in the plan panel that includes this requirement, and present it for approval. When I approve the final revision, create a suitable repository-free project and an assigned task for yourself, copy the plan into that task, and have it save the note in its output document and finish. Do not create the project or task before approval.`,
|
|
1,
|
|
);
|
|
const draft = await api.get<Plan>(
|
|
`/api/issues/${issue!.id}/documents/plan`,
|
|
);
|
|
expect(draft.body).toContain(draftMarker);
|
|
await noTasks();
|
|
await input.capture(
|
|
"chat-plan-draft",
|
|
"Draft plan in the conversation",
|
|
"chat-plan-draft.png",
|
|
);
|
|
const initialInteractions = await api.get<
|
|
Array<{
|
|
status: string;
|
|
kind: string;
|
|
payload?: {
|
|
target?: { revisionId?: string };
|
|
rejectLabel?: string;
|
|
};
|
|
}>
|
|
>(`/api/issues/${issue!.id}/interactions`);
|
|
const initialApproval = initialInteractions.find(
|
|
(row) =>
|
|
row.status === "pending" &&
|
|
row.kind === "request_confirmation" &&
|
|
row.payload?.target?.revisionId === draft.latestRevisionId,
|
|
);
|
|
expect(
|
|
initialApproval,
|
|
"draft has a revision-bound approval",
|
|
).toBeTruthy();
|
|
const reviseButton = page
|
|
.getByRole("button", {
|
|
name: initialApproval!.payload?.rejectLabel ?? "Reject",
|
|
exact: true,
|
|
})
|
|
.last();
|
|
await reviseButton.click();
|
|
await page
|
|
.getByTestId("plan-revision-composer")
|
|
.last()
|
|
.locator('[contenteditable="true"],textarea')
|
|
.first()
|
|
.fill(
|
|
`Revise the plan: the finished welcome note itself must contain the exact phrase ${marker} instead of ${draftMarker}. Include that requirement in the revised plan. The execution task should save that welcome note in its output document. Present this revised plan for approval; wait for that approval before handing it off as agreed.`,
|
|
);
|
|
await reviseButton.click();
|
|
await idle(2);
|
|
const revised = await api.get<Plan>(
|
|
`/api/issues/${issue!.id}/documents/plan`,
|
|
);
|
|
expect(revised.body).toContain(marker);
|
|
// A revision-history section may quote the superseded requirement.
|
|
// The executed output below must use only the accepted requirement.
|
|
expect(revised.latestRevisionId).not.toBe(draft.latestRevisionId);
|
|
acceptedPlan = revised;
|
|
await noTasks();
|
|
const interactions = await api.get<
|
|
Array<{
|
|
id: string;
|
|
status: string;
|
|
kind: string;
|
|
payload?: {
|
|
target?: { revisionId?: string };
|
|
acceptLabel?: string;
|
|
};
|
|
}>
|
|
>(`/api/issues/${issue!.id}/interactions`);
|
|
const approval = interactions.find(
|
|
(row) =>
|
|
row.status === "pending" &&
|
|
row.kind === "request_confirmation" &&
|
|
row.payload?.target?.revisionId === revised.latestRevisionId,
|
|
);
|
|
expect(approval, "approval targets the revised plan").toBeTruthy();
|
|
await input.capture(
|
|
"chat-plan-revised",
|
|
"Revised plan before handoff",
|
|
"chat-plan-revised.png",
|
|
);
|
|
await page
|
|
.getByRole("button", {
|
|
name: approval!.payload?.acceptLabel ?? "Approve",
|
|
exact: true,
|
|
})
|
|
.last()
|
|
.click();
|
|
await idle(4);
|
|
await input.evidence("chat-plan-revisions.json", {
|
|
draft,
|
|
revised,
|
|
approval,
|
|
source: await api.get(`/api/issues/${issue!.id}/documents/plan`),
|
|
});
|
|
} else {
|
|
repositoryCatalogBefore = await api.get(
|
|
`/api/companies/${f.company.id}/project-repositories`,
|
|
);
|
|
expect(
|
|
repositoryCatalogBefore!.repositories.filter((repository) =>
|
|
expectedRepositoryUrls.includes(repository.url),
|
|
),
|
|
).toHaveLength(0);
|
|
await turn(
|
|
`Create a project called Repository Discussion ${nonce} for work spanning https://github.com/octocat/Hello-World and https://github.com/octocat/Spoon-Knife. These existing public repositories are not in our catalog; register both URLs. Then make one assigned task for yourself to write a two-sentence description of the intended project in an output document, containing ${marker}, and complete that task. No code changes or remote repository creation are needed.`,
|
|
2,
|
|
);
|
|
}
|
|
const children = await tasks();
|
|
expect(children).toHaveLength(1);
|
|
const child = children[0]!;
|
|
await pollUntil({
|
|
label: `execution task ${child.id} completes`,
|
|
deadlineAt: Date.now() + 240_000,
|
|
intervalMs: 1000,
|
|
load: async () => ({
|
|
task: await api.get<ChatIssue>(`/api/issues/${child.id}`),
|
|
runs: await api.get<ChatRun[]>(`/api/issues/${child.id}/runs`),
|
|
}),
|
|
accept: (state) => state.task.status === "done",
|
|
reject: (state) => chatTaskCompletionFailure(state.task, state.runs),
|
|
});
|
|
runs = await allRuns();
|
|
input.observe(issue!, runs);
|
|
const plan =
|
|
caseId === "plan-handoff"
|
|
? await api.get<Plan>(`/api/issues/${child.id}/documents/plan`)
|
|
: null;
|
|
const taskRuns = runs.filter(
|
|
(run) => run.contextSnapshot?.issueId === child.id,
|
|
);
|
|
if (plan) {
|
|
assertChatHandoff(child, plan, taskRuns, issue!);
|
|
expect(plan.body).toContain(marker);
|
|
const sourcePlan = await api.get<Plan>(
|
|
`/api/issues/${issue!.id}/documents/plan`,
|
|
);
|
|
expect(sourcePlan.body).toBe(acceptedPlan!.body);
|
|
expect(sourcePlan.latestRevisionId).toBe(
|
|
acceptedPlan!.latestRevisionId,
|
|
);
|
|
} else assertChatTaskHandoff(child, taskRuns, issue!);
|
|
const output = await readChatOutputDocument(api, child.id, marker);
|
|
assertChatExecutionOutput(
|
|
output.body,
|
|
marker,
|
|
caseId === "plan-handoff" ? draftMarker : undefined,
|
|
);
|
|
await input.evidence("chat-execution-output.json", {
|
|
taskId: child.id,
|
|
document: output,
|
|
revisions: await api.get(
|
|
`/api/issues/${child.id}/documents/${encodeURIComponent(output.key)}/revisions`,
|
|
),
|
|
executionRunIds: taskRuns.map((run) => run.id),
|
|
});
|
|
expect(
|
|
(await comments())
|
|
.filter((c) => c.authorAgentId)
|
|
.map((c) => c.body)
|
|
.join("\n"),
|
|
).toMatch(new RegExp(`${child.id}|${child.identifier}`));
|
|
const projects = await api.get<
|
|
Array<{
|
|
id: string;
|
|
name: string;
|
|
workspaces: Array<{ id: string; name: string; repoUrl?: string }>;
|
|
}>
|
|
>(`/api/companies/${f.company.id}/projects`);
|
|
expect(projects).toHaveLength(1);
|
|
if (existingProject) {
|
|
expect(child.projectId).toBe(existingProject.id);
|
|
await expect(
|
|
page.getByRole("article", { name: /Project created:/ }),
|
|
).toHaveCount(0);
|
|
} else {
|
|
await expect(
|
|
page.getByRole("article", { name: /Project created:/ }),
|
|
).toHaveCount(1);
|
|
if (caseId === "multi-repository") {
|
|
expect(projects[0]!.workspaces.map((w) => w.repoUrl).sort()).toEqual(
|
|
[...expectedRepositoryUrls].sort(),
|
|
);
|
|
// URL registration is persisted as project workspaces; the discovery
|
|
// catalog continues to reflect authorized GitHub connections only.
|
|
expect(
|
|
projects[0]!.workspaces.every((workspace) => Boolean(workspace.id)),
|
|
).toBe(true);
|
|
expect(
|
|
new Set(projects[0]!.workspaces.map((workspace) => workspace.id))
|
|
.size,
|
|
).toBe(2);
|
|
const persistedProject = await api.get<{
|
|
workspaces: Array<{ id: string; repoUrl?: string }>;
|
|
}>(`/api/projects/${projects[0]!.id}`);
|
|
expect(
|
|
persistedProject.workspaces.map(({ id, repoUrl }) => ({
|
|
id,
|
|
repoUrl,
|
|
})),
|
|
).toEqual(
|
|
projects[0]!.workspaces.map(({ id, repoUrl }) => ({ id, repoUrl })),
|
|
);
|
|
await input.evidence("chat-repository-registration.json", {
|
|
catalogBefore: repositoryCatalogBefore,
|
|
projectId: projects[0]!.id,
|
|
registeredWorkspaces: persistedProject.workspaces,
|
|
});
|
|
const projectCard = page.getByRole("article", {
|
|
name: /Project created:/,
|
|
});
|
|
// Repository labels may be customized; verify the actual destinations.
|
|
for (const repositoryUrl of expectedRepositoryUrls) {
|
|
await expect(
|
|
projectCard.locator(`a[href="${repositoryUrl}"]`),
|
|
).toBeVisible();
|
|
}
|
|
} else
|
|
expect(projects[0]!.workspaces.filter((w) => w.repoUrl)).toHaveLength(
|
|
0,
|
|
);
|
|
await page.reload({ waitUntil: "domcontentloaded", timeout: 60_000 });
|
|
await expect(
|
|
page.getByRole("article", { name: /Project created:/ }),
|
|
).toHaveCount(1);
|
|
}
|
|
await input.evidence("chat-handoff.json", {
|
|
source: issue!,
|
|
task: child,
|
|
plan,
|
|
output,
|
|
projects,
|
|
});
|
|
}
|
|
await idle(execution.task.expectedRunCount);
|
|
expect(runs.filter((run) => !isResetRun(run))).toHaveLength(
|
|
execution.task.expectedRunCount,
|
|
);
|
|
for (const run of runs.filter((run) => !isResetRun(run))) {
|
|
expect(run.runtimeMode).toBe(execution.profile.expectedRuntimeMode);
|
|
expect(run.status).toBe(
|
|
caseId === "stop-new-resume" && run.status === "cancelled"
|
|
? "cancelled"
|
|
: "succeeded",
|
|
);
|
|
}
|
|
await input.evidence("api-state.json", {
|
|
issue: issue!,
|
|
runs,
|
|
runGroups: {
|
|
resets: runs.filter(isResetRun).map((run) => run.id),
|
|
cancelled: runs
|
|
.filter((run) => run.status === "cancelled")
|
|
.map((run) => run.id),
|
|
conversation: runs
|
|
.filter(
|
|
(run) =>
|
|
!isResetRun(run) && run.contextSnapshot?.issueId === issue!.id,
|
|
)
|
|
.map((run) => run.id),
|
|
handoff: runs
|
|
.filter((run) => run.contextSnapshot?.issueId !== issue!.id)
|
|
.map((run) => run.id),
|
|
},
|
|
comments: await comments(),
|
|
activity: await api.get(`/api/issues/${issue!.id}/activity`),
|
|
runEvidence: await Promise.all(
|
|
runs.map((run) => collectChatRunEvidence(api, run)),
|
|
),
|
|
});
|
|
await input.capture(
|
|
"final-state",
|
|
"Chat waiting after its verified workflow",
|
|
"final-state.png",
|
|
);
|
|
return { issue: issue!, runs };
|
|
} finally {
|
|
await api.patch("/api/instance/settings/experimental", {
|
|
enableAgentChat: settings.enableAgentChat,
|
|
enableClassicTaskInterface: settings.enableClassicTaskInterface,
|
|
});
|
|
}
|
|
}
|