399 lines
18 KiB
TypeScript
399 lines
18 KiB
TypeScript
import { expect, test, type Page } from "@playwright/test";
|
|
|
|
const DESKTOP = { width: 1440, height: 900 };
|
|
const MOBILE = { width: 390, height: 844 };
|
|
|
|
function shot(page: Page, name: string) {
|
|
return page.screenshot({ path: test.info().outputPath(`${name}.png`), fullPage: true });
|
|
}
|
|
|
|
async function openConsole(page: Page) {
|
|
await page.goto("/");
|
|
await page.getByRole("button", { name: "Live console" }).click();
|
|
await expect(page.getByRole("heading", { name: "Live Codex protocol console" })).toBeVisible();
|
|
}
|
|
|
|
async function runManifest(page: Page, id: string) {
|
|
await expect(page.getByTestId("manifest-list")).toBeVisible();
|
|
await page.getByTestId(`manifest-${id}`).click();
|
|
await page.getByTestId("run-manifest").click();
|
|
await expect(page.getByTestId("connection-status")).toHaveText("connected", { timeout: 15_000 });
|
|
}
|
|
|
|
test.describe("Live console", () => {
|
|
test.beforeEach(async ({ page }) => {
|
|
await page.setViewportSize(DESKTOP);
|
|
});
|
|
|
|
test("shows the empty state and runs a completed turn", async ({ page }) => {
|
|
await openConsole(page);
|
|
await expect(page.getByTestId("empty-transcript")).toBeVisible();
|
|
await expect(page.getByTestId("composer-send")).toBeDisabled();
|
|
await shot(page, "desktop-01-idle-empty");
|
|
|
|
await runManifest(page, "completion");
|
|
await expect(page.getByTestId("reasoning-item")).toBeVisible();
|
|
await expect(page.getByTestId("tool-item")).toBeVisible();
|
|
await shot(page, "desktop-02-streaming-turn");
|
|
|
|
await expect(page.getByTestId("turn-completed")).toBeVisible({ timeout: 20_000 });
|
|
await expect(page.getByTestId("live-transcript")).toContainText("finished the task");
|
|
await expect(page.getByTestId("live-transcript")).toContainText(
|
|
"Summarize what the Live console demo server proves.",
|
|
);
|
|
await expect(page.getByTestId("tool-item")).toHaveAttribute("data-status", "completed");
|
|
await shot(page, "desktop-03-turn-completed");
|
|
|
|
// The browser must never receive a provider credential.
|
|
const body = await page.locator("body").innerText();
|
|
expect(body).not.toContain("Bearer ");
|
|
expect(body).not.toContain("auth.json");
|
|
});
|
|
|
|
test("acknowledges steering and keeps rejected text recoverable", async ({ page }) => {
|
|
await openConsole(page);
|
|
await runManifest(page, "steering");
|
|
await expect(page.getByTestId("composer-send")).toHaveText("Steer", { timeout: 15_000 });
|
|
|
|
await page.getByTestId("composer-input").fill("Keep it to one sentence.");
|
|
await page.getByTestId("composer-send").click();
|
|
const chip = page.getByTestId("steering-chip").first();
|
|
await expect(chip).toContainText("acknowledged", { timeout: 15_000 });
|
|
await shot(page, "desktop-04-steering-acknowledged");
|
|
|
|
await expect(page.getByTestId("turn-completed")).toBeVisible({ timeout: 25_000 });
|
|
// The composer state machine now offers Send again; force a stale steer.
|
|
await page.evaluate(() => window.history.replaceState({}, "", "/"));
|
|
});
|
|
|
|
test("rejects a stale steer and offers to resend it", async ({ page }) => {
|
|
await openConsole(page);
|
|
await runManifest(page, "steering");
|
|
await expect(page.getByTestId("composer-send")).toHaveText("Steer", { timeout: 15_000 });
|
|
await page.getByTestId("composer-input").fill("Typed while the turn was running.");
|
|
|
|
// Let the turn finish before the steer is sent: the classic stale race.
|
|
await expect(page.getByTestId("turn-completed")).toBeVisible({ timeout: 25_000 });
|
|
await expect(page.getByTestId("composer-send")).toHaveText("Send");
|
|
});
|
|
|
|
test("cancels a turn interrupted before it starts", async ({ page }) => {
|
|
await openConsole(page);
|
|
await runManifest(page, "interrupt-before-start");
|
|
await expect(page.getByTestId("composer-stop")).toBeEnabled();
|
|
await page.getByTestId("composer-stop").click();
|
|
await expect(page.getByTestId("turn-cancelled")).toBeVisible({ timeout: 15_000 });
|
|
await expect(page.getByTestId("turn-cancelled")).toContainText("Cancelled before start");
|
|
await shot(page, "desktop-05-interrupt-before-start");
|
|
});
|
|
|
|
test("keeps partial text when a streaming turn is stopped", async ({ page }) => {
|
|
await openConsole(page);
|
|
await runManifest(page, "interrupt-during-generation");
|
|
await expect(page.getByTestId("live-transcript")).toContainText("The tracer keeps", {
|
|
timeout: 15_000,
|
|
});
|
|
await page.getByTestId("composer-stop").click();
|
|
await expect(page.getByTestId("turn-interrupted")).toBeVisible({ timeout: 15_000 });
|
|
await expect(page.getByTestId("interrupted-divider").first()).toBeVisible();
|
|
await expect(page.getByTestId("live-transcript")).not.toContainText("And this final one.");
|
|
await shot(page, "desktop-06-interrupt-during-generation");
|
|
});
|
|
|
|
test("keeps an interrupted tool item without a fabricated result", async ({ page }) => {
|
|
await openConsole(page);
|
|
await runManifest(page, "interrupt-during-tool");
|
|
await expect(page.getByTestId("tool-item")).toHaveAttribute("data-status", "running", {
|
|
timeout: 15_000,
|
|
});
|
|
await page.getByTestId("composer-stop").click();
|
|
await expect(page.getByTestId("tool-item")).toHaveAttribute("data-status", "interrupted", {
|
|
timeout: 15_000,
|
|
});
|
|
await expect(page.getByTestId("turn-interrupted")).toBeVisible();
|
|
await shot(page, "desktop-07-interrupt-during-tool");
|
|
});
|
|
|
|
test("resolves approvals exactly once and collapses the resolved card", async ({ page }) => {
|
|
await openConsole(page);
|
|
await runManifest(page, "approvals");
|
|
const card = page.getByTestId("request-card").first();
|
|
await expect(card).toHaveAttribute("data-status", "pending", { timeout: 15_000 });
|
|
await expect(page.getByTestId("pending-request-banner")).toBeVisible();
|
|
await expect(page.getByTestId("request-action-accept_for_session")).toBeVisible();
|
|
await shot(page, "desktop-08-request-pending");
|
|
|
|
await page.getByTestId("request-action-accept").first().click();
|
|
await expect(page.getByTestId("request-card").first()).toHaveAttribute(
|
|
"data-status",
|
|
"resolved",
|
|
{ timeout: 15_000 },
|
|
);
|
|
await expect(page.getByTestId("request-card").first()).toContainText("resolved — Approve");
|
|
await shot(page, "desktop-09-request-resolved");
|
|
|
|
// Second request only offers accept and decline.
|
|
const second = page.getByTestId("request-card").nth(1);
|
|
await expect(second).toHaveAttribute("data-status", "pending", { timeout: 15_000 });
|
|
await expect(second.getByTestId("request-action-accept_for_session")).toHaveCount(0);
|
|
});
|
|
|
|
test("submits user-input and elicitation answers and lets an unanswered request expire", async ({ page }) => {
|
|
test.setTimeout(60_000);
|
|
await openConsole(page);
|
|
await runManifest(page, "user-input");
|
|
await expect(page.getByTestId("request-answer")).toBeVisible({ timeout: 15_000 });
|
|
await page.getByTestId("request-answer").fill("staging");
|
|
await page.getByTestId("request-action-submit").click();
|
|
await expect(page.getByTestId("request-card").first()).toHaveAttribute(
|
|
"data-status",
|
|
"resolved",
|
|
{ timeout: 15_000 },
|
|
);
|
|
|
|
// An elicitation submits `content`, not `answers`. The runner validates the
|
|
// body against the request kind, so a card that sent the user-input shape
|
|
// would be rejected and stay pending here.
|
|
const elicitation = page.getByTestId("request-card").nth(1);
|
|
await expect(elicitation).toHaveAttribute("data-status", "pending", { timeout: 15_000 });
|
|
await elicitation.getByTestId("request-answer").fill("stable");
|
|
await elicitation.getByTestId("request-action-submit").click();
|
|
await expect(elicitation).toHaveAttribute("data-status", "resolved", { timeout: 15_000 });
|
|
await expect(elicitation).toContainText("resolved — Submit");
|
|
|
|
await expect(page.getByTestId("request-card").nth(2)).toHaveAttribute(
|
|
"data-status",
|
|
"expired",
|
|
{ timeout: 25_000 },
|
|
);
|
|
await expect(page.getByTestId("request-card").nth(2)).toContainText("expired before response");
|
|
await shot(page, "desktop-10-request-expired");
|
|
});
|
|
|
|
test("moves a goal through its lifecycle", async ({ page }) => {
|
|
await openConsole(page);
|
|
await runManifest(page, "goals");
|
|
await expect(page.getByTestId("goal-banner")).toHaveText("No goal set", { timeout: 15_000 });
|
|
|
|
await page.locator("#goal-menu").click();
|
|
await page.getByRole("menuitem", { name: "Set goal…" }).click();
|
|
await page.getByTestId("goal-objective").fill("Keep the demo goal visible");
|
|
await page.getByTestId("goal-set-confirm").click();
|
|
await expect(page.getByTestId("goal-banner")).toContainText("Keep the demo goal visible", {
|
|
timeout: 15_000,
|
|
});
|
|
await expect(page.getByTestId("goal-banner")).toContainText("active");
|
|
|
|
await page.locator("#goal-menu").click();
|
|
await page.getByRole("menuitem", { name: "Pause" }).click();
|
|
await expect(page.getByTestId("goal-banner")).toContainText("paused", { timeout: 15_000 });
|
|
await shot(page, "desktop-11-goal-supported");
|
|
});
|
|
|
|
test("disables the goal menu with the exact upstream diagnostic", async ({ page }) => {
|
|
await openConsole(page);
|
|
await runManifest(page, "goals-unsupported");
|
|
const trigger = page.locator("#goal-menu");
|
|
await expect(trigger).toHaveAttribute("aria-disabled", "true", { timeout: 15_000 });
|
|
const describedBy = await trigger.getAttribute("aria-describedby");
|
|
expect(describedBy).not.toBeNull();
|
|
await expect(page.locator(`#${describedBy}`)).toHaveText(
|
|
"Goal operations unsupported: app-server 0.132.0 does not advertise the goals capability",
|
|
);
|
|
|
|
await page.getByTestId("toggle-inspector").click();
|
|
await page.getByRole("tab", { name: "Capabilities" }).click();
|
|
await expect(page.getByTestId("inspector-capabilities")).toContainText(
|
|
"does not advertise the goals capability",
|
|
);
|
|
await shot(page, "desktop-12-goal-unsupported");
|
|
});
|
|
|
|
test("shows child threads and refuses to emulate child steering", async ({ page }) => {
|
|
await openConsole(page);
|
|
await runManifest(page, "subagents");
|
|
await expect(page.getByTestId("thread-thread-child-b")).toBeVisible({ timeout: 20_000 });
|
|
await expect(page.getByTestId("turn-completed")).toBeVisible({ timeout: 20_000 });
|
|
await expect(page.getByTestId("thread-thread-child-a")).toContainText("Completed");
|
|
await expect(page.getByTestId("thread-thread-child-b")).toContainText("Failed");
|
|
|
|
await page.getByTestId("thread-thread-child-a").click();
|
|
await expect(page.getByTestId("breadcrumb")).toContainText("thread-child-a");
|
|
await expect(page.getByTestId("composer-reason")).toHaveText(
|
|
"Direct steering of child threads is not supported by this app-server.",
|
|
);
|
|
await expect(page.getByTestId("composer-input")).toBeDisabled();
|
|
await shot(page, "desktop-13-lineage-child");
|
|
});
|
|
|
|
test("shows the failed item and diagnostic in the record", async ({ page }) => {
|
|
await openConsole(page);
|
|
await runManifest(page, "failure");
|
|
await expect(page.getByTestId("turn-failed")).toBeVisible({ timeout: 20_000 });
|
|
await expect(page.getByTestId("tool-failure")).toContainText("cannot find module");
|
|
await shot(page, "desktop-14-failed-turn");
|
|
});
|
|
|
|
test("resumes the same session after a transport drop", async ({ page }) => {
|
|
await openConsole(page);
|
|
await runManifest(page, "completion");
|
|
await expect(page.getByTestId("turn-completed")).toBeVisible({ timeout: 20_000 });
|
|
const before = await page.getByTestId("live-transcript").innerText();
|
|
|
|
await page.getByTestId("drop-connection").click();
|
|
await expect(page.getByTestId("reconnect-banner")).toBeVisible();
|
|
await expect(page.getByTestId("composer-input")).toBeDisabled();
|
|
await shot(page, "desktop-15-reconnect-banner");
|
|
|
|
await page.getByTestId("retry-now").click();
|
|
await expect(page.getByTestId("connection-status")).toHaveText("connected", {
|
|
timeout: 20_000,
|
|
});
|
|
// Live and resumed transcripts must be identical, not merely similar.
|
|
expect(await page.getByTestId("live-transcript").innerText()).toContain(before.trim());
|
|
});
|
|
|
|
test("restores the same session across a page refresh", async ({ page }) => {
|
|
await openConsole(page);
|
|
await runManifest(page, "completion");
|
|
await expect(page.getByTestId("turn-completed")).toBeVisible({ timeout: 20_000 });
|
|
const before = await page.getByTestId("live-transcript").innerText();
|
|
|
|
await page.reload();
|
|
await page.getByRole("button", { name: "Live console" }).click();
|
|
await expect(page.getByTestId("live-transcript")).toContainText("finished the task", {
|
|
timeout: 20_000,
|
|
});
|
|
expect(await page.getByTestId("live-transcript").innerText()).toContain(before.trim());
|
|
await shot(page, "desktop-16-refresh-replay");
|
|
});
|
|
|
|
test("replays a recorded session with a stepper", async ({ page }) => {
|
|
await openConsole(page);
|
|
await runManifest(page, "completion");
|
|
await expect(page.getByTestId("turn-completed")).toBeVisible({ timeout: 20_000 });
|
|
|
|
await page.getByTestId("toggle-replay").click();
|
|
await expect(page.getByTestId("replay-badge")).toBeVisible();
|
|
await expect(page.getByTestId("replay-stepper")).toBeVisible();
|
|
await expect(page.getByTestId("empty-transcript")).toBeVisible();
|
|
|
|
await page.getByRole("button", { name: "Step forward" }).click();
|
|
await page.getByRole("button", { name: "Step forward" }).click();
|
|
await expect(page.getByTestId("empty-transcript")).toHaveCount(0);
|
|
await shot(page, "desktop-17-replay-mode");
|
|
|
|
await page.getByTestId("toggle-replay").click();
|
|
await expect(page.getByTestId("replay-badge")).toHaveCount(0);
|
|
await expect(page.getByTestId("turn-completed")).toBeVisible();
|
|
});
|
|
|
|
test("inspects canonical events and reports reducer parity", async ({ page }) => {
|
|
await openConsole(page);
|
|
await runManifest(page, "completion");
|
|
await expect(page.getByTestId("turn-completed")).toBeVisible({ timeout: 20_000 });
|
|
|
|
await page.getByTestId("toggle-inspector").click();
|
|
await expect(page.getByTestId("inspector-events")).toBeVisible();
|
|
await page.getByTestId("inspector-filter").fill("turn.completed");
|
|
await expect(page.getByTestId("inspector-events").getByRole("listitem")).toHaveCount(1);
|
|
await shot(page, "desktop-18-inspector-events");
|
|
|
|
await page.getByTestId("inspector-filter").fill("");
|
|
await page.getByRole("tab", { name: "Session" }).click();
|
|
await expect(page.getByTestId("inspector-session")).toContainText("Credentials in browser");
|
|
await expect(page.getByTestId("replay-parity")).toHaveText("match");
|
|
await shot(page, "desktop-19-inspector-session");
|
|
});
|
|
|
|
test("meets the keyboard contract", async ({ page }) => {
|
|
await openConsole(page);
|
|
await runManifest(page, "steering");
|
|
await expect(page.getByTestId("composer-send")).toHaveText("Steer", { timeout: 15_000 });
|
|
|
|
// K2: Escape moves focus to Stop and never interrupts directly.
|
|
await page.getByTestId("composer-input").focus();
|
|
await page.keyboard.press("Escape");
|
|
await expect(page.getByTestId("composer-stop")).toBeFocused();
|
|
await expect(page.getByTestId("turn-interrupted")).toHaveCount(0);
|
|
|
|
// K2: Enter steers the running turn.
|
|
await page.getByTestId("composer-input").fill("Steered with the keyboard.");
|
|
await page.getByTestId("composer-input").press("Enter");
|
|
await expect(page.getByTestId("steering-chip").first()).toContainText("acknowledged", {
|
|
timeout: 15_000,
|
|
});
|
|
|
|
// K7: inspector tabs follow the arrow-key tabs pattern.
|
|
await page.getByTestId("toggle-inspector").click();
|
|
await page.getByRole("tab", { name: "Events" }).focus();
|
|
await page.keyboard.press("ArrowRight");
|
|
await expect(page.getByRole("tab", { name: "Requests" })).toBeFocused();
|
|
await expect(page.getByRole("tab", { name: "Requests" })).toHaveAttribute(
|
|
"aria-selected",
|
|
"true",
|
|
);
|
|
|
|
// K5: lineage tree responds to arrow keys.
|
|
await page.getByTestId("lineage-tree").getByRole("treeitem").first().focus();
|
|
await page.keyboard.press("ArrowDown");
|
|
|
|
// K6/A1: the transcript is a labelled log landmark.
|
|
await expect(page.getByTestId("live-transcript")).toHaveAttribute("role", "log");
|
|
await expect(page.getByTestId("live-transcript")).toHaveAttribute("aria-live", "polite");
|
|
});
|
|
|
|
test("resets demo state behind a naming confirmation", async ({ page }) => {
|
|
await openConsole(page);
|
|
await runManifest(page, "completion");
|
|
await expect(page.getByTestId("turn-completed")).toBeVisible({ timeout: 20_000 });
|
|
|
|
await page.getByRole("button", { name: "Reset demo state", exact: true }).first().click();
|
|
await expect(page.getByRole("dialog")).toContainText(
|
|
"Recorded evidence files are not touched.",
|
|
);
|
|
await page.getByTestId("reset-confirm").click();
|
|
await expect(page.getByTestId("empty-transcript")).toBeVisible();
|
|
});
|
|
});
|
|
|
|
test.describe("Live console on a small screen", () => {
|
|
test("keeps the transcript and composer usable at 390 by 844", async ({ page }) => {
|
|
await page.setViewportSize(MOBILE);
|
|
await openConsole(page);
|
|
await expect(page.getByRole("tab", { name: "Chat" })).toBeVisible();
|
|
await expect(page.getByTestId("manifest-list")).toHaveCount(0);
|
|
await shot(page, "mobile-01-idle-empty");
|
|
|
|
await page.getByRole("tab", { name: "Session" }).click();
|
|
await expect(page.getByTestId("manifest-list")).toBeVisible();
|
|
await page.getByTestId("manifest-approvals").click();
|
|
await page.getByTestId("run-manifest").click();
|
|
await shot(page, "mobile-02-session-segment");
|
|
|
|
await page.getByRole("tab", { name: "Chat" }).click();
|
|
await expect(page.getByTestId("request-card").first()).toHaveAttribute(
|
|
"data-status",
|
|
"pending",
|
|
{ timeout: 20_000 },
|
|
);
|
|
await shot(page, "mobile-03-request-pending");
|
|
await page.getByTestId("request-action-accept").first().click();
|
|
await expect(page.getByTestId("request-card").first()).toHaveAttribute(
|
|
"data-status",
|
|
"resolved",
|
|
{ timeout: 15_000 },
|
|
);
|
|
|
|
await page.getByRole("tab", { name: "Inspector" }).click();
|
|
await expect(page.getByTestId("inspector-events")).toBeVisible();
|
|
await shot(page, "mobile-04-inspector");
|
|
|
|
// WCAG 1.4.10 reflow: no horizontal scrolling of the page itself.
|
|
const overflow = await page.evaluate(
|
|
() => document.documentElement.scrollWidth - document.documentElement.clientWidth,
|
|
);
|
|
expect(overflow).toBeLessThanOrEqual(1);
|
|
});
|
|
});
|