import { expect, test, type Page } from "@playwright/test"; const DESKTOP = { width: 1440, height: 900 }; const MOBILE = { width: 390, height: 844 }; async function screenshotPair(page: Page, name: string, mobilePanel = "chat") { await page.setViewportSize(DESKTOP); await page.screenshot({ path: test.info().outputPath(`reference-${name}-1440x900.png`) }); await page.setViewportSize(MOBILE); await expect(page.getByTestId("composer-input")).toBeVisible(); if (mobilePanel !== "chat") { await page.getByRole("button", { name: "Menu" }).click(); await page.getByRole("menuitem", { name: mobilePanel === "session" ? "Session controls" : "Protocol inspector" }).click(); } await expect(page.locator('[data-slot="runner-console-app"]')).toBeVisible(); await page.screenshot({ path: test.info().outputPath(`reference-${name}-390x844.png`) }); await page.setViewportSize(DESKTOP); } async function miniScreenshotPair(page: Page, name: string) { await page.setViewportSize(DESKTOP); await page.screenshot({ path: test.info().outputPath(`mini-${name}-1440x900.png`) }); await page.setViewportSize(MOBILE); await page.screenshot({ path: test.info().outputPath(`mini-${name}-390x844.png`) }); await page.setViewportSize(DESKTOP); } async function openReference(page: Page) { await page.setViewportSize(DESKTOP); await page.goto("/reference-console/"); await expect(page.getByRole("heading", { level: 1 })).toBeVisible(); await expect(page.getByTestId("console-version")).toHaveText("🖇️ v0.1.2"); await expect(page.getByTestId("manifest-list")).toBeVisible(); } test("starts as a regular chat while keeping protocol debug available", async ({ page }) => { await openReference(page); await expect(page.getByRole("heading", { name: "Codex protocol chat" })).toBeVisible(); await page.getByTestId("composer-input").fill("Hello through the protocol"); await page.getByTestId("composer-send").click(); await expect(page.getByTestId("runner-transcript")).toContainText( "normal chat carried through the runner protocol", { timeout: 15_000 }, ); await expect(page.locator("body")).not.toContainText("Live console boundary: operate only"); await page.getByTestId("toggle-inspector").click(); await expect(page.getByRole("complementary", { name: "Protocol inspector" })).toBeVisible(); }); async function runManifest(page: Page, id: string) { await page.getByTestId(`manifest-${id}`).click(); await page.getByTestId("run-manifest").click(); await expect(page.getByTestId("connection-status")).toHaveText("connected", { timeout: 15_000 }); } test.describe("SDK reference console evidence", () => { test("captures idle and streaming states at both canonical viewports", async ({ page }) => { await openReference(page); await expect(page.getByTestId("empty-transcript")).toBeVisible(); await screenshotPair(page, "idle-empty"); await runManifest(page, "completion"); await expect(page.getByTestId("reasoning-item")).toBeVisible(); await expect(page.getByTestId("composer-send")).toHaveText("Steer"); const streamingAnswer = page.locator('.pcr-message[data-role="assistant"] .pcr-message-text'); await expect(streamingAnswer).toBeVisible({ timeout: 15_000 }); const firstFrame = await streamingAnswer.innerText(); await expect.poll(async () => (await streamingAnswer.innerText()).length).toBeGreaterThan(firstFrame.length); await expect(page.locator('.pcr-message[data-role="assistant"]')).toHaveAttribute("data-state", /streaming|settled/); await screenshotPair(page, "streaming-turn"); await expect(page.getByTestId("turn-completed")).toBeVisible({ timeout: 20_000 }); const responseVisibility = await page.locator('.pcr-message[data-role="assistant"]').last().evaluate((response) => { const viewport = response.closest('[data-testid="runner-transcript"]'); if (!(viewport instanceof HTMLElement)) return false; const responseBox = response.getBoundingClientRect(); const viewportBox = viewport.getBoundingClientRect(); return responseBox.top >= viewportBox.top && responseBox.top < viewportBox.bottom; }); expect(responseVisibility).toBe(true); expect(await page.locator("body").innerText()).not.toContain("Bearer "); expect(await page.locator("body").innerText()).not.toContain("auth.json"); const terminal = page.locator('[data-testid="tool-item"][data-view="terminal"]').first(); await terminal.locator(":scope > summary").click(); await expect(terminal).toContainText("Input"); await expect(terminal).toContainText("Output"); const debugDetails = terminal.getByTestId("tool-debug-details"); await debugDetails.locator("summary").click(); await expect(debugDetails).toContainText("sourceEventId"); await expect(debugDetails).toContainText("item.completed"); await screenshotPair(page, "terminal-debug"); }); test("captures steering and interrupt states", async ({ page }) => { await openReference(page); await runManifest(page, "steering"); await expect(page.getByTestId("composer-send")).toHaveText("Steer"); await page.getByTestId("composer-input").fill("Keep it short."); await page.getByTestId("composer-send").click(); await expect(page.getByTestId("steering-chip")).toContainText("acknowledged", { timeout: 15_000 }); await screenshotPair(page, "steering-acknowledged"); await page.getByRole("button", { name: "Reset demo state", exact: true }).click(); await page.getByTestId("reset-confirm").click(); await runManifest(page, "interrupt-during-generation"); await expect(page.getByTestId("runner-transcript")).toContainText("The tracer keeps", { timeout: 15_000 }); await page.getByTestId("composer-stop").click(); await expect(page.getByTestId("turn-interrupted")).toBeVisible({ timeout: 15_000 }); await screenshotPair(page, "interrupt"); }); test("captures request, goal, reconnect, replay, and failure states", async ({ page }) => { await openReference(page); await runManifest(page, "approvals"); await expect(page.getByTestId("request-card").first()).toHaveAttribute("data-status", "pending", { timeout: 15_000 }); await screenshotPair(page, "request-pending"); await page.getByTestId("request-action-accept").first().click(); await expect(page.getByTestId("request-card").first()).toHaveAttribute("data-status", "resolved", { timeout: 15_000 }); await screenshotPair(page, "request-resolved"); await page.getByRole("button", { name: "Reset demo state", exact: true }).click(); await page.getByTestId("reset-confirm").click(); await runManifest(page, "goals"); await page.getByRole("button", { name: "Goal", exact: true }).click(); await page.getByRole("menuitem", { name: "Set goal…" }).click(); await page.getByTestId("goal-objective").fill("Prove the SDK lifecycle."); await page.getByTestId("goal-set-confirm").click(); await expect(page.getByTestId("goal-banner")).toContainText("Prove the SDK lifecycle."); await screenshotPair(page, "goal-set", "session"); await page.setViewportSize(DESKTOP); await page.getByTestId("drop-connection").click(); await expect(page.getByTestId("reconnect-banner")).toBeVisible(); await screenshotPair(page, "reconnect-banner"); await expect(page.getByTestId("connection-status")).toHaveText("connected", { timeout: 10_000 }); await page.getByTestId("toggle-replay").click(); await expect(page.getByTestId("replay-controls")).toBeVisible(); await page.getByRole("button", { name: "Step forward" }).click(); await screenshotPair(page, "replay-mode"); await page.setViewportSize(DESKTOP); await page.getByTestId("toggle-replay").click(); await page.getByRole("button", { name: "Reset demo state", exact: true }).click(); await page.getByTestId("reset-confirm").click(); await runManifest(page, "failure"); await expect(page.getByTestId("turn-failed")).toBeVisible({ timeout: 20_000 }); await screenshotPair(page, "failed-turn"); }); }); test.describe("SDK mini consumer evidence", () => { test("uses public render, token, request, reconnect, and replay extension points", async ({ page }) => { await page.goto("/mini-consumer/"); await expect(page.getByRole("heading", { name: "Runner mini consumer" })).toBeVisible(); await miniScreenshotPair(page, "idle"); await page.getByRole("button", { name: "Start selected flow" }).click(); await expect(page.getByText("Mini renderer:").first()).toBeVisible({ timeout: 15_000 }); await miniScreenshotPair(page, "streaming-custom-renderer-token"); await page.getByTestId("composer-input").fill("Keep the mini answer short."); await expect(page.getByTestId("composer-send")).toHaveText("Steer"); await page.getByTestId("composer-send").click(); await expect(page.getByTestId("mini-steering-chip")).toContainText("acknowledged", { timeout: 15_000 }); await expect(page.getByTestId("turn-completed")).toBeVisible({ timeout: 20_000 }); await page.getByLabel("Demo manifest").selectOption("interrupt-during-generation"); await page.getByRole("button", { name: "Start selected flow" }).click(); await expect(page.getByTestId("composer-send")).toHaveText("Steer", { timeout: 15_000 }); await page.getByTestId("composer-stop").click(); await expect(page.getByTestId("turn-interrupted")).toBeVisible({ timeout: 15_000 }); await page.getByLabel("Demo manifest").selectOption("approvals"); await page.getByRole("button", { name: "Start selected flow" }).click(); await expect(page.getByTestId("request-card").first()).toHaveAttribute("data-status", "pending", { timeout: 15_000 }); await miniScreenshotPair(page, "request-pending"); await page.getByTestId("request-action-accept").first().click(); await expect(page.getByTestId("request-card").first()).toHaveAttribute("data-status", "resolved", { timeout: 15_000 }); await page.getByTestId("mini-goal-set").click(); await expect(page.getByTestId("mini-goal-state")).toContainText("Exercise the public SDK.", { timeout: 15_000 }); await page.getByTestId("mini-goal-clear").click(); await expect(page.getByTestId("mini-goal-state")).toContainText("none", { timeout: 15_000 }); await page.getByRole("button", { name: "Drop connection" }).click(); await expect(page.getByTestId("reconnect-banner")).toBeVisible(); await miniScreenshotPair(page, "reconnect"); await expect(page.getByTestId("reconnect-banner")).toHaveCount(0, { timeout: 10_000 }); await page.getByRole("button", { name: "Enter replay" }).click(); await expect(page.getByTestId("replay-controls")).toBeVisible(); await page.getByTestId("replay-position").press("ArrowRight"); await expect(page.getByTestId("mini-replay-parity")).toContainText("match"); await miniScreenshotPair(page, "replay"); }); }); test("meets keyboard, accessibility, responsive, and grid contracts", async ({ page }) => { await openReference(page); await runManifest(page, "completion"); await expect(page.getByTestId("turn-completed")).toBeVisible({ timeout: 20_000 }); await page.getByTestId("composer-input").fill("first"); await page.getByTestId("composer-input").press("Shift+Enter"); await expect(page.getByTestId("composer-input")).toHaveValue("first\n"); await page.getByTestId("toggle-inspector").click(); await page.getByRole("tab", { name: "Events" }).focus(); await page.keyboard.press("ArrowRight"); await expect(page.getByRole("tab", { name: "Requests" })).toBeFocused(); const goalMenu = page.getByRole("button", { name: "Goal", exact: true }); await goalMenu.focus(); await page.keyboard.press("ArrowDown"); await expect(page.getByRole("menu")).toBeVisible(); await expect(page.getByRole("menuitem").first()).toBeFocused(); await page.keyboard.press("Escape"); await expect(goalMenu).toBeFocused(); await expect(page.getByRole("menu")).toHaveCount(0); await goalMenu.press("ArrowDown"); await expect(page.getByRole("menu")).toBeVisible(); await expect(page.getByRole("menuitem").first()).toBeFocused(); await page.getByRole("menuitem", { name: "Set goal…" }).click(); await expect(page.getByRole("dialog", { name: "Set goal" })).toBeVisible(); await page.keyboard.press("Escape"); await expect(page.getByRole("dialog", { name: "Set goal" })).not.toBeVisible(); await page.getByTestId("toggle-replay").click(); await page.getByTestId("replay-position").focus(); await page.keyboard.press("ArrowRight"); await expect(page.getByTestId("replay-position")).toHaveValue("1"); await page.keyboard.press("Space"); await expect(page.getByTestId("replay-controls").getByRole("button", { name: "Pause", exact: true })).toBeVisible(); await page.getByTestId("toggle-replay").click(); const terminal = page.locator('[data-testid="tool-item"][data-view="terminal"]').first(); await terminal.locator(":scope > summary").click(); await terminal.getByTestId("tool-debug-details").locator("summary").click(); expect(await page.getByRole("log").count()).toBe(1); expect(await page.locator('[aria-live="polite"]').count()).toBe(1); const unnamed = await page.locator("button").evaluateAll((buttons) => buttons.filter((button) => (button.getAttribute("aria-label") ?? button.textContent ?? "").trim().length === 0).length); expect(unnamed).toBe(0); const boxes = await page.evaluate(() => { const root = document.querySelector('[data-slot="runner-console-app"]')!; const rail = document.querySelector(".pcr-console-rail")!; const center = document.querySelector(".pcr-console-center")!; return { root: root.getBoundingClientRect(), rail: rail.getBoundingClientRect(), center: center.getBoundingClientRect() }; }); expect(boxes.rail.right).toBeLessThanOrEqual(boxes.center.left); expect(boxes.center.right).toBeLessThanOrEqual(boxes.root.right); await page.setViewportSize(MOBILE); await expect(page.getByTestId("composer-input")).toBeVisible(); await expect(page.getByRole("button", { name: "Menu" })).toBeVisible(); await page.getByRole("button", { name: "Menu" }).click(); await expect(page.getByRole("menuitem", { name: "Session controls" })).toBeVisible(); await expect(page.getByRole("menuitem", { name: "Protocol inspector" })).toBeVisible(); await page.keyboard.press("Escape"); expect(await page.getByRole("log").count()).toBe(1); await expect(terminal.getByTestId("tool-debug-details")).toContainText("sourceEventId"); const overflow = await page.evaluate(() => document.documentElement.scrollWidth - document.documentElement.clientWidth); expect(overflow).toBeLessThanOrEqual(1); });