266 lines
14 KiB
TypeScript
266 lines
14 KiB
TypeScript
import { expect, test, type Page } from "@playwright/test";
|
|
|
|
const DESKTOP = { width: 1440, height: 900 };
|
|
const MOBILE = { width: 390, height: 844 };
|
|
|
|
async function screenshotPair(page: Page, name: string, mobilePanel = "chat") {
|
|
await page.setViewportSize(DESKTOP);
|
|
await page.screenshot({ path: test.info().outputPath(`reference-${name}-1440x900.png`) });
|
|
await page.setViewportSize(MOBILE);
|
|
await expect(page.getByTestId("composer-input")).toBeVisible();
|
|
if (mobilePanel !== "chat") {
|
|
await page.getByRole("button", { name: "Menu" }).click();
|
|
await page.getByRole("menuitem", { name: mobilePanel === "session" ? "Session controls" : "Protocol inspector" }).click();
|
|
}
|
|
await expect(page.locator('[data-slot="runner-console-app"]')).toBeVisible();
|
|
await page.screenshot({ path: test.info().outputPath(`reference-${name}-390x844.png`) });
|
|
await page.setViewportSize(DESKTOP);
|
|
}
|
|
|
|
async function miniScreenshotPair(page: Page, name: string) {
|
|
await page.setViewportSize(DESKTOP);
|
|
await page.screenshot({ path: test.info().outputPath(`mini-${name}-1440x900.png`) });
|
|
await page.setViewportSize(MOBILE);
|
|
await page.screenshot({ path: test.info().outputPath(`mini-${name}-390x844.png`) });
|
|
await page.setViewportSize(DESKTOP);
|
|
}
|
|
|
|
async function openReference(page: Page) {
|
|
await page.setViewportSize(DESKTOP);
|
|
await page.goto("/reference-console/");
|
|
await expect(page.getByRole("heading", { level: 1 })).toBeVisible();
|
|
await expect(page.getByTestId("console-version")).toHaveText("🖇️ v0.1.2");
|
|
await expect(page.getByTestId("manifest-list")).toBeVisible();
|
|
}
|
|
|
|
test("starts as a regular chat while keeping protocol debug available", async ({ page }) => {
|
|
await openReference(page);
|
|
await expect(page.getByRole("heading", { name: "Codex protocol chat" })).toBeVisible();
|
|
await page.getByTestId("composer-input").fill("Hello through the protocol");
|
|
await page.getByTestId("composer-send").click();
|
|
await expect(page.getByTestId("runner-transcript")).toContainText(
|
|
"normal chat carried through the runner protocol",
|
|
{ timeout: 15_000 },
|
|
);
|
|
await expect(page.locator("body")).not.toContainText("Live console boundary: operate only");
|
|
await page.getByTestId("toggle-inspector").click();
|
|
await expect(page.getByRole("complementary", { name: "Protocol inspector" })).toBeVisible();
|
|
});
|
|
|
|
async function runManifest(page: Page, id: string) {
|
|
await page.getByTestId(`manifest-${id}`).click();
|
|
await page.getByTestId("run-manifest").click();
|
|
await expect(page.getByTestId("connection-status")).toHaveText("connected", { timeout: 15_000 });
|
|
}
|
|
|
|
test.describe("SDK reference console evidence", () => {
|
|
test("captures idle and streaming states at both canonical viewports", async ({ page }) => {
|
|
await openReference(page);
|
|
await expect(page.getByTestId("empty-transcript")).toBeVisible();
|
|
await screenshotPair(page, "idle-empty");
|
|
|
|
await runManifest(page, "completion");
|
|
await expect(page.getByTestId("reasoning-item")).toBeVisible();
|
|
await expect(page.getByTestId("composer-send")).toHaveText("Steer");
|
|
const streamingAnswer = page.locator('.pcr-message[data-role="assistant"] .pcr-message-text');
|
|
await expect(streamingAnswer).toBeVisible({ timeout: 15_000 });
|
|
const firstFrame = await streamingAnswer.innerText();
|
|
await expect.poll(async () => (await streamingAnswer.innerText()).length).toBeGreaterThan(firstFrame.length);
|
|
await expect(page.locator('.pcr-message[data-role="assistant"]')).toHaveAttribute("data-state", /streaming|settled/);
|
|
await screenshotPair(page, "streaming-turn");
|
|
await expect(page.getByTestId("turn-completed")).toBeVisible({ timeout: 20_000 });
|
|
const responseVisibility = await page.locator('.pcr-message[data-role="assistant"]').last().evaluate((response) => {
|
|
const viewport = response.closest('[data-testid="runner-transcript"]');
|
|
if (!(viewport instanceof HTMLElement)) return false;
|
|
const responseBox = response.getBoundingClientRect();
|
|
const viewportBox = viewport.getBoundingClientRect();
|
|
return responseBox.top >= viewportBox.top && responseBox.top < viewportBox.bottom;
|
|
});
|
|
expect(responseVisibility).toBe(true);
|
|
expect(await page.locator("body").innerText()).not.toContain("Bearer ");
|
|
expect(await page.locator("body").innerText()).not.toContain("auth.json");
|
|
const terminal = page.locator('[data-testid="tool-item"][data-view="terminal"]').first();
|
|
await terminal.locator(":scope > summary").click();
|
|
await expect(terminal).toContainText("Input");
|
|
await expect(terminal).toContainText("Output");
|
|
const debugDetails = terminal.getByTestId("tool-debug-details");
|
|
await debugDetails.locator("summary").click();
|
|
await expect(debugDetails).toContainText("sourceEventId");
|
|
await expect(debugDetails).toContainText("item.completed");
|
|
await screenshotPair(page, "terminal-debug");
|
|
});
|
|
|
|
test("captures steering and interrupt states", async ({ page }) => {
|
|
await openReference(page);
|
|
await runManifest(page, "steering");
|
|
await expect(page.getByTestId("composer-send")).toHaveText("Steer");
|
|
await page.getByTestId("composer-input").fill("Keep it short.");
|
|
await page.getByTestId("composer-send").click();
|
|
await expect(page.getByTestId("steering-chip")).toContainText("acknowledged", { timeout: 15_000 });
|
|
await screenshotPair(page, "steering-acknowledged");
|
|
|
|
await page.getByRole("button", { name: "Reset demo state", exact: true }).click();
|
|
await page.getByTestId("reset-confirm").click();
|
|
await runManifest(page, "interrupt-during-generation");
|
|
await expect(page.getByTestId("runner-transcript")).toContainText("The tracer keeps", { timeout: 15_000 });
|
|
await page.getByTestId("composer-stop").click();
|
|
await expect(page.getByTestId("turn-interrupted")).toBeVisible({ timeout: 15_000 });
|
|
await screenshotPair(page, "interrupt");
|
|
});
|
|
|
|
test("captures request, goal, reconnect, replay, and failure states", async ({ page }) => {
|
|
await openReference(page);
|
|
await runManifest(page, "approvals");
|
|
await expect(page.getByTestId("request-card").first()).toHaveAttribute("data-status", "pending", { timeout: 15_000 });
|
|
await screenshotPair(page, "request-pending");
|
|
await page.getByTestId("request-action-accept").first().click();
|
|
await expect(page.getByTestId("request-card").first()).toHaveAttribute("data-status", "resolved", { timeout: 15_000 });
|
|
await screenshotPair(page, "request-resolved");
|
|
|
|
await page.getByRole("button", { name: "Reset demo state", exact: true }).click();
|
|
await page.getByTestId("reset-confirm").click();
|
|
await runManifest(page, "goals");
|
|
await page.getByRole("button", { name: "Goal", exact: true }).click();
|
|
await page.getByRole("menuitem", { name: "Set goal…" }).click();
|
|
await page.getByTestId("goal-objective").fill("Prove the SDK lifecycle.");
|
|
await page.getByTestId("goal-set-confirm").click();
|
|
await expect(page.getByTestId("goal-banner")).toContainText("Prove the SDK lifecycle.");
|
|
await screenshotPair(page, "goal-set", "session");
|
|
|
|
await page.setViewportSize(DESKTOP);
|
|
await page.getByTestId("drop-connection").click();
|
|
await expect(page.getByTestId("reconnect-banner")).toBeVisible();
|
|
await screenshotPair(page, "reconnect-banner");
|
|
await expect(page.getByTestId("connection-status")).toHaveText("connected", { timeout: 10_000 });
|
|
|
|
await page.getByTestId("toggle-replay").click();
|
|
await expect(page.getByTestId("replay-controls")).toBeVisible();
|
|
await page.getByRole("button", { name: "Step forward" }).click();
|
|
await screenshotPair(page, "replay-mode");
|
|
|
|
await page.setViewportSize(DESKTOP);
|
|
await page.getByTestId("toggle-replay").click();
|
|
await page.getByRole("button", { name: "Reset demo state", exact: true }).click();
|
|
await page.getByTestId("reset-confirm").click();
|
|
await runManifest(page, "failure");
|
|
await expect(page.getByTestId("turn-failed")).toBeVisible({ timeout: 20_000 });
|
|
await screenshotPair(page, "failed-turn");
|
|
});
|
|
});
|
|
|
|
test.describe("SDK mini consumer evidence", () => {
|
|
test("uses public render, token, request, reconnect, and replay extension points", async ({ page }) => {
|
|
await page.goto("/mini-consumer/");
|
|
await expect(page.getByRole("heading", { name: "Runner mini consumer" })).toBeVisible();
|
|
await miniScreenshotPair(page, "idle");
|
|
|
|
await page.getByRole("button", { name: "Start selected flow" }).click();
|
|
await expect(page.getByText("Mini renderer:").first()).toBeVisible({ timeout: 15_000 });
|
|
await miniScreenshotPair(page, "streaming-custom-renderer-token");
|
|
|
|
await page.getByTestId("composer-input").fill("Keep the mini answer short.");
|
|
await expect(page.getByTestId("composer-send")).toHaveText("Steer");
|
|
await page.getByTestId("composer-send").click();
|
|
await expect(page.getByTestId("mini-steering-chip")).toContainText("acknowledged", { timeout: 15_000 });
|
|
|
|
await expect(page.getByTestId("turn-completed")).toBeVisible({ timeout: 20_000 });
|
|
await page.getByLabel("Demo manifest").selectOption("interrupt-during-generation");
|
|
await page.getByRole("button", { name: "Start selected flow" }).click();
|
|
await expect(page.getByTestId("composer-send")).toHaveText("Steer", { timeout: 15_000 });
|
|
await page.getByTestId("composer-stop").click();
|
|
await expect(page.getByTestId("turn-interrupted")).toBeVisible({ timeout: 15_000 });
|
|
|
|
await page.getByLabel("Demo manifest").selectOption("approvals");
|
|
await page.getByRole("button", { name: "Start selected flow" }).click();
|
|
await expect(page.getByTestId("request-card").first()).toHaveAttribute("data-status", "pending", { timeout: 15_000 });
|
|
await miniScreenshotPair(page, "request-pending");
|
|
await page.getByTestId("request-action-accept").first().click();
|
|
await expect(page.getByTestId("request-card").first()).toHaveAttribute("data-status", "resolved", { timeout: 15_000 });
|
|
|
|
await page.getByTestId("mini-goal-set").click();
|
|
await expect(page.getByTestId("mini-goal-state")).toContainText("Exercise the public SDK.", { timeout: 15_000 });
|
|
await page.getByTestId("mini-goal-clear").click();
|
|
await expect(page.getByTestId("mini-goal-state")).toContainText("none", { timeout: 15_000 });
|
|
|
|
await page.getByRole("button", { name: "Drop connection" }).click();
|
|
await expect(page.getByTestId("reconnect-banner")).toBeVisible();
|
|
await miniScreenshotPair(page, "reconnect");
|
|
await expect(page.getByTestId("reconnect-banner")).toHaveCount(0, { timeout: 10_000 });
|
|
|
|
await page.getByRole("button", { name: "Enter replay" }).click();
|
|
await expect(page.getByTestId("replay-controls")).toBeVisible();
|
|
await page.getByTestId("replay-position").press("ArrowRight");
|
|
await expect(page.getByTestId("mini-replay-parity")).toContainText("match");
|
|
await miniScreenshotPair(page, "replay");
|
|
});
|
|
});
|
|
|
|
test("meets keyboard, accessibility, responsive, and grid contracts", async ({ page }) => {
|
|
await openReference(page);
|
|
await runManifest(page, "completion");
|
|
await expect(page.getByTestId("turn-completed")).toBeVisible({ timeout: 20_000 });
|
|
|
|
await page.getByTestId("composer-input").fill("first");
|
|
await page.getByTestId("composer-input").press("Shift+Enter");
|
|
await expect(page.getByTestId("composer-input")).toHaveValue("first\n");
|
|
|
|
await page.getByTestId("toggle-inspector").click();
|
|
await page.getByRole("tab", { name: "Events" }).focus();
|
|
await page.keyboard.press("ArrowRight");
|
|
await expect(page.getByRole("tab", { name: "Requests" })).toBeFocused();
|
|
|
|
const goalMenu = page.getByRole("button", { name: "Goal", exact: true });
|
|
await goalMenu.focus();
|
|
await page.keyboard.press("ArrowDown");
|
|
await expect(page.getByRole("menu")).toBeVisible();
|
|
await expect(page.getByRole("menuitem").first()).toBeFocused();
|
|
await page.keyboard.press("Escape");
|
|
await expect(goalMenu).toBeFocused();
|
|
await expect(page.getByRole("menu")).toHaveCount(0);
|
|
|
|
await goalMenu.press("ArrowDown");
|
|
await expect(page.getByRole("menu")).toBeVisible();
|
|
await expect(page.getByRole("menuitem").first()).toBeFocused();
|
|
await page.getByRole("menuitem", { name: "Set goal…" }).click();
|
|
await expect(page.getByRole("dialog", { name: "Set goal" })).toBeVisible();
|
|
await page.keyboard.press("Escape");
|
|
await expect(page.getByRole("dialog", { name: "Set goal" })).not.toBeVisible();
|
|
|
|
await page.getByTestId("toggle-replay").click();
|
|
await page.getByTestId("replay-position").focus();
|
|
await page.keyboard.press("ArrowRight");
|
|
await expect(page.getByTestId("replay-position")).toHaveValue("1");
|
|
await page.keyboard.press("Space");
|
|
await expect(page.getByTestId("replay-controls").getByRole("button", { name: "Pause", exact: true })).toBeVisible();
|
|
await page.getByTestId("toggle-replay").click();
|
|
const terminal = page.locator('[data-testid="tool-item"][data-view="terminal"]').first();
|
|
await terminal.locator(":scope > summary").click();
|
|
await terminal.getByTestId("tool-debug-details").locator("summary").click();
|
|
|
|
expect(await page.getByRole("log").count()).toBe(1);
|
|
expect(await page.locator('[aria-live="polite"]').count()).toBe(1);
|
|
const unnamed = await page.locator("button").evaluateAll((buttons) => buttons.filter((button) => (button.getAttribute("aria-label") ?? button.textContent ?? "").trim().length === 0).length);
|
|
expect(unnamed).toBe(0);
|
|
|
|
const boxes = await page.evaluate(() => {
|
|
const root = document.querySelector<HTMLElement>('[data-slot="runner-console-app"]')!;
|
|
const rail = document.querySelector<HTMLElement>(".pcr-console-rail")!;
|
|
const center = document.querySelector<HTMLElement>(".pcr-console-center")!;
|
|
return { root: root.getBoundingClientRect(), rail: rail.getBoundingClientRect(), center: center.getBoundingClientRect() };
|
|
});
|
|
expect(boxes.rail.right).toBeLessThanOrEqual(boxes.center.left);
|
|
expect(boxes.center.right).toBeLessThanOrEqual(boxes.root.right);
|
|
|
|
await page.setViewportSize(MOBILE);
|
|
await expect(page.getByTestId("composer-input")).toBeVisible();
|
|
await expect(page.getByRole("button", { name: "Menu" })).toBeVisible();
|
|
await page.getByRole("button", { name: "Menu" }).click();
|
|
await expect(page.getByRole("menuitem", { name: "Session controls" })).toBeVisible();
|
|
await expect(page.getByRole("menuitem", { name: "Protocol inspector" })).toBeVisible();
|
|
await page.keyboard.press("Escape");
|
|
expect(await page.getByRole("log").count()).toBe(1);
|
|
await expect(terminal.getByTestId("tool-debug-details")).toContainText("sourceEventId");
|
|
const overflow = await page.evaluate(() => document.documentElement.scrollWidth - document.documentElement.clientWidth);
|
|
expect(overflow).toBeLessThanOrEqual(1);
|
|
});
|