From 99697c2c51edf004c18f2c4b62a8b9c2b141252b Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 6 Sep 2026 19:41:12 -0500 Subject: [PATCH] fix(evals): open tool evidence in the correct inspector tab Co-Authored-By: Paperclip --- .../devtools/issue-thread/src/App.tsx | 5 +++++ .../devtools/issue-thread/src/EvidencePanel.tsx | 7 +++++-- .../docs/runner-protocol-live-evals.md | 3 +++ .../scripts/verify-runner-evalbook-viewer.mjs | 17 ++++++++++++++++- 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/packages/paperclip-runner/devtools/issue-thread/src/App.tsx b/packages/paperclip-runner/devtools/issue-thread/src/App.tsx index 3cf8d53440..c511c631fc 100644 --- a/packages/paperclip-runner/devtools/issue-thread/src/App.tsx +++ b/packages/paperclip-runner/devtools/issue-thread/src/App.tsx @@ -12,6 +12,7 @@ import type { CapabilityDevtoolsSnapshot } from "../../../src/devtools"; import { capabilityDenialCount } from "../../../src/issue-thread/types"; import { Composer } from "./Composer"; import { EvidencePanel } from "./EvidencePanel"; +import type { CapabilityDevtoolsTab } from "./DevtoolsInspector"; import { Icon } from "./Icons"; import { IssueHeader } from "./IssueHeader"; import { applyFakeInteractionResponse } from "./fake-store"; @@ -333,6 +334,7 @@ export function App() { const [panelOpen, setPanelOpen] = useState(() => embeddedEval !== null || readStoredFlag(route.surface === "chat" ? CHAT_PANEL_OPEN_KEY : PANEL_OPEN_KEY, false)); const [panelWidth, setPanelWidth] = useState(() => readStoredNumber(PANEL_WIDTH_KEY, 384)); + const [devtoolsTab, setDevtoolsTab] = useState(embeddedEval !== null ? "eval" : "evidence"); const [segment, setSegment] = useState<"thread" | "evidence">(route.segment); const [openSections, setOpenSections] = useState(["tools"]); const [selectedTurnId, setSelectedTurnId] = useState("all"); @@ -572,6 +574,7 @@ export function App() { const openEvidence = useCallback( (section: CapabilityEvidenceSectionId, recordId: string) => { setPanelOpen(true); + setDevtoolsTab("evidence"); setSegment("evidence"); setSelectedTurnId("all"); setOpenSections((current) => (current.includes(section) ? current : [...current, section])); @@ -1497,6 +1500,8 @@ export function App() { {showPanel ? ( { diff --git a/packages/paperclip-runner/devtools/issue-thread/src/EvidencePanel.tsx b/packages/paperclip-runner/devtools/issue-thread/src/EvidencePanel.tsx index 8c5bed14ee..efc0169c65 100644 --- a/packages/paperclip-runner/devtools/issue-thread/src/EvidencePanel.tsx +++ b/packages/paperclip-runner/devtools/issue-thread/src/EvidencePanel.tsx @@ -114,6 +114,8 @@ export interface EvidencePanelProps { snapshot: CapabilityIssueThreadSnapshot; devtools?: CapabilityDevtoolsSnapshot | null; evalReport?: EvalInspectorReport | null; + devtoolsTab: CapabilityDevtoolsTab; + onDevtoolsTabChange: (tab: CapabilityDevtoolsTab) => void; onForkRevision?: (revision: number) => void; layout: "side" | "overlay" | "segment"; width: number; @@ -331,11 +333,12 @@ function Section({ } export function EvidencePanel(props: EvidencePanelProps) { - const [devtoolsTab, setDevtoolsTab] = useState(props.evalReport ? "eval" : "evidence"); const { snapshot, devtools, evalReport, + devtoolsTab, + onDevtoolsTabChange, onForkRevision = () => undefined, layout, width, @@ -394,7 +397,7 @@ export function EvidencePanel(props: EvidencePanelProps) { {devtools === null ? ( evalReport ? :

Loading company state…

) : ( - + )} ) : null} diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index fd00a9ac8d..9ff718f324 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -27,6 +27,9 @@ Before uploading, the report job runs the actual built application in Chromium against representative passing, failing and missing-recording pages in both reports. It checks initial rendering, tool expansion, read-only controls, navigation, reload, a narrow viewport and the public no-API-request boundary. +In full-evidence reports, a tool's **View in Evidence** link selects the Evidence +tab and highlights its record, including when reopening that same record after +switching tabs. The browser check exercises this cross-link too. Run the same check locally with `node packages/paperclip-runner/scripts/verify-runner-evalbook-viewer.mjs --report-root /path/to/report`. Add `--screenshots /path/to/proof` for visual evidence. Public screenshots are diff --git a/packages/paperclip-runner/scripts/verify-runner-evalbook-viewer.mjs b/packages/paperclip-runner/scripts/verify-runner-evalbook-viewer.mjs index 5e5b01e30e..059f43fac8 100644 --- a/packages/paperclip-runner/scripts/verify-runner-evalbook-viewer.mjs +++ b/packages/paperclip-runner/scripts/verify-runner-evalbook-viewer.mjs @@ -4,7 +4,7 @@ import assert from "node:assert/strict"; import { createServer } from "node:http"; import { readFile, readdir, mkdir } from "node:fs/promises"; import { resolve, join, extname, sep } from "node:path"; -import { chromium } from "@playwright/test"; +import { chromium, expect } from "@playwright/test"; const arg = (name) => process.argv[process.argv.indexOf(name) + 1]; if (!process.argv.includes("--report-root")) @@ -110,6 +110,21 @@ try { }) .count(), ); + else if (payload.devtools) { + const evidenceLink = page + .locator('[data-thread-item="tool_activity"][open]') + .first() + .getByRole("button", { name: "View in Evidence", exact: true }); + const evidenceTab = page.getByRole("tab", { name: "Evidence", exact: true }); + await evidenceLink.click(); + await expect(evidenceTab).toHaveAttribute("aria-selected", "true"); + await expect(page.locator('[data-highlighted="true"]').first()).toBeVisible(); + // Reopening the same record after selecting another tab must work too. + await page.getByRole("tab", { name: "Eval", exact: true }).click(); + await evidenceLink.click(); + await expect(evidenceTab).toHaveAttribute("aria-selected", "true"); + await page.getByRole("tab", { name: "Eval", exact: true }).click(); + } } if (payload.publication) { assert.ok(