From 26f1f6a76bd5dbaa55e849e78a460beb0018c37b Mon Sep 17 00:00:00 2001 From: ethernet Date: Thu, 23 Jul 2026 01:27:38 -0400 Subject: [PATCH] feat(desktop): improve tool call detail views (#69868) * fix(desktop): improve fallback tool-call details Show failed image-generation calls through the normal fallback row, remove duplicate normal-mode web-search JSON, and format Technical Mode payloads as readable JSON. * feat(desktop): render terminal tool calls as transcripts Show terminal commands with a prompt and exit status, then reveal ANSI-safe stdout and stderr in the expanded tool row. * fix(desktop): reconcile tool calls by command Match context-only tool starts with command-bearing completions when their IDs differ, preventing stale duplicate terminal rows. Show the web-search query above its result cards. --- .../assistant-ui/thread/message-parts.tsx | 11 +- .../assistant-ui/thread/streaming.test.tsx | 61 ++++++++- .../assistant-ui/tool/fallback-model.test.ts | 32 +++++ .../assistant-ui/tool/fallback-model/index.ts | 20 ++- .../assistant-ui/tool/fallback-model/types.ts | 6 + .../assistant-ui/tool/fallback.test.ts | 15 ++- .../components/assistant-ui/tool/fallback.tsx | 124 ++++++++++++------ apps/desktop/src/lib/chat-messages.test.ts | 30 +++++ apps/desktop/src/lib/chat-messages.ts | 4 +- 9 files changed, 254 insertions(+), 49 deletions(-) diff --git a/apps/desktop/src/components/assistant-ui/thread/message-parts.tsx b/apps/desktop/src/components/assistant-ui/thread/message-parts.tsx index d1f0848766417..d578710b3d162 100644 --- a/apps/desktop/src/components/assistant-ui/thread/message-parts.tsx +++ b/apps/desktop/src/components/assistant-ui/thread/message-parts.tsx @@ -14,12 +14,21 @@ import { ActivityTimerText } from '@/components/chat/activity-timer-text' import { DisclosureRow } from '@/components/chat/disclosure-row' import { GeneratedImage } from '@/components/chat/generated-image-result' import { useI18n } from '@/i18n' +import { generatedImageFromResult } from '@/lib/generated-images' import { useEnterAnimation } from '@/lib/use-enter-animation' import { cn } from '@/lib/utils' -const ImageGenerateTool: FC = ({ args, result }) => { +const ImageGenerateTool: FC = props => { + const { args, result } = props const aspectRatio = typeof args?.aspect_ratio === 'string' ? args.aspect_ratio : undefined + // The image card owns successful generations. Failed or malformed results + // still need the normal tool row: it extracts the error text and gives the + // user an honest, expandable failure rather than silently dropping the call. + if (result !== undefined && !generatedImageFromResult(result)) { + return + } + return (
diff --git a/apps/desktop/src/components/assistant-ui/thread/streaming.test.tsx b/apps/desktop/src/components/assistant-ui/thread/streaming.test.tsx index 8c31a9185f1df..c9d5dc53baea4 100644 --- a/apps/desktop/src/components/assistant-ui/thread/streaming.test.tsx +++ b/apps/desktop/src/components/assistant-ui/thread/streaming.test.tsx @@ -217,7 +217,10 @@ function assistantTodoMessage( } as ThreadMessage } -function assistantImageMessage(running = false): ThreadMessage { +function assistantImageMessage( + running = false, + result: unknown = { image: 'https://cdn.example/cat.png', success: true } +): ThreadMessage { return { id: `assistant-image-${running ? 'running' : 'done'}`, role: 'assistant', @@ -228,7 +231,7 @@ function assistantImageMessage(running = false): ThreadMessage { toolName: 'image_generate', args: { prompt: 'draw a cat' }, argsText: JSON.stringify({ prompt: 'draw a cat' }), - ...(running ? {} : { result: { image: 'https://cdn.example/cat.png', success: true } }) + ...(running ? {} : { result }) } ], status: running ? { type: 'running' } : { type: 'complete', reason: 'stop' }, @@ -243,6 +246,32 @@ function assistantImageMessage(running = false): ThreadMessage { } as ThreadMessage } +function assistantTerminalMessage(): ThreadMessage { + return { + id: 'assistant-terminal-1', + role: 'assistant', + content: [ + { + type: 'tool-call', + toolCallId: 'terminal-1', + toolName: 'terminal', + args: { command: 'npm run check --workspace=apps/desktop' }, + argsText: JSON.stringify({ command: 'npm run check --workspace=apps/desktop' }), + result: { exit_code: 0, stdout: 'all checks passed' } + } + ], + status: { type: 'complete', reason: 'stop' }, + createdAt, + metadata: { + unstable_state: null, + unstable_annotations: [], + unstable_data: [], + steps: [], + custom: {} + } + } as ThreadMessage +} + interface StreamingControls { emitSecond: () => void complete: () => void @@ -634,4 +663,32 @@ describe('assistant-ui streaming renderer', () => { expect(container.querySelector('[data-slot="aui_generated-image"]')).toBeTruthy() expect(screen.queryByRole('status', { name: /rendering image/i })).toBeNull() }) + + it('uses the normal tool row for failed image generations instead of dropping their error payload', async () => { + const { container } = render( + + ) + + fireEvent.click(container.querySelector('[data-tool-row] button')!) + + await waitFor(() => { + expect(container.textContent).toContain('FAL rejected the prompt') + }) + expect(container.querySelector('[data-slot="aui_generated-image"]')).toBeNull() + expect(container.textContent).not.toContain('"success":false') + }) + + it('shows the command prompt and exit code for terminal calls', async () => { + const { container } = render() + + fireEvent.click(container.querySelector('[data-tool-row] button')!) + + await waitFor(() => { + expect(container.textContent).toContain('$ npm run check --workspace=apps/desktop') + expect(container.textContent).toContain('exit 0') + expect(container.textContent).toContain('all checks passed') + }) + }) }) diff --git a/apps/desktop/src/components/assistant-ui/tool/fallback-model.test.ts b/apps/desktop/src/components/assistant-ui/tool/fallback-model.test.ts index c3f5361191d16..275cc13fb4f64 100644 --- a/apps/desktop/src/components/assistant-ui/tool/fallback-model.test.ts +++ b/apps/desktop/src/components/assistant-ui/tool/fallback-model.test.ts @@ -76,6 +76,38 @@ describe('buildToolView terminal exit-code status', () => { 'error' ) }) + + it('keeps the command and exit code for the terminal transcript', () => { + const view = buildToolView( + part({ + args: { command: 'npm run check --workspace=apps/desktop' }, + result: { exit_code: 0, output: 'done' }, + toolName: 'terminal' + }), + '' + ) + + expect(view.terminalCommand).toBe('npm run check --workspace=apps/desktop') + expect(view.terminalExitCode).toBe(0) + }) +}) + +describe('buildToolView web-search query', () => { + it('keeps the query separate from structured search results', () => { + const view = buildToolView( + part({ + args: { query: 'Hermes Agent Desktop tool calls' }, + result: { web: [{ snippet: 'Desktop docs', title: 'Hermes docs', url: 'https://example.com/docs' }] }, + toolName: 'web_search' + }), + '' + ) + + expect(view.searchQuery).toBe('Hermes Agent Desktop tool calls') + expect(view.searchHits).toEqual([ + { snippet: 'Desktop docs', title: 'Hermes docs', url: 'https://example.com/docs' } + ]) + }) }) describe('buildToolView browser_navigate title', () => { diff --git a/apps/desktop/src/components/assistant-ui/tool/fallback-model/index.ts b/apps/desktop/src/components/assistant-ui/tool/fallback-model/index.ts index a1e8720431cf6..dde24c1b42093 100644 --- a/apps/desktop/src/components/assistant-ui/tool/fallback-model/index.ts +++ b/apps/desktop/src/components/assistant-ui/tool/fallback-model/index.ts @@ -128,6 +128,14 @@ function readFileDisplayTarget(args: Record, result: Record): string { + return ( + firstStringField(args, ['context', 'preview']) || + firstStringField(args, ['command', 'code']) || + contextValue(args) + ) +} + const TOOL_META: Record = { browser_click: { icon: 'globe', @@ -1315,10 +1323,7 @@ function dynamicTitle( } if (part.toolName === 'terminal' || part.toolName === 'execute_code') { - const command = - firstStringField(args, ['context', 'preview']) || - firstStringField(args, ['command', 'code']) || - contextValue(args) + const command = shellCommand(args) if (command) { const action = @@ -1384,6 +1389,8 @@ export function buildToolView(part: ToolPart, inlineDiff: string): ToolView { const searchHits = part.toolName === 'web_search' && status !== 'error' ? extractSearchResults(part.result) : undefined + const searchQuery = + part.toolName === 'web_search' ? firstStringField(argsRecord, ['search_term', 'query']) || contextValue(argsRecord) : '' const resultCount = status === 'error' ? null : toolResultCount(part, argsRecord, resultRecord) @@ -1398,6 +1405,8 @@ export function buildToolView(part: ToolPart, inlineDiff: string): ToolView { // field — otherwise the merged `detail` already covers it and double- // rendering would duplicate output. const hasSplitStreams = rendersAnsi && (Boolean(stdout) || Boolean(stderrRaw)) + const terminalCommand = part.toolName === 'terminal' ? shellCommand(argsRecord) : undefined + const terminalExitCode = part.toolName === 'terminal' ? numericField(resultRecord, 'exit_code') : undefined return { countLabel: resultCount ? formatCountLabel(resultCount) : undefined, @@ -1409,8 +1418,11 @@ export function buildToolView(part: ToolPart, inlineDiff: string): ToolView { inlineDiff, previewTarget: toolPreviewTarget(part.toolName, argsRecord, resultRecord), rendersAnsi: rendersAnsi || undefined, + searchQuery: searchQuery || undefined, searchHits: searchHits?.length ? searchHits : undefined, stderr: hasSplitStreams ? stderrRaw || undefined : undefined, + terminalCommand, + terminalExitCode, stdout: hasSplitStreams ? stdout || undefined : undefined, status, subtitle, diff --git a/apps/desktop/src/components/assistant-ui/tool/fallback-model/types.ts b/apps/desktop/src/components/assistant-ui/tool/fallback-model/types.ts index e61140a12a6e3..66afaf48406fa 100644 --- a/apps/desktop/src/components/assistant-ui/tool/fallback-model/types.ts +++ b/apps/desktop/src/components/assistant-ui/tool/fallback-model/types.ts @@ -40,11 +40,17 @@ export interface ToolView { * (terminal/execute_code) so the renderer knows to run them through * the ANSI parser instead of printing them as literals. */ rendersAnsi?: boolean + /** Original query, shown above structured web-search results. */ + searchQuery?: string searchHits?: SearchResultRow[] /** When the backend reports stderr as a separate stream (terminal / * execute_code), the renderer shows it as its own labeled, neutrally * tinted block under stdout — distinct from an error tone. */ stderr?: string + /** Terminal-only command shown as the prompt in the expanded transcript. */ + terminalCommand?: string + /** Terminal-only process exit code, when the backend reported one. */ + terminalExitCode?: number /** When set, the renderer uses stdout+stderr as separate sections and * ignores the merged `detail`. */ stdout?: string diff --git a/apps/desktop/src/components/assistant-ui/tool/fallback.test.ts b/apps/desktop/src/components/assistant-ui/tool/fallback.test.ts index 53d0822419333..16855afc73bba 100644 --- a/apps/desktop/src/components/assistant-ui/tool/fallback.test.ts +++ b/apps/desktop/src/components/assistant-ui/tool/fallback.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' -import { shouldBoundToolGroup, UNBOUNDABLE_TOOLS } from './fallback' +import { shouldBoundToolGroup, technicalTrace, UNBOUNDABLE_TOOLS } from './fallback' describe('shouldBoundToolGroup', () => { it('bounds long runs of ordinary tool calls', () => { @@ -22,3 +22,16 @@ describe('UNBOUNDABLE_TOOLS', () => { expect(UNBOUNDABLE_TOOLS.has('image_generate')).toBe(true) }) }) + +describe('technicalTrace', () => { + it('indents object payloads and persisted JSON strings', () => { + expect(technicalTrace({ offset: 2, path: '/tmp/demo.txt' }, '{"success":true,"lines":["a","b"]}')).toBe( + 'Arguments:\n{\n "offset": 2,\n "path": "/tmp/demo.txt"\n}\n\nResult:\n{\n "success": true,\n "lines": [\n "a",\n "b"\n ]\n}' + ) + }) + + it('leaves scalar strings untouched', () => { + expect(technicalTrace(undefined, 'plain text')).toBe('Result:\nplain text') + expect(technicalTrace(undefined, '"already quoted"')).toBe('Result:\n"already quoted"') + }) +}) diff --git a/apps/desktop/src/components/assistant-ui/tool/fallback.tsx b/apps/desktop/src/components/assistant-ui/tool/fallback.tsx index bf4cfaed6d0ed..0a90c69856072 100644 --- a/apps/desktop/src/components/assistant-ui/tool/fallback.tsx +++ b/apps/desktop/src/components/assistant-ui/tool/fallback.tsx @@ -62,7 +62,6 @@ import { type ToolStatus, type ToolTitleAction } from './fallback-model' -import { prettyJson } from './fallback-model/format' // `true` when a ToolEntry is rendered inside an embedding wrapper that owns // the per-row chrome (timer / preview). The flat ToolGroupSlot sets this @@ -104,23 +103,39 @@ interface ToolStatusCopy { statusRunning: string } -function rawTechnicalTrace(args: unknown, result: unknown): string { - const parts = [args, result] - .filter(value => value !== undefined && value !== null) - .map(value => { - if (typeof value === 'string') { - return value - } +function prettyTechnicalValue(value: unknown): string { + if (typeof value === 'string') { + const trimmed = value.trim() - try { - return JSON.stringify(value) - } catch { - return String(value) - } - }) - .filter(Boolean) + if (!trimmed.startsWith('{') && !trimmed.startsWith('[')) { + return value + } - return clampForDisplay(parts.join('\n')) + try { + const parsed = JSON.parse(value) + + return parsed && typeof parsed === 'object' ? JSON.stringify(parsed, null, 2) : value + } catch { + return value + } + } + + try { + return JSON.stringify(value, null, 2) + } catch { + return String(value) + } +} + +export function technicalTrace(args: unknown, result: unknown): string { + const parts = [ + ['Arguments', args], + ['Result', result] + ] + .filter(([, value]) => value !== undefined && value !== null) + .map(([label, value]) => `${label}:\n${prettyTechnicalValue(value)}`) + + return clampForDisplay(parts.join('\n\n')) } function statusGlyph(status: ToolStatus, copy: ToolStatusCopy): ReactNode { @@ -363,7 +378,8 @@ function ToolEntry({ part }: ToolEntryProps) { const showDetail = !view.inlineDiff && - ((view.status === 'error' && Boolean(detailSections.summary || detailSections.body)) || + (Boolean(view.stdout || view.stderr) || + (view.status === 'error' && Boolean(detailSections.summary || detailSections.body)) || (view.status !== 'error' && Boolean(view.detail) && !detailMatchesTitle && !detailMatchesSubtitle)) const renderDetailAsCode = @@ -373,21 +389,16 @@ function ToolEntry({ part }: ToolEntryProps) { const hasSearchHits = Boolean(view.searchHits?.length) const searchResultsLabel = part.toolName === 'web_search' ? 'Search results' : view.detailLabel - // Only web_search renders the raw JSON drilldown, so serialize the result - // lazily here instead of prettyJson-ing every tool's result in buildToolView. - const rawResult = useMemo( - () => (part.toolName === 'web_search' && toolViewMode !== 'technical' ? prettyJson(part.result) : ''), - [part.toolName, part.result, toolViewMode] - ) - - const showRawSearchDrilldown = - part.toolName === 'web_search' && - part.result !== undefined && - toolViewMode !== 'technical' && - Boolean(rawResult.trim()) - const hasExpandableContent = Boolean( - view.imageUrl || view.inlineDiff || showDetail || hasSearchHits || toolViewMode === 'technical' + view.imageUrl || + view.inlineDiff || + showDetail || + hasSearchHits || + view.stdout || + view.stderr || + view.terminalCommand || + view.terminalExitCode !== undefined || + toolViewMode === 'technical' ) // copyAction reads the uncapped view.detail; clampForDisplay below only bounds @@ -511,6 +522,9 @@ function ToolEntry({ part }: ToolEntryProps) { text={copyAction.text} /> )} + {part.toolName === 'terminal' && toolViewMode !== 'technical' && ( + + )} {view.imageUrl && (
@@ -518,6 +532,12 @@ function ToolEntry({ part }: ToolEntryProps) { )} {hasSearchHits && view.searchHits && (
+ {view.searchQuery && ( +

+ Search + {view.searchQuery} +

+ )} {searchResultsLabel &&

{searchResultsLabel}

}
@@ -596,22 +616,16 @@ function ToolEntry({ part }: ToolEntryProps) { )}
))} - {showRawSearchDrilldown && ( -
- {copy.rawResponse} - 
{rawResult}
-
- )} {toolViewMode === 'technical' && !(isFileEdit && view.inlineDiff) && ( 
-              {rawTechnicalTrace(part.args, part.result)}
+              {technicalTrace(part.args, part.result)}
)} {toolViewMode === 'technical' && isFileEdit && view.inlineDiff && (
Tool payload 
-                {rawTechnicalTrace(part.args, part.result)}
+                {technicalTrace(part.args, part.result)}
)} @@ -621,6 +635,38 @@ function ToolEntry({ part }: ToolEntryProps) { ) } +interface TerminalTranscriptProps { + command?: string + exitCode?: number +} + +function TerminalTranscript({ command, exitCode }: TerminalTranscriptProps) { + if (!command && exitCode === undefined) { + return null + } + + return ( +
+ {command && ( + + $ + {command} + + )} + {exitCode !== undefined && ( + + exit {exitCode} + + )} +
+ ) +} + // A back-to-back run of this many tool calls collapses into the bounded, // auto-scrolling window; fewer than this stays a plain inline stack. const TOOL_GROUP_SCROLL_THRESHOLD = 3 diff --git a/apps/desktop/src/lib/chat-messages.test.ts b/apps/desktop/src/lib/chat-messages.test.ts index e63fae09cb89d..07234feac817e 100644 --- a/apps/desktop/src/lib/chat-messages.test.ts +++ b/apps/desktop/src/lib/chat-messages.test.ts @@ -538,6 +538,36 @@ describe('upsertToolPart', () => { expect(summaries).toEqual(['Did 5 searches', 'Did 5 searches']) }) + it('pairs a terminal completion with its context-only start when event IDs differ', () => { + const started = upsertToolPart( + [], + { context: 'echo "Hello from the terminal"', name: 'terminal', tool_id: 'terminal-start' }, + 'running' + ) + + const completed = upsertToolPart( + started, + { + args: { command: 'echo "Hello from the terminal"' }, + name: 'terminal', + result: { exit_code: 0, stdout: 'Hello from the terminal' }, + tool_id: 'terminal-complete' + }, + 'complete' + ) + + const terminalParts = completed.filter( + (part): part is Extract => part.type === 'tool-call' && part.toolName === 'terminal' + ) + + expect(terminalParts).toHaveLength(1) + expect(terminalParts[0]?.toolCallId).toBe('terminal-complete') + expect(terminalParts[0] && 'result' in terminalParts[0] ? terminalParts[0].result : undefined).toMatchObject({ + exit_code: 0, + stdout: 'Hello from the terminal' + }) + }) + it('preserves query args when completion payload omits context', () => { const started = upsertToolPart( [], diff --git a/apps/desktop/src/lib/chat-messages.ts b/apps/desktop/src/lib/chat-messages.ts index 06ef904f2ddad..4af96dce36bbd 100644 --- a/apps/desktop/src/lib/chat-messages.ts +++ b/apps/desktop/src/lib/chat-messages.ts @@ -360,7 +360,7 @@ function toolPayloadMatchValues(payload: GatewayEventPayload | undefined): strin // `clarify.request` (a fresh request id) must correlate with the `tool.start` // row (the model's tool_call_id) so the two ids don't produce a duplicate // clarify card — same correlation ClarifyToolPending uses for request↔args. - const query = firstStringField(payloadArgs, ['search_term', 'query', 'question']) + const query = firstStringField(payloadArgs, ['search_term', 'query', 'question', 'command', 'code', 'path']) const context = typeof payload?.context === 'string' ? payload.context.trim() : '' const preview = typeof payload?.preview === 'string' ? payload.preview.trim() : '' @@ -373,7 +373,7 @@ function toolPartMatchValues(part: ChatMessagePart): string[] { } const args = part.args as Record - const query = firstStringField(args, ['search_term', 'query', 'question']) + const query = firstStringField(args, ['search_term', 'query', 'question', 'command', 'code', 'path']) const context = typeof args.context === 'string' ? args.context.trim() : '' const preview = typeof args.preview === 'string' ? args.preview.trim() : ''