From 2c94e3fb63b36106bc36ab825093cb8b7d9cd3c7 Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Sat, 8 Aug 2026 15:37:34 -0500 Subject: [PATCH 1/3] refactor(gateway): one helper for prefixing per-turn notes onto model input The speech-interrupted and reaction notes each hand-rolled the same string / multimodal-list prepend. Collapse both onto _prepend_note, which also gives the "model input only, never persisted, cache-safe" contract a single place to be written down. --- tui_gateway/server.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/tui_gateway/server.py b/tui_gateway/server.py index 4b58252b34d30..c0bda0f8d35e1 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -9513,6 +9513,24 @@ def _start_notification_poller(sid: str, session: dict) -> threading.Event: return stop +def _prepend_note(run_message: Any, note: str) -> Any: + """Prefix a per-turn note onto the MODEL INPUT, leaving the prompt alone. + + Everything the model needs to know about the turn but the user did not + type — an interrupted reply, reactions, the surface they typed into — + arrives this way. persist_user_message keeps the clean prompt, so no + scaffolding reaches the transcript, and annotating the NEW turn never + rewrites an already-sent message, so the cached prefix survives. + """ + if not note: + return run_message + if isinstance(run_message, str): + return f"{note}\n\n{run_message}" + if isinstance(run_message, list): + return [{"type": "text", "text": note}, *run_message] + return run_message + + def _run_prompt_submit( rid, sid: str, @@ -9765,21 +9783,10 @@ def _run_prompt_submit( from tools.tts_streaming import SPEECH_INTERRUPTED_NOTE, take_speech_interrupted if take_speech_interrupted(): - if isinstance(run_message, str): - run_message = f"{SPEECH_INTERRUPTED_NOTE}\n\n{run_message}" - elif isinstance(run_message, list): - run_message = [{"type": "text", "text": SPEECH_INTERRUPTED_NOTE}, *run_message] + run_message = _prepend_note(run_message, SPEECH_INTERRUPTED_NOTE) - # Reactions the user added since the last turn ride the MODEL INPUT - # only (same enrichment channel as the speech-interrupted note); - # persist_user_message below stays the clean prompt, so no - # scaffolding reaches the transcript. Cache-safe: annotating the - # NEW turn never rewrites an already-sent message. - if reaction_notes := _pending_reaction_notes(session): - if isinstance(run_message, str): - run_message = f"{reaction_notes}\n\n{run_message}" - elif isinstance(run_message, list): - run_message = [{"type": "text", "text": reaction_notes}, *run_message] + # Reactions the user added since the last turn. + run_message = _prepend_note(run_message, _pending_reaction_notes(session)) def _stream(delta): with session["history_lock"]: From e24bac49fa0c5debb610443213756a728fbe1d86 Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Sat, 8 Aug 2026 15:37:41 -0500 Subject: [PATCH 2/3] feat(desktop): tell the agent when it is floating in HUD mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In HUD mode Hermes is a strip over the app the user is actually working in, so "what's under you?" or "look up the weather" is almost always about that app — but the agent had no way to know it was floating, and answered from its own browser and panes instead. The desktop tags a HUD submit with `surface: 'hud'` and the gateway turns that into a per-turn note pointing at read_window_below, and at carrying the work out in the app underneath. It rides the model-bound message beside the reaction and speech-interrupted notes rather than the system prompt: one session can be driven from the app window on one turn and the HUD on the next, and the system prompt has to stay byte-stable. Every tool the note names is checked against the agent's own schema first, so a session without computer_use or read_window_below is never pointed at a tool it cannot call. --- agent/prompt_builder.py | 50 +++++++ .../hooks/use-prompt-actions/index.test.tsx | 43 ++++++ .../hooks/use-prompt-actions/submit.ts | 6 + tests/tui_gateway/test_hud_surface_note.py | 139 ++++++++++++++++++ tui_gateway/methods_prompt.py | 5 + tui_gateway/server.py | 13 ++ 6 files changed, 256 insertions(+) create mode 100644 tests/tui_gateway/test_hud_surface_note.py diff --git a/agent/prompt_builder.py b/agent/prompt_builder.py index 1a855981231eb..81f9816e468c6 100644 --- a/agent/prompt_builder.py +++ b/agent/prompt_builder.py @@ -696,6 +696,56 @@ STEER_CHANNEL_NOTE += ( "because it remains in the conversation history." ) + +def hud_surface_note(valid_tool_names: "set[str] | None" = None) -> str: + """Per-turn note for a message typed into the desktop's floating HUD. + + HUD mode is a strip of Hermes floating over another application, so the + user is rarely asking about Hermes — they are asking about the thing behind + it, and the work they want done usually belongs in that app rather than in + a surface of our own. Left to itself the model answers from its own + browser and panes, which is the wrong half of the screen. + + It is a per-turn fact, not a platform: one desktop session can be driven + from the app window on one turn and the HUD on the next. So it rides the + model-bound message beside the reaction / speech-interrupted notes rather + than the system prompt, which has to stay byte-stable for a conversation's + whole life. + + Each sentence is gated on the tool it names — naming a tool outside this + agent's schema invites a hallucinated call — and the note as a whole is + withheld without the one it rests on. + """ + names = valid_tool_names or set() + if "read_window_below" not in names: + return "" + + sentences = [ + "[Note: this message came from HUD mode — a small floating Hermes " + "window sitting over whatever the user is actually working in, so an " + 'unqualified "this" or "here" usually means the app behind the HUD ' + "rather than anything inside Hermes. read_window_below identifies " + "that app." + ] + if "computer_use" in names: + sentences.append( + "Prefer carrying the work out in that same app — computer_use " + "takes its name in `app` — over pulling the task into a surface " + "of your own." + ) + if "browser_navigate" in names: + sentences.append( + "When the app underneath is a browser, that means driving the " + "user's browser rather than opening yours with " + "browser_navigate." + ) + sentences.append( + "This is a prior, not a rule: when the request names its own target, " + "follow the request.]" + ) + return " ".join(sentences) + + # Model name substrings that should use the 'developer' role instead of # 'system' for the system prompt. OpenAI's newer models (GPT-5, Codex) # give stronger instruction-following weight to the 'developer' role. diff --git a/apps/desktop/src/app/session/hooks/use-prompt-actions/index.test.tsx b/apps/desktop/src/app/session/hooks/use-prompt-actions/index.test.tsx index c8e2ef1d5e060..8af22a06e8371 100644 --- a/apps/desktop/src/app/session/hooks/use-prompt-actions/index.test.tsx +++ b/apps/desktop/src/app/session/hooks/use-prompt-actions/index.test.tsx @@ -8,6 +8,7 @@ import { textPart } from '@/lib/chat-messages' import { createClientSessionState } from '@/lib/chat-runtime' import { $composerAttachments, $composerDraft, type ComposerAttachment, setComposerDraft } from '@/store/composer' import { $queuedPromptsBySession, getQueuedPrompts } from '@/store/composer-queue' +import { $hudMode } from '@/store/hud' import { $notifications, clearNotifications } from '@/store/notifications' import { $busy, @@ -323,6 +324,48 @@ function renderedSeedTexts(seeds: Record[]): string[] { }) } +describe('usePromptActions HUD surface', () => { + // The HUD floats over the app the user is really working in, so the gateway + // turns this flag into a per-turn hint: look at the window underneath before + // reaching for Hermes's own browser and panes. + afterEach(() => { + cleanup() + $hudMode.set(false) + vi.restoreAllMocks() + }) + + async function submitFromHud(hud: boolean) { + $hudMode.set(hud) + + const submitted: (Record | undefined)[] = [] + + const requestGateway = vi.fn(async (method: string, params?: Record) => { + if (method === 'prompt.submit') { + submitted.push(params) + } + + return {} as never + }) + + let handle: HarnessHandle | null = null + await actRender( + (handle = h)} refreshSessions={async () => undefined} requestGateway={requestGateway} /> + ) + + await handle!.submitText("what's under you rn?") + + return submitted[0] + } + + it('tags a message typed into the HUD', async () => { + expect(await submitFromHud(true)).toMatchObject({ surface: 'hud' }) + }) + + it('says nothing about the surface from the app window', async () => { + expect(await submitFromHud(false)).not.toHaveProperty('surface') + }) +}) + describe('usePromptActions slash session targeting', () => { const STORED_SESSION_ID = 'stored-db-xyz789' const RECOVERED_SESSION_ID = 'rt-recovered-456' diff --git a/apps/desktop/src/app/session/hooks/use-prompt-actions/submit.ts b/apps/desktop/src/app/session/hooks/use-prompt-actions/submit.ts index 5e48b28903044..f5712e8aea972 100644 --- a/apps/desktop/src/app/session/hooks/use-prompt-actions/submit.ts +++ b/apps/desktop/src/app/session/hooks/use-prompt-actions/submit.ts @@ -18,6 +18,7 @@ import { type ComposerAttachment, terminalContextBlocksFromDraft } from '@/store/composer' +import { $hudMode } from '@/store/hud' import { clearNotifications, notify, notifyError } from '@/store/notifications' import { requestDesktopOnboarding } from '@/store/onboarding' import { @@ -619,6 +620,11 @@ export function useSubmitPrompt(deps: SubmitPromptDeps) { session_id: targetId, text, ...(interrupted && { interrupted }), + // Typed into the floating HUD, which means the user is looking at + // another app rather than at Hermes. The gateway turns this into a + // per-turn hint so the agent reads the window underneath and works + // in it, instead of reaching for its own browser and panes. + ...($hudMode.get() && { surface: 'hud' }), // A queue drain is a "run after" message, never a live-turn // correction. The flag tells the gateway's busy path to hold it for // the next turn untouched — without it, losing the settle race diff --git a/tests/tui_gateway/test_hud_surface_note.py b/tests/tui_gateway/test_hud_surface_note.py new file mode 100644 index 0000000000000..2f791fa004e45 --- /dev/null +++ b/tests/tui_gateway/test_hud_surface_note.py @@ -0,0 +1,139 @@ +"""HUD mode reaches the model as a per-turn note, not as a platform hint. + +The floating HUD sits over whatever the user is really working in, so a request +typed there is usually about that other app — and usually wants to be carried +out IN that app. Without the note the model answers from its own browser and +panes, which is the wrong half of the screen. + +The same desktop session can be driven from the app window on one turn and the +HUD on the next, so this cannot live in the system prompt — that has to stay +byte-stable for the life of a conversation. It rides the model-bound message +instead, beside the reaction and speech-interrupted notes. +""" + +import threading +import types + +import pytest + +from agent.prompt_builder import hud_surface_note +from tui_gateway import server + +FULL_KIT = {"read_window_below", "computer_use", "browser_navigate"} + + +def _session(*, tools=FULL_KIT, **extra): + return { + "agent": types.SimpleNamespace(valid_tool_names=set(tools)), + "session_key": "session-key", + "history": [], + "history_lock": threading.Lock(), + "history_version": 0, + "running": False, + "transport": None, + "attached_images": [], + **extra, + } + + +class TestNoteContents: + """Every tool the note names has to be one this agent actually has.""" + + def test_points_at_the_window_below_and_at_working_in_it(self): + note = hud_surface_note(FULL_KIT) + + assert "read_window_below" in note + assert "computer_use" in note + assert "browser_navigate" in note + + def test_no_note_at_all_without_the_tool_it_rests_on(self): + assert hud_surface_note({"computer_use", "browser_navigate"}) == "" + + def test_identifies_the_app_even_when_it_cannot_drive_it(self): + note = hud_surface_note({"read_window_below"}) + + assert "read_window_below" in note + assert "computer_use" not in note + + def test_browser_preference_needs_a_browser_to_prefer_over(self): + note = hud_surface_note({"read_window_below", "computer_use"}) + + assert "computer_use" in note + assert "browser_navigate" not in note + + def test_no_tools_at_all(self): + assert hud_surface_note(None) == "" + + +class TestTurnRouting: + def test_hud_turn_gets_the_note(self): + assert server._hud_surface_note(_session(client_surface="hud")) == hud_surface_note(FULL_KIT) + + def test_app_window_turn_gets_nothing(self): + assert server._hud_surface_note(_session(client_surface="")) == "" + + def test_session_that_never_reported_a_surface_gets_nothing(self): + """Every other client (TUI, dashboard, messaging) omits the field.""" + assert server._hud_surface_note(_session()) == "" + + def test_survives_an_agent_with_no_toolset_at_all(self): + session = _session(client_surface="hud") + session["agent"] = types.SimpleNamespace() + + assert server._hud_surface_note(session) == "" + + +class TestPrepending: + """Notes ride the model input; the persisted prompt stays what was typed.""" + + def test_plain_text_turn(self): + assert server._prepend_note("go to x", "[Note: hi]") == "[Note: hi]\n\ngo to x" + + def test_multimodal_turn_keeps_its_parts(self): + parts = [{"type": "text", "text": "go to x"}, {"type": "image_url", "image_url": {}}] + + assert server._prepend_note(parts, "[Note: hi]") == [ + {"type": "text", "text": "[Note: hi]"}, + *parts, + ] + + def test_nothing_to_say_leaves_the_message_untouched(self): + parts = [{"type": "text", "text": "go to x"}] + + assert server._prepend_note("go to x", "") == "go to x" + assert server._prepend_note(parts, "") is parts + + +class TestSurfaceRecording: + """``prompt.submit`` stamps the window each message was typed into.""" + + @pytest.fixture + def busy_session(self): + # A running session takes the busy path, which returns before any of + # the agent/DB machinery — enough to observe what submit recorded. + session = _session(running=True) + server._sessions["sid"] = session + yield session + server._sessions.pop("sid", None) + + def _submit(self, **params): + return server._methods["prompt.submit"]( + "r1", {"session_id": "sid", "text": "what is this?", "queued": True, **params} + ) + + def test_hud_submit_is_recorded(self, busy_session): + self._submit(surface="hud") + + assert busy_session["client_surface"] == "hud" + + def test_the_next_app_window_submit_clears_it(self, busy_session): + """A stale 'hud' would tell the model the user is still floating.""" + self._submit(surface="hud") + self._submit() + + assert busy_session["client_surface"] == "" + + def test_an_unknown_surface_is_not_hud(self, busy_session): + self._submit(surface="pet-overlay") + + assert busy_session["client_surface"] == "" diff --git a/tui_gateway/methods_prompt.py b/tui_gateway/methods_prompt.py index 021fa9aee2c13..3344a0c15065d 100644 --- a/tui_gateway/methods_prompt.py +++ b/tui_gateway/methods_prompt.py @@ -113,6 +113,11 @@ def _(rid, params: dict) -> dict: return err if (limit_message := _ensure_active_session_slot(sid, session)) is not None: return _err(rid, 4090, limit_message) + # Which desktop window this message was typed into. One session can be + # driven from the app window and the HUD in turn, so it is rewritten on + # every submit — a HUD message must not leave the next app-window message + # claiming the user is still floating over another app. + session["client_surface"] = "hud" if params.get("surface") == "hud" else "" if truncate_user_ordinal is not None and isinstance(text, str): # A rewind/regenerate replays a turn from what the transcript shows. A # skill turn shows its invocation, so re-expand it here — otherwise diff --git a/tui_gateway/server.py b/tui_gateway/server.py index c0bda0f8d35e1..74c576d63b645 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -9513,6 +9513,15 @@ def _start_notification_poller(sid: str, session: dict) -> threading.Event: return stop +def _hud_surface_note(session: dict) -> str: + """The HUD-mode note for this turn, or "" when it was not typed there.""" + if session.get("client_surface") != "hud": + return "" + from agent.prompt_builder import hud_surface_note + + return hud_surface_note(getattr(session.get("agent"), "valid_tool_names", None)) + + def _prepend_note(run_message: Any, note: str) -> Any: """Prefix a per-turn note onto the MODEL INPUT, leaving the prompt alone. @@ -9788,6 +9797,10 @@ def _run_prompt_submit( # Reactions the user added since the last turn. run_message = _prepend_note(run_message, _pending_reaction_notes(session)) + # Which window the message was typed into. HUD mode is per-turn + # state, so it cannot live in the (byte-stable) system prompt. + run_message = _prepend_note(run_message, _hud_surface_note(session)) + def _stream(delta): with session["history_lock"]: _append_inflight_delta(session, delta) From 0665cd4b5b5567e07662c9371795ecba1bcb7998 Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Sat, 8 Aug 2026 16:21:15 -0500 Subject: [PATCH 3/3] style(hud): tighten the surface-note comments and test helper Comment wording only, plus the desktop test's boolean parameter becomes an 'app' | 'hud' union so the call site says which window it means. --- agent/prompt_builder.py | 4 ++-- .../hooks/use-prompt-actions/index.test.tsx | 14 +++++++------- .../app/session/hooks/use-prompt-actions/submit.ts | 7 +++---- tui_gateway/methods_prompt.py | 8 ++++---- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/agent/prompt_builder.py b/agent/prompt_builder.py index 81f9816e468c6..304437170bdd4 100644 --- a/agent/prompt_builder.py +++ b/agent/prompt_builder.py @@ -706,8 +706,8 @@ def hud_surface_note(valid_tool_names: "set[str] | None" = None) -> str: a surface of our own. Left to itself the model answers from its own browser and panes, which is the wrong half of the screen. - It is a per-turn fact, not a platform: one desktop session can be driven - from the app window on one turn and the HUD on the next. So it rides the + It is a per-turn fact, not a platform — one desktop session can be driven + from the app window on one turn and the HUD on the next — so it rides the model-bound message beside the reaction / speech-interrupted notes rather than the system prompt, which has to stay byte-stable for a conversation's whole life. diff --git a/apps/desktop/src/app/session/hooks/use-prompt-actions/index.test.tsx b/apps/desktop/src/app/session/hooks/use-prompt-actions/index.test.tsx index 8af22a06e8371..69e3d76e0c110 100644 --- a/apps/desktop/src/app/session/hooks/use-prompt-actions/index.test.tsx +++ b/apps/desktop/src/app/session/hooks/use-prompt-actions/index.test.tsx @@ -324,18 +324,18 @@ function renderedSeedTexts(seeds: Record[]): string[] { }) } +// The HUD floats over the app the user is really working in, so the gateway +// turns this flag into a per-turn hint: read the window underneath and work in +// it, rather than reaching for Hermes's own browser and panes. describe('usePromptActions HUD surface', () => { - // The HUD floats over the app the user is really working in, so the gateway - // turns this flag into a per-turn hint: look at the window underneath before - // reaching for Hermes's own browser and panes. afterEach(() => { cleanup() $hudMode.set(false) vi.restoreAllMocks() }) - async function submitFromHud(hud: boolean) { - $hudMode.set(hud) + async function submitFrom(window: 'app' | 'hud') { + $hudMode.set(window === 'hud') const submitted: (Record | undefined)[] = [] @@ -358,11 +358,11 @@ describe('usePromptActions HUD surface', () => { } it('tags a message typed into the HUD', async () => { - expect(await submitFromHud(true)).toMatchObject({ surface: 'hud' }) + expect(await submitFrom('hud')).toMatchObject({ surface: 'hud' }) }) it('says nothing about the surface from the app window', async () => { - expect(await submitFromHud(false)).not.toHaveProperty('surface') + expect(await submitFrom('app')).not.toHaveProperty('surface') }) }) diff --git a/apps/desktop/src/app/session/hooks/use-prompt-actions/submit.ts b/apps/desktop/src/app/session/hooks/use-prompt-actions/submit.ts index f5712e8aea972..be6264748eaa5 100644 --- a/apps/desktop/src/app/session/hooks/use-prompt-actions/submit.ts +++ b/apps/desktop/src/app/session/hooks/use-prompt-actions/submit.ts @@ -620,10 +620,9 @@ export function useSubmitPrompt(deps: SubmitPromptDeps) { session_id: targetId, text, ...(interrupted && { interrupted }), - // Typed into the floating HUD, which means the user is looking at - // another app rather than at Hermes. The gateway turns this into a - // per-turn hint so the agent reads the window underneath and works - // in it, instead of reaching for its own browser and panes. + // Typed into the floating HUD, so the user is looking at another app + // rather than at Hermes. The gateway turns this into a per-turn hint + // to read the window underneath and work in it. ...($hudMode.get() && { surface: 'hud' }), // A queue drain is a "run after" message, never a live-turn // correction. The flag tells the gateway's busy path to hold it for diff --git a/tui_gateway/methods_prompt.py b/tui_gateway/methods_prompt.py index 3344a0c15065d..f76e2f65b2341 100644 --- a/tui_gateway/methods_prompt.py +++ b/tui_gateway/methods_prompt.py @@ -113,10 +113,10 @@ def _(rid, params: dict) -> dict: return err if (limit_message := _ensure_active_session_slot(sid, session)) is not None: return _err(rid, 4090, limit_message) - # Which desktop window this message was typed into. One session can be - # driven from the app window and the HUD in turn, so it is rewritten on - # every submit — a HUD message must not leave the next app-window message - # claiming the user is still floating over another app. + # Which desktop window this message was typed into. Rewritten on every + # submit, because one session can be driven from the app window and the HUD + # in turn: a stale "hud" would tell the model the user is still floating + # over another app when they are back in Hermes. session["client_surface"] = "hud" if params.get("surface") == "hud" else "" if truncate_user_ordinal is not None and isinstance(text, str): # A rewind/regenerate replays a turn from what the transcript shows. A