Merge pull request #82007 from NousResearch/bb/hud-surface-note

The agent knows when it's floating in HUD mode, and looks at the app underneath

agent/prompt_builder.py

@@ -696,6 +696,56 @@ STEER_CHANNEL_NOTE += (
     "because it remains in the conversation history."
 )
 
+
+def hud_surface_note(valid_tool_names: "set[str] | None" = None) -> str:
+    """Per-turn note for a message typed into the desktop's floating HUD.
+
+    HUD mode is a strip of Hermes floating over another application, so the
+    user is rarely asking about Hermes — they are asking about the thing behind
+    it, and the work they want done usually belongs in that app rather than in
+    a surface of our own. Left to itself the model answers from its own
+    browser and panes, which is the wrong half of the screen.
+
+    It is a per-turn fact, not a platform — one desktop session can be driven
+    from the app window on one turn and the HUD on the next — so it rides the
+    model-bound message beside the reaction / speech-interrupted notes rather
+    than the system prompt, which has to stay byte-stable for a conversation's
+    whole life.
+
+    Each sentence is gated on the tool it names — naming a tool outside this
+    agent's schema invites a hallucinated call — and the note as a whole is
+    withheld without the one it rests on.
+    """
+    names = valid_tool_names or set()
+    if "read_window_below" not in names:
+        return ""
+
+    sentences = [
+        "[Note: this message came from HUD mode — a small floating Hermes "
+        "window sitting over whatever the user is actually working in, so an "
+        'unqualified "this" or "here" usually means the app behind the HUD '
+        "rather than anything inside Hermes. read_window_below identifies "
+        "that app."
+    ]
+    if "computer_use" in names:
+        sentences.append(
+            "Prefer carrying the work out in that same app — computer_use "
+            "takes its name in `app` — over pulling the task into a surface "
+            "of your own."
+        )
+        if "browser_navigate" in names:
+            sentences.append(
+                "When the app underneath is a browser, that means driving the "
+                "user's browser rather than opening yours with "
+                "browser_navigate."
+            )
+    sentences.append(
+        "This is a prior, not a rule: when the request names its own target, "
+        "follow the request.]"
+    )
+    return " ".join(sentences)
+
+
 # Model name substrings that should use the 'developer' role instead of
 # 'system' for the system prompt.  OpenAI's newer models (GPT-5, Codex)
 # give stronger instruction-following weight to the 'developer' role.

apps/desktop/src/app/session/hooks/use-prompt-actions/index.test.tsx

@@ -8,6 +8,7 @@ import { textPart } from '@/lib/chat-messages'
 import { createClientSessionState } from '@/lib/chat-runtime'
 import { $composerAttachments, $composerDraft, type ComposerAttachment, setComposerDraft } from '@/store/composer'
 import { $queuedPromptsBySession, getQueuedPrompts } from '@/store/composer-queue'
+import { $hudMode } from '@/store/hud'
 import { $notifications, clearNotifications } from '@/store/notifications'
 import {
   $busy,
@@ -323,6 +324,48 @@ function renderedSeedTexts(seeds: Record<string, unknown>[]): string[] {
   })
 }
 
+// The HUD floats over the app the user is really working in, so the gateway
+// turns this flag into a per-turn hint: read the window underneath and work in
+// it, rather than reaching for Hermes's own browser and panes.
+describe('usePromptActions HUD surface', () => {
+  afterEach(() => {
+    cleanup()
+    $hudMode.set(false)
+    vi.restoreAllMocks()
+  })
+
+  async function submitFrom(window: 'app' | 'hud') {
+    $hudMode.set(window === 'hud')
+
+    const submitted: (Record<string, unknown> | undefined)[] = []
+
+    const requestGateway = vi.fn(async (method: string, params?: Record<string, unknown>) => {
+      if (method === 'prompt.submit') {
+        submitted.push(params)
+      }
+
+      return {} as never
+    })
+
+    let handle: HarnessHandle | null = null
+    await actRender(
+      <Harness onReady={h => (handle = h)} refreshSessions={async () => undefined} requestGateway={requestGateway} />
+    )
+
+    await handle!.submitText("what's under you rn?")
+
+    return submitted[0]
+  }
+
+  it('tags a message typed into the HUD', async () => {
+    expect(await submitFrom('hud')).toMatchObject({ surface: 'hud' })
+  })
+
+  it('says nothing about the surface from the app window', async () => {
+    expect(await submitFrom('app')).not.toHaveProperty('surface')
+  })
+})
+
 describe('usePromptActions slash session targeting', () => {
   const STORED_SESSION_ID = 'stored-db-xyz789'
   const RECOVERED_SESSION_ID = 'rt-recovered-456'

apps/desktop/src/app/session/hooks/use-prompt-actions/submit.ts

@@ -18,6 +18,7 @@ import {
   type ComposerAttachment,
   terminalContextBlocksFromDraft
 } from '@/store/composer'
+import { $hudMode } from '@/store/hud'
 import { clearNotifications, notify, notifyError } from '@/store/notifications'
 import { requestDesktopOnboarding } from '@/store/onboarding'
 import {
@@ -619,6 +620,10 @@ export function useSubmitPrompt(deps: SubmitPromptDeps) {
           session_id: targetId,
           text,
           ...(interrupted && { interrupted }),
+          // Typed into the floating HUD, so the user is looking at another app
+          // rather than at Hermes. The gateway turns this into a per-turn hint
+          // to read the window underneath and work in it.
+          ...($hudMode.get() && { surface: 'hud' }),
           // A queue drain is a "run after" message, never a live-turn
           // correction. The flag tells the gateway's busy path to hold it for
           // the next turn untouched — without it, losing the settle race

tests/tui_gateway/test_hud_surface_note.py

@@ -0,0 +1,139 @@
+"""HUD mode reaches the model as a per-turn note, not as a platform hint.
+
+The floating HUD sits over whatever the user is really working in, so a request
+typed there is usually about that other app — and usually wants to be carried
+out IN that app. Without the note the model answers from its own browser and
+panes, which is the wrong half of the screen.
+
+The same desktop session can be driven from the app window on one turn and the
+HUD on the next, so this cannot live in the system prompt — that has to stay
+byte-stable for the life of a conversation. It rides the model-bound message
+instead, beside the reaction and speech-interrupted notes.
+"""
+
+import threading
+import types
+
+import pytest
+
+from agent.prompt_builder import hud_surface_note
+from tui_gateway import server
+
+FULL_KIT = {"read_window_below", "computer_use", "browser_navigate"}
+
+
+def _session(*, tools=FULL_KIT, **extra):
+    return {
+        "agent": types.SimpleNamespace(valid_tool_names=set(tools)),
+        "session_key": "session-key",
+        "history": [],
+        "history_lock": threading.Lock(),
+        "history_version": 0,
+        "running": False,
+        "transport": None,
+        "attached_images": [],
+        **extra,
+    }
+
+
+class TestNoteContents:
+    """Every tool the note names has to be one this agent actually has."""
+
+    def test_points_at_the_window_below_and_at_working_in_it(self):
+        note = hud_surface_note(FULL_KIT)
+
+        assert "read_window_below" in note
+        assert "computer_use" in note
+        assert "browser_navigate" in note
+
+    def test_no_note_at_all_without_the_tool_it_rests_on(self):
+        assert hud_surface_note({"computer_use", "browser_navigate"}) == ""
+
+    def test_identifies_the_app_even_when_it_cannot_drive_it(self):
+        note = hud_surface_note({"read_window_below"})
+
+        assert "read_window_below" in note
+        assert "computer_use" not in note
+
+    def test_browser_preference_needs_a_browser_to_prefer_over(self):
+        note = hud_surface_note({"read_window_below", "computer_use"})
+
+        assert "computer_use" in note
+        assert "browser_navigate" not in note
+
+    def test_no_tools_at_all(self):
+        assert hud_surface_note(None) == ""
+
+
+class TestTurnRouting:
+    def test_hud_turn_gets_the_note(self):
+        assert server._hud_surface_note(_session(client_surface="hud")) == hud_surface_note(FULL_KIT)
+
+    def test_app_window_turn_gets_nothing(self):
+        assert server._hud_surface_note(_session(client_surface="")) == ""
+
+    def test_session_that_never_reported_a_surface_gets_nothing(self):
+        """Every other client (TUI, dashboard, messaging) omits the field."""
+        assert server._hud_surface_note(_session()) == ""
+
+    def test_survives_an_agent_with_no_toolset_at_all(self):
+        session = _session(client_surface="hud")
+        session["agent"] = types.SimpleNamespace()
+
+        assert server._hud_surface_note(session) == ""
+
+
+class TestPrepending:
+    """Notes ride the model input; the persisted prompt stays what was typed."""
+
+    def test_plain_text_turn(self):
+        assert server._prepend_note("go to x", "[Note: hi]") == "[Note: hi]\n\ngo to x"
+
+    def test_multimodal_turn_keeps_its_parts(self):
+        parts = [{"type": "text", "text": "go to x"}, {"type": "image_url", "image_url": {}}]
+
+        assert server._prepend_note(parts, "[Note: hi]") == [
+            {"type": "text", "text": "[Note: hi]"},
+            *parts,
+        ]
+
+    def test_nothing_to_say_leaves_the_message_untouched(self):
+        parts = [{"type": "text", "text": "go to x"}]
+
+        assert server._prepend_note("go to x", "") == "go to x"
+        assert server._prepend_note(parts, "") is parts
+
+
+class TestSurfaceRecording:
+    """``prompt.submit`` stamps the window each message was typed into."""
+
+    @pytest.fixture
+    def busy_session(self):
+        # A running session takes the busy path, which returns before any of
+        # the agent/DB machinery — enough to observe what submit recorded.
+        session = _session(running=True)
+        server._sessions["sid"] = session
+        yield session
+        server._sessions.pop("sid", None)
+
+    def _submit(self, **params):
+        return server._methods["prompt.submit"](
+            "r1", {"session_id": "sid", "text": "what is this?", "queued": True, **params}
+        )
+
+    def test_hud_submit_is_recorded(self, busy_session):
+        self._submit(surface="hud")
+
+        assert busy_session["client_surface"] == "hud"
+
+    def test_the_next_app_window_submit_clears_it(self, busy_session):
+        """A stale 'hud' would tell the model the user is still floating."""
+        self._submit(surface="hud")
+        self._submit()
+
+        assert busy_session["client_surface"] == ""
+
+    def test_an_unknown_surface_is_not_hud(self, busy_session):
+        self._submit(surface="pet-overlay")
+
+        assert busy_session["client_surface"] == ""

tui_gateway/methods_prompt.py

@@ -113,6 +113,11 @@ def _(rid, params: dict) -> dict:
         return err
     if (limit_message := _ensure_active_session_slot(sid, session)) is not None:
         return _err(rid, 4090, limit_message)
+    # Which desktop window this message was typed into. Rewritten on every
+    # submit, because one session can be driven from the app window and the HUD
+    # in turn: a stale "hud" would tell the model the user is still floating
+    # over another app when they are back in Hermes.
+    session["client_surface"] = "hud" if params.get("surface") == "hud" else ""
     if truncate_user_ordinal is not None and isinstance(text, str):
         # A rewind/regenerate replays a turn from what the transcript shows. A
         # skill turn shows its invocation, so re-expand it here — otherwise

tui_gateway/server.py

@@ -9513,6 +9513,33 @@ def _start_notification_poller(sid: str, session: dict) -> threading.Event:
     return stop
 
 
+def _hud_surface_note(session: dict) -> str:
+    """The HUD-mode note for this turn, or "" when it was not typed there."""
+    if session.get("client_surface") != "hud":
+        return ""
+    from agent.prompt_builder import hud_surface_note
+
+    return hud_surface_note(getattr(session.get("agent"), "valid_tool_names", None))
+
+
+def _prepend_note(run_message: Any, note: str) -> Any:
+    """Prefix a per-turn note onto the MODEL INPUT, leaving the prompt alone.
+
+    Everything the model needs to know about the turn but the user did not
+    type — an interrupted reply, reactions, the surface they typed into —
+    arrives this way. persist_user_message keeps the clean prompt, so no
+    scaffolding reaches the transcript, and annotating the NEW turn never
+    rewrites an already-sent message, so the cached prefix survives.
+    """
+    if not note:
+        return run_message
+    if isinstance(run_message, str):
+        return f"{note}\n\n{run_message}"
+    if isinstance(run_message, list):
+        return [{"type": "text", "text": note}, *run_message]
+    return run_message
+
+
 def _run_prompt_submit(
     rid,
     sid: str,
@@ -9765,21 +9792,14 @@ def _run_prompt_submit(
             from tools.tts_streaming import SPEECH_INTERRUPTED_NOTE, take_speech_interrupted
 
             if take_speech_interrupted():
-                if isinstance(run_message, str):
-                    run_message = f"{SPEECH_INTERRUPTED_NOTE}\n\n{run_message}"
-                elif isinstance(run_message, list):
-                    run_message = [{"type": "text", "text": SPEECH_INTERRUPTED_NOTE}, *run_message]
+                run_message = _prepend_note(run_message, SPEECH_INTERRUPTED_NOTE)
 
-            # Reactions the user added since the last turn ride the MODEL INPUT
-            # only (same enrichment channel as the speech-interrupted note);
-            # persist_user_message below stays the clean prompt, so no
-            # scaffolding reaches the transcript. Cache-safe: annotating the
-            # NEW turn never rewrites an already-sent message.
-            if reaction_notes := _pending_reaction_notes(session):
-                if isinstance(run_message, str):
-                    run_message = f"{reaction_notes}\n\n{run_message}"
-                elif isinstance(run_message, list):
-                    run_message = [{"type": "text", "text": reaction_notes}, *run_message]
+            # Reactions the user added since the last turn.
+            run_message = _prepend_note(run_message, _pending_reaction_notes(session))
+
+            # Which window the message was typed into. HUD mode is per-turn
+            # state, so it cannot live in the (byte-stable) system prompt.
+            run_message = _prepend_note(run_message, _hud_surface_note(session))
 
             def _stream(delta):
                 with session["history_lock"]: