From 86b50c6a294ee8097444d33105b862f91989f628 Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Sat, 8 Aug 2026 19:17:30 -0500 Subject: [PATCH] fix(desktop): keep earlier HUD windows in scope for the turn The HUD-mode note tells the model that an unqualified "this" means the app behind the strip. It says nothing about the app that was behind it a minute ago, and the user drags the strip from app to app mid-thought: parked over Spotify, "pause that and play X here" is one request spanning two apps, and only the second half has a window under it. Those earlier windows are already in context as read_window_below results, so the note only has to say they still count. Without it the latest window reads as the only one and half the request is silently dropped. No new tool names, so the existing gating tests cover it unchanged. --- agent/prompt_builder.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/agent/prompt_builder.py b/agent/prompt_builder.py index 304437170bdd4..dc583c7529580 100644 --- a/agent/prompt_builder.py +++ b/agent/prompt_builder.py @@ -712,6 +712,13 @@ def hud_surface_note(valid_tool_names: "set[str] | None" = None) -> str: than the system prompt, which has to stay byte-stable for a conversation's whole life. + The same is true one level down: the app underneath changes as the user + drags the strip around, and they carry a thought across the move ("pause + that and play X here"). Earlier windows are already in context as + read_window_below results, so the note only has to say they still count — + without that, the latest window reads as the only one and half of a + two-app request is silently dropped. + Each sentence is gated on the tool it names — naming a tool outside this agent's schema invites a hallucinated call — and the note as a whole is withheld without the one it rests on. @@ -725,7 +732,11 @@ def hud_surface_note(valid_tool_names: "set[str] | None" = None) -> str: "window sitting over whatever the user is actually working in, so an " 'unqualified "this" or "here" usually means the app behind the HUD ' "rather than anything inside Hermes. read_window_below identifies " - "that app." + "that app.", + "They move the HUD from app to app mid-conversation, so one you " + "identified on an earlier turn is still a live target: a reference " + "that does not fit the window below may name one from a turn or two " + "ago, and a single message can span both.", ] if "computer_use" in names: sentences.append(