From 406501fd97721e0784fcc0f73f940130809a3238 Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Fri, 7 Aug 2026 22:27:28 -0500 Subject: [PATCH] =?UTF-8?q?feat(agent):=20read=5Fwindow=5Fbelow=20tool=20?= =?UTF-8?q?=E2=80=94=20which=20OS=20window=20is=20underneath=20the=20deskt?= =?UTF-8?q?op=20app?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Desktop-gated (desktop_ui toolset) metadata-only window awareness: the agent can ask which application window sits directly behind the Hermes window (app, title, bounds — never pixels). Rides the same blocking bridge as read_terminal: the gateway emits window.read.request and the renderer answers window.read.respond. --- agent/agent_init.py | 2 + agent/agent_runtime_helpers.py | 11 ++++- agent/tool_executor.py | 19 ++++++++ run_agent.py | 2 + tests/tools/test_read_window_tool.py | 50 +++++++++++++++++++++ tools/read_window_tool.py | 67 ++++++++++++++++++++++++++++ toolsets.py | 1 + tui_gateway/methods_prompt.py | 9 ++++ tui_gateway/server.py | 11 +++++ 9 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 tests/tools/test_read_window_tool.py create mode 100644 tools/read_window_tool.py diff --git a/agent/agent_init.py b/agent/agent_init.py index d038c76183fd6..0cb21e9446539 100644 --- a/agent/agent_init.py +++ b/agent/agent_init.py @@ -493,6 +493,7 @@ def init_agent( clarify_callback: callable = None, read_terminal_callback: callable = None, read_preview_callback: callable = None, + read_window_below_callback: callable = None, step_callback: callable = None, stream_delta_callback: callable = None, interim_assistant_callback: callable = None, @@ -767,6 +768,7 @@ def init_agent( agent.clarify_callback = clarify_callback agent.read_terminal_callback = read_terminal_callback agent.read_preview_callback = read_preview_callback + agent.read_window_below_callback = read_window_below_callback agent.step_callback = step_callback agent.stream_delta_callback = stream_delta_callback agent.interim_assistant_callback = interim_assistant_callback diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py index 4a1e969e928ae..e64fd9ad72309 100644 --- a/agent/agent_runtime_helpers.py +++ b/agent/agent_runtime_helpers.py @@ -99,7 +99,7 @@ def _ra(): AGENT_RUNTIME_POST_HOOK_TOOL_NAMES = frozenset( - {"todo", "session_search", "memory", "clarify", "read_terminal", "read_preview", "delegate_task"} + {"todo", "session_search", "memory", "clarify", "read_terminal", "read_preview", "read_window_below", "delegate_task"} ) @@ -3000,6 +3000,15 @@ def invoke_tool(agent, function_name: str, function_args: dict, effective_task_i ), next_args, ) + elif function_name == "read_window_below": + def _execute(next_args: dict) -> Any: + from tools.read_window_tool import read_window_below_tool as _read_window_below_tool + return _finish_agent_tool( + _read_window_below_tool( + callback=getattr(agent, "read_window_below_callback", None), + ), + next_args, + ) elif function_name == "delegate_task": def _execute(next_args: dict) -> Any: return _finish_agent_tool(agent._dispatch_delegate_task(next_args), next_args) diff --git a/agent/tool_executor.py b/agent/tool_executor.py index 80830fbed544d..9645199ef012d 100644 --- a/agent/tool_executor.py +++ b/agent/tool_executor.py @@ -1880,6 +1880,25 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('read_preview', function_args, tool_duration, result=function_result)}") + elif function_name == "read_window_below": + def _execute(next_args: dict) -> Any: + from tools.read_window_tool import read_window_below_tool as _read_window_below_tool + return _read_window_below_tool( + callback=getattr(agent, "read_window_below_callback", None), + ) + function_result, function_args, middleware_trace, _execution_blocked, _execution_dispatched = _managed_values(_run_agent_tool_execution_middleware( + agent, + function_name=function_name, + function_args=function_args, + effective_task_id=effective_task_id, + tool_call_id=getattr(tool_call, "id", "") or "", + execute=_execute, + scope_block=_ts_scope_block, + display_index=i, + )) + tool_duration = time.time() - tool_start_time + if agent._should_emit_quiet_tool_messages(): + agent._vprint(f" {_get_cute_tool_message_impl('read_window_below', function_args, tool_duration, result=function_result)}") elif function_name == "delegate_task": tasks_arg = function_args.get("tasks") if tasks_arg and isinstance(tasks_arg, list): diff --git a/run_agent.py b/run_agent.py index 0aad92ec0ce62..a8bbc515eaaa3 100644 --- a/run_agent.py +++ b/run_agent.py @@ -470,6 +470,7 @@ class AIAgent: clarify_callback: callable = None, read_terminal_callback: callable = None, read_preview_callback: callable = None, + read_window_below_callback: callable = None, step_callback: callable = None, stream_delta_callback: callable = None, interim_assistant_callback: callable = None, @@ -556,6 +557,7 @@ class AIAgent: clarify_callback=clarify_callback, read_terminal_callback=read_terminal_callback, read_preview_callback=read_preview_callback, + read_window_below_callback=read_window_below_callback, step_callback=step_callback, stream_delta_callback=stream_delta_callback, interim_assistant_callback=interim_assistant_callback, diff --git a/tests/tools/test_read_window_tool.py b/tests/tools/test_read_window_tool.py new file mode 100644 index 0000000000000..e63c7c61e13ea --- /dev/null +++ b/tests/tools/test_read_window_tool.py @@ -0,0 +1,50 @@ +"""Tests for the GUI-surface ``read_window_below`` tool.""" + +import json + +from tools import read_window_tool as rw +from tools.registry import registry + + +def test_lives_in_the_gui_surface_toolset(monkeypatch): + """Mirrors read_terminal: scoped by toolset, not by the backend's env.""" + monkeypatch.delenv("HERMES_DESKTOP", raising=False) + entry = registry.get_entry("read_window_below") + + assert entry is not None + assert entry.toolset == "desktop_ui" + assert entry.check_fn is None + + +def test_requires_callback(): + """Outside the desktop GUI there is no bridge — a clear error, no crash.""" + result = json.loads(rw.read_window_below_tool(callback=None)) + assert "desktop" in result["error"] + + +def test_empty_answer_means_unavailable(): + result = json.loads(rw.read_window_below_tool(callback=lambda: "")) + assert "error" in result + + +def test_passes_json_through(): + payload = { + "window": {"app": "Figma", "title": "", "bounds": {"x": 0, "y": 38, "width": 1470, "height": 870}, "id": 13937}, + "frontmost": {"app": "Figma", "title": ""}, + "platform": "darwin", + } + result = json.loads(rw.read_window_below_tool(callback=lambda: json.dumps(payload))) + assert result == payload + + +def test_wraps_non_json_text(): + result = json.loads(rw.read_window_below_tool(callback=lambda: "plain words")) + assert result == {"text": "plain words"} + + +def test_callback_failure_is_reported(): + def _boom(): + raise RuntimeError("renderer went away") + + result = json.loads(rw.read_window_below_tool(callback=_boom)) + assert "renderer went away" in result["error"] diff --git a/tools/read_window_tool.py b/tools/read_window_tool.py new file mode 100644 index 0000000000000..8dc5725394222 --- /dev/null +++ b/tools/read_window_tool.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Read which OS window sits directly underneath the Hermes desktop window. + +The window list lives with the OS, so this tool round-trips through the +gateway's blocking-prompt bridge — the same one `read_terminal` uses: +tui_gateway emits ``window.read.request``, the desktop renderer asks its main +process (which owns native window enumeration) and answers with +``window.read.respond``. This module is just schema + a thin dispatcher over +the platform-injected callback. +""" + +import json +from typing import Callable, Optional + +from tools.registry import registry, tool_error + + +def read_window_below_tool(callback: Optional[Callable] = None) -> str: + """Return the window underneath the Hermes window as a JSON string.""" + if callback is None: + return tool_error( + "read_window_below is only available in the Hermes desktop app." + ) + + try: + raw = callback() + except Exception as exc: + return tool_error(f"Failed to read the window below: {exc}") + + if not raw: + return tool_error( + "Could not determine the window underneath (the desktop app did " + "not answer, or window enumeration is unavailable on this system)." + ) + + # Desktop answers with a JSON object; pass it through, else wrap the raw text. + try: + return json.dumps(json.loads(raw), ensure_ascii=False) + except (TypeError, ValueError): + return json.dumps({"text": str(raw)}, ensure_ascii=False) + + +READ_WINDOW_BELOW_SCHEMA = { + "name": "read_window_below", + "description": ( + "Identify the application window directly underneath (behind) the " + "Hermes desktop window — what the user is working in behind this app. " + "Returns JSON: {window: {app, title, bounds{x,y,width,height}, id}, " + "frontmost: {app, title}, platform}. `title` may be empty when the OS " + "withholds window titles (e.g. macOS without the Screen Recording " + "permission — never prompted for, noted in `note`). Metadata only; " + "this never captures pixels or content of other windows." + ), + "parameters": { + "type": "object", + "properties": {}, + }, +} + + +registry.register( + name="read_window_below", + toolset="desktop_ui", + schema=READ_WINDOW_BELOW_SCHEMA, + handler=lambda args, **kw: read_window_below_tool(callback=kw.get("callback")), + emoji="🪟", +) diff --git a/toolsets.py b/toolsets.py index e713858c236c2..f7f7939257b50 100644 --- a/toolsets.py +++ b/toolsets.py @@ -273,6 +273,7 @@ TOOLSETS = { "tools": [ "read_terminal", "close_terminal", "open_preview", "read_preview", + "read_window_below", "focus_pane", "react_to_message", ], "includes": [] diff --git a/tui_gateway/methods_prompt.py b/tui_gateway/methods_prompt.py index 15b9328a109f1..021fa9aee2c13 100644 --- a/tui_gateway/methods_prompt.py +++ b/tui_gateway/methods_prompt.py @@ -936,6 +936,15 @@ def _(rid, params: dict) -> dict: return _respond(rid, params, "text", allow_expired=True) +@method("window.read.respond") +def _(rid, params: dict) -> dict: + # `text` is a JSON string describing the OS window underneath the Hermes + # window (read_window_below tool). allow_expired=True for the same reason + # as terminal.read: the tool's bounded wait can expire while the renderer's + # round-trip to the main process is still in flight. + return _respond(rid, params, "text", allow_expired=True) + + @method("sudo.respond") def _(rid, params: dict) -> dict: return _respond(rid, params, "password", allow_expired=True) diff --git a/tui_gateway/server.py b/tui_gateway/server.py index 1a6f84d04aac0..67083a69269f6 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -3240,6 +3240,7 @@ def _block(event: str, sid: str, payload: dict, timeout: float | None = 300) -> "clarify.request", "terminal.read.request", "preview.read.request", + "window.read.request", }: _emit( f"{event.removesuffix('.request')}.expire", @@ -5809,6 +5810,16 @@ def _agent_cbs(sid: str) -> dict: {k: v for k, v in (("start", start), ("count", count)) if v is not None}, timeout=45, ), + # read_window_below tool (desktop GUI): the renderer asks its main + # process (which owns native window enumeration) which OS window sits + # directly underneath the Hermes window, and answers + # window.read.respond with the serialized metadata. + "read_window_below_callback": lambda: _block( + "window.read.request", + sid, + {}, + timeout=30, + ), } # Interim assistant commentary (text alongside tool calls, or the attempted