From 1706502aa70485440a64127475f780c193784d6d Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 12 Aug 2026 21:53:11 -0700 Subject: [PATCH] feat(computer_use): spill full element tree to a cache file and report numeric bounds_scale (#85047) --- tests/tools/test_computer_use.py | 92 +++++++++++++++++++++ tools/computer_use/tool.py | 136 ++++++++++++++++++++++++++++++- 2 files changed, 227 insertions(+), 1 deletion(-) diff --git a/tests/tools/test_computer_use.py b/tests/tools/test_computer_use.py index ee2e676af9215..7dc07239f95c5 100644 --- a/tests/tools/test_computer_use.py +++ b/tests/tools/test_computer_use.py @@ -2268,3 +2268,95 @@ class TestEscalationEnrichment: payload = _action_payload(self._refusal()) assert payload["escalation"]["alternative"] == "page" assert payload["verdict"]["decision"] == "escalate" + + +class TestElementSpillFile: + """Detail dropped from the in-context capture must be recoverable on disk.""" + + def _dense_capture(self): + from tools.computer_use.backend import CaptureResult, UIElement + + elems = [ + UIElement(index=i, role="Document", label=f"msg {i}: " + "x" * 2000, + bounds=(100 + i, 200, 3600, 60), app="chrome.exe") + for i in range(120) + ] + return CaptureResult(mode="som", width=1455, height=791, png_b64=None, + elements=elems, app="chrome.exe", + window_title="Discord", png_bytes_len=0) + + def test_spill_file_holds_full_untruncated_tree(self, tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + from tools.computer_use.tool import _capture_response + + out = json.loads(_capture_response(self._dense_capture())) + assert "elements_file" in out + assert str(out["elements_file"]) in out["summary"] + spill = json.loads( + open(out["elements_file"], encoding="utf-8").read()) + # Everything the in-context response dropped is in the file: + assert spill["total_elements"] == 120 + assert len(spill["elements"]) == 120 # beyond max_elements cap + assert len(spill["elements"][0]["label"]) > 2000 # beyond label cap + assert spill["elements"][119]["label"].startswith("msg 119") + + def test_no_spill_when_nothing_dropped(self, tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + from tools.computer_use.backend import CaptureResult, UIElement + from tools.computer_use.tool import _capture_response + + cap = CaptureResult(mode="som", width=1455, height=791, png_b64=None, + elements=[UIElement(index=0, role="Button", + label="OK", + bounds=(10, 10, 50, 20), + app="")], + app="X", window_title="t", png_bytes_len=0) + out = json.loads(_capture_response(cap)) + assert "elements_file" not in out + + def test_spill_pruning_bounds_cache_growth(self, tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + from tools.computer_use import tool as cu_tool + + cap = self._dense_capture() + for _ in range(cu_tool._MAX_SPILL_FILES + 5): + assert cu_tool._spill_elements_to_file(cap) is not None + cache = tmp_path / "cache" / "computer_use" + assert len(list(cache.glob("elements_*.json"))) <= cu_tool._MAX_SPILL_FILES + + def test_spill_failure_never_breaks_capture(self, monkeypatch): + from tools.computer_use import tool as cu_tool + + monkeypatch.setattr(cu_tool, "_spill_elements_to_file", + lambda cap: None) + out = json.loads(cu_tool._capture_response(self._dense_capture())) + # Capture still succeeds and stays budget-capped without the file. + assert out["truncated_elements"] == 20 + assert "elements_file" not in out + + +class TestBoundsScaleField: + def test_scale_reported_when_spaces_diverge(self, tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + from tools.computer_use.backend import CaptureResult, UIElement + from tools.computer_use.tool import _capture_response + + # Live repro geometry: 1455x791 screenshot, native bounds to 3799. + elems = [UIElement(index=0, role="Button", label="Close", + bounds=(3730, 0, 69, 60), app="")] + cap = CaptureResult(mode="som", width=1455, height=791, png_b64=None, + elements=elems, app="chrome.exe", + window_title="", png_bytes_len=0) + out = json.loads(_capture_response(cap)) + assert out["bounds_scale"] == pytest.approx(3799 / 1455, abs=0.01) + assert f"~{out['bounds_scale']}x" in out["summary"] + + def test_no_scale_when_spaces_match(self): + from tools.computer_use.backend import UIElement + from tools.computer_use.tool import _bounds_scale + + elems = [UIElement(index=0, role="Button", label="OK", + bounds=(10, 10, 50, 20), app="")] + assert _bounds_scale(elems, 1455, 791) is None + assert _bounds_scale([], 1455, 791) is None + assert _bounds_scale(elems, 0, 0) is None diff --git a/tools/computer_use/tool.py b/tools/computer_use/tool.py index c6d9178ddb81c..714facf7276a0 100644 --- a/tools/computer_use/tool.py +++ b/tools/computer_use/tool.py @@ -988,6 +988,20 @@ def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEME response_width = image_dimensions[0] if image_dimensions else cap.width response_height = image_dimensions[1] if image_dimensions else cap.height bounds_note = _bounds_space_note(visible_elements, response_width, response_height) + bounds_scale = _bounds_scale(visible_elements, response_width, response_height) + if bounds_note and bounds_scale: + bounds_note += ( + f"; estimated scale ~{bounds_scale}x (screenshot position x " + f"{bounds_scale} ≈ native coordinate)" + ) + # When the in-context response drops detail (capped labels / capped element + # array), spill the complete tree to a cache file so the model can read or + # grep the full text on demand instead of losing it entirely. + elements_file = ( + _spill_elements_to_file(cap) + if _capture_lost_detail(cap, visible_elements, truncated_elements) + else None + ) image_too_small = bool( image_dimensions and ( @@ -1009,6 +1023,12 @@ def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEME ] if bounds_note: summary_lines.append(f" ({bounds_note})") + if elements_file: + summary_lines.append( + f" (full element tree with untruncated labels saved to " + f"{elements_file} — read_file/search_files it if you need " + "dropped label text or elements beyond the cap)" + ) if element_index: summary_lines.extend(element_index) # Multimodal and AX paths both reference `summary`; build it once up-front @@ -1036,6 +1056,7 @@ def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEME cap, summary, visible_elements=visible_elements, truncated_elements=truncated_elements, + elements_file=elements_file, ) if routed is not None: return routed @@ -1069,6 +1090,10 @@ def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEME } if truncated_elements: payload["truncated_elements"] = truncated_elements + if elements_file: + payload["elements_file"] = elements_file + if bounds_scale: + payload["bounds_scale"] = bounds_scale return json.dumps(payload) # Prefer the explicit MIME type cua-driver attaches to its image @@ -1092,7 +1117,9 @@ def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEME ], "text_summary": summary, "meta": {"mode": cap.mode, "width": response_width, "height": response_height, - "elements": total_elements, "png_bytes": cap.png_bytes_len}, + "elements": total_elements, "png_bytes": cap.png_bytes_len, + **({"elements_file": elements_file} if elements_file else {}), + **({"bounds_scale": bounds_scale} if bounds_scale else {})}, } # AX-only (or image-missing fallback): text path actually carries the # `elements` array, so the truncation note applies here. @@ -1114,6 +1141,10 @@ def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEME } if truncated_elements: payload["truncated_elements"] = truncated_elements + if elements_file: + payload["elements_file"] = elements_file + if bounds_scale: + payload["bounds_scale"] = bounds_scale return json.dumps(payload) @@ -1228,6 +1259,7 @@ def _route_capture_through_aux_vision( *, visible_elements: Optional[List[UIElement]] = None, truncated_elements: int = 0, + elements_file: Optional[str] = None, ) -> Optional[str]: """Pre-analyse the captured PNG via ``vision_analyze`` and return a text result. @@ -1339,6 +1371,8 @@ def _route_capture_through_aux_vision( } if truncated_elements: payload["truncated_elements"] = truncated_elements + if elements_file: + payload["elements_file"] = elements_file return json.dumps(payload) @@ -1409,6 +1443,106 @@ def _format_elements(elements: List[UIElement], max_lines: int = 40) -> List[str # page content — captures are not a text-extraction surface. _MAX_ELEMENT_LABEL_CHARS = 120 +# Keep at most this many spilled element-tree files in the cache dir. Each +# capture of a dense UI can spill; without pruning the cache grows unbounded. +_MAX_SPILL_FILES = 20 + + +def _spill_elements_to_file(cap: CaptureResult) -> Optional[str]: + """Write the FULL element tree (untruncated labels) to a cache file. + + The in-context response caps labels at ``_MAX_ELEMENT_LABEL_CHARS`` and + the array at ``max_elements`` to protect the tool-result budget, but the + dropped text is sometimes exactly what the task needs (reading a chat + transcript or document text exposed through the AX tree). Spilling the + complete tree to disk gives the model an escape hatch — read_file / + search_files against the returned path — without paying the full tree + into context on every capture. + + Returns the absolute path, or None on any failure (spilling is an + enhancement; a capture must never fail because the cache dir is + unwritable). + """ + try: + import uuid as _uuid + + from hermes_constants import get_hermes_dir + + cache_dir = get_hermes_dir("cache/computer_use", "computer_use_cache") + cache_dir.mkdir(parents=True, exist_ok=True) + # Prune oldest spills beyond the cap (best-effort). + try: + spills = sorted( + cache_dir.glob("elements_*.json"), + key=lambda p: p.stat().st_mtime, + ) + for stale in spills[: max(0, len(spills) - (_MAX_SPILL_FILES - 1))]: + stale.unlink(missing_ok=True) + except Exception: + pass + path = cache_dir / f"elements_{_uuid.uuid4().hex}.json" + payload = { + "app": cap.app, + "window_title": cap.window_title, + "total_elements": len(cap.elements), + "elements": [ + { + "index": e.index, + "role": e.role, + "label": e.label, # full, untruncated + "bounds": list(e.bounds), + "app": e.app, + } + for e in cap.elements + ], + } + path.write_text( + json.dumps(payload, ensure_ascii=False, indent=1), + encoding="utf-8", + ) + return str(path) + except Exception as exc: # pragma: no cover - defensive + logger.debug("computer_use: element spill failed: %s", exc) + return None + + +def _capture_lost_detail( + cap: CaptureResult, visible_elements: List[UIElement], truncated_elements: int, +) -> bool: + """True when the in-context response drops information the full tree has.""" + if truncated_elements: + return True + return any( + len(e.label) > _MAX_ELEMENT_LABEL_CHARS for e in visible_elements + ) + + +def _bounds_scale( + elements: List[UIElement], image_width: int, image_height: int, +) -> Optional[float]: + """Estimated native-bounds → screenshot-pixel scale factor, or None. + + Only meaningful when the two spaces diverge (same condition as + ``_bounds_space_note``). Uses the larger of the two axis ratios so the + estimate is driven by the axis with real extent data. Rounded to 2 + decimals — this is a heuristic for mapping screenshot positions to + native coordinates, not display-metrics ground truth. + """ + if not elements or image_width <= 0 or image_height <= 0: + return None + max_x = 0 + max_y = 0 + for e in elements: + try: + x, y, w, h = e.bounds + except (TypeError, ValueError): + continue + max_x = max(max_x, int(x) + int(w)) + max_y = max(max_y, int(y) + int(h)) + if max_x <= image_width * 1.05 and max_y <= image_height * 1.05: + return None + return round(max(max_x / image_width, max_y / image_height), 2) + def _bounds_space_note( elements: List[UIElement], image_width: int, image_height: int,