fix(cli): scrub lone surrogates before oneshot stdout write

Prevent UnicodeEncodeError when model text contains U+D800-range
surrogates by sanitizing to U+FFFD before writing to UTF-8 stdout.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
rainbowgits 2026-08-06 17:00:13 +03:00 committed by Teknium
parent 45aa902c18
commit 8b799fa77d
2 changed files with 48 additions and 0 deletions

View File

@ -277,6 +277,15 @@ def run_oneshot(
_write_usage_file(usage_file, result)
# Model text can contain lone UTF-16 surrogates (invalid in UTF-8). Writing
# those to a real stdout TextIO raises UnicodeEncodeError and aborts with
# exit 1 after the turn already completed — scrub to U+FFFD first.
# See #80366.
if response:
from agent.message_sanitization import _sanitize_surrogates
response = _sanitize_surrogates(response)
if response:
real_stdout.write(response)
if not response.endswith("\n"):

View File

@ -0,0 +1,39 @@
"""Oneshot stdout must survive lone UTF-16 surrogates in model text (#80366)."""
from __future__ import annotations
import subprocess
import sys
import textwrap
from pathlib import Path
def test_oneshot_replaces_lone_surrogate_and_exits_zero():
"""hermes -z must print U+FFFD and exit 0 when the model returns U+D800."""
program = textwrap.dedent(
"""
import hermes_cli.oneshot as oneshot
dirty = "answer \\ud800 here"
oneshot._run_agent = lambda *args, **kwargs: (
dirty,
{"final_response": dirty, "failed": False, "partial": False, "completed": True},
)
raise SystemExit(oneshot.run_oneshot("hello"))
"""
)
result = subprocess.run(
[sys.executable, "-c", program],
cwd=Path(__file__).resolve().parents[2],
capture_output=True,
timeout=30,
check=False,
)
assert result.returncode == 0, result.stderr.decode("utf-8", errors="replace")
# U+FFFD as UTF-8; no raw surrogate bytes
assert "\ufffd".encode("utf-8") in result.stdout
assert b"answer " in result.stdout
assert b" here\n" in result.stdout
assert b"Traceback" not in result.stderr