From 6e30aa2a3cbc777a9cac2d7c4faf8de64668d674 Mon Sep 17 00:00:00 2001
From: PRATHAMESH75 <prathamesh290504@gmail.com>
Date: Mon, 20 Jul 2026 19:10:49 +0530
Subject: [PATCH] fix(skills): tolerate non-UTF-8 bytes in hub lock.json
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

_read_hub_installed_names() reads ~/.hermes/skills/.hub/lock.json with a
strict utf-8 decode. Hub skill descriptions can carry Windows-1252
typographic bytes (em-dash 0x97, smart quotes, bullets) as single high
bytes; read_text(encoding="utf-8") then raises UnicodeDecodeError, which
is a ValueError sibling not caught by the function's
except (OSError, json.JSONDecodeError). It escapes and 500s the whole
/api/skills endpoint, blanking the desktop Skills panel.

Decode with errors="replace" so the offending byte degrades to U+FFFD
and the structurally valid JSON — and every other skill — stays readable.

Fixes #68053
---
 tests/tools/test_hub_lock_non_utf8_68053.py | 65 +++++++++++++++++++++
 tools/skill_usage.py                        | 10 +++-
 2 files changed, 74 insertions(+), 1 deletion(-)
 create mode 100644 tests/tools/test_hub_lock_non_utf8_68053.py

diff --git a/tests/tools/test_hub_lock_non_utf8_68053.py b/tests/tools/test_hub_lock_non_utf8_68053.py
new file mode 100644
index 0000000000000..509b4aefc5ed3
--- /dev/null
+++ b/tests/tools/test_hub_lock_non_utf8_68053.py
@@ -0,0 +1,65 @@
+"""Regression test for #68053 — hub lock.json with Windows-1252 bytes.
+
+`_read_hub_installed_names()` reads `~/.hermes/skills/.hub/lock.json` with a
+strict UTF-8 decode. A hub skill description carrying a Windows-1252 typographic
+byte (em-dash `0x97`, smart quotes, bullets) makes `read_text(encoding="utf-8")`
+raise `UnicodeDecodeError` — a `ValueError` sibling that is NOT caught by the
+function's `except (OSError, json.JSONDecodeError)`, so it escapes and returns
+HTTP 500 from the entire `/api/skills` endpoint, blanking the desktop Skills
+panel. The fix decodes with `errors="replace"`, so the offending byte degrades
+to U+FFFD and every skill name in the (structurally valid) lock stays readable.
+"""
+from __future__ import annotations
+
+import importlib
+from pathlib import Path
+
+import pytest
+
+
+@pytest.fixture
+def skills_home(tmp_path, monkeypatch):
+    home = tmp_path / ".hermes"
+    home.mkdir()
+    (home / "skills").mkdir()
+    monkeypatch.setattr(Path, "home", lambda: tmp_path)
+    monkeypatch.setenv("HERMES_HOME", str(home))
+    import tools.skill_usage as mod
+    importlib.reload(mod)
+    return home
+
+
+def _write_hub_lock(home: Path, raw: bytes) -> None:
+    hub_dir = home / "skills" / ".hub"
+    hub_dir.mkdir(parents=True, exist_ok=True)
+    (hub_dir / "lock.json").write_bytes(raw)
+
+
+def test_windows_1252_em_dash_does_not_raise(skills_home):
+    """A 0x97 em-dash byte in a description must not blow up the reader."""
+    import tools.skill_usage as mod
+
+    # Valid JSON structure, but the description value contains a raw cp1252
+    # em-dash byte (0x97) instead of the UTF-8 sequence.
+    raw = (
+        b'{"installed": {"supply-chain": '
+        b'{"description": "guidance: safe \x97 3 finding(s)"}}}'
+    )
+    _write_hub_lock(skills_home, raw)
+
+    names = mod._read_hub_installed_names()
+
+    # The skill name is still recovered; no UnicodeDecodeError / 500.
+    assert "supply-chain" in names
+
+
+def test_clean_utf8_lock_still_read(skills_home):
+    """A well-formed UTF-8 lock keeps working unchanged."""
+    import tools.skill_usage as mod
+
+    raw = '{"installed": {"alpha": {}, "beta": {}}}'.encode("utf-8")
+    _write_hub_lock(skills_home, raw)
+
+    names = mod._read_hub_installed_names()
+
+    assert {"alpha", "beta"} <= names
diff --git a/tools/skill_usage.py b/tools/skill_usage.py
index dcdca87f81288..ee5a35d2ce793 100644
--- a/tools/skill_usage.py
+++ b/tools/skill_usage.py
@@ -210,7 +210,15 @@ def _read_hub_installed_names() -> Set[str]:
     if not lock_path.exists():
         return set()
     try:
-        data = json.loads(lock_path.read_text(encoding="utf-8"))
+        # Tolerate non-UTF-8 bytes in the lock file. Hub descriptions can carry
+        # Windows-1252 typographic chars (em-dash 0x97, smart quotes, bullets)
+        # written as single high bytes; a strict utf-8 read raises
+        # UnicodeDecodeError, which is a ValueError sibling (not OSError/
+        # JSONDecodeError) so it escapes the handler below and 500s the whole
+        # /api/skills endpoint. errors="replace" degrades the offending byte to
+        # U+FFFD, keeping the (structurally valid) JSON — and every other
+        # skill — readable. See #68053.
+        data = json.loads(lock_path.read_text(encoding="utf-8", errors="replace"))
         if isinstance(data, dict):
             installed = data.get("installed") or {}
             if isinstance(installed, dict):