From 780e0980773a875322abd720e5e126a4fe448e7b Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 15 Jul 2026 17:16:00 -0700 Subject: [PATCH] fix: widen UTF-8 BOM tolerance to all sibling frontmatter parsers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit fixes the canonical agent/skill_utils.parse_frontmatter. Six more modules reimplement the '---' fence check locally and had the same bug: - tools/skill_manager_tool.py _validate_frontmatter — rejected BOM'd skill_manage create/edit content outright - tools/skills_hub.py GitHubSource._parse_frontmatter_quick and OptionalSkillSource._parse_frontmatter — hub browse/install metadata - hermes_cli/skills_hub.py — local skill install validation - gateway/run.py — skill slug discovery for disabled-skill hints - agent/prompt_builder.py _strip_yaml_frontmatter — BOM'd context files (AGENTS.md) leaked raw frontmatter into the system prompt - tools/blueprints.py _split_frontmatter — str.lstrip() does not strip U+FEFF (not whitespace), so the existing lstrip never covered it Sibling-surface regression tests added. Bug class also fixed upstream in cline/cline#12218 (found by the weekly Cline PR scout). --- agent/prompt_builder.py | 1 + gateway/run.py | 1 + hermes_cli/skills_hub.py | 1 + tests/agent/test_skill_utils.py | 40 +++++++++++++++++++++++++++++++++ tools/blueprints.py | 2 +- tools/skill_manager_tool.py | 3 +++ tools/skills_hub.py | 2 ++ 7 files changed, 49 insertions(+), 1 deletion(-) diff --git a/agent/prompt_builder.py b/agent/prompt_builder.py index 1c1977b034dc7..910dccc7f4934 100644 --- a/agent/prompt_builder.py +++ b/agent/prompt_builder.py @@ -114,6 +114,7 @@ def _strip_yaml_frontmatter(content: str) -> str: strip it so only the human-readable markdown body is injected into the system prompt. """ + content = content.lstrip("\ufeff") # tolerate UTF-8 BOM (Windows editors) if content.startswith("---"): end = content.find("\n---", 3) if end != -1: diff --git a/gateway/run.py b/gateway/run.py index 1f0e194a1e818..275a8ed6de239 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -2344,6 +2344,7 @@ def _skill_slug_from_frontmatter(skill_md: Path) -> tuple[str | None, str | None content = skill_md.read_text(encoding="utf-8", errors="replace") except Exception: return None, None + content = content.lstrip("\ufeff") # tolerate UTF-8 BOM (Windows editors) if not content.startswith("---"): return None, None end = content.find("\n---", 3) diff --git a/hermes_cli/skills_hub.py b/hermes_cli/skills_hub.py index 66fb903ab3e0c..5c8653b3bac81 100644 --- a/hermes_cli/skills_hub.py +++ b/hermes_cli/skills_hub.py @@ -1461,6 +1461,7 @@ def do_publish(skill_path: str, target: str = "github", repo: str = "", # Validate the skill import yaml skill_md = (path / "SKILL.md").read_text(encoding="utf-8") + skill_md = skill_md.lstrip("\ufeff") # tolerate UTF-8 BOM (Windows editors) fm = {} if skill_md.startswith("---"): import re diff --git a/tests/agent/test_skill_utils.py b/tests/agent/test_skill_utils.py index 0de3672b6f393..471acd4041951 100644 --- a/tests/agent/test_skill_utils.py +++ b/tests/agent/test_skill_utils.py @@ -474,3 +474,43 @@ class TestParseFrontmatterBOM: fm, _ = parse_frontmatter(raw) assert fm["name"] == "my-skill" assert fm["platforms"] == ["macos"] + + +class TestBOMToleranceSiblingSites: + """The BOM fix must cover every independent frontmatter parser, not just + the canonical ``parse_frontmatter`` — several modules reimplement the + ``---`` fence check locally.""" + + SKILL = "---\nname: bom-skill\ndescription: Saved by Notepad\n---\n\n# Body\n" + + def test_skill_manager_validate_accepts_bom(self): + from tools.skill_manager_tool import _validate_frontmatter + + assert _validate_frontmatter("\ufeff" + self.SKILL) is None + + def test_prompt_builder_strips_bom_frontmatter(self): + # A BOM'd context file (AGENTS.md etc.) must not leak raw + # frontmatter into the system prompt. + from agent.prompt_builder import _strip_yaml_frontmatter + + out = _strip_yaml_frontmatter("\ufeff---\nfoo: bar\n---\nBody text\n") + assert out.strip() == "Body text" + + def test_blueprints_split_frontmatter_bom(self): + # str.lstrip() does NOT strip U+FEFF (it is not whitespace), so the + # pre-existing lstrip() in _split_frontmatter never covered it. + from tools.blueprints import _split_frontmatter + + fm = _split_frontmatter("\ufeff---\nname: bp\n---\nbody") + assert fm is not None + assert fm.get("name") == "bp" + + def test_skills_hub_parsers_accept_bom(self): + from tools.skills_hub import GitHubSource, OptionalSkillSource + + for parser in ( + GitHubSource._parse_frontmatter_quick, + OptionalSkillSource._parse_frontmatter, + ): + fm = parser("\ufeff" + self.SKILL) + assert fm.get("name") == "bom-skill", parser.__qualname__ diff --git a/tools/blueprints.py b/tools/blueprints.py index 7e4c5591a088c..700a17fcd7629 100644 --- a/tools/blueprints.py +++ b/tools/blueprints.py @@ -73,7 +73,7 @@ def _split_frontmatter(text: str) -> Optional[Dict[str, Any]]: """Return the parsed YAML frontmatter mapping, or None if absent/invalid.""" if not isinstance(text, str): return None - stripped = text.lstrip() + stripped = text.lstrip("\ufeff").lstrip() # BOM is not whitespace; strip explicitly if not stripped.startswith("---"): return None # Find the closing fence after the opening one. diff --git a/tools/skill_manager_tool.py b/tools/skill_manager_tool.py index 3045474b1c94f..eaf30dd41ad0d 100644 --- a/tools/skill_manager_tool.py +++ b/tools/skill_manager_tool.py @@ -529,6 +529,9 @@ def _validate_frontmatter(content: str) -> Optional[str]: if not content.strip(): return "Content cannot be empty." + # Tolerate a leading UTF-8 BOM (Windows editors) before the fence. + content = content.lstrip("\ufeff") + if not content.startswith("---"): return "SKILL.md must start with YAML frontmatter (---). See existing skills for format." diff --git a/tools/skills_hub.py b/tools/skills_hub.py index 755b32fcbbe25..2db5491889b5a 100644 --- a/tools/skills_hub.py +++ b/tools/skills_hub.py @@ -1165,6 +1165,7 @@ class GitHubSource(SkillSource): @staticmethod def _parse_frontmatter_quick(content: str) -> dict: """Parse YAML frontmatter from SKILL.md content.""" + content = content.lstrip("\ufeff") # tolerate UTF-8 BOM (Windows editors) if not content.startswith("---"): return {} match = re.search(r'\n---\s*\n', content[3:]) @@ -3317,6 +3318,7 @@ class OptionalSkillSource(SkillSource): @staticmethod def _parse_frontmatter(content: str) -> dict: """Parse YAML frontmatter from SKILL.md content.""" + content = content.lstrip("\ufeff") # tolerate UTF-8 BOM (Windows editors) if not content.startswith("---"): return {} match = re.search(r'\n---\s*\n', content[3:])