test(eval): strip ANSI before help-output substring asserts (v0.55.0 CI fix)

3 macOS CI failures from the v0.55.0 push — Rich wraps option names
with ANSI escapes when the terminal is narrow (macOS CI runners
default to a smaller width than Linux/Windows), so substring searches
like `"--goal" in result.output` fail because the actual output
contains `\x1b[1;36m-\x1b[0m\x1b[1;36m-goal\x1b[0m`.

Project precedent: v0.53.5 / v0.53.6 / v0.53.8 / v0.53.9 all hit the
same pattern; tests/test_auto_tuning.py and tests/test_eval_platform.py
already ship `_ANSI_RE` + `_strip_ansi` helpers.

Failures fixed:
  tests/test_v0550.py::TestCLIPlumbing::test_eval_design_help
  tests/test_v0550.py::TestEvalAgainst::test_against_help
  tests/test_v0550_followups.py::TestEvalAgainst::test_against_cli_help_lists_flag

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Alpamys 2026-05-15 13:10:53 +05:00
parent 58d7d510bf
commit 04c504e761
2 changed files with 25 additions and 7 deletions

View File

@ -50,6 +50,17 @@ from soup_cli.utils.eval_lock_coverage import (
POSIX_ONLY = pytest.mark.skipif(os.name == "nt", reason="POSIX-only symlink test")
# Rich wraps option names with ANSI escapes when the terminal is narrow
# (macOS CI runners default to a smaller width than Linux/Windows), so
# substring searches like `"--goal" in output` fail without stripping.
# Project precedent: tests/test_auto_tuning.py, test_eval_platform.py,
# v0.53.5 / v0.53.6 / v0.53.8 / v0.53.9 CI hardening commits.
_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
def _strip_ansi(text: str) -> str:
return _ANSI_RE.sub("", text)
# ---------------------------------------------------------------------------
# Part A — Eval design from data
@ -760,7 +771,7 @@ class TestCLIPlumbing:
from soup_cli.commands.eval import app
result = self.runner.invoke(app, ["design", "--help"])
assert result.exit_code == 0, result.output
assert "--goal" in result.output
assert "--goal" in _strip_ansi(result.output)
def test_eval_design_end_to_end(self, tmp_path, monkeypatch):
from soup_cli.commands.eval import app
@ -901,9 +912,10 @@ class TestEvalAgainst:
from soup_cli.commands.eval import app
result = self.runner.invoke(app, ["against", "--help"])
assert result.exit_code == 0, (result.output, repr(result.exception))
assert "--candidate" in result.output
assert "--metric" in result.output
assert "--json-only" in result.output
out = _strip_ansi(result.output)
assert "--candidate" in out
assert "--metric" in out
assert "--json-only" in out
def test_against_requires_candidate(self):
# `--candidate` is a required option; omitting must fail.

View File

@ -502,15 +502,21 @@ class TestEvalAgainst:
tracker.get_metric_series("run-1", "")
def test_against_cli_help_lists_flag(self):
import re as _re
from typer.testing import CliRunner
from soup_cli.commands.eval import app
runner = CliRunner()
result = runner.invoke(app, ["against", "--help"])
assert result.exit_code == 0, (result.output, repr(result.exception))
assert "--candidate" in result.output
assert "--metric" in result.output
assert "--json-only" in result.output
# Strip Rich ANSI escapes — macOS CI uses a narrower terminal that
# wraps option names, breaking bare substring searches (precedent:
# v0.53.5/v0.53.6/v0.53.8/v0.53.9 CI hardening commits).
out = _re.sub(r"\x1b\[[0-9;]*m", "", result.output)
assert "--candidate" in out
assert "--metric" in out
assert "--json-only" in out
# ---------------------------------------------------------------------------