144 lines
5.1 KiB
Python
144 lines
5.1 KiB
Python
"""Tests for how a unified run is reported to Discord and the job summary.
|
|
|
|
Discord rejects an over-long payload with a 400, which loses the whole
|
|
notification, so the size behavior here is worth pinning down.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from tests.unified.runner import (
|
|
DISCORD_MAX_CONTENT,
|
|
RunArtifact,
|
|
RunArtifacts,
|
|
StepFailure,
|
|
TestOutcome,
|
|
artifact_line,
|
|
artifact_lines,
|
|
clamp_lines,
|
|
failure_lines,
|
|
gha_run_lines,
|
|
)
|
|
|
|
_PREFIX = "unified-test-results/2026-09-03/1123-merge-abc1234-33779689337-1"
|
|
|
|
|
|
def _presigned(name: str, token_len: int) -> RunArtifact:
|
|
"""A presigned URL of realistic shape; OIDC session tokens dominate its length."""
|
|
url = (
|
|
f"https://honcho-unified-tests.s3.amazonaws.com/{_PREFIX}/{name}"
|
|
"?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Expires=259200"
|
|
f"&X-Amz-Security-Token={'t' * token_len}&X-Amz-Signature={'0' * 64}"
|
|
)
|
|
return RunArtifact(key=f"{_PREFIX}/{name}", url=url)
|
|
|
|
|
|
def _discord_lines(artifacts: RunArtifacts) -> list[str]:
|
|
"""Mirror of the Discord report the runner assembles."""
|
|
return [
|
|
"⚠️ **Unified Test Results**",
|
|
"Results: 35/41 passed, 6/41 failed",
|
|
"Execution time: 1015.42s",
|
|
*artifact_line("View Complete Results", artifacts.results),
|
|
*gha_run_lines(),
|
|
*([f"Reasoning traces: `{artifacts.traces.key}`"] if artifacts.traces else []),
|
|
]
|
|
|
|
|
|
@pytest.fixture
|
|
def in_actions(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setenv("GITHUB_RUN_ID", "33779689337")
|
|
monkeypatch.setenv("GITHUB_REPOSITORY", "plastic-labs/honcho")
|
|
|
|
|
|
def test_clamp_lines_leaves_a_short_report_alone() -> None:
|
|
lines = ["one", "two", "three"]
|
|
assert clamp_lines(lines, DISCORD_MAX_CONTENT) == "one\ntwo\nthree"
|
|
|
|
|
|
def test_clamp_lines_drops_the_longest_line_not_the_last() -> None:
|
|
"""The Actions link is short and leads everywhere; a presigned URL is neither."""
|
|
lines = ["head", "x" * 100, "[View GHA](url)"]
|
|
assert clamp_lines(lines, 40) == "head\n[View GHA](url)"
|
|
|
|
|
|
def test_clamp_lines_preserves_display_order() -> None:
|
|
lines = ["a", "y" * 50, "b", "c"]
|
|
assert clamp_lines(lines, 10) == "a\nb\nc"
|
|
|
|
|
|
@pytest.mark.usefixtures("in_actions")
|
|
@pytest.mark.parametrize("token_len", [0, 400, 900, 1400, 1800])
|
|
def test_discord_report_never_exceeds_the_webhook_limit(token_len: int) -> None:
|
|
"""Regression: six judge verdicts plus two presigned URLs returned a 400."""
|
|
artifacts = RunArtifacts(
|
|
results=_presigned("results.json", token_len),
|
|
traces=_presigned("unified-reasoning-traces.jsonl", token_len),
|
|
)
|
|
sent = clamp_lines(_discord_lines(artifacts), DISCORD_MAX_CONTENT)
|
|
assert len(sent) <= DISCORD_MAX_CONTENT
|
|
|
|
|
|
@pytest.mark.usefixtures("in_actions")
|
|
@pytest.mark.parametrize("token_len", [0, 400, 900, 1400, 1800])
|
|
def test_actions_link_always_survives_clamping(token_len: int) -> None:
|
|
"""However long the presigned URLs get, the run stays reachable."""
|
|
artifacts = RunArtifacts(
|
|
results=_presigned("results.json", token_len),
|
|
traces=_presigned("unified-reasoning-traces.jsonl", token_len),
|
|
)
|
|
sent = clamp_lines(_discord_lines(artifacts), DISCORD_MAX_CONTENT)
|
|
assert (
|
|
"[View GHA](https://github.com/plastic-labs/honcho/actions/runs/33779689337)"
|
|
in sent
|
|
)
|
|
|
|
|
|
@pytest.mark.usefixtures("in_actions")
|
|
def test_discord_report_omits_per_test_failures() -> None:
|
|
"""Failure detail belongs in the job summary the Actions link points at."""
|
|
artifacts = RunArtifacts(results=_presigned("results.json", 400))
|
|
assert not any("**Failures**" in line for line in _discord_lines(artifacts))
|
|
|
|
|
|
def test_gha_lines_are_empty_outside_actions(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.delenv("GITHUB_RUN_ID", raising=False)
|
|
monkeypatch.delenv("GITHUB_REPOSITORY", raising=False)
|
|
assert gha_run_lines() == []
|
|
|
|
|
|
def test_artifact_line_falls_back_to_the_key_when_presigning_failed() -> None:
|
|
assert artifact_line("Traces", RunArtifact(key="k/x.jsonl")) == [
|
|
"Traces: `k/x.jsonl`"
|
|
]
|
|
assert artifact_line("Traces", None) == []
|
|
|
|
|
|
def test_job_summary_keeps_both_signed_links() -> None:
|
|
artifacts = RunArtifacts(
|
|
results=_presigned("results.json", 900),
|
|
traces=_presigned("unified-reasoning-traces.jsonl", 900),
|
|
)
|
|
lines = artifact_lines(artifacts)
|
|
assert len(lines) == 2
|
|
assert all("https://" in line for line in lines)
|
|
|
|
|
|
def test_failure_lines_reports_every_failure_in_full() -> None:
|
|
reason = "LLM Judge failed: " + "the model did not recall the fact. " * 20
|
|
results = {
|
|
"a.json": TestOutcome("FAIL", 1.0, StepFailure(4, "query", reason)),
|
|
"b.json": TestOutcome("PASS", 1.0),
|
|
"c.json": TestOutcome("INVALID SCHEMA", 0.1),
|
|
}
|
|
lines = failure_lines(results)
|
|
assert lines[:2] == ["", "**Failures**"]
|
|
assert len(lines) == 4 # blank, header, and one bullet per non-PASS
|
|
assert reason in lines[2] # untruncated
|
|
assert "INVALID SCHEMA" in lines[3] # falls back to status when no StepFailure
|
|
|
|
|
|
def test_failure_lines_empty_when_everything_passed() -> None:
|
|
assert failure_lines({"a.json": TestOutcome("PASS", 1.0)}) == []
|