diff --git a/src/routers/sessions.py b/src/routers/sessions.py index f0d5b056..41f2fb67 100644 --- a/src/routers/sessions.py +++ b/src/routers/sessions.py @@ -248,6 +248,19 @@ def _select_summary_for_context( token_limit - short_len, ) + if short_summary or long_summary: + # A summary exists but none fits. The caller sees `summary: null`, which + # is indistinguishable from "this session has no summary", so say so. + # `token_limit` here is already net of the representation and peer card, + # which is usually why the budget is smaller than the request suggests. + logger.info( + "Summary dropped: budget %s too small (short=%s, long=%s, limit=%s)", + summary_budget, + short_len or None, + long_len or None, + token_limit, + ) + return None, 0, token_limit diff --git a/src/utils/summarizer.py b/src/utils/summarizer.py index 2abdb0f9..1bf4bd4f 100644 --- a/src/utils/summarizer.py +++ b/src/utils/summarizer.py @@ -886,12 +886,21 @@ async def get_session_context( ) messages_tokens = token_limit - latest_short_summary["token_count"] messages_start_id = latest_short_summary["message_id"] + elif latest_short_summary or latest_long_summary: + # A summary exists but does not fit the 40% allocation. The caller + # receives `summary: null`, which is indistinguishable from a session + # that has none, so this is reported rather than left at debug. + logger.info( + "Summary dropped: budget %s too small (short=%s, long=%s, limit=%s)", + summary_tokens_limit, + short_len or None, + long_len or None, + token_limit, + ) else: logger.debug( - "No summary available for get_context call with token limit %s, returning empty string. Normal if brand-new session. long_summary_len: %s, short_summary_len: %s", + "No summary for get_context with token limit %s. Normal for a new session.", token_limit, - long_len, - short_len, ) # Get recent messages after summary diff --git a/tests/routes/test_session_context_summary.py b/tests/routes/test_session_context_summary.py new file mode 100644 index 00000000..7c4f0fcc --- /dev/null +++ b/tests/routes/test_session_context_summary.py @@ -0,0 +1,167 @@ +"""How `get_context` decides whether to serve a session summary. + +Two paths, and which one runs depends on `peer_target`: + +- Without it, `summarizer.get_session_context` gives the summary 40% of the + requested `tokens`. +- With it, `sessions._select_summary_for_context` gives it 40% of what remains + *after* the peer representation and card are subtracted, so an observer with + many observations can starve a perfectly valid summary. + +Either way the caller just sees `summary: null`, indistinguishable from a +session that has none, which is why both paths now log when they drop one. +""" + +from __future__ import annotations + +import datetime as dt + +import pytest +from fastapi.testclient import TestClient +from nanoid import generate as generate_nanoid +from sqlalchemy.ext.asyncio import AsyncSession + +from src import schemas +from src.models import Peer, Workspace +from src.routers.sessions import ( + _select_summary_for_context, # pyright: ignore[reportPrivateUsage] +) +from src.utils.summarizer import ( + Summary, + SummaryType, + _save_summary, # pyright: ignore[reportPrivateUsage] +) + +# Measured from CI run 33779689337: 12 messages from one peer produce 12 +# explicit observations costing ~1176 tokens. Only the `peer_target` path pays +# this, and the unified `config_summary` fixtures do not take that path. +_FIXTURE_REPRESENTATION_TOKENS = 1176 +_SHORT_SUMMARY_CAP = 1000 # SUMMARY.MAX_TOKENS_SHORT default + + +def _summary_schema(token_count: int) -> schemas.Summary: + return schemas.Summary( + content="A summary of the conversation so far.", + message_id=1, + summary_type="short", + created_at=dt.datetime.now(dt.UTC).isoformat(), + token_count=token_count, + message_public_id="msg_public", + ) + + +def _stored_summary(token_count: int) -> Summary: + return Summary( + content="A summary of the conversation so far. " * 5, + message_id=1, + summary_type=SummaryType.SHORT.value, + created_at=dt.datetime.now(dt.UTC).isoformat(), + token_count=token_count, + message_public_id="msg_public", + ) + + +def test_representation_can_exhaust_the_budget_entirely() -> None: + """A large representation can leave a negative budget on the observer path.""" + adjusted = 400 - _FIXTURE_REPRESENTATION_TOKENS + assert adjusted < 0 + chosen, _, _ = _select_summary_for_context( + _summary_schema(99), None, adjusted, True + ) + assert chosen is None + + +def test_a_conforming_summary_can_still_be_dropped() -> None: + """With that representation, 2500 leaves 529 — under `SUMMARY.MAX_TOKENS_SHORT`.""" + adjusted = 2500 - _FIXTURE_REPRESENTATION_TOKENS + chosen, _, _ = _select_summary_for_context( + _summary_schema(_SHORT_SUMMARY_CAP), None, adjusted, True + ) + assert chosen is None + + +def test_fixture_limit_fits_any_conforming_summary() -> None: + """4000 leaves room even when a representation is subtracted.""" + adjusted = 4000 - _FIXTURE_REPRESENTATION_TOKENS + assert int(adjusted * 0.4) >= _SHORT_SUMMARY_CAP + chosen, _, _ = _select_summary_for_context( + _summary_schema(_SHORT_SUMMARY_CAP), None, adjusted, True + ) + assert chosen is not None + + +def test_zero_token_summary_is_never_served() -> None: + chosen, _, _ = _select_summary_for_context(_summary_schema(0), None, 4000, True) + assert chosen is None + + +def test_dropped_summary_is_logged_not_silent( + caplog: pytest.LogCaptureFixture, +) -> None: + """`summary: null` is indistinguishable from 'no summary exists' otherwise.""" + with caplog.at_level("INFO", logger="src.routers.sessions"): + _select_summary_for_context(_summary_schema(900), None, 1000, True) + assert "Summary dropped" in caplog.text + + +def test_no_log_when_the_session_simply_has_no_summary( + caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level("INFO", logger="src.routers.sessions"): + _select_summary_for_context(None, None, 1000, True) + assert "Summary dropped" not in caplog.text + + +@pytest.mark.parametrize("with_observer", [False, True]) +async def test_a_stored_summary_is_served( + client: TestClient, + sample_data: tuple[Workspace, Peer], + db_session: AsyncSession, + with_observer: bool, +) -> None: + """Retrieval itself works: a saved summary comes back through the route.""" + workspace, peer = sample_data + session_id = str(generate_nanoid()) + client.post( + f"/v3/workspaces/{workspace.name}/sessions", + json={"id": session_id, "peers": {peer.name: {}}}, + ) + await _save_summary(db_session, _stored_summary(99), workspace.name, session_id) + await db_session.commit() + + url = ( + f"/v3/workspaces/{workspace.name}/sessions/{session_id}/context" + "?summary=true&tokens=4000" + ) + if with_observer: + url += f"&peer_target={peer.name}" + + data = client.get(url).json() + assert data["summary"] is not None + assert data["summary"]["token_count"] == 99 + + +async def test_fixture_path_ignores_representation_budget( + client: TestClient, + sample_data: tuple[Workspace, Peer], + db_session: AsyncSession, +) -> None: + """Without `peer_target`, the summary gets 40% of `tokens` outright. + + The unified `config_summary` fixtures set `observer_peer_id`, but the runner + does not forward it to `get_context`, so this is the path they exercise. + """ + workspace, peer = sample_data + session_id = str(generate_nanoid()) + client.post( + f"/v3/workspaces/{workspace.name}/sessions", + json={"id": session_id, "peers": {peer.name: {}}}, + ) + await _save_summary(db_session, _stored_summary(99), workspace.name, session_id) + await db_session.commit() + + url = f"/v3/workspaces/{workspace.name}/sessions/{session_id}/context" + data = client.get(f"{url}?summary=true&tokens=2500").json() + + assert data["summary"] is not None + assert data.get("peer_representation") is None diff --git a/tests/unified/runner.py b/tests/unified/runner.py index c9e9d2e3..8fa0374c 100644 --- a/tests/unified/runner.py +++ b/tests/unified/runner.py @@ -5,9 +5,10 @@ import os import sys import threading import time -from datetime import datetime, timezone +from dataclasses import dataclass +from datetime import UTC, datetime from pathlib import Path -from typing import Any +from typing import Any, ClassVar import httpx from anthropic import AsyncAnthropic @@ -76,28 +77,167 @@ class TestExecutionError(Exception): pass -async def send_discord_message(webhook_url: str, message: str) -> None: - """Send a message to Discord via webhook.""" +# Discord rejects a webhook payload whose content exceeds this with a 400. +DISCORD_MAX_CONTENT = 2000 + + +def clamp_lines(lines: list[str], limit: int) -> str: + """Join `lines` within `limit`, dropping the longest ones first if needed. + + Whole lines rather than characters: a presigned URL cut in half is useless + and renders as broken markdown. Longest-first rather than last-first because + the only lines that can blow the budget are presigned URLs — dropping one of + those keeps every short, always-valid link, the Actions run link above all, + instead of losing them to a long URL that merely came first. + """ + kept = list(range(len(lines))) + + def size() -> int: + return sum(len(lines[i]) for i in kept) + max(0, len(kept) - 1) + + while kept and size() > limit: + kept.remove(max(kept, key=lambda i: len(lines[i]))) + return "\n".join(lines[i] for i in sorted(kept)) + + +async def send_discord_message(webhook_url: str, lines: list[str]) -> None: + """Send a report to Discord via webhook. + + Clamped to Discord's content limit here rather than at the call site: a + presigned URL carries an OIDC session token and can run past a thousand + characters on its own, and a 400 loses the whole notification. + """ try: async with httpx.AsyncClient() as client: - response = await client.post(webhook_url, json={"content": message}) + content = clamp_lines(lines, DISCORD_MAX_CONTENT) + response = await client.post(webhook_url, json={"content": content}) response.raise_for_status() logger.info("Discord notification sent successfully") except Exception: logger.exception("Failed to send Discord notification") +@dataclass +class StepFailure: + """Why a test stopped: the step that raised, and what it said.""" + + step_index: int + step_type: str + message: str + + def describe(self) -> str: + return f"step {self.step_index} ({self.step_type}): {self.message}" + + +@dataclass +class TestOutcome: + """One test's result. `failure` carries the reason whenever status isn't PASS.""" + + # Not a pytest case despite the name; keeps collection from warning on it. + __test__: ClassVar[bool] = False + + status: str + duration: float + failure: StepFailure | None = None + + +@dataclass +class RunArtifact: + """One uploaded file: its S3 key, and a presigned URL when one could be made.""" + + key: str + url: str | None = None + + +@dataclass +class RunArtifacts: + """Artifacts published for a run. Any field is None when its upload failed.""" + + results: RunArtifact | None = None + traces: RunArtifact | None = None + + +# 3 days. Long enough to survive a weekend before someone reads the report. +PRESIGN_EXPIRY_SECONDS = 259200 + + +def presign(s3_client: Any, bucket: str, key: str) -> RunArtifact: + """Wrap an uploaded key with a presigned URL, or just the key if signing fails.""" + try: + url: str = s3_client.generate_presigned_url( + "get_object", + Params={"Bucket": bucket, "Key": key}, + ExpiresIn=PRESIGN_EXPIRY_SECONDS, + ) + return RunArtifact(key=key, url=url) + except Exception as e: + logger.warning(f"Could not generate S3 presigned URL for {key}: {e}") + return RunArtifact(key=key) + + +def artifact_line(label: str, artifact: RunArtifact | None) -> list[str]: + """One markdown line for an artifact: a link when presigned, the key otherwise.""" + if artifact is None: + return [] + if artifact.url: + return [f"[{label}]({artifact.url}) — `{artifact.key}`"] + return [f"{label}: `{artifact.key}`"] + + +def artifact_lines(artifacts: RunArtifacts) -> list[str]: + """Both uploaded artifacts. The reasoning traces carry the full prompts and + model outputs for the run, which is what a failure usually needs to diagnose. + """ + return artifact_line("View Complete Results", artifacts.results) + artifact_line( + "Reasoning traces", artifacts.traces + ) + + +def gha_run_lines() -> list[str]: + """Link to this run's Actions page, which hosts the job summary. + + That summary carries the per-test failure reasons in full, so the Discord + message can stay short and point at it instead of restating them. + """ + run_id = os.getenv("GITHUB_RUN_ID") + repository = os.getenv("GITHUB_REPOSITORY") + if not run_id or not repository: + return [] + server = os.getenv("GITHUB_SERVER_URL", "https://github.com") + return [f"[View GHA]({server}/{repository}/actions/runs/{run_id})"] + + +def failure_lines(results: dict[str, "TestOutcome"]) -> list[str]: + """One markdown bullet per failing test, naming the step and the reason.""" + failed = [(name, o) for name, o in results.items() if o.status != "PASS"] + if not failed: + return [] + lines = ["", "**Failures**"] + for name, outcome in failed: + reason = outcome.failure.describe() if outcome.failure else outcome.status + lines.append(f"- `{name}` — {reason}") + return lines + + +def write_job_summary(lines: list[str]) -> None: + """Append a markdown block to the GitHub Actions job summary; a no-op locally.""" + summary_path = os.getenv("GITHUB_STEP_SUMMARY") + if not summary_path: + return + try: + with open(summary_path, "a", encoding="utf-8") as handle: + handle.write("\n".join(lines) + "\n") + except OSError as e: + logger.warning(f"Could not write job summary: {e}") + + async def save_results_to_s3( - results: dict[str, tuple[str, float]], + results: dict[str, TestOutcome], failed_count: int, total_count: int, execution_time: float, -) -> tuple[str | None, str | None]: - """Save comprehensive test results to S3. - - Returns: - Tuple of (presigned_url, s3_key). Either or both may be None if upload/URL generation fails. - """ +) -> RunArtifacts: + """Save comprehensive test results and reasoning traces to S3.""" try: import boto3 @@ -112,13 +252,13 @@ async def save_results_to_s3( credentials = session.get_credentials() # pyright: ignore if not credentials: logger.warning("No AWS credentials available, skipping S3 upload") - return None, None + return RunArtifacts() except Exception as e: logger.warning(f"Could not verify AWS credentials: {e}, skipping S3 upload") - return None, None + return RunArtifacts() # Create comprehensive results object - timestamp = datetime.now(timezone.utc).isoformat() + timestamp = datetime.now(UTC).isoformat() github_run_id = os.getenv("GITHUB_RUN_ID", "local") github_run_attempt = os.getenv("GITHUB_RUN_ATTEMPT", "1") github_sha = os.getenv("GITHUB_SHA", "unknown") @@ -141,16 +281,27 @@ async def save_results_to_s3( "tests": [ { "name": name, - "status": status, - "duration": duration, + "status": outcome.status, + "duration": outcome.duration, + # The reason a test failed lives only in the job log otherwise, + # where secret masking can render it unreadable. + "failure": ( + { + "step_index": outcome.failure.step_index, + "step_type": outcome.failure.step_type, + "message": outcome.failure.message, + } + if outcome.failure + else None + ), } - for name, (status, duration) in results.items() + for name, outcome in results.items() ], } # One "folder" per run: /// holding results.json plus # the reasoning-trace file(s), so a run's summary and full LLM I/O live together. - date_str = datetime.now(timezone.utc).strftime("%Y-%m-%d") + date_str = datetime.now(UTC).strftime("%Y-%m-%d") sha_short = github_sha[:7] if github_sha != "unknown" else "unknown" ref_slug = github_ref.replace("/", "-") # branch names may contain "/" run_slug = f"{ref_slug}-{sha_short}-{github_run_id}-{github_run_attempt}" @@ -169,6 +320,7 @@ async def save_results_to_s3( # Upload the reasoning traces (full LLM/deriver I/O) captured this run. The # API and deriver both append to REASONING_TRACES_FILE (file-locked). Use # upload_file so large trace files stream via multipart instead of buffering. + traces: RunArtifact | None = None traces_path_str = os.getenv("REASONING_TRACES_FILE") if traces_path_str: traces_path = Path(traces_path_str) @@ -182,6 +334,7 @@ async def save_results_to_s3( ExtraArgs={"ContentType": "application/x-ndjson"}, ) logger.info(f"Saved reasoning traces to S3 key {traces_key}") + traces = presign(s3_client, s3_bucket, traces_key) except Exception as e: logger.error( f"Failed to upload reasoning traces: {e}", exc_info=True @@ -191,20 +344,13 @@ async def save_results_to_s3( f"REASONING_TRACES_FILE={traces_path} is missing or empty; no traces uploaded" ) - try: - url: str = s3_client.generate_presigned_url( # pyright: ignore - "get_object", - Params={"Bucket": s3_bucket, "Key": results_key}, - ExpiresIn=259200, # 3 days - ) - return url, results_key # pyright: ignore - except Exception as e: - logger.warning(f"Could not generate S3 presigned URL: {e}") - return None, results_key + return RunArtifacts( + results=presign(s3_client, s3_bucket, results_key), traces=traces + ) except Exception as e: logger.error(f"Failed to save results to S3: {e}", exc_info=True) - return None, None + return RunArtifacts() class UnifiedTestExecutor: @@ -248,7 +394,10 @@ class UnifiedTestExecutor: ) return response - async def execute(self, test_def: TestDefinition, test_name: str) -> bool: + async def execute( + self, test_def: TestDefinition, test_name: str + ) -> StepFailure | None: + """Run every step. Returns None on success, or the failure that stopped it.""" logger.info(f"Starting test: {test_name}") # 1. Apply workspace config if present @@ -264,10 +413,12 @@ class UnifiedTestExecutor: await self.execute_step(step) except Exception as e: logger.error(f"Step {i + 1} failed: {e}", exc_info=False) - return False + return StepFailure( + step_index=i + 1, step_type=step.step_type, message=str(e) + ) logger.info(f"Test {test_name} PASSED") - return True + return None async def execute_step(self, step: Any): if isinstance(step, SetWorkspaceConfigAction): @@ -357,7 +508,9 @@ class UnifiedTestExecutor: if step.duration: await asyncio.sleep(step.duration) if step.target == "queue_empty": - # Flush mode is enabled by default in the harness (DERIVER_FLUSH_ENABLED=true) + # Flush is process-wide, not per-step: the harness starts the + # deriver with DERIVER_FLUSH_ENABLED=true so batches never wait + # on the token threshold. See tests/bench/harness.py. await self.wait_for_queue(step.timeout) elif isinstance(step, ScheduleDreamAction): @@ -692,7 +845,7 @@ class UnifiedTestRunner: raise ValueError("tests_dir must be set if test_file is not") test_files = sorted(list(self.tests_dir.glob("*.json"))) - results: dict[str, tuple[str, float]] = {} + results: dict[str, TestOutcome] = {} logger.info(f"Found {len(test_files)} test(s)") @@ -721,23 +874,28 @@ class UnifiedTestRunner: workspace_id=f"test_{test_name}_{int(time.time())}", ) - success = await executor.execute(test_def, test_name) + failure = await executor.execute(test_def, test_name) test_duration = time.time() - test_start_time - results[test_file.name] = ( - "PASS" if success else "FAIL", - test_duration, + results[test_file.name] = TestOutcome( + status="PASS" if failure is None else "FAIL", + duration=test_duration, + failure=failure, ) except ValidationError as e: logger.error(f"Schema validation failed for {test_file}: {e}") test_duration = time.time() - test_start_time - results[test_file.name] = ("INVALID SCHEMA", test_duration) + results[test_file.name] = TestOutcome( + status="INVALID SCHEMA", duration=test_duration + ) except Exception as e: logger.error( f"Test {test_file.name} failed with error: {e}", exc_info=True ) test_duration = time.time() - test_start_time - results[test_file.name] = (f"ERROR: {str(e)}", test_duration) + results[test_file.name] = TestOutcome( + status=f"ERROR: {str(e)}", duration=test_duration + ) total_suite_time = time.time() - suite_start_time @@ -752,16 +910,18 @@ class UnifiedTestRunner: # Calculate max name length for alignment max_name_length = max(len(name) for name in results) if results else 0 - for name, (status, duration) in results.items(): - duration_str = f"({duration:.2f}s)" - if status == "PASS": + for name, outcome in results.items(): + duration_str = f"({outcome.duration:.2f}s)" + if outcome.status == "PASS": print( - f"{name:<{max_name_length}} {GREEN}{status:<15}{RESET} {duration_str}" + f"{name:<{max_name_length}} {GREEN}{outcome.status:<15}{RESET} {duration_str}" ) else: print( - f"{name:<{max_name_length}} {RED}{status:<15}{RESET} {duration_str}" + f"{name:<{max_name_length}} {RED}{outcome.status:<15}{RESET} {duration_str}" ) + if outcome.failure: + print(f"{'':<{max_name_length}} {outcome.failure.describe()}") failed_count += 1 print("=" * 60) @@ -771,30 +931,46 @@ class UnifiedTestRunner: # 5. Save results and send notifications # Always attempt S3 upload - save_results_to_s3 will check for credentials - url: str | None - s3_key: str | None - url, s3_key = await save_results_to_s3( + artifacts = await save_results_to_s3( results, failed_count, total_count, total_suite_time ) - # 6. Send Discord notification + # 6. Report the run: GitHub job summary, then Discord. + passed_count = total_count - failed_count + status_emoji = "✅" if failed_count == 0 else "⚠️" + headline = ( + f"Results: {passed_count}/{total_count} passed, " + f"{failed_count}/{total_count} failed" + ) + + write_job_summary( + [ + f"## {status_emoji} Unified Test Results", + "", + headline, + "", + f"Execution time: {total_suite_time:.2f}s", + *failure_lines(results), + "", + *artifact_lines(artifacts), + ] + ) + discord_webhook_url = os.getenv("TEST_DISCORD_WEBHOOK_URL") if discord_webhook_url: - passed_count = total_count - failed_count - status_emoji = "✅" if failed_count == 0 else "⚠️" - message_lines = [ f"{status_emoji} **Unified Test Results**", - f"Results: {passed_count}/{total_count} passed, {failed_count}/{total_count} failed", + headline, f"Execution time: {total_suite_time:.2f}s", + *artifact_line("View Complete Results", artifacts.results), + *gha_run_lines(), + *( + [f"Reasoning traces: `{artifacts.traces.key}`"] + if artifacts.traces + else [] + ), ] - if s3_key: - message_lines.append(f"File: `{s3_key}`") - if url: - message_lines.append(f"[View Complete Results]({url})") - message = "\n".join(message_lines) - - await send_discord_message(discord_webhook_url, message) + await send_discord_message(discord_webhook_url, message_lines) return failed_count diff --git a/tests/unified/schema.py b/tests/unified/schema.py index 31e30946..bb54da71 100644 --- a/tests/unified/schema.py +++ b/tests/unified/schema.py @@ -1,7 +1,7 @@ import datetime from typing import Annotated, Any, Literal -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field from src.config import ReasoningLevel from src.schemas import ( @@ -14,6 +14,8 @@ from src.schemas import ( class TestStep(BaseModel): + model_config = ConfigDict(extra="forbid") # pyright: ignore + description: str | None = None @@ -89,10 +91,6 @@ class WaitAction(TestStep): ) target: Literal["queue_empty"] = "queue_empty" timeout: int = 60 - flush: bool = Field( - False, - description="Enable flush mode to bypass batch token threshold before waiting", - ) # --- Dream Actions --- diff --git a/tests/unified/test_cases/config_deriver_hierarchy.json b/tests/unified/test_cases/config_deriver_hierarchy.json index 1ab883db..8eb7bc9d 100644 --- a/tests/unified/test_cases/config_deriver_hierarchy.json +++ b/tests/unified/test_cases/config_deriver_hierarchy.json @@ -34,8 +34,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", @@ -86,8 +85,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", diff --git a/tests/unified/test_cases/config_message_positive_override.json b/tests/unified/test_cases/config_message_positive_override.json index 911f21b5..a9a46623 100644 --- a/tests/unified/test_cases/config_message_positive_override.json +++ b/tests/unified/test_cases/config_message_positive_override.json @@ -36,8 +36,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", diff --git a/tests/unified/test_cases/config_peercard_control.json b/tests/unified/test_cases/config_peercard_control.json index 11db312a..04145446 100644 --- a/tests/unified/test_cases/config_peercard_control.json +++ b/tests/unified/test_cases/config_peercard_control.json @@ -38,8 +38,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", diff --git a/tests/unified/test_cases/config_summary_control.json b/tests/unified/test_cases/config_summary_control.json index f71cdeee..69eb9ffd 100644 --- a/tests/unified/test_cases/config_summary_control.json +++ b/tests/unified/test_cases/config_summary_control.json @@ -77,16 +77,15 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", "target": "get_context", "session_id": "session_summary", "summary": true, - "max_tokens": 400, - "observer_peer_id": "eve", + "max_tokens": 4000, + "description": "Unscoped read, so the summary gets 40% of max_tokens. Naming an observer would route through the peer_target path, where the representation and peer card are subtracted from the budget first.", "assertions": [ { "assertion_type": "llm_judge", diff --git a/tests/unified/test_cases/config_summary_control_deriver_off.json b/tests/unified/test_cases/config_summary_control_deriver_off.json index 7a788990..32642457 100644 --- a/tests/unified/test_cases/config_summary_control_deriver_off.json +++ b/tests/unified/test_cases/config_summary_control_deriver_off.json @@ -76,16 +76,15 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", "target": "get_context", "session_id": "session_summary", "summary": true, - "max_tokens": 400, - "observer_peer_id": "eve", + "max_tokens": 4000, + "description": "Unscoped read, so the summary gets 40% of max_tokens. Naming an observer would route through the peer_target path, where the representation and peer card are subtracted from the budget first.", "assertions": [ { "assertion_type": "llm_judge", diff --git a/tests/unified/test_cases/dialectic_reasoning_levels.json b/tests/unified/test_cases/dialectic_reasoning_levels.json index 509e0c39..13cf9780 100644 --- a/tests/unified/test_cases/dialectic_reasoning_levels.json +++ b/tests/unified/test_cases/dialectic_reasoning_levels.json @@ -45,8 +45,7 @@ { "step_type": "wait", "target": "queue_empty", - "timeout": 120, - "flush": true + "timeout": 120 }, { "step_type": "query", diff --git a/tests/unified/test_cases/dialectic_structured_output.json b/tests/unified/test_cases/dialectic_structured_output.json index 59cc1365..bfcb094e 100644 --- a/tests/unified/test_cases/dialectic_structured_output.json +++ b/tests/unified/test_cases/dialectic_structured_output.json @@ -80,8 +80,7 @@ { "step_type": "wait", "target": "queue_empty", - "timeout": 180, - "flush": true + "timeout": 180 }, { "step_type": "query", diff --git a/tests/unified/test_cases/dialectic_tool_calls.json b/tests/unified/test_cases/dialectic_tool_calls.json index 0ae3b923..77845a11 100644 --- a/tests/unified/test_cases/dialectic_tool_calls.json +++ b/tests/unified/test_cases/dialectic_tool_calls.json @@ -80,8 +80,7 @@ { "step_type": "wait", "target": "queue_empty", - "timeout": 180, - "flush": true + "timeout": 180 }, { "step_type": "query", diff --git a/tests/unified/test_cases/dream_knowledge_updates_and_patterns.json b/tests/unified/test_cases/dream_knowledge_updates_and_patterns.json index 7a6f1764..28e262b7 100644 --- a/tests/unified/test_cases/dream_knowledge_updates_and_patterns.json +++ b/tests/unified/test_cases/dream_knowledge_updates_and_patterns.json @@ -44,8 +44,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "add_messages", @@ -75,8 +74,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "add_messages", @@ -114,8 +112,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "schedule_dream", @@ -127,8 +124,7 @@ { "step_type": "wait", "target": "queue_empty", - "timeout": 180, - "flush": true + "timeout": 180 }, { "step_type": "query", @@ -148,7 +144,6 @@ "target": "get_representation", "observer_peer_id": "assistant", "observed_peer_id": "maya", - "session_id": "maya_life_story", "assertions": [ { "assertion_type": "llm_judge", diff --git a/tests/unified/test_cases/longmem_ancash.json b/tests/unified/test_cases/longmem_ancash.json index ef455eb8..52ed0523 100644 --- a/tests/unified/test_cases/longmem_ancash.json +++ b/tests/unified/test_cases/longmem_ancash.json @@ -75,8 +75,7 @@ { "step_type": "wait", "target": "queue_empty", - "timeout": 180, - "flush": true + "timeout": 180 }, { "step_type": "query", diff --git a/tests/unified/test_cases/longmem_ancash_directional.json b/tests/unified/test_cases/longmem_ancash_directional.json index 2871a9a0..b27adf04 100644 --- a/tests/unified/test_cases/longmem_ancash_directional.json +++ b/tests/unified/test_cases/longmem_ancash_directional.json @@ -74,8 +74,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", diff --git a/tests/unified/test_cases/longmem_ancash_no_session.json b/tests/unified/test_cases/longmem_ancash_no_session.json index b0462d4a..97c7292c 100644 --- a/tests/unified/test_cases/longmem_ancash_no_session.json +++ b/tests/unified/test_cases/longmem_ancash_no_session.json @@ -74,8 +74,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", diff --git a/tests/unified/test_cases/longmem_giftcard.json b/tests/unified/test_cases/longmem_giftcard.json index 6ef43463..e1fa8fe6 100644 --- a/tests/unified/test_cases/longmem_giftcard.json +++ b/tests/unified/test_cases/longmem_giftcard.json @@ -3664,7 +3664,7 @@ { "step_type": "wait", "target": "queue_empty", - "flush": true + "timeout": 600 }, { "step_type": "query", diff --git a/tests/unified/test_cases/longmem_plank.json b/tests/unified/test_cases/longmem_plank.json index bbdcd765..a9426bbc 100644 --- a/tests/unified/test_cases/longmem_plank.json +++ b/tests/unified/test_cases/longmem_plank.json @@ -154,8 +154,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", diff --git a/tests/unified/test_cases/longmem_triple_7161e7e2_single-session-assistant.json b/tests/unified/test_cases/longmem_triple_7161e7e2_single-session-assistant.json index 1564b289..c9ed3d39 100644 --- a/tests/unified/test_cases/longmem_triple_7161e7e2_single-session-assistant.json +++ b/tests/unified/test_cases/longmem_triple_7161e7e2_single-session-assistant.json @@ -3719,7 +3719,7 @@ { "step_type": "wait", "target": "queue_empty", - "flush": true + "timeout": 600 }, { "step_type": "query", diff --git a/tests/unified/test_cases/longmem_triple_e47becba_single-session-user.json b/tests/unified/test_cases/longmem_triple_e47becba_single-session-user.json index 48911d89..01165256 100644 --- a/tests/unified/test_cases/longmem_triple_e47becba_single-session-user.json +++ b/tests/unified/test_cases/longmem_triple_e47becba_single-session-user.json @@ -3833,8 +3833,7 @@ { "step_type": "wait", "target": "queue_empty", - "timeout": 600, - "flush": true + "timeout": 600 }, { "step_type": "query", diff --git a/tests/unified/test_cases/longmem_triple_gpt4_59149c77_temporal-reasoning.json b/tests/unified/test_cases/longmem_triple_gpt4_59149c77_temporal-reasoning.json index ded356dd..427f6b6d 100644 --- a/tests/unified/test_cases/longmem_triple_gpt4_59149c77_temporal-reasoning.json +++ b/tests/unified/test_cases/longmem_triple_gpt4_59149c77_temporal-reasoning.json @@ -3431,7 +3431,7 @@ { "step_type": "wait", "target": "queue_empty", - "flush": true + "timeout": 600 }, { "step_type": "query", diff --git a/tests/unified/test_cases/message_deriver_disabled.json b/tests/unified/test_cases/message_deriver_disabled.json index 315c0995..fa6937c9 100644 --- a/tests/unified/test_cases/message_deriver_disabled.json +++ b/tests/unified/test_cases/message_deriver_disabled.json @@ -40,8 +40,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", diff --git a/tests/unified/test_cases/observation_2peer_bidirectional.json b/tests/unified/test_cases/observation_2peer_bidirectional.json index 17d85a68..ce453889 100644 --- a/tests/unified/test_cases/observation_2peer_bidirectional.json +++ b/tests/unified/test_cases/observation_2peer_bidirectional.json @@ -40,8 +40,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", diff --git a/tests/unified/test_cases/observation_2peer_both_observe_me_false.json b/tests/unified/test_cases/observation_2peer_both_observe_me_false.json index 5e33adb2..c12c2e50 100644 --- a/tests/unified/test_cases/observation_2peer_both_observe_me_false.json +++ b/tests/unified/test_cases/observation_2peer_both_observe_me_false.json @@ -40,8 +40,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", diff --git a/tests/unified/test_cases/observation_2peer_default.json b/tests/unified/test_cases/observation_2peer_default.json index ba771615..debd86a9 100644 --- a/tests/unified/test_cases/observation_2peer_default.json +++ b/tests/unified/test_cases/observation_2peer_default.json @@ -40,8 +40,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", diff --git a/tests/unified/test_cases/observation_2peer_observe_me_false_blocks_observation.json b/tests/unified/test_cases/observation_2peer_observe_me_false_blocks_observation.json index f153c5ea..cf1f7c80 100644 --- a/tests/unified/test_cases/observation_2peer_observe_me_false_blocks_observation.json +++ b/tests/unified/test_cases/observation_2peer_observe_me_false_blocks_observation.json @@ -40,8 +40,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", diff --git a/tests/unified/test_cases/observation_2peer_observe_me_false_but_can_still_observe_others.json b/tests/unified/test_cases/observation_2peer_observe_me_false_but_can_still_observe_others.json index a5c3de23..b5e606aa 100644 --- a/tests/unified/test_cases/observation_2peer_observe_me_false_but_can_still_observe_others.json +++ b/tests/unified/test_cases/observation_2peer_observe_me_false_but_can_still_observe_others.json @@ -40,8 +40,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", diff --git a/tests/unified/test_cases/observation_2peer_unidirectional_alice_observes_bob.json b/tests/unified/test_cases/observation_2peer_unidirectional_alice_observes_bob.json index 944efacb..7c1c8703 100644 --- a/tests/unified/test_cases/observation_2peer_unidirectional_alice_observes_bob.json +++ b/tests/unified/test_cases/observation_2peer_unidirectional_alice_observes_bob.json @@ -40,8 +40,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", diff --git a/tests/unified/test_cases/observation_2peer_unidirectional_bob_observes_alice.json b/tests/unified/test_cases/observation_2peer_unidirectional_bob_observes_alice.json index 99538fa4..c7d411ba 100644 --- a/tests/unified/test_cases/observation_2peer_unidirectional_bob_observes_alice.json +++ b/tests/unified/test_cases/observation_2peer_unidirectional_bob_observes_alice.json @@ -40,8 +40,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", diff --git a/tests/unified/test_cases/observation_3peer_all_observe_each_other.json b/tests/unified/test_cases/observation_3peer_all_observe_each_other.json index e201309c..fa6cae29 100644 --- a/tests/unified/test_cases/observation_3peer_all_observe_each_other.json +++ b/tests/unified/test_cases/observation_3peer_all_observe_each_other.json @@ -52,8 +52,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", diff --git a/tests/unified/test_cases/observation_3peer_circular.json b/tests/unified/test_cases/observation_3peer_circular.json index 4ae12689..f7c95ddf 100644 --- a/tests/unified/test_cases/observation_3peer_circular.json +++ b/tests/unified/test_cases/observation_3peer_circular.json @@ -52,8 +52,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", diff --git a/tests/unified/test_cases/observation_3peer_multiple_observers_one_observed.json b/tests/unified/test_cases/observation_3peer_multiple_observers_one_observed.json index 8449df1f..08768e84 100644 --- a/tests/unified/test_cases/observation_3peer_multiple_observers_one_observed.json +++ b/tests/unified/test_cases/observation_3peer_multiple_observers_one_observed.json @@ -48,8 +48,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", @@ -130,8 +129,7 @@ { "step_type": "wait", "target": "queue_empty", - "timeout": 180, - "flush": true + "timeout": 180 }, { "step_type": "query", diff --git a/tests/unified/test_cases/observation_3peer_one_observer_multiple_observed.json b/tests/unified/test_cases/observation_3peer_one_observer_multiple_observed.json index e329b4bc..9a2d060b 100644 --- a/tests/unified/test_cases/observation_3peer_one_observer_multiple_observed.json +++ b/tests/unified/test_cases/observation_3peer_one_observer_multiple_observed.json @@ -48,8 +48,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", diff --git a/tests/unified/test_cases/observation_3peer_selective_observation.json b/tests/unified/test_cases/observation_3peer_selective_observation.json index b0c08325..f39fef80 100644 --- a/tests/unified/test_cases/observation_3peer_selective_observation.json +++ b/tests/unified/test_cases/observation_3peer_selective_observation.json @@ -48,8 +48,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", diff --git a/tests/unified/test_cases/observation_4peer_complex_matrix.json b/tests/unified/test_cases/observation_4peer_complex_matrix.json index 23a4c70b..17d723a2 100644 --- a/tests/unified/test_cases/observation_4peer_complex_matrix.json +++ b/tests/unified/test_cases/observation_4peer_complex_matrix.json @@ -52,8 +52,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", diff --git a/tests/unified/test_cases/observation_asymmetric_visibility.json b/tests/unified/test_cases/observation_asymmetric_visibility.json index 24710477..46acf71a 100644 --- a/tests/unified/test_cases/observation_asymmetric_visibility.json +++ b/tests/unified/test_cases/observation_asymmetric_visibility.json @@ -40,8 +40,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "create_session", @@ -77,8 +76,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", diff --git a/tests/unified/test_cases/observation_isolation_between_sessions.json b/tests/unified/test_cases/observation_isolation_between_sessions.json index 6ed6c0eb..dddb8eab 100644 --- a/tests/unified/test_cases/observation_isolation_between_sessions.json +++ b/tests/unified/test_cases/observation_isolation_between_sessions.json @@ -40,8 +40,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "create_session", @@ -77,8 +76,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", diff --git a/tests/unified/test_cases/peer_isolation_test.json b/tests/unified/test_cases/peer_isolation_test.json index 2c87fced..54bc3e8e 100644 --- a/tests/unified/test_cases/peer_isolation_test.json +++ b/tests/unified/test_cases/peer_isolation_test.json @@ -48,8 +48,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "create_session", @@ -81,8 +80,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", diff --git a/tests/unified/test_cases/scope_confines_recall.json b/tests/unified/test_cases/scope_confines_recall.json index 77589ca8..725cd376 100644 --- a/tests/unified/test_cases/scope_confines_recall.json +++ b/tests/unified/test_cases/scope_confines_recall.json @@ -55,8 +55,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", diff --git a/tests/unified/test_cases/session_deriver_disabled.json b/tests/unified/test_cases/session_deriver_disabled.json index 2c613fb7..0af60379 100644 --- a/tests/unified/test_cases/session_deriver_disabled.json +++ b/tests/unified/test_cases/session_deriver_disabled.json @@ -30,8 +30,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", diff --git a/tests/unified/test_cases/workspace_chat_cross_peer.json b/tests/unified/test_cases/workspace_chat_cross_peer.json index 3d474c1d..a611948e 100644 --- a/tests/unified/test_cases/workspace_chat_cross_peer.json +++ b/tests/unified/test_cases/workspace_chat_cross_peer.json @@ -68,8 +68,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", diff --git a/tests/unified/test_cases/workspace_chat_from_observations.json b/tests/unified/test_cases/workspace_chat_from_observations.json index f40eae04..8142317b 100644 --- a/tests/unified/test_cases/workspace_chat_from_observations.json +++ b/tests/unified/test_cases/workspace_chat_from_observations.json @@ -52,8 +52,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", diff --git a/tests/unified/test_cases/workspace_deriver_disabled.json b/tests/unified/test_cases/workspace_deriver_disabled.json index ce1a054c..b30e9953 100644 --- a/tests/unified/test_cases/workspace_deriver_disabled.json +++ b/tests/unified/test_cases/workspace_deriver_disabled.json @@ -30,8 +30,7 @@ }, { "step_type": "wait", - "target": "queue_empty", - "flush": true + "target": "queue_empty" }, { "step_type": "query", diff --git a/tests/unified/test_reporting.py b/tests/unified/test_reporting.py new file mode 100644 index 00000000..48734232 --- /dev/null +++ b/tests/unified/test_reporting.py @@ -0,0 +1,143 @@ +"""Tests for how a unified run is reported to Discord and the job summary. + +Discord rejects an over-long payload with a 400, which loses the whole +notification, so the size behavior here is worth pinning down. +""" + +from __future__ import annotations + +import pytest + +from tests.unified.runner import ( + DISCORD_MAX_CONTENT, + RunArtifact, + RunArtifacts, + StepFailure, + TestOutcome, + artifact_line, + artifact_lines, + clamp_lines, + failure_lines, + gha_run_lines, +) + +_PREFIX = "unified-test-results/2026-09-03/1123-merge-abc1234-33779689337-1" + + +def _presigned(name: str, token_len: int) -> RunArtifact: + """A presigned URL of realistic shape; OIDC session tokens dominate its length.""" + url = ( + f"https://honcho-unified-tests.s3.amazonaws.com/{_PREFIX}/{name}" + "?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Expires=259200" + f"&X-Amz-Security-Token={'t' * token_len}&X-Amz-Signature={'0' * 64}" + ) + return RunArtifact(key=f"{_PREFIX}/{name}", url=url) + + +def _discord_lines(artifacts: RunArtifacts) -> list[str]: + """Mirror of the Discord report the runner assembles.""" + return [ + "⚠️ **Unified Test Results**", + "Results: 35/41 passed, 6/41 failed", + "Execution time: 1015.42s", + *artifact_line("View Complete Results", artifacts.results), + *gha_run_lines(), + *([f"Reasoning traces: `{artifacts.traces.key}`"] if artifacts.traces else []), + ] + + +@pytest.fixture +def in_actions(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GITHUB_RUN_ID", "33779689337") + monkeypatch.setenv("GITHUB_REPOSITORY", "plastic-labs/honcho") + + +def test_clamp_lines_leaves_a_short_report_alone() -> None: + lines = ["one", "two", "three"] + assert clamp_lines(lines, DISCORD_MAX_CONTENT) == "one\ntwo\nthree" + + +def test_clamp_lines_drops_the_longest_line_not_the_last() -> None: + """The Actions link is short and leads everywhere; a presigned URL is neither.""" + lines = ["head", "x" * 100, "[View GHA](url)"] + assert clamp_lines(lines, 40) == "head\n[View GHA](url)" + + +def test_clamp_lines_preserves_display_order() -> None: + lines = ["a", "y" * 50, "b", "c"] + assert clamp_lines(lines, 10) == "a\nb\nc" + + +@pytest.mark.usefixtures("in_actions") +@pytest.mark.parametrize("token_len", [0, 400, 900, 1400, 1800]) +def test_discord_report_never_exceeds_the_webhook_limit(token_len: int) -> None: + """Regression: six judge verdicts plus two presigned URLs returned a 400.""" + artifacts = RunArtifacts( + results=_presigned("results.json", token_len), + traces=_presigned("unified-reasoning-traces.jsonl", token_len), + ) + sent = clamp_lines(_discord_lines(artifacts), DISCORD_MAX_CONTENT) + assert len(sent) <= DISCORD_MAX_CONTENT + + +@pytest.mark.usefixtures("in_actions") +@pytest.mark.parametrize("token_len", [0, 400, 900, 1400, 1800]) +def test_actions_link_always_survives_clamping(token_len: int) -> None: + """However long the presigned URLs get, the run stays reachable.""" + artifacts = RunArtifacts( + results=_presigned("results.json", token_len), + traces=_presigned("unified-reasoning-traces.jsonl", token_len), + ) + sent = clamp_lines(_discord_lines(artifacts), DISCORD_MAX_CONTENT) + assert ( + "[View GHA](https://github.com/plastic-labs/honcho/actions/runs/33779689337)" + in sent + ) + + +@pytest.mark.usefixtures("in_actions") +def test_discord_report_omits_per_test_failures() -> None: + """Failure detail belongs in the job summary the Actions link points at.""" + artifacts = RunArtifacts(results=_presigned("results.json", 400)) + assert not any("**Failures**" in line for line in _discord_lines(artifacts)) + + +def test_gha_lines_are_empty_outside_actions(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("GITHUB_RUN_ID", raising=False) + monkeypatch.delenv("GITHUB_REPOSITORY", raising=False) + assert gha_run_lines() == [] + + +def test_artifact_line_falls_back_to_the_key_when_presigning_failed() -> None: + assert artifact_line("Traces", RunArtifact(key="k/x.jsonl")) == [ + "Traces: `k/x.jsonl`" + ] + assert artifact_line("Traces", None) == [] + + +def test_job_summary_keeps_both_signed_links() -> None: + artifacts = RunArtifacts( + results=_presigned("results.json", 900), + traces=_presigned("unified-reasoning-traces.jsonl", 900), + ) + lines = artifact_lines(artifacts) + assert len(lines) == 2 + assert all("https://" in line for line in lines) + + +def test_failure_lines_reports_every_failure_in_full() -> None: + reason = "LLM Judge failed: " + "the model did not recall the fact. " * 20 + results = { + "a.json": TestOutcome("FAIL", 1.0, StepFailure(4, "query", reason)), + "b.json": TestOutcome("PASS", 1.0), + "c.json": TestOutcome("INVALID SCHEMA", 0.1), + } + lines = failure_lines(results) + assert lines[:2] == ["", "**Failures**"] + assert len(lines) == 4 # blank, header, and one bullet per non-PASS + assert reason in lines[2] # untruncated + assert "INVALID SCHEMA" in lines[3] # falls back to status when no StepFailure + + +def test_failure_lines_empty_when_everything_passed() -> None: + assert failure_lines({"a.json": TestOutcome("PASS", 1.0)}) == []