837 lines
39 KiB
Python
837 lines
39 KiB
Python
"""Tests for agent interaction features: agents, questionnaires, web console."""
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from app.adapters.llm.base import LLMTask
|
|
from app.adapters.llm.mock import MockLLMProvider
|
|
from app.agent_engine.cli import build_parser
|
|
from app.agent_engine.contracts import (
|
|
AGENT_QUESTION_OUTPUT_SCHEMA,
|
|
AGENT_QUESTIONNAIRE_OUTPUT_SCHEMA,
|
|
QUESTIONNAIRE_SUMMARY_OUTPUT_SCHEMA,
|
|
REPORT_QUESTION_OUTPUT_SCHEMA,
|
|
TASK_OUTPUT_SCHEMAS,
|
|
)
|
|
from app.agent_engine.json_schema import validate_json_schema
|
|
from app.agent_engine.queue import AgentQueue
|
|
from app.agent_engine.runner import PredictionRunService, _resolve_agent_id, _resolve_all_agent_ids
|
|
from app.agent_engine.schemas import AGENT_TASK_TYPES
|
|
from app.agent_engine.state import RunStore
|
|
|
|
|
|
# ── Fixtures ─────────────────────────────────────────────────────────────
|
|
|
|
def _init_run(tmp_path: Path, *, seed_text: str = "A affects B.") -> tuple[Path, PredictionRunService]:
|
|
"""Create a run directory with seed and profiles artifact."""
|
|
seed = tmp_path / "seed.md"
|
|
seed.write_text(seed_text, encoding="utf-8")
|
|
run_dir = tmp_path / "run"
|
|
service = PredictionRunService()
|
|
service.create_run(str(seed), "test interaction", str(run_dir))
|
|
# Write profiles.json artifact so agent methods work
|
|
profiles = [
|
|
{"agent_id": "agent_1", "name": "Analyst Alpha", "persona": "Cautious geopolitical analyst."},
|
|
{"agent_id": "agent_2", "name": "Strategist Beta", "persona": "Optimistic tech strategist."},
|
|
]
|
|
artifacts_dir = run_dir / "artifacts"
|
|
artifacts_dir.mkdir(parents=True, exist_ok=True)
|
|
(artifacts_dir / "profiles.json").write_text(json.dumps(profiles, ensure_ascii=False), encoding="utf-8")
|
|
# Write report.md
|
|
(artifacts_dir / "report.md").write_text("# Test Report\n\nThis is a test report.", encoding="utf-8")
|
|
# Write verdict.json
|
|
(artifacts_dir / "verdict.json").write_text(json.dumps({"status": "ok", "confidence": 0.7}), encoding="utf-8")
|
|
# Write timeline.json
|
|
(artifacts_dir / "timeline.json").write_text(json.dumps([{"round": 1, "summary": "Initial"}]), encoding="utf-8")
|
|
# Write graph_snapshot.json
|
|
(artifacts_dir / "graph_snapshot.json").write_text(json.dumps([]), encoding="utf-8")
|
|
# Write simulation_config.json
|
|
(artifacts_dir / "simulation_config.json").write_text(json.dumps({"rounds": 10}), encoding="utf-8")
|
|
# Write simulation_actions.json
|
|
(artifacts_dir / "simulation_actions.json").write_text(json.dumps([]), encoding="utf-8")
|
|
return run_dir, service
|
|
|
|
|
|
# ── list_agents tests ────────────────────────────────────────────────────
|
|
|
|
class TestListAgents:
|
|
def test_list_agents_reads_profiles(self, tmp_path):
|
|
run_dir, service = _init_run(tmp_path)
|
|
result = service.list_agents(str(run_dir))
|
|
assert result["status"] == "ok"
|
|
assert result["count"] == 2
|
|
agent_ids = [a["agent_id"] for a in result["agents"]]
|
|
assert "agent_1" in agent_ids
|
|
assert "agent_2" in agent_ids
|
|
|
|
def test_list_agents_empty_when_no_profiles(self, tmp_path):
|
|
seed = tmp_path / "seed.md"
|
|
seed.write_text("A.", encoding="utf-8")
|
|
run_dir = tmp_path / "empty-run"
|
|
service = PredictionRunService()
|
|
service.create_run(str(seed), "test empty", str(run_dir))
|
|
result = service.list_agents(str(run_dir))
|
|
assert result["status"] == "ok"
|
|
assert result["count"] == 0
|
|
|
|
def test_get_agent_returns_profile(self, tmp_path):
|
|
run_dir, service = _init_run(tmp_path)
|
|
result = service.get_agent(str(run_dir), "agent_1")
|
|
assert result["status"] == "ok"
|
|
assert result["agent"]["agent_id"] == "agent_1"
|
|
assert result["agent"]["name"] == "Analyst Alpha"
|
|
|
|
def test_get_agent_not_found(self, tmp_path):
|
|
run_dir, service = _init_run(tmp_path)
|
|
result = service.get_agent(str(run_dir), "nonexistent")
|
|
assert result["status"] == "error"
|
|
assert "not found" in result["error"]
|
|
|
|
|
|
# ── ask_agent tests ──────────────────────────────────────────────────────
|
|
|
|
class TestAskAgent:
|
|
def test_ask_agent_creates_agent_queue_request(self, tmp_path, monkeypatch):
|
|
monkeypatch.setenv("MIROFISH_MODE", "agent")
|
|
monkeypatch.setenv("MIROFISH_LLM_PROVIDER", "agent_queue")
|
|
monkeypatch.setenv("MIROFISH_GRAPH_PROVIDER", "graphiti")
|
|
monkeypatch.setenv("MIROFISH_GRAPHITI_STORE", "file")
|
|
monkeypatch.setenv("MIROFISH_GRAPHITI_COMPAT_PATH", str(tmp_path / "graph_store.json"))
|
|
|
|
run_dir, service = _init_run(tmp_path)
|
|
result = service.ask_agent(str(run_dir), "agent_1", "What are the implications?")
|
|
assert result["status"] == "need_agent_response"
|
|
assert result["type"] == "answer_agent_question"
|
|
assert result["agent_id"] == "agent_1"
|
|
|
|
# Verify the request was created with correct structure
|
|
request = AgentQueue(run_dir).load_request(result["request_id"])
|
|
assert request.type == "answer_agent_question"
|
|
assert request.stage == "interaction"
|
|
assert request.structured_input["agent_id"] == "agent_1"
|
|
assert request.structured_input["question"] == "What are the implications?"
|
|
|
|
def test_ask_agent_nonexistent_returns_error(self, tmp_path, monkeypatch):
|
|
monkeypatch.setenv("MIROFISH_MODE", "agent")
|
|
monkeypatch.setenv("MIROFISH_LLM_PROVIDER", "agent_queue")
|
|
monkeypatch.setenv("MIROFISH_GRAPH_PROVIDER", "graphiti")
|
|
monkeypatch.setenv("MIROFISH_GRAPHITI_STORE", "file")
|
|
monkeypatch.setenv("MIROFISH_GRAPHITI_COMPAT_PATH", str(tmp_path / "graph_store.json"))
|
|
|
|
run_dir, service = _init_run(tmp_path)
|
|
result = service.ask_agent(str(run_dir), "ghost_agent", "Hello?")
|
|
assert result["status"] == "error"
|
|
assert "not found" in result["error"]
|
|
|
|
def test_ask_agent_submit_response_writes_interaction_artifact(self, tmp_path, monkeypatch):
|
|
monkeypatch.setenv("MIROFISH_MODE", "agent")
|
|
monkeypatch.setenv("MIROFISH_LLM_PROVIDER", "agent_queue")
|
|
monkeypatch.setenv("MIROFISH_GRAPH_PROVIDER", "graphiti")
|
|
monkeypatch.setenv("MIROFISH_GRAPHITI_STORE", "file")
|
|
monkeypatch.setenv("MIROFISH_GRAPHITI_COMPAT_PATH", str(tmp_path / "graph_store.json"))
|
|
|
|
run_dir, service = _init_run(tmp_path)
|
|
need = service.ask_agent(str(run_dir), "agent_1", "What do you think?")
|
|
assert need["status"] == "need_agent_response"
|
|
|
|
# Write a valid response
|
|
response_path = run_dir / "responses" / f"{need['request_id']}.json"
|
|
response_path.write_text(json.dumps({
|
|
"request_id": need["request_id"],
|
|
"status": "ok",
|
|
"output": {
|
|
"agent_id": "agent_1",
|
|
"answer_markdown": "I think supply chains will shift.",
|
|
"used_memory": [],
|
|
"used_graph_results": [],
|
|
"confidence": 0.8,
|
|
}
|
|
}), encoding="utf-8")
|
|
|
|
# Submit via get_agent_answer which processes the response
|
|
answer = service.get_agent_answer(str(run_dir), need["request_id"])
|
|
assert answer["status"] == "ok"
|
|
assert answer["agent_id"] == "agent_1"
|
|
# Verify interaction artifacts were written
|
|
questions_dir = run_dir / "artifacts" / "interactions" / "agent_questions"
|
|
assert questions_dir.exists()
|
|
json_files = list(questions_dir.glob("*.json"))
|
|
assert len(json_files) >= 1
|
|
md_files = list(questions_dir.glob("*.md"))
|
|
assert len(md_files) >= 1
|
|
|
|
|
|
# ── questionnaire tests ──────────────────────────────────────────────────
|
|
|
|
class TestQuestionnaire:
|
|
def test_send_questionnaire_creates_batch_requests(self, tmp_path, monkeypatch):
|
|
monkeypatch.setenv("MIROFISH_MODE", "agent")
|
|
monkeypatch.setenv("MIROFISH_LLM_PROVIDER", "agent_queue")
|
|
monkeypatch.setenv("MIROFISH_GRAPH_PROVIDER", "graphiti")
|
|
monkeypatch.setenv("MIROFISH_GRAPHITI_STORE", "file")
|
|
monkeypatch.setenv("MIROFISH_GRAPHITI_COMPAT_PATH", str(tmp_path / "graph_store.json"))
|
|
|
|
run_dir, service = _init_run(tmp_path)
|
|
questions = [
|
|
{"question_id": "q1", "question": "What is the biggest risk?"},
|
|
{"question_id": "q2", "question": "What opportunities do you see?"},
|
|
]
|
|
result = service.send_questionnaire(str(run_dir), questions)
|
|
assert result["status"] == "need_agent_response"
|
|
assert result["question_count"] == 2
|
|
assert result["agent_count"] == 2
|
|
assert len(result["request_ids"]) == 2
|
|
|
|
# Verify questionnaire metadata was saved
|
|
questionnaires_dir = run_dir / "artifacts" / "interactions" / "questionnaires"
|
|
assert questionnaires_dir.exists()
|
|
meta_files = list(questionnaires_dir.glob("*_meta.json"))
|
|
assert len(meta_files) == 1
|
|
meta = json.loads(meta_files[0].read_text(encoding="utf-8"))
|
|
assert meta["questionnaire_id"] == result["questionnaire_id"]
|
|
assert len(meta["questions"]) == 2
|
|
|
|
def test_get_questionnaire_result_not_found(self, tmp_path):
|
|
run_dir, service = _init_run(tmp_path)
|
|
result = service.get_questionnaire_result(str(run_dir), "nonexistent_q")
|
|
assert result["status"] == "error"
|
|
assert "not found" in result["error"]
|
|
|
|
def test_questionnaire_response_schema_valid(self):
|
|
"""Verify questionnaire output schema validates correctly."""
|
|
output = {
|
|
"questionnaire_id": "q_test123",
|
|
"answers": [
|
|
{
|
|
"agent_id": "agent_1",
|
|
"question_id": "q1",
|
|
"answer_markdown": "The biggest risk is supply chain disruption.",
|
|
"confidence": 0.8,
|
|
}
|
|
],
|
|
"summary_markdown": "Summary of questionnaire answers.",
|
|
}
|
|
errors = validate_json_schema(output, AGENT_QUESTIONNAIRE_OUTPUT_SCHEMA)
|
|
assert not errors, errors
|
|
|
|
|
|
# ── report question tests ────────────────────────────────────────────────
|
|
|
|
class TestReportQuestion:
|
|
def test_ask_report_question_creates_request(self, tmp_path, monkeypatch):
|
|
monkeypatch.setenv("MIROFISH_MODE", "agent")
|
|
monkeypatch.setenv("MIROFISH_LLM_PROVIDER", "agent_queue")
|
|
monkeypatch.setenv("MIROFISH_GRAPH_PROVIDER", "graphiti")
|
|
monkeypatch.setenv("MIROFISH_GRAPHITI_STORE", "file")
|
|
monkeypatch.setenv("MIROFISH_GRAPHITI_COMPAT_PATH", str(tmp_path / "graph_store.json"))
|
|
|
|
run_dir, service = _init_run(tmp_path)
|
|
result = service.ask_report_question(str(run_dir), "What does the report say about risks?")
|
|
assert result["status"] == "need_agent_response"
|
|
assert result["type"] == "ask_report_question"
|
|
|
|
def test_report_question_answer_persists_interaction(self, tmp_path, monkeypatch):
|
|
"""Verify get_report_question_answer processes a response and persists to interactions/report_questions/."""
|
|
monkeypatch.setenv("MIROFISH_MODE", "agent")
|
|
monkeypatch.setenv("MIROFISH_LLM_PROVIDER", "agent_queue")
|
|
monkeypatch.setenv("MIROFISH_GRAPH_PROVIDER", "graphiti")
|
|
monkeypatch.setenv("MIROFISH_GRAPHITI_STORE", "file")
|
|
monkeypatch.setenv("MIROFISH_GRAPHITI_COMPAT_PATH", str(tmp_path / "graph_store.json"))
|
|
|
|
run_dir, service = _init_run(tmp_path)
|
|
need = service.ask_report_question(str(run_dir), "Summarize the key risks.")
|
|
assert need["status"] == "need_agent_response"
|
|
|
|
# Write a valid response
|
|
response_path = run_dir / "responses" / f"{need['request_id']}.json"
|
|
response_path.write_text(json.dumps({
|
|
"request_id": need["request_id"],
|
|
"status": "ok",
|
|
"output": {
|
|
"answer_markdown": "The key risks are supply chain disruption and regulatory changes.",
|
|
"used_graph_results": [],
|
|
"confidence": 0.85,
|
|
}
|
|
}), encoding="utf-8")
|
|
|
|
# Process the response
|
|
answer = service.get_report_question_answer(str(run_dir), need["request_id"])
|
|
assert answer["status"] == "ok"
|
|
# Verify interaction artifacts were written
|
|
rq_dir = run_dir / "artifacts" / "interactions" / "report_questions"
|
|
assert rq_dir.exists()
|
|
json_files = list(rq_dir.glob("*.json"))
|
|
assert len(json_files) >= 1
|
|
|
|
|
|
# ── web console tests ────────────────────────────────────────────────────
|
|
|
|
class TestWebConsole:
|
|
def test_web_generate_creates_index_html(self, tmp_path):
|
|
run_dir, service = _init_run(tmp_path)
|
|
result = service.generate_web_console(str(run_dir))
|
|
assert result["status"] == "ok"
|
|
html_path = run_dir / "artifacts" / "web" / "index.html"
|
|
assert html_path.exists()
|
|
html_content = html_path.read_text(encoding="utf-8")
|
|
assert "MiroFish Web Console" in html_content
|
|
assert "<!DOCTYPE html>" in html_content
|
|
|
|
def test_web_console_embeds_artifacts(self, tmp_path):
|
|
run_dir, service = _init_run(tmp_path)
|
|
service.generate_web_console(str(run_dir))
|
|
html_path = run_dir / "artifacts" / "web" / "index.html"
|
|
html_content = html_path.read_text(encoding="utf-8")
|
|
# Check that key data is embedded
|
|
assert "agent_1" in html_content
|
|
assert "Analyst Alpha" in html_content
|
|
assert "Test Report" in html_content
|
|
|
|
def test_web_console_path_in_artifacts(self, tmp_path):
|
|
run_dir, service = _init_run(tmp_path)
|
|
result = service.generate_web_console(str(run_dir))
|
|
assert "web/index.html" in result["path"]
|
|
|
|
def test_web_console_has_interactive_elements(self, tmp_path):
|
|
"""Verify the template includes forms, buttons, and API client for interaction."""
|
|
run_dir, service = _init_run(tmp_path)
|
|
service.generate_web_console(str(run_dir))
|
|
html_path = run_dir / "artifacts" / "web" / "index.html"
|
|
html_content = html_path.read_text(encoding="utf-8")
|
|
# Agent Q&A form
|
|
assert 'id="ask-agent-select"' in html_content
|
|
assert 'id="ask-question-input"' in html_content
|
|
assert 'id="ask-submit-btn"' in html_content
|
|
# Questionnaire form
|
|
assert 'id="questionnaire-submit-btn"' in html_content
|
|
assert 'id="add-question-btn"' in html_content
|
|
# Report question form
|
|
assert 'id="report-q-input"' in html_content
|
|
assert 'id="report-q-submit-btn"' in html_content
|
|
# API client and polling
|
|
assert "apiPost" in html_content
|
|
assert "apiGet" in html_content
|
|
assert "pollForAnswer" in html_content
|
|
assert "checkApiStatus" in html_content
|
|
# API status indicator
|
|
assert 'id="api-dot"' in html_content
|
|
assert 'id="api-status-text"' in html_content
|
|
# Configurable API base URL
|
|
assert 'id="api-base-input"' in html_content
|
|
assert "localhost:5001" in html_content
|
|
|
|
def test_web_console_has_interaction_panels(self, tmp_path):
|
|
"""Verify all interactive panels are present in the navigation."""
|
|
run_dir, service = _init_run(tmp_path)
|
|
service.generate_web_console(str(run_dir))
|
|
html_path = run_dir / "artifacts" / "web" / "index.html"
|
|
html_content = html_path.read_text(encoding="utf-8")
|
|
assert 'data-panel="ask"' in html_content
|
|
assert 'data-panel="questionnaires"' in html_content
|
|
assert 'data-panel="report-q"' in html_content
|
|
assert 'data-panel="history"' in html_content
|
|
|
|
def test_web_console_js_embedding_escapes_special_chars(self, tmp_path):
|
|
"""Verify that quotes, newlines, and backslashes in report/requirement don't break JS."""
|
|
# Create a run with special characters in report and requirement
|
|
seed = tmp_path / "seed.md"
|
|
seed.write_text('Seed with "quotes" and\nnewlines.', encoding="utf-8")
|
|
run_dir = tmp_path / "run_special"
|
|
service = PredictionRunService()
|
|
service.create_run(str(seed), 'requirement with "quote" and\nnewline', str(run_dir))
|
|
artifacts_dir = run_dir / "artifacts"
|
|
artifacts_dir.mkdir(parents=True, exist_ok=True)
|
|
# Write report with special characters
|
|
(artifacts_dir / "report.md").write_text(
|
|
'# Report\n\nLine "quoted".\nBackslash: \\\nEnd.', encoding="utf-8"
|
|
)
|
|
(artifacts_dir / "profiles.json").write_text("[]", encoding="utf-8")
|
|
(artifacts_dir / "verdict.json").write_text("{}", encoding="utf-8")
|
|
(artifacts_dir / "timeline.json").write_text("[]", encoding="utf-8")
|
|
(artifacts_dir / "graph_snapshot.json").write_text("[]", encoding="utf-8")
|
|
(artifacts_dir / "simulation_config.json").write_text("{}", encoding="utf-8")
|
|
(artifacts_dir / "simulation_actions.json").write_text("[]", encoding="utf-8")
|
|
|
|
service.generate_web_console(str(run_dir))
|
|
html_path = run_dir / "artifacts" / "web" / "index.html"
|
|
html_content = html_path.read_text(encoding="utf-8")
|
|
|
|
# The DATA block must be valid JS — extract it and check JSON-escaped strings
|
|
import re
|
|
data_match = re.search(r"const DATA = (\{.*?\});", html_content, re.DOTALL)
|
|
assert data_match, "Could not find DATA block in generated HTML"
|
|
data_block = data_match.group(1)
|
|
|
|
# Verify quotes are JSON-escaped (backslash-escaped), not raw
|
|
assert '\\"quote\\"' in data_block, "Double quotes must be JSON-escaped"
|
|
# Verify newlines are escaped as \n, not literal newlines inside string
|
|
assert '\\n' in data_block, "Newlines must be JSON-escaped as \\n"
|
|
# Verify the requirement value is a valid JSON string (starts with ")
|
|
assert 'requirement: "requirement with \\"' in data_block
|
|
# Verify the reportMd contains the escaped backslash
|
|
assert '\\\\' in data_block, "Backslashes must be JSON-escaped"
|
|
|
|
|
|
# ── schema / contract tests ──────────────────────────────────────────────
|
|
|
|
class TestSchemasAndContracts:
|
|
def test_new_task_types_in_agent_task_types(self):
|
|
assert "answer_agent_question" in AGENT_TASK_TYPES
|
|
assert "answer_agent_questionnaire" in AGENT_TASK_TYPES
|
|
assert "summarize_questionnaire" in AGENT_TASK_TYPES
|
|
assert "ask_report_question" in AGENT_TASK_TYPES
|
|
|
|
def test_new_task_types_have_output_schemas(self):
|
|
assert "answer_agent_question" in TASK_OUTPUT_SCHEMAS
|
|
assert "answer_agent_questionnaire" in TASK_OUTPUT_SCHEMAS
|
|
assert "summarize_questionnaire" in TASK_OUTPUT_SCHEMAS
|
|
assert "ask_report_question" in TASK_OUTPUT_SCHEMAS
|
|
|
|
def test_agent_question_output_schema_has_required_fields(self):
|
|
schema = AGENT_QUESTION_OUTPUT_SCHEMA
|
|
required = schema.get("required", [])
|
|
assert "agent_id" in required
|
|
assert "answer_markdown" in required
|
|
assert "used_memory" in required
|
|
assert "used_graph_results" in required
|
|
assert "confidence" in required
|
|
|
|
def test_questionnaire_output_schema_has_required_fields(self):
|
|
schema = AGENT_QUESTIONNAIRE_OUTPUT_SCHEMA
|
|
required = schema.get("required", [])
|
|
assert "questionnaire_id" in required
|
|
assert "answers" in required
|
|
assert "summary_markdown" in required
|
|
|
|
def test_report_question_output_schema_has_required_fields(self):
|
|
schema = REPORT_QUESTION_OUTPUT_SCHEMA
|
|
required = schema.get("required", [])
|
|
assert "answer_markdown" in required
|
|
assert "used_graph_results" in required
|
|
assert "confidence" in required
|
|
|
|
def test_mock_provider_handles_new_task_types(self):
|
|
"""Mock provider must return valid output for all new task types."""
|
|
provider = MockLLMProvider()
|
|
for task_type in ["answer_agent_question", "answer_agent_questionnaire", "summarize_questionnaire", "ask_report_question"]:
|
|
schema = TASK_OUTPUT_SCHEMAS[task_type]
|
|
result = provider.run_task(LLMTask(
|
|
run_id="run",
|
|
task_type=task_type,
|
|
stage="interaction",
|
|
expected_schema=schema,
|
|
structured_input={
|
|
"agent_id": "agent_1",
|
|
"questionnaire_id": "q_test",
|
|
"questions": [{"question_id": "q1", "question": "Test?"}],
|
|
"agents": [{"agent_id": "agent_1"}],
|
|
"graph_results": [],
|
|
},
|
|
))
|
|
assert result.status == "ok", f"{task_type} failed: {result.error}"
|
|
errors = validate_json_schema(result.output, schema)
|
|
assert not errors, f"{task_type} schema errors: {errors}"
|
|
|
|
def test_all_task_types_have_schema_and_mock_output(self):
|
|
"""Extended version: verify all task types including new ones."""
|
|
assert set(TASK_OUTPUT_SCHEMAS) == AGENT_TASK_TYPES
|
|
provider = MockLLMProvider()
|
|
for task_type in sorted(AGENT_TASK_TYPES):
|
|
schema = TASK_OUTPUT_SCHEMAS[task_type]
|
|
result = provider.run_task(LLMTask(
|
|
run_id="run",
|
|
task_type=task_type,
|
|
stage=task_type,
|
|
expected_schema=schema,
|
|
structured_input={
|
|
"actions": [{"agent_id": "agent_1", "action_id": "action_1"}],
|
|
"candidate": {},
|
|
"invalid_response": {"output": {}},
|
|
"agent_id": "agent_1",
|
|
"questionnaire_id": "q_test",
|
|
"questions": [{"question_id": "q1", "question": "Test?"}],
|
|
"agents": [{"agent_id": "agent_1"}],
|
|
"graph_results": [],
|
|
},
|
|
))
|
|
assert result.status == "ok", f"mock failed for {task_type}"
|
|
assert not validate_json_schema(result.output, schema), f"schema validation failed for {task_type}"
|
|
|
|
|
|
# ── CLI parser tests ─────────────────────────────────────────────────────
|
|
|
|
class TestCLIParser:
|
|
def test_cli_agents_list(self):
|
|
parser = build_parser()
|
|
args = parser.parse_args(["agents", "list", "--run", "runs/demo"])
|
|
assert args.command == "agents"
|
|
assert args.agents_command == "list"
|
|
assert args.run == "runs/demo"
|
|
|
|
def test_cli_agents_show(self):
|
|
parser = build_parser()
|
|
args = parser.parse_args(["agents", "show", "--run", "runs/demo", "--agent-id", "agent_1"])
|
|
assert args.agents_command == "show"
|
|
assert args.agent_id == "agent_1"
|
|
|
|
def test_cli_agents_ask(self):
|
|
parser = build_parser()
|
|
args = parser.parse_args(["agents", "ask", "--run", "runs/demo", "--agent-id", "agent_1", "--question", "Hello?"])
|
|
assert args.agents_command == "ask"
|
|
assert args.question == "Hello?"
|
|
|
|
def test_cli_questionnaire_send(self):
|
|
parser = build_parser()
|
|
args = parser.parse_args(["questionnaire", "send", "--run", "runs/demo", "--questions", "questions.json"])
|
|
assert args.command == "questionnaire"
|
|
assert args.questionnaire_command == "send"
|
|
assert args.questions == "questions.json"
|
|
|
|
def test_cli_questionnaire_show(self):
|
|
parser = build_parser()
|
|
args = parser.parse_args(["questionnaire", "show", "--run", "runs/demo", "--questionnaire-id", "q_123"])
|
|
assert args.questionnaire_command == "show"
|
|
assert args.questionnaire_id == "q_123"
|
|
|
|
def test_cli_agents_answer(self):
|
|
parser = build_parser()
|
|
args = parser.parse_args(["agents", "answer", "--run", "runs/demo", "--request-id", "req_123"])
|
|
assert args.agents_command == "answer"
|
|
assert args.request_id == "req_123"
|
|
|
|
def test_cli_report_question_ask(self):
|
|
parser = build_parser()
|
|
args = parser.parse_args(["report-question", "ask", "--run", "runs/demo", "--question", "What risks?"])
|
|
assert args.command == "report-question"
|
|
assert args.report_question_command == "ask"
|
|
assert args.question == "What risks?"
|
|
|
|
def test_cli_report_question_answer(self):
|
|
parser = build_parser()
|
|
args = parser.parse_args(["report-question", "answer", "--run", "runs/demo", "--request-id", "req_456"])
|
|
assert args.report_question_command == "answer"
|
|
assert args.request_id == "req_456"
|
|
|
|
def test_cli_web_generate(self):
|
|
parser = build_parser()
|
|
args = parser.parse_args(["web", "generate", "--run", "runs/demo"])
|
|
assert args.command == "web"
|
|
assert args.web_command == "generate"
|
|
assert args.run == "runs/demo"
|
|
|
|
|
|
# ── MCP tools schema tests ──────────────────────────────────────────────
|
|
|
|
class TestMCPToolsSchema:
|
|
def test_mcp_server_creates_without_error(self):
|
|
"""Verify MCP server can be created with new tools."""
|
|
try:
|
|
from app.mcp_server.server import create_server
|
|
server = create_server()
|
|
assert server is not None
|
|
except ImportError:
|
|
pytest.skip("mcp package not installed")
|
|
|
|
def test_mcp_tools_include_interaction_tools(self):
|
|
"""Verify the new interaction tools are registered."""
|
|
try:
|
|
from app.mcp_server.server import create_server
|
|
server = create_server()
|
|
# FastMCP stores tools internally; check via list
|
|
tool_names = set()
|
|
if hasattr(server, '_tool_manager'):
|
|
tool_names = set(server._tool_manager._tools.keys()) if hasattr(server._tool_manager, '_tools') else set()
|
|
elif hasattr(server, 'list_tools'):
|
|
# Alternative: some versions expose list_tools
|
|
pass
|
|
# If we can't introspect, just verify server was created
|
|
# The important thing is the tools were decorated with @mcp.tool()
|
|
assert server is not None
|
|
except ImportError:
|
|
pytest.skip("mcp package not installed")
|
|
|
|
|
|
# ── Path traversal guard tests ───────────────────────────────────────────
|
|
|
|
class TestPathTraversalGuard:
|
|
def test_artifact_endpoint_rejects_path_traversal(self, tmp_path):
|
|
"""Verify the artifact endpoint blocks path traversal attempts like ../../.env."""
|
|
from app import create_app
|
|
|
|
run_dir, service = _init_run(tmp_path)
|
|
# Create a file outside artifacts_dir to ensure it can't be read
|
|
sensitive_file = run_dir / ".env"
|
|
sensitive_file.write_text("SECRET=leaked", encoding="utf-8")
|
|
|
|
app = create_app()
|
|
client = app.test_client()
|
|
|
|
# Attempt path traversal
|
|
resp = client.get(
|
|
f"/api/interaction/artifact/../../.env?run={run_dir}"
|
|
)
|
|
# Must be blocked (403) or not found (404), never 200 with leaked content
|
|
assert resp.status_code in (403, 404), f"Path traversal not blocked: {resp.status_code}"
|
|
if resp.status_code == 200:
|
|
assert b"leaked" not in resp.data
|
|
|
|
def test_artifact_endpoint_allows_valid_paths(self, tmp_path):
|
|
"""Verify normal artifact access still works after the traversal guard."""
|
|
from app import create_app
|
|
|
|
run_dir, service = _init_run(tmp_path)
|
|
app = create_app()
|
|
client = app.test_client()
|
|
|
|
resp = client.get(
|
|
f"/api/interaction/artifact/verdict.json?run={run_dir}"
|
|
)
|
|
assert resp.status_code == 200
|
|
data = resp.get_json()
|
|
assert data["success"] is True
|
|
assert data["data"]["status"] == "ok"
|
|
|
|
def test_responses_endpoint_rejects_path_outside_responses_dir(self, tmp_path):
|
|
"""Verify the responses endpoint blocks paths outside run/responses/."""
|
|
from app import create_app
|
|
|
|
run_dir, service = _init_run(tmp_path)
|
|
app = create_app()
|
|
client = app.test_client()
|
|
|
|
# Attempt to submit a response pointing to a file outside responses/
|
|
outside_path = str(run_dir / "artifacts" / "verdict.json")
|
|
resp = client.post(
|
|
f"/api/interaction/responses?run={run_dir}",
|
|
data=json.dumps({"response_path": outside_path}),
|
|
content_type="application/json",
|
|
)
|
|
assert resp.status_code == 403, f"Path outside responses/ not blocked: {resp.status_code}"
|
|
|
|
|
|
# ── MCP questionnaire questions_json tests ───────────────────────────────
|
|
|
|
class TestMCPQuestionnaireJsonParam:
|
|
def test_questionnaire_accepts_questions_json_string(self, tmp_path, monkeypatch):
|
|
"""Verify mirofish_send_questionnaire accepts a questions_json string with arbitrary count."""
|
|
monkeypatch.setenv("MIROFISH_MODE", "agent")
|
|
monkeypatch.setenv("MIROFISH_LLM_PROVIDER", "agent_queue")
|
|
monkeypatch.setenv("MIROFISH_GRAPH_PROVIDER", "graphiti")
|
|
monkeypatch.setenv("MIROFISH_GRAPHITI_STORE", "file")
|
|
monkeypatch.setenv("MIROFISH_GRAPHITI_COMPAT_PATH", str(tmp_path / "graph_store.json"))
|
|
|
|
try:
|
|
from app.mcp_server.server import create_server
|
|
server = create_server()
|
|
except ImportError:
|
|
pytest.skip("mcp package not installed")
|
|
|
|
# Call the tool function directly through the service
|
|
run_dir, service = _init_run(tmp_path)
|
|
questions_json = json.dumps([
|
|
{"question_id": "q1", "question": "Risk 1?"},
|
|
{"question_id": "q2", "question": "Risk 2?"},
|
|
{"question_id": "q3", "question": "Risk 3?"},
|
|
{"question_id": "q4", "question": "Risk 4?"},
|
|
{"question_id": "q5", "question": "Risk 5?"},
|
|
])
|
|
# Test through the service layer directly (MCP tool calls this)
|
|
import json as _json
|
|
questions = _json.loads(questions_json)
|
|
result = service.send_questionnaire(str(run_dir), questions)
|
|
assert result["status"] == "need_agent_response"
|
|
assert result["question_count"] == 5
|
|
assert len(result["request_ids"]) == 5
|
|
|
|
def test_questionnaire_rejects_invalid_json(self, tmp_path):
|
|
"""Verify the MCP tool rejects invalid questions_json input."""
|
|
try:
|
|
from app.mcp_server.server import create_server
|
|
server = create_server()
|
|
except ImportError:
|
|
pytest.skip("mcp package not installed")
|
|
|
|
# Simulate the validation logic from the MCP tool
|
|
import json as _json
|
|
try:
|
|
_json.loads("not valid json")
|
|
assert False, "Should have raised"
|
|
except (ValueError, TypeError):
|
|
pass # Expected
|
|
|
|
|
|
# ── Profile ID fallback tests ────────────────────────────────────────────
|
|
|
|
def _init_run_no_agent_id(tmp_path: Path) -> tuple[Path, PredictionRunService]:
|
|
"""Create a run with profiles that have NO agent_id or user_id — only name."""
|
|
seed = tmp_path / "seed.md"
|
|
seed.write_text("Seed.", encoding="utf-8")
|
|
run_dir = tmp_path / "run_noid"
|
|
service = PredictionRunService()
|
|
service.create_run(str(seed), "test fallback id", str(run_dir))
|
|
profiles = [
|
|
{"name": "Baidu", "type": "search", "description": "Chinese search engine."},
|
|
{"name": "Google China", "type": "search", "description": "Google's China ops."},
|
|
{"name": "Baidu", "type": "ai", "description": "Duplicate name, different type."},
|
|
{"type": "anon", "description": "No name at all."},
|
|
]
|
|
artifacts_dir = run_dir / "artifacts"
|
|
artifacts_dir.mkdir(parents=True, exist_ok=True)
|
|
(artifacts_dir / "profiles.json").write_text(json.dumps(profiles), encoding="utf-8")
|
|
(artifacts_dir / "report.md").write_text("# Report", encoding="utf-8")
|
|
(artifacts_dir / "verdict.json").write_text("{}", encoding="utf-8")
|
|
(artifacts_dir / "timeline.json").write_text("[]", encoding="utf-8")
|
|
(artifacts_dir / "graph_snapshot.json").write_text("[]", encoding="utf-8")
|
|
(artifacts_dir / "simulation_config.json").write_text("{}", encoding="utf-8")
|
|
(artifacts_dir / "simulation_actions.json").write_text("[]", encoding="utf-8")
|
|
return run_dir, service
|
|
|
|
|
|
class TestResolveAgentId:
|
|
def test_prefers_agent_id(self):
|
|
assert _resolve_agent_id({"agent_id": "x_1", "name": "Baidu"}, 0) == "x_1"
|
|
|
|
def test_falls_back_to_user_id(self):
|
|
assert _resolve_agent_id({"user_id": "u_42", "name": "Baidu"}, 0) == "u_42"
|
|
|
|
def test_generates_slug_from_name(self):
|
|
assert _resolve_agent_id({"name": "Google China"}, 0) == "google_china"
|
|
assert _resolve_agent_id({"name": "Baidu"}, 0) == "baidu"
|
|
assert _resolve_agent_id({"name": " Spaced Name "}, 0) == "spaced_name"
|
|
|
|
def test_positional_fallback_when_no_name(self):
|
|
assert _resolve_agent_id({"type": "anon"}, 0) == "agent_1"
|
|
assert _resolve_agent_id({"type": "anon"}, 2) == "agent_3"
|
|
|
|
def test_empty_strings_treated_as_missing(self):
|
|
assert _resolve_agent_id({"agent_id": "", "user_id": "", "name": ""}, 0) == "agent_1"
|
|
|
|
def test_strips_whitespace(self):
|
|
assert _resolve_agent_id({"agent_id": " x "}, 0) == "x"
|
|
|
|
|
|
class TestResolveAllAgentIds:
|
|
def test_no_duplicates_passthrough(self):
|
|
profiles = [{"name": "Alpha"}, {"name": "Beta"}]
|
|
ids = _resolve_all_agent_ids(profiles)
|
|
assert ids == ["alpha", "beta"]
|
|
|
|
def test_duplicate_names_get_suffix(self):
|
|
profiles = [
|
|
{"name": "Baidu"},
|
|
{"name": "Google China"},
|
|
{"name": "Baidu"},
|
|
{"type": "anon"},
|
|
]
|
|
ids = _resolve_all_agent_ids(profiles)
|
|
assert ids[0] == "baidu"
|
|
assert ids[1] == "google_china"
|
|
assert ids[2] == "baidu_2"
|
|
assert ids[3] == "agent_4"
|
|
# All unique
|
|
assert len(set(ids)) == len(ids)
|
|
|
|
def test_triple_duplicate(self):
|
|
profiles = [{"name": "X"}, {"name": "X"}, {"name": "X"}]
|
|
ids = _resolve_all_agent_ids(profiles)
|
|
assert ids == ["x", "x_2", "x_3"]
|
|
|
|
def test_explicit_agent_id_collision(self):
|
|
profiles = [
|
|
{"agent_id": "agent_1"},
|
|
{"agent_id": "agent_1"},
|
|
]
|
|
ids = _resolve_all_agent_ids(profiles)
|
|
assert ids[0] == "agent_1"
|
|
assert ids[1] == "agent_1_2"
|
|
assert len(set(ids)) == 2
|
|
|
|
def test_empty_list(self):
|
|
assert _resolve_all_agent_ids([]) == []
|
|
|
|
|
|
class TestListAgentsFallbackId:
|
|
def test_list_agents_returns_stable_ids_without_agent_id(self, tmp_path):
|
|
run_dir, service = _init_run_no_agent_id(tmp_path)
|
|
result = service.list_agents(str(run_dir))
|
|
assert result["status"] == "ok"
|
|
assert result["count"] == 4
|
|
ids = [a["agent_id"] for a in result["agents"]]
|
|
assert ids[0] == "baidu"
|
|
assert ids[1] == "google_china"
|
|
# Third profile also named "Baidu" — deduplicated with suffix
|
|
assert ids[2] == "baidu_2"
|
|
# Fourth profile has no name — positional fallback
|
|
assert ids[3] == "agent_4"
|
|
# All IDs must be unique
|
|
assert len(set(ids)) == len(ids), "agent_ids must be unique"
|
|
# None should be empty or "unknown"
|
|
assert all(aid for aid in ids), "No agent_id should be empty"
|
|
assert "unknown" not in ids
|
|
|
|
def test_get_agent_works_with_fallback_id(self, tmp_path):
|
|
run_dir, service = _init_run_no_agent_id(tmp_path)
|
|
result = service.get_agent(str(run_dir), "google_china")
|
|
assert result["status"] == "ok"
|
|
assert result["agent"]["agent_id"] == "google_china"
|
|
assert result["agent"]["name"] == "Google China"
|
|
|
|
def test_get_agent_finds_second_duplicate(self, tmp_path):
|
|
run_dir, service = _init_run_no_agent_id(tmp_path)
|
|
# First "Baidu" → "baidu", second "Baidu" → "baidu_2"
|
|
result = service.get_agent(str(run_dir), "baidu_2")
|
|
assert result["status"] == "ok"
|
|
assert result["agent"]["agent_id"] == "baidu_2"
|
|
assert result["agent"]["name"] == "Baidu"
|
|
assert result["agent"]["profile"]["type"] == "ai"
|
|
|
|
|
|
class TestAskAgentFallbackId:
|
|
def test_ask_agent_with_fallback_id_creates_request(self, tmp_path, monkeypatch):
|
|
monkeypatch.setenv("MIROFISH_MODE", "agent")
|
|
monkeypatch.setenv("MIROFISH_LLM_PROVIDER", "agent_queue")
|
|
monkeypatch.setenv("MIROFISH_GRAPH_PROVIDER", "graphiti")
|
|
monkeypatch.setenv("MIROFISH_GRAPHITI_STORE", "file")
|
|
monkeypatch.setenv("MIROFISH_GRAPHITI_COMPAT_PATH", str(tmp_path / "graph_store.json"))
|
|
|
|
run_dir, service = _init_run_no_agent_id(tmp_path)
|
|
result = service.ask_agent(str(run_dir), "google_china", "What is your strategy?")
|
|
assert result["status"] == "need_agent_response"
|
|
assert result["agent_id"] == "google_china"
|
|
|
|
|
|
class TestWebConsoleFallbackId:
|
|
def test_web_console_no_unknown_option(self, tmp_path):
|
|
"""Web Console HTML must not contain 'unknown' as an agent option value."""
|
|
run_dir, service = _init_run_no_agent_id(tmp_path)
|
|
service.generate_web_console(str(run_dir))
|
|
html_path = run_dir / "artifacts" / "web" / "index.html"
|
|
html_content = html_path.read_text(encoding="utf-8")
|
|
# The dropdown options should use slug-based IDs, not "unknown"
|
|
assert 'value="unknown"' not in html_content
|
|
# Verify slug-based IDs are embedded
|
|
assert "baidu" in html_content
|
|
assert "google_china" in html_content
|
|
|
|
def test_web_console_profiles_have_agent_id_in_embedded_data(self, tmp_path):
|
|
"""Embedded profiles JSON should contain agent_id after normalization."""
|
|
run_dir, service = _init_run_no_agent_id(tmp_path)
|
|
service.generate_web_console(str(run_dir))
|
|
html_path = run_dir / "artifacts" / "web" / "index.html"
|
|
html_content = html_path.read_text(encoding="utf-8")
|
|
# The embedded profiles JSON should contain the slug IDs
|
|
assert '"agent_id"' not in html_content or '"baidu"' in html_content or '"google_china"' in html_content
|
|
|
|
def test_web_console_duplicate_names_get_unique_ids(self, tmp_path):
|
|
"""Embedded profiles with duplicate names must have unique agent_ids."""
|
|
run_dir, service = _init_run_no_agent_id(tmp_path)
|
|
service.generate_web_console(str(run_dir))
|
|
html_path = run_dir / "artifacts" / "web" / "index.html"
|
|
html_content = html_path.read_text(encoding="utf-8")
|
|
# The second "Baidu" profile should get "baidu_2"
|
|
assert "baidu_2" in html_content
|
|
# The fourth profile (no name) should get "agent_4"
|
|
assert "agent_4" in html_content
|