feat(interviews): Delphi subagent (3 rounds: open, rate, revise) + convergence metrics
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Christian Moellmann
parent
75762ccc18
commit
5d7111b54e
|
|
@ -0,0 +1,195 @@
|
|||
from __future__ import annotations
|
||||
import json
|
||||
import statistics
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
import yaml
|
||||
from app.models.interview import (
|
||||
DelphiOpenResponse, DelphiRatingResponse,
|
||||
)
|
||||
from app.services.interviews.base import StakeholderInterviewer, PersonaRecord
|
||||
|
||||
|
||||
class DelphiSubagent:
|
||||
def __init__(self, llm, memory, instrument_path: Path, language: str = "de"):
|
||||
with Path(instrument_path).open("r", encoding="utf-8") as f:
|
||||
self.instrument = yaml.safe_load(f)
|
||||
self.interviewer = StakeholderInterviewer(llm=llm, memory=memory, language=language)
|
||||
self.llm = llm
|
||||
self.language = language
|
||||
|
||||
# --- Round 1: open questions ---
|
||||
def _r1_schema(self) -> str:
|
||||
return json.dumps({
|
||||
"answers": {q["question_id"]: "<string>" for q in self.instrument["questions"]}
|
||||
}, ensure_ascii=False)
|
||||
|
||||
def _r1_prompt(self) -> str:
|
||||
lines = ["Bitte beantworten Sie offen:" if self.language == "de" else "Please answer openly:"]
|
||||
for q in self.instrument["questions"]:
|
||||
txt = q["de"] if self.language == "de" else q["en"]
|
||||
lines.append(f"[{q['question_id']}] {txt}")
|
||||
return "\n".join(lines)
|
||||
|
||||
def _r1_validate(self, raw: dict) -> Optional[dict]:
|
||||
if not isinstance(raw, dict): return None
|
||||
ans = raw.get("answers")
|
||||
if not isinstance(ans, dict): return None
|
||||
required = {q["question_id"] for q in self.instrument["questions"]}
|
||||
if not required.issubset(ans.keys()): return None
|
||||
return raw
|
||||
|
||||
def administer_round1(self, persona: PersonaRecord) -> DelphiOpenResponse:
|
||||
raw = self.interviewer.ask_in_character(
|
||||
persona, user_prompt=self._r1_prompt(),
|
||||
schema_hint=self._r1_schema(), validate=self._r1_validate,
|
||||
)
|
||||
return DelphiOpenResponse(agent_id=persona.agent_id, round=1,
|
||||
answers={k: str(v) for k, v in raw["answers"].items()})
|
||||
|
||||
# --- Round 2: rate themes ---
|
||||
def _r2_schema(self, theme_ids: list[str]) -> str:
|
||||
return json.dumps({
|
||||
"ratings": {tid: {"importance": "<int 1-5>", "plausibility": "<int 1-5>"} for tid in theme_ids}
|
||||
}, ensure_ascii=False)
|
||||
|
||||
def _r2_prompt(self, themes: list[dict]) -> str:
|
||||
head = "Bewerten Sie jedes Thema nach Wichtigkeit (1-5) und Plausibilität (1-5):" if self.language == "de" \
|
||||
else "Rate each theme on importance (1-5) and plausibility (1-5):"
|
||||
body = [f"- [{t['theme_id']}] {t['label']}" for t in themes]
|
||||
return head + "\n" + "\n".join(body)
|
||||
|
||||
def _r2_validate(self, theme_ids: list[str]):
|
||||
def v(raw: dict) -> Optional[dict]:
|
||||
if not isinstance(raw, dict): return None
|
||||
ratings = raw.get("ratings", {})
|
||||
if set(ratings.keys()) != set(theme_ids): return None
|
||||
for tid, r in ratings.items():
|
||||
if not isinstance(r, dict): return None
|
||||
for key in ("importance", "plausibility"):
|
||||
if not isinstance(r.get(key), int) or not 1 <= r[key] <= 5: return None
|
||||
return raw
|
||||
return v
|
||||
|
||||
def administer_round2(self, persona: PersonaRecord, themes: list[dict]) -> DelphiRatingResponse:
|
||||
theme_ids = [t["theme_id"] for t in themes]
|
||||
raw = self.interviewer.ask_in_character(
|
||||
persona, user_prompt=self._r2_prompt(themes),
|
||||
schema_hint=self._r2_schema(theme_ids), validate=self._r2_validate(theme_ids),
|
||||
)
|
||||
return DelphiRatingResponse(agent_id=persona.agent_id, round=2,
|
||||
ratings={k: dict(v) for k, v in raw["ratings"].items()})
|
||||
|
||||
# --- Round 3: revise after seeing group stats ---
|
||||
def administer_round3(
|
||||
self, persona: PersonaRecord, themes: list[dict], group_stats: dict, own_r2: DelphiRatingResponse
|
||||
) -> DelphiRatingResponse:
|
||||
theme_ids = [t["theme_id"] for t in themes]
|
||||
head = ("Sie sehen unten die anonymisierten Gruppenwerte (Median, IQR). "
|
||||
"Bitte überarbeiten Sie Ihre Bewertungen, wenn Sie möchten, und begründen Sie kurz.") \
|
||||
if self.language == "de" else \
|
||||
("Below are the anonymised group values (median, IQR). "
|
||||
"Please revise your ratings if you wish and add a short justification.")
|
||||
ctx_lines = []
|
||||
for t in themes:
|
||||
tid = t["theme_id"]
|
||||
gs = group_stats.get(tid, {})
|
||||
own = own_r2.ratings.get(tid, {})
|
||||
ctx_lines.append(
|
||||
f"[{tid}] {t['label']} — group importance median={gs.get('imp_median')}, "
|
||||
f"IQR={gs.get('imp_iqr')}; plausibility median={gs.get('plaus_median')}, "
|
||||
f"IQR={gs.get('plaus_iqr')}. Your R2: imp={own.get('importance')}, plaus={own.get('plausibility')}."
|
||||
)
|
||||
prompt = head + "\n\n" + "\n".join(ctx_lines)
|
||||
schema = json.dumps({
|
||||
"ratings": {tid: {"importance": "<int 1-5>", "plausibility": "<int 1-5>"} for tid in theme_ids},
|
||||
"justification": "<string>",
|
||||
}, ensure_ascii=False)
|
||||
|
||||
def validate(raw):
|
||||
if not isinstance(raw, dict): return None
|
||||
ratings = raw.get("ratings", {})
|
||||
if set(ratings.keys()) != set(theme_ids): return None
|
||||
for r in ratings.values():
|
||||
if not isinstance(r, dict): return None
|
||||
for key in ("importance", "plausibility"):
|
||||
if not isinstance(r.get(key), int) or not 1 <= r[key] <= 5: return None
|
||||
return raw
|
||||
|
||||
raw = self.interviewer.ask_in_character(persona, user_prompt=prompt,
|
||||
schema_hint=schema, validate=validate)
|
||||
return DelphiRatingResponse(
|
||||
agent_id=persona.agent_id, round=3,
|
||||
ratings={k: dict(v) for k, v in raw["ratings"].items()},
|
||||
justification=raw.get("justification"),
|
||||
)
|
||||
|
||||
|
||||
def extract_themes(round1: list[DelphiOpenResponse], llm) -> list[dict]:
|
||||
text_blocks = []
|
||||
for r in round1:
|
||||
for qid, ans in r.answers.items():
|
||||
text_blocks.append(f"[agent {r.agent_id} {qid}] {ans}")
|
||||
schema = json.dumps({"themes": [{"theme_id": "<string>", "label": "<short string>"}]}, ensure_ascii=False)
|
||||
messages = [
|
||||
{"role": "system", "content":
|
||||
"You extract distinct thematic codes from open-ended German fisheries survey responses. "
|
||||
f"Return JSON ONLY matching: {schema}. Use stable theme_ids of form theme_0, theme_1, …"},
|
||||
{"role": "user", "content": "Responses:\n" + "\n".join(text_blocks) + "\n\nReturn up to 12 distinct themes."},
|
||||
]
|
||||
raw = llm.chat_json(messages=messages, temperature=0.0)
|
||||
themes = raw.get("themes", []) if isinstance(raw, dict) else []
|
||||
out = []
|
||||
for i, t in enumerate(themes):
|
||||
if isinstance(t, dict) and "label" in t:
|
||||
out.append({"theme_id": t.get("theme_id") or f"theme_{i}", "label": str(t["label"])})
|
||||
return out
|
||||
|
||||
|
||||
def _iqr(xs: list[float]) -> float:
|
||||
if not xs: return 0.0
|
||||
xs = sorted(xs)
|
||||
q1 = statistics.quantiles(xs, n=4)[0] if len(xs) >= 4 else xs[0]
|
||||
q3 = statistics.quantiles(xs, n=4)[2] if len(xs) >= 4 else xs[-1]
|
||||
return q3 - q1
|
||||
|
||||
|
||||
def convergence_metrics(r2: list[DelphiRatingResponse], r3: list[DelphiRatingResponse]) -> dict:
|
||||
by_r2 = {r.agent_id: r for r in r2}
|
||||
by_r3 = {r.agent_id: r for r in r3}
|
||||
themes: set[str] = set()
|
||||
for r in r2 + r3:
|
||||
themes.update(r.ratings.keys())
|
||||
out: dict[str, dict] = {}
|
||||
for t in sorted(themes):
|
||||
imp_r2 = [by_r2[a].ratings[t]["importance"] for a in by_r2 if t in by_r2[a].ratings]
|
||||
imp_r3 = [by_r3[a].ratings[t]["importance"] for a in by_r3 if t in by_r3[a].ratings]
|
||||
plaus_r2 = [by_r2[a].ratings[t]["plausibility"] for a in by_r2 if t in by_r2[a].ratings]
|
||||
plaus_r3 = [by_r3[a].ratings[t]["plausibility"] for a in by_r3 if t in by_r3[a].ratings]
|
||||
out[t] = {
|
||||
"imp_median_r2": statistics.median(imp_r2) if imp_r2 else None,
|
||||
"imp_median_r3": statistics.median(imp_r3) if imp_r3 else None,
|
||||
"imp_iqr_r2": _iqr(imp_r2),
|
||||
"imp_iqr_r3": _iqr(imp_r3),
|
||||
"delta_iqr_importance": _iqr(imp_r3) - _iqr(imp_r2),
|
||||
"plaus_iqr_r2": _iqr(plaus_r2),
|
||||
"plaus_iqr_r3": _iqr(plaus_r3),
|
||||
"delta_iqr_plausibility": _iqr(plaus_r3) - _iqr(plaus_r2),
|
||||
}
|
||||
return out
|
||||
|
||||
|
||||
def group_stats_from_r2(r2: list[DelphiRatingResponse]) -> dict:
|
||||
themes: set[str] = set()
|
||||
for r in r2: themes.update(r.ratings.keys())
|
||||
stats: dict[str, dict] = {}
|
||||
for t in themes:
|
||||
imps = [r.ratings[t]["importance"] for r in r2 if t in r.ratings]
|
||||
plauss = [r.ratings[t]["plausibility"] for r in r2 if t in r.ratings]
|
||||
stats[t] = {
|
||||
"imp_median": statistics.median(imps) if imps else None,
|
||||
"imp_iqr": _iqr(imps),
|
||||
"plaus_median": statistics.median(plauss) if plauss else None,
|
||||
"plaus_iqr": _iqr(plauss),
|
||||
}
|
||||
return stats
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
name: delphi_v1
|
||||
version: "1.0"
|
||||
language_default: de
|
||||
rounds: 3
|
||||
questions:
|
||||
- {question_id: q1, de: "Welche drei Faktoren werden die deutsche Fischerei bis 2040 am stärksten prägen?", en: "Which three factors will most shape German fisheries by 2040?"}
|
||||
- {question_id: q2, de: "Welche Akteurinnen und Akteure sind heute entscheidend, werden aber unterschätzt?", en: "Which actors are decisive today but underestimated?"}
|
||||
- {question_id: q3, de: "Was sollte sich in den nächsten fünf Jahren ändern, damit die Fischerei eine Zukunft hat?", en: "What should change in the next five years for fisheries to have a future?"}
|
||||
- {question_id: q4, de: "Welcher Trend macht Ihnen am meisten Hoffnung – und welcher am meisten Sorge?", en: "Which trend gives you most hope — and which most concern?"}
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
from pathlib import Path
|
||||
from app.services.interviews.base import PersonaRecord, MemoryDigest
|
||||
from app.services.interviews.delphi import (
|
||||
DelphiSubagent, extract_themes, convergence_metrics,
|
||||
)
|
||||
|
||||
INSTRUMENT = Path(__file__).resolve().parents[2] / "scripts" / "instruments" / "delphi_v1.yaml"
|
||||
|
||||
class _Mem:
|
||||
def get_digest(self, agent_id, max_chars=2000):
|
||||
return MemoryDigest(text="x", available=True)
|
||||
|
||||
class _R1LLM:
|
||||
def chat_json(self, messages, temperature=0.0, max_tokens=None, **kw):
|
||||
return {"answers": {
|
||||
"q1": "Klimawandel, Quoten, Generationswechsel",
|
||||
"q2": "MSC, Aquakultur",
|
||||
"q3": "Russland, EU-Politik",
|
||||
"q4": "Verbraucherpreise",
|
||||
}}
|
||||
|
||||
class _R2LLM:
|
||||
def chat_json(self, messages, temperature=0.0, max_tokens=None, **kw):
|
||||
return {"ratings": {f"theme_{i}": {"importance": 4, "plausibility": 3} for i in range(5)}}
|
||||
|
||||
class _ExtractLLM:
|
||||
def chat_json(self, messages, temperature=0.0, max_tokens=None, **kw):
|
||||
return {"themes": [
|
||||
{"theme_id": "theme_0", "label": "Klimawandel"},
|
||||
{"theme_id": "theme_1", "label": "Quoten"},
|
||||
{"theme_id": "theme_2", "label": "MSC"},
|
||||
{"theme_id": "theme_3", "label": "EU-Politik"},
|
||||
{"theme_id": "theme_4", "label": "Generationswechsel"},
|
||||
]}
|
||||
|
||||
def test_delphi_round1_open():
|
||||
sub = DelphiSubagent(llm=_R1LLM(), memory=_Mem(), instrument_path=INSTRUMENT)
|
||||
persona = PersonaRecord(agent_id=2, name="A", persona="p")
|
||||
resp = sub.administer_round1(persona)
|
||||
assert resp.round == 1
|
||||
assert len(resp.answers) == 4
|
||||
|
||||
def test_extract_themes_aggregates():
|
||||
from app.models.interview import DelphiOpenResponse
|
||||
r1 = [DelphiOpenResponse(agent_id=i, answers={"q1": "Klimawandel", "q2": "MSC"}) for i in range(3)]
|
||||
themes = extract_themes(r1, llm=_ExtractLLM())
|
||||
assert len(themes) == 5
|
||||
assert all("theme_id" in t for t in themes)
|
||||
|
||||
def test_convergence_metrics():
|
||||
from app.models.interview import DelphiRatingResponse
|
||||
r2 = [DelphiRatingResponse(agent_id=i, round=2,
|
||||
ratings={"t1": {"importance": 3, "plausibility": 3}}) for i in range(5)]
|
||||
r3 = [DelphiRatingResponse(agent_id=i, round=3,
|
||||
ratings={"t1": {"importance": 4, "plausibility": 4}}) for i in range(5)]
|
||||
conv = convergence_metrics(r2, r3)
|
||||
assert "t1" in conv
|
||||
assert conv["t1"]["delta_iqr_importance"] is not None
|
||||
Loading…
Reference in New Issue