feat(dashboard): add Agent Insights page — live AI-agent usage analytics
New /api/agent-insights endpoint aggregates real data (chat-history, cost-history, agent-time, agent-registry, router-keywords) to show: - which agent you chat with most (user turns) - which agent spends the most time on tasks (derived agent-time) - which agent is best suited for what (registry desc + router keywords + roles) - which models you use most per agent (cost-history) Dashboard page (agent-insights.js) with stat cards, two Chart.js bar charts, suite cards, models-per-agent table, and full per-agent breakdown. Verified live in browser against real data.
This commit is contained in:
parent
042fd288f8
commit
bc8094be77
|
|
@ -91,6 +91,8 @@ const api = {
|
|||
// Agent Time Monitor
|
||||
getAgentTime: (recompute = false) => api.get(`/api/agent-time${recompute ? '?recompute=true' : ''}`),
|
||||
recomputeAgentTime: () => api.post('/api/agent-time/recompute', {}),
|
||||
// Agent Insights (AI usage analytics)
|
||||
getAgentInsights: (recompute = false) => api.get(`/api/agent-insights${recompute ? '?recompute=true' : ''}`),
|
||||
// Agent Registry
|
||||
getAgents: () => api.get('/api/agents'),
|
||||
registerAgent: (data) => api.post('/api/agents/register', data),
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@
|
|||
<a href="#agent-health" class="nav-item" data-page="agent-health"><span class="nav-icon">🏥</span><span class="nav-label">Agent Health</span><span class="nav-badge" id="agentHealthCount">0</span></a>
|
||||
<a href="#smart-router" class="nav-item" data-page="smart-router"><span class="nav-icon">🧭</span><span class="nav-label">Smart Router</span></a>
|
||||
<a href="#learning-analytics" class="nav-item" data-page="learning-analytics"><span class="nav-icon">📊</span><span class="nav-label">Learning Analytics</span></a>
|
||||
<a href="#agent-insights" class="nav-item" data-page="agent-insights"><span class="nav-icon">🧠</span><span class="nav-label">Agent Insights</span></a>
|
||||
<a href="#session-replay" class="nav-item" data-page="session-replay"><span class="nav-icon">🔄</span><span class="nav-label">Session Replay</span></a>
|
||||
<a href="#agent-time" class="nav-item" data-page="agent-time"><span class="nav-icon">⏱</span><span class="nav-label">Agent Time</span></a>
|
||||
<div class="sidebar-section"><div class="sidebar-section-label">Management</div></div>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,193 @@
|
|||
// Agent Insights — live AI-agent usage analytics.
|
||||
// Answers: who I chat with most, who spends the most time, who's best
|
||||
// suited for what, and which models I use per agent. All derived from
|
||||
// real data via /api/agent-insights.
|
||||
|
||||
function fmtDuration(sec) {
|
||||
sec = Math.max(0, Math.round(sec || 0));
|
||||
const h = Math.floor(sec / 3600);
|
||||
const m = Math.floor((sec % 3600) / 60);
|
||||
const s = sec % 60;
|
||||
if (h > 0) return `${h}h ${m}m`;
|
||||
if (m > 0) return `${m}m ${s}s`;
|
||||
return `${s}s`;
|
||||
}
|
||||
|
||||
function agentColor(agent) {
|
||||
const palette = {
|
||||
opencode: '#6c5ce7', hermes: '#00b894', gemini: '#0984e3',
|
||||
jarvis: '#fd79a8', kilocode: '#e17055', codex: '#fdcb6e',
|
||||
system: '#636e72', test: '#b2bec3',
|
||||
};
|
||||
return palette[agent] || '#74b9ff';
|
||||
}
|
||||
|
||||
let _charts = {};
|
||||
function destroyCharts() {
|
||||
Object.values(_charts).forEach(c => { try { c.destroy(); } catch (e) {} });
|
||||
_charts = {};
|
||||
}
|
||||
|
||||
async function renderAgentInsights() {
|
||||
const content = document.getElementById('pageContent');
|
||||
content.innerHTML = `
|
||||
<div class="page-header">
|
||||
<div class="page-header-left">
|
||||
<h1 class="page-title">Agent Insights</h1>
|
||||
<p class="page-subtitle">Live analytics on how you actually use your AI agents</p>
|
||||
</div>
|
||||
<div class="flex gap-2">
|
||||
<button class="btn btn-secondary" onclick="renderAgentInsights()">↻ Refresh</button>
|
||||
<button class="btn btn-primary" onclick="recomputeAndRenderInsights()">⟳ Recompute time</button>
|
||||
</div>
|
||||
</div>
|
||||
<div id="insightsBody"><div class="loading"><div class="loading-spinner"></div><span>Analyzing agent usage…</span></div></div>
|
||||
`;
|
||||
|
||||
let data;
|
||||
try {
|
||||
data = await api.getAgentInsights();
|
||||
} catch (err) {
|
||||
document.getElementById('insightsBody').innerHTML = `
|
||||
<div class="empty-state"><div class="empty-state-icon">⚠</div>
|
||||
<div class="empty-state-title">Could not load insights</div>
|
||||
<div class="empty-state-desc">${escapeHtml(err.message)}</div></div>`;
|
||||
return;
|
||||
}
|
||||
|
||||
const t = data.totals || {};
|
||||
const pa = data.per_agent || {};
|
||||
const body = document.getElementById('insightsBody');
|
||||
|
||||
body.innerHTML = `
|
||||
<div class="grid grid-4 mb-4">
|
||||
<div class="card stat-card"><div class="stat-icon purple">💬</div><div class="stat-value">${(t.total_user_turns||0).toLocaleString()}</div><div class="stat-label">Your chat turns</div></div>
|
||||
<div class="card stat-card"><div class="stat-icon blue">⏱</div><div class="stat-value">${fmtDuration(t.total_time_seconds)}</div><div class="stat-label">Total agent time</div></div>
|
||||
<div class="card stat-card"><div class="stat-icon green">🤖</div><div class="stat-value">${t.agents_observed||0}</div><div class="stat-label">Agents observed</div></div>
|
||||
<div class="card stat-card"><div class="stat-icon ${t.total_cost>0?'yellow':'green'}">💰</div><div class="stat-value">$${(t.total_cost||0).toFixed(4)}</div><div class="stat-label">Tracked cost</div></div>
|
||||
</div>
|
||||
|
||||
<div class="grid grid-2 mb-4">
|
||||
<div class="card">
|
||||
<div class="card-header"><span class="card-title">💬 Most chatted-with agents</span></div>
|
||||
<div class="chart-container"><canvas id="chatsChart"></canvas></div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div class="card-header"><span class="card-title">⏱ Agents spending the most time</span></div>
|
||||
<div class="chart-container"><canvas id="timeChart"></canvas></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card mb-4">
|
||||
<div class="card-header"><span class="card-title">🎯 Best suited for (derived from registry + router keywords + roles)</span></div>
|
||||
<div id="suiteGrid" class="grid grid-3"></div>
|
||||
</div>
|
||||
|
||||
<div class="card mb-4">
|
||||
<div class="card-header"><span class="card-title">🧠 Models used per agent</span></div>
|
||||
<div id="modelsTable"></div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<div class="card-header"><span class="card-title">📊 Per-agent breakdown</span></div>
|
||||
<div id="breakdownTable"></div>
|
||||
</div>
|
||||
|
||||
<p class="page-subtitle mt-3" style="font-size:11px;opacity:.6">Generated ${formatDate(data.generated_at)} · data: chat-history, cost-history, agent-time, agent-registry, router-keywords</p>
|
||||
`;
|
||||
|
||||
// Charts
|
||||
const chats = (data.most_chatted || []).slice(0, 8);
|
||||
const times = (data.most_time || []).slice(0, 8);
|
||||
|
||||
_charts.chats = new Chart(document.getElementById('chatsChart'), {
|
||||
type: 'bar',
|
||||
data: {
|
||||
labels: chats.map(c => c.display_name || c.agent),
|
||||
datasets: [{
|
||||
label: 'Your chat turns',
|
||||
data: chats.map(c => c.user_turns),
|
||||
backgroundColor: chats.map(c => agentColor(c.agent)),
|
||||
borderRadius: 6,
|
||||
}],
|
||||
},
|
||||
options: { responsive: true, maintainAspectRatio: false, plugins: { legend: { display: false } }, scales: { y: { beginAtZero: true, ticks: { precision: 0 } } } },
|
||||
});
|
||||
|
||||
_charts.time = new Chart(document.getElementById('timeChart'), {
|
||||
type: 'bar',
|
||||
data: {
|
||||
labels: times.map(c => c.display_name || c.agent),
|
||||
datasets: [{
|
||||
label: 'Time spent',
|
||||
data: times.map(c => Math.round(c.time_seconds / 60)),
|
||||
backgroundColor: times.map(c => agentColor(c.agent)),
|
||||
borderRadius: 6,
|
||||
}],
|
||||
},
|
||||
options: { responsive: true, maintainAspectRatio: false, plugins: { legend: { display: false }, tooltip: { callbacks: { label: (ctx) => `${ctx.parsed.y} min` } } }, scales: { y: { beginAtZero: true, title: { display: true, text: 'minutes' } } } },
|
||||
});
|
||||
|
||||
// Suite cards
|
||||
const suiteGrid = document.getElementById('suiteGrid');
|
||||
const suiteAgents = Object.values(pa)
|
||||
.filter(a => (a.suite && (a.suite.description || a.suite.keywords?.length)))
|
||||
.sort((a, b) => (b.user_turns + b.time_seconds / 60) - (a.user_turns + a.time_seconds / 60));
|
||||
if (suiteAgents.length === 0) {
|
||||
suiteGrid.innerHTML = '<div class="empty-state" style="padding:16px">No role data available yet.</div>';
|
||||
} else {
|
||||
suiteGrid.innerHTML = suiteAgents.map(a => {
|
||||
const s = a.suite || {};
|
||||
const kws = (s.keywords || []).slice(0, 8).map(k => `<span class="badge badge-accent" style="font-size:10px;margin:2px">${escapeHtml(k)}</span>`).join('');
|
||||
return `
|
||||
<div class="card" style="border-left:4px solid ${agentColor(a.agent)}">
|
||||
<div class="flex items-center gap-2 mb-2">
|
||||
<span class="badge" style="background:${agentColor(a.agent)};color:#fff">${escapeHtml(a.display_name || a.agent)}</span>
|
||||
<span style="font-size:11px;color:var(--text-muted)">${a.user_turns} chats · ${fmtDuration(a.time_seconds)}</span>
|
||||
</div>
|
||||
<div style="font-size:12px;color:var(--text-muted);margin-bottom:6px">${escapeHtml(s.description || s.role_hint || '—')}</div>
|
||||
<div>${kws || '<span style="font-size:11px;opacity:.5">no keywords</span>'}</div>
|
||||
</div>`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
// Models per agent
|
||||
const modelsRows = Object.values(pa)
|
||||
.filter(a => a.top_models && a.top_models.length)
|
||||
.sort((a, b) => b.tokens - a.tokens);
|
||||
document.getElementById('modelsTable').innerHTML = modelsRows.length
|
||||
? `<div class="table-wrapper"><table>
|
||||
<thead><tr><th>Agent</th><th>Top models (by uses)</th><th>Tokens</th><th>Cost</th></tr></thead>
|
||||
<tbody>${modelsRows.map(a => `
|
||||
<tr>
|
||||
<td><span class="badge" style="background:${agentColor(a.agent)};color:#fff">${escapeHtml(a.display_name || a.agent)}</span></td>
|
||||
<td>${a.top_models.slice(0, 4).map(([m, c]) => `<span class="badge badge-accent" style="font-size:11px;margin:2px">${escapeHtml(m)} <b>×${c}</b></span>`).join(' ')}</td>
|
||||
<td>${(a.tokens || 0).toLocaleString()}</td>
|
||||
<td>$${(a.cost || 0).toFixed(4)}</td>
|
||||
</tr>`).join('')}</tbody>
|
||||
</table></div>`
|
||||
: '<div class="empty-state" style="padding:16px">No cost/model data recorded yet.</div>';
|
||||
|
||||
// Full breakdown
|
||||
const rows = Object.values(pa).sort((a, b) => b.user_turns - a.user_turns);
|
||||
document.getElementById('breakdownTable').innerHTML = `
|
||||
<div class="table-wrapper"><table>
|
||||
<thead><tr><th>Agent</th><th>Chat turns</th><th>Msgs</th><th>Time</th><th>Sessions</th><th>Top model</th><th>First seen</th><th>Last seen</th></tr></thead>
|
||||
<tbody>${rows.map(a => `
|
||||
<tr>
|
||||
<td><span class="badge" style="background:${agentColor(a.agent)};color:#fff">${escapeHtml(a.display_name || a.agent)}</span></td>
|
||||
<td><b>${a.user_turns}</b></td>
|
||||
<td>${a.chat_messages}</td>
|
||||
<td>${fmtDuration(a.time_seconds)}</td>
|
||||
<td>${a.sessions}</td>
|
||||
<td>${a.top_models && a.top_models.length ? escapeHtml(a.top_models[0][0]) : '—'}</td>
|
||||
<td style="font-size:11px">${a.first_seen ? formatDate(a.first_seen) : '—'}</td>
|
||||
<td style="font-size:11px">${a.last_seen ? formatDate(a.last_seen) : '—'}</td>
|
||||
</tr>`).join('')}</tbody>
|
||||
</table></div>`;
|
||||
}
|
||||
|
||||
async function recomputeAndRenderInsights() {
|
||||
try { await api.recomputeAgentTime(); } catch (e) {}
|
||||
await renderAgentInsights();
|
||||
}
|
||||
|
|
@ -117,6 +117,7 @@ const PAGE_TITLES = {
|
|||
scheduler: { title: 'Scheduler', breadcrumb: 'Automated workflows' },
|
||||
audit: { title: 'Audit Log', breadcrumb: 'System activity trail' },
|
||||
cost: { title: 'Cost Analytics', breadcrumb: 'Usage & spending' },
|
||||
'agent-insights': { title: 'Agent Insights', breadcrumb: 'AI usage analytics' },
|
||||
plugins: { title: 'Plugin Registry', breadcrumb: 'Manage plugins' },
|
||||
backups: { title: 'Backups', breadcrumb: 'Disaster recovery' },
|
||||
prompts: { title: 'Prompt Library', breadcrumb: 'Reusable templates' },
|
||||
|
|
|
|||
168
server.py
168
server.py
|
|
@ -2341,6 +2341,174 @@ def agent_time_skin():
|
|||
body = f"{report.get('total_human', '0h 0m 0s')}\n{total} {per}"
|
||||
return Response(content=body, media_type="text/plain; charset=utf-8")
|
||||
|
||||
|
||||
# ─── Routes: Agent Insights (AI usage analytics) ──────────────────
|
||||
# Aggregates real usage data across chat history, cost history, derived
|
||||
# agent-time, the agent registry, and router keywords to answer:
|
||||
# - which agent do I chat with most
|
||||
# - which agent spends the most time completing tasks
|
||||
# - which agent seems best suited for what
|
||||
# - which models I use most with each agent
|
||||
# All source data is read-only; no writes happen on this endpoint.
|
||||
|
||||
ROUTER_KEYWORDS_FILE = BASE_DIR / "data" / "router-keywords.json"
|
||||
|
||||
# Human-readable role hints keyed by agent name (falls back to registry
|
||||
# description / router keywords when unknown).
|
||||
AGENT_ROLE_HINTS = {
|
||||
"opencode": "Code generation, file ops, DevOps/infra, git, software engineering",
|
||||
"hermes": "Persistent memory, cron scheduling, messaging channels, skill hub, coordination",
|
||||
"gemini": "Web research, multi-modal/image/PDF analysis, reasoning, data analysis",
|
||||
"jarvis": "Local-first personal AI: deep research, knowledge, memory, general reasoning",
|
||||
"kilocode": "AI coding assistant: implement, refactor, programming tasks",
|
||||
"codex": "OpenAI Codex: coding, debugging, tests, builds",
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/agent-insights")
|
||||
def get_agent_insights(recompute: bool = False):
|
||||
# Pull the four required datasets from disk.
|
||||
chat = load_chat_history().get("messages", [])
|
||||
cost = read_json(
|
||||
BASE_DIR / "data" / "cost-history.json",
|
||||
{"entries": []},
|
||||
).get("entries", [])
|
||||
registry = load_agent_registry()
|
||||
router_kw = read_json(ROUTER_KEYWORDS_FILE, {})
|
||||
|
||||
if recompute or not AGENT_TIME_REPORT.exists():
|
||||
time_report = _recompute_agent_time()
|
||||
else:
|
||||
try:
|
||||
time_report = json.loads(AGENT_TIME_REPORT.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
time_report = _recompute_agent_time()
|
||||
time_agents = time_report.get("agents", {})
|
||||
|
||||
# 1) Chat volume per agent (count user+assistant messages; pair count
|
||||
# = number of user turns, which is the most meaningful "chat" metric).
|
||||
chat_counts = {}
|
||||
user_turns = {}
|
||||
first_chat = {}
|
||||
last_chat = {}
|
||||
for m in chat:
|
||||
a = m.get("agent") or "unknown"
|
||||
chat_counts[a] = chat_counts.get(a, 0) + 1
|
||||
if m.get("role") == "user":
|
||||
user_turns[a] = user_turns.get(a, 0) + 1
|
||||
ts = m.get("timestamp")
|
||||
if ts:
|
||||
if a not in first_chat or ts < first_chat[a]:
|
||||
first_chat[a] = ts
|
||||
if a not in last_chat or ts > last_chat[a]:
|
||||
last_chat[a] = ts
|
||||
|
||||
# 2) Time per agent (from derived agent-time report).
|
||||
time_per_agent = {
|
||||
a: {
|
||||
"total_seconds": d.get("total_seconds", 0),
|
||||
"sessions": d.get("sessions", 0),
|
||||
"touches": d.get("touches", 0),
|
||||
}
|
||||
for a, d in time_agents.items()
|
||||
}
|
||||
|
||||
# 3) Models used per agent (from cost history).
|
||||
models_per_agent = {}
|
||||
tokens_per_agent = {}
|
||||
cost_per_agent = {}
|
||||
for e in cost:
|
||||
a = e.get("agent") or "unknown"
|
||||
model = e.get("model") or "unknown"
|
||||
models_per_agent.setdefault(a, {})
|
||||
models_per_agent[a][model] = models_per_agent[a].get(model, 0) + 1
|
||||
tokens_per_agent[a] = tokens_per_agent.get(a, 0) + (e.get("tokens", 0) or 0)
|
||||
cost_per_agent[a] = cost_per_agent.get(a, 0) + (e.get("cost", 0) or 0)
|
||||
|
||||
# 4) Best-suited-for: combine registry description, router keywords,
|
||||
# and AGENTS.md role hints into a per-agent "suite" summary.
|
||||
suite = {}
|
||||
for name in set(list(registry.keys()) + list(router_kw.keys()) +
|
||||
list(chat_counts.keys()) + list(time_per_agent.keys())):
|
||||
desc = (registry.get(name, {}) or {}).get("description", "")
|
||||
role = AGENT_ROLE_HINTS.get(name, "")
|
||||
kws = router_kw.get(name, [])
|
||||
suite[name] = {
|
||||
"description": desc or role,
|
||||
"role_hint": role,
|
||||
"keywords": kws,
|
||||
}
|
||||
|
||||
# Build a per-agent consolidated view.
|
||||
all_agents = set(
|
||||
list(chat_counts.keys()) + list(time_per_agent.keys())
|
||||
+ list(models_per_agent.keys()) + list(suite.keys())
|
||||
)
|
||||
per_agent = {}
|
||||
for a in all_agents:
|
||||
per_agent[a] = {
|
||||
"agent": a,
|
||||
"display_name": (registry.get(a, {}) or {}).get("display_name", a),
|
||||
"chat_messages": chat_counts.get(a, 0),
|
||||
"user_turns": user_turns.get(a, 0),
|
||||
"time_seconds": time_per_agent.get(a, {}).get("total_seconds", 0),
|
||||
"sessions": time_per_agent.get(a, {}).get("sessions", 0),
|
||||
"touches": time_per_agent.get(a, {}).get("touches", 0),
|
||||
"tokens": tokens_per_agent.get(a, 0),
|
||||
"cost": round(cost_per_agent.get(a, 0), 6),
|
||||
"top_models": sorted(
|
||||
models_per_agent.get(a, {}).items(),
|
||||
key=lambda kv: kv[1], reverse=True,
|
||||
),
|
||||
"suite": suite.get(a, {}),
|
||||
"first_seen": first_chat.get(a),
|
||||
"last_seen": last_chat.get(a),
|
||||
}
|
||||
|
||||
# Rankings
|
||||
by_chats = sorted(
|
||||
per_agent.values(), key=lambda x: x["user_turns"], reverse=True
|
||||
)
|
||||
by_time = sorted(
|
||||
per_agent.values(), key=lambda x: x["time_seconds"], reverse=True
|
||||
)
|
||||
|
||||
# Most-used model overall per agent already in top_models; also a global
|
||||
# model popularity map.
|
||||
global_models = {}
|
||||
for a, models in models_per_agent.items():
|
||||
for m, c in models.items():
|
||||
global_models[m] = global_models.get(m, 0) + c
|
||||
|
||||
return {
|
||||
"generated_at": get_timestamp(),
|
||||
"totals": {
|
||||
"total_chat_messages": sum(chat_counts.values()),
|
||||
"total_user_turns": sum(user_turns.values()),
|
||||
"total_time_seconds": time_report.get("total_seconds", 0),
|
||||
"total_time_human": time_report.get("total_human", "0h 0m 0s"),
|
||||
"total_tokens": sum(tokens_per_agent.values()),
|
||||
"total_cost": round(sum(cost_per_agent.values()), 6),
|
||||
"agents_observed": len(per_agent),
|
||||
},
|
||||
"most_chatted": [
|
||||
{"agent": x["agent"], "display_name": x["display_name"],
|
||||
"user_turns": x["user_turns"], "chat_messages": x["chat_messages"]}
|
||||
for x in by_chats if x["user_turns"] > 0
|
||||
],
|
||||
"most_time": [
|
||||
{"agent": x["agent"], "display_name": x["display_name"],
|
||||
"time_seconds": x["time_seconds"], "sessions": x["sessions"],
|
||||
"touches": x["touches"]}
|
||||
for x in by_time if x["time_seconds"] > 0
|
||||
],
|
||||
"global_models": sorted(
|
||||
global_models.items(), key=lambda kv: kv[1], reverse=True
|
||||
),
|
||||
"per_agent": per_agent,
|
||||
}
|
||||
|
||||
|
||||
# ─── Favicon ──────────────────────────────────────────────────────
|
||||
|
||||
FAVICON_SVG = '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32"><defs><linearGradient id="g" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" stop-color="#6c5ce7"/><stop offset="100%" stop-color="#fd79a8"/></linearGradient></defs><rect width="32" height="32" rx="8" fill="url(#g)"/><polygon points="16,6 24,11 24,21 16,26 8,21 8,11" fill="none" stroke="white" stroke-width="2" stroke-linejoin="round"/><circle cx="16" cy="16" r="3" fill="white"/></svg>'
|
||||
|
|
|
|||
Loading…
Reference in New Issue