256 lines
10 KiB
JavaScript
256 lines
10 KiB
JavaScript
/**
|
|
* Local voice provider for God's Eye View (Solendra, 9 Sep 2026).
|
|
*
|
|
* Answers upstream issue #212: drive the SAME 28 voice tools with a local LLM
|
|
* (Ollama) or Anthropic instead of the OpenAI Realtime API. The browser does
|
|
* speech-to-text (Web Speech API) and text-to-speech (speechSynthesis); this
|
|
* module only turns a transcript + tool schemas into tool calls.
|
|
*
|
|
* Endpoints (installed by installLocalVoiceMiddleware):
|
|
* GET /api/local-voice/status → { configured, provider, model, lang }
|
|
* POST /api/local-voice/turn → { text, toolCalls:[{ id, name, args }] }
|
|
*
|
|
* Conversation format is provider-neutral so the client keeps ONE history:
|
|
* { role:'user', content }
|
|
* { role:'assistant', content, toolCalls:[{ id, name, args }] }
|
|
* { role:'tool', id, name, content } (content = JSON string of the result)
|
|
*
|
|
* Env: LOCAL_VOICE_PROVIDER=ollama|anthropic|auto (default auto),
|
|
* OLLAMA_URL (default http://127.0.0.1:11434), OLLAMA_MODEL (default qwen3),
|
|
* ANTHROPIC_API_KEY, ANTHROPIC_MODEL (default claude-sonnet-5),
|
|
* LOCAL_VOICE_LANG (BCP-47 hint for the browser recognizer, default navigator language).
|
|
*/
|
|
|
|
const MAX_HISTORY = 24;
|
|
const MAX_BODY_BYTES = 256 * 1024;
|
|
const DEFAULT_OLLAMA_URL = 'http://127.0.0.1:11434';
|
|
const DEFAULT_OLLAMA_MODEL = 'qwen3';
|
|
const DEFAULT_ANTHROPIC_MODEL = 'claude-sonnet-5';
|
|
const ANTHROPIC_VERSION = '2023-06-01';
|
|
|
|
export function resolveLocalVoiceProvider(env = process.env) {
|
|
const requested = String(env.LOCAL_VOICE_PROVIDER || 'auto').trim().toLowerCase();
|
|
const anthropicKey = String(env.ANTHROPIC_API_KEY || '').trim();
|
|
const ollamaUrl = String(env.OLLAMA_URL || '').trim();
|
|
const lang = String(env.LOCAL_VOICE_LANG || '').trim() || null;
|
|
let provider = null;
|
|
if (requested === 'anthropic' && anthropicKey) provider = 'anthropic';
|
|
else if (requested === 'ollama' && (ollamaUrl || requested === 'ollama')) provider = 'ollama';
|
|
else if (requested === 'auto') provider = anthropicKey ? 'anthropic' : (ollamaUrl ? 'ollama' : null);
|
|
if (!provider) return { configured: false, provider: null, model: null, lang };
|
|
if (provider === 'anthropic') {
|
|
return { configured: true, provider, model: String(env.ANTHROPIC_MODEL || DEFAULT_ANTHROPIC_MODEL).trim(), lang, apiKey: anthropicKey };
|
|
}
|
|
return {
|
|
configured: true,
|
|
provider,
|
|
model: String(env.OLLAMA_MODEL || DEFAULT_OLLAMA_MODEL).trim(),
|
|
lang,
|
|
url: (ollamaUrl || DEFAULT_OLLAMA_URL).replace(/\/+$/, ''),
|
|
};
|
|
}
|
|
|
|
/** Realtime-style tool ({type:'function',name,description,parameters}) → Ollama/OpenAI chat tool. */
|
|
export function toOllamaTools(tools) {
|
|
return (tools || []).map((t) => ({
|
|
type: 'function',
|
|
function: { name: t.name, description: t.description || '', parameters: t.parameters || { type: 'object', properties: {} } },
|
|
}));
|
|
}
|
|
|
|
/** Realtime-style tool → Anthropic tool. */
|
|
export function toAnthropicTools(tools) {
|
|
return (tools || []).map((t) => ({
|
|
name: t.name,
|
|
description: t.description || '',
|
|
input_schema: t.parameters || { type: 'object', properties: {} },
|
|
}));
|
|
}
|
|
|
|
export function sanitizeHistory(messages) {
|
|
if (!Array.isArray(messages)) return [];
|
|
const clean = [];
|
|
for (const m of messages) {
|
|
if (!m || typeof m !== 'object') continue;
|
|
const role = m.role === 'assistant' || m.role === 'tool' ? m.role : 'user';
|
|
const content = typeof m.content === 'string' ? m.content.slice(0, 8000) : '';
|
|
if (role === 'assistant') {
|
|
const toolCalls = Array.isArray(m.toolCalls)
|
|
? m.toolCalls
|
|
.filter((c) => c && typeof c.name === 'string')
|
|
.map((c, i) => ({ id: String(c.id || `call_${i}`), name: c.name, args: c.args && typeof c.args === 'object' ? c.args : {} }))
|
|
: [];
|
|
clean.push({ role, content, toolCalls });
|
|
} else if (role === 'tool') {
|
|
clean.push({ role, id: String(m.id || ''), name: String(m.name || ''), content });
|
|
} else {
|
|
clean.push({ role, content });
|
|
}
|
|
}
|
|
return clean.slice(-MAX_HISTORY);
|
|
}
|
|
|
|
function toOllamaMessages(instructions, history) {
|
|
const out = [{ role: 'system', content: instructions }];
|
|
for (const m of history) {
|
|
if (m.role === 'assistant') {
|
|
const msg = { role: 'assistant', content: m.content || '' };
|
|
if (m.toolCalls.length) msg.tool_calls = m.toolCalls.map((c) => ({ function: { name: c.name, arguments: c.args } }));
|
|
out.push(msg);
|
|
} else if (m.role === 'tool') {
|
|
out.push({ role: 'tool', content: m.content || '{}' });
|
|
} else {
|
|
out.push({ role: 'user', content: m.content });
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function toAnthropicMessages(history) {
|
|
const out = [];
|
|
let pendingResults = null;
|
|
const flushResults = () => {
|
|
if (pendingResults && pendingResults.length) out.push({ role: 'user', content: pendingResults });
|
|
pendingResults = null;
|
|
};
|
|
for (const m of history) {
|
|
if (m.role === 'tool') {
|
|
if (!pendingResults) pendingResults = [];
|
|
pendingResults.push({ type: 'tool_result', tool_use_id: m.id, content: m.content || '{}' });
|
|
continue;
|
|
}
|
|
flushResults();
|
|
if (m.role === 'assistant') {
|
|
const blocks = [];
|
|
if (m.content) blocks.push({ type: 'text', text: m.content });
|
|
for (const c of m.toolCalls) blocks.push({ type: 'tool_use', id: c.id, name: c.name, input: c.args });
|
|
if (blocks.length) out.push({ role: 'assistant', content: blocks });
|
|
} else {
|
|
out.push({ role: 'user', content: [{ type: 'text', text: m.content || '…' }] });
|
|
}
|
|
}
|
|
flushResults();
|
|
// Anthropic requires alternating roles starting with user; merge adjacent same-role messages.
|
|
const merged = [];
|
|
for (const m of out) {
|
|
const last = merged[merged.length - 1];
|
|
if (last && last.role === m.role) last.content = last.content.concat(m.content);
|
|
else merged.push(m);
|
|
}
|
|
if (merged.length && merged[0].role !== 'user') merged.unshift({ role: 'user', content: [{ type: 'text', text: '…' }] });
|
|
return merged;
|
|
}
|
|
|
|
async function ollamaTurn(config, instructions, tools, history) {
|
|
const response = await fetch(`${config.url}/api/chat`, {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({
|
|
model: config.model,
|
|
stream: false,
|
|
think: false,
|
|
messages: toOllamaMessages(instructions, history),
|
|
tools: toOllamaTools(tools),
|
|
options: { temperature: 0.2 },
|
|
}),
|
|
});
|
|
if (!response.ok) throw new Error(`Ollama ${response.status}: ${(await response.text()).slice(0, 200)}`);
|
|
const data = await response.json();
|
|
const message = data?.message || {};
|
|
const toolCalls = Array.isArray(message.tool_calls)
|
|
? message.tool_calls.map((c, i) => ({
|
|
id: String(c.id || `call_${Date.now()}_${i}`),
|
|
name: c.function?.name || '',
|
|
args: parseArgs(c.function?.arguments),
|
|
})).filter((c) => c.name)
|
|
: [];
|
|
return { text: String(message.content || '').trim(), toolCalls, model: data?.model || config.model };
|
|
}
|
|
|
|
async function anthropicTurn(config, instructions, tools, history) {
|
|
const response = await fetch('https://api.anthropic.com/v1/messages', {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json',
|
|
'x-api-key': config.apiKey,
|
|
'anthropic-version': ANTHROPIC_VERSION,
|
|
},
|
|
body: JSON.stringify({
|
|
model: config.model,
|
|
max_tokens: 600,
|
|
system: instructions,
|
|
tools: toAnthropicTools(tools),
|
|
messages: toAnthropicMessages(history),
|
|
}),
|
|
});
|
|
if (!response.ok) throw new Error(`Anthropic ${response.status}: ${(await response.text()).slice(0, 200)}`);
|
|
const data = await response.json();
|
|
const blocks = Array.isArray(data?.content) ? data.content : [];
|
|
const text = blocks.filter((b) => b.type === 'text').map((b) => b.text).join(' ').trim();
|
|
const toolCalls = blocks.filter((b) => b.type === 'tool_use').map((b) => ({ id: b.id, name: b.name, args: b.input || {} }));
|
|
return { text, toolCalls, model: data?.model || config.model };
|
|
}
|
|
|
|
function parseArgs(value) {
|
|
if (value && typeof value === 'object') return value;
|
|
if (typeof value === 'string' && value.trim()) {
|
|
try { return JSON.parse(value); } catch { return {}; }
|
|
}
|
|
return {};
|
|
}
|
|
|
|
async function readJson(req, maxBytes) {
|
|
const chunks = [];
|
|
let size = 0;
|
|
for await (const chunk of req) {
|
|
size += chunk.length;
|
|
if (size > maxBytes) throw new Error('body too large');
|
|
chunks.push(chunk);
|
|
}
|
|
const text = Buffer.concat(chunks).toString('utf8');
|
|
return text ? JSON.parse(text) : {};
|
|
}
|
|
|
|
function sendJson(res, status, payload) {
|
|
res.statusCode = status;
|
|
res.setHeader('Content-Type', 'application/json');
|
|
res.end(JSON.stringify(payload));
|
|
}
|
|
|
|
/**
|
|
* @param {import('connect').Server} middlewares Vite's connect app.
|
|
* @param {{ tools: object[], instructions: string, env?: object }} options
|
|
*/
|
|
export function installLocalVoiceMiddleware(middlewares, { tools, instructions, env = process.env }) {
|
|
const localInstructions = `${instructions}\n`
|
|
+ 'You are running on a local/private model, not OpenAI. Reply in the language the operator speaks (Spanish when they speak Spanish). '
|
|
+ 'Keep spoken replies to one short sentence. When a tool is needed, call it; after tool results, confirm in one sentence.';
|
|
|
|
middlewares.use('/api/local-voice/status', (req, res) => {
|
|
const config = resolveLocalVoiceProvider(env);
|
|
sendJson(res, 200, { configured: config.configured, provider: config.provider, model: config.model, lang: config.lang });
|
|
});
|
|
|
|
middlewares.use('/api/local-voice/turn', async (req, res) => {
|
|
if (req.method !== 'POST') return sendJson(res, 405, { error: 'Method not allowed' });
|
|
const config = resolveLocalVoiceProvider(env);
|
|
if (!config.configured) return sendJson(res, 503, { error: 'No local voice provider configured (set OLLAMA_URL or ANTHROPIC_API_KEY)' });
|
|
let body;
|
|
try {
|
|
body = await readJson(req, MAX_BODY_BYTES);
|
|
} catch (error) {
|
|
return sendJson(res, 400, { error: `Bad request: ${error.message}` });
|
|
}
|
|
const history = sanitizeHistory(body.messages);
|
|
if (!history.length) return sendJson(res, 400, { error: 'messages required' });
|
|
try {
|
|
const turn = config.provider === 'anthropic'
|
|
? await anthropicTurn(config, localInstructions, tools, history)
|
|
: await ollamaTurn(config, localInstructions, tools, history);
|
|
sendJson(res, 200, { ...turn, provider: config.provider });
|
|
} catch (error) {
|
|
sendJson(res, 502, { error: String(error?.message || error).slice(0, 300), provider: config.provider });
|
|
}
|
|
});
|
|
}
|