From 1dc6fca6422efb099e7a7ff6d9fd789caa2c3722 Mon Sep 17 00:00:00 2001 From: TD Date: Wed, 9 Sep 2026 19:19:45 +0200 Subject: [PATCH] feat(voice): typed command box, on-screen replies and help answers for the local provider --- server/localVoice.js | 4 +++- src/voice/localVoice.js | 52 +++++++++++++++++++++++++++++++++++++++-- style.css | 36 ++++++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 3 deletions(-) diff --git a/server/localVoice.js b/server/localVoice.js index 00abe99..af251a7 100644 --- a/server/localVoice.js +++ b/server/localVoice.js @@ -224,7 +224,9 @@ function sendJson(res, status, payload) { export function installLocalVoiceMiddleware(middlewares, { tools, instructions, env = process.env }) { const localInstructions = `${instructions}\n` + 'You are running on a local/private model, not OpenAI. Reply in the language the operator speaks (Spanish when they speak Spanish). ' - + 'Keep spoken replies to one short sentence. When a tool is needed, call it; after tool results, confirm in one sentence.'; + + 'Keep spoken replies to one short sentence. When a tool is needed, call it; after tool results, confirm in one sentence. ' + + 'If the operator asks what you can do, what to say, or how to zoom / rotate / orbit / follow / enter the cockpit / change layers or styles, ' + + 'answer without calling a tool: a short list (max 8 lines) of example commands drawn from your tools, in their language.'; middlewares.use('/api/local-voice/status', (req, res) => { const config = resolveLocalVoiceProvider(env); diff --git a/src/voice/localVoice.js b/src/voice/localVoice.js index b4163fb..9f78b09 100644 --- a/src/voice/localVoice.js +++ b/src/voice/localVoice.js @@ -48,6 +48,10 @@ export class LocalVoiceController { this.spaceKeyHeld = false; this.pushToTalkKeyHeld = false; this.lastError = null; + this.panel = createLocalPanel(this.ui?.root, { + onSubmit: (text) => this.sendTextCommand(text), + placeholder: this.lang.toLowerCase().startsWith('es') ? 'Escribe una orden · «?» = ayuda' : "Type a command · '?' for help", + }); this.setStatus('idle'); } @@ -123,6 +127,7 @@ export class LocalVoiceController { } stop({ removeUi = false } = {}) { + if (removeUi) this.panel?.root?.remove(); this.active = false; this.stopRecognizer(); try { window.speechSynthesis?.cancel(); } catch { /* ignore */ } @@ -131,12 +136,24 @@ export class LocalVoiceController { } sendTextCommand(text) { - const clean = String(text || '').trim(); + let clean = String(text || '').trim(); if (!clean) return false; + if (clean === '?' || clean === 'help' || clean === 'ayuda') { + clean = this.lang.toLowerCase().startsWith('es') + ? '¿Qué puedo decirte? Resume en pocas líneas las órdenes que entiendes: navegar, zoom, girar/orbitar, seguir aviones o barcos, cabina, capas, estilos, radio.' + : 'What can I say? Summarize in a few lines the commands you understand: navigate, zoom, orbit/rotate, follow aircraft or ships, cockpit, layers, styles, radio.'; + } this.handleUtterance(clean, { typed: true }); return true; } + showReply(text, { error = false } = {}) { + if (!this.panel?.reply) return; + this.panel.reply.hidden = !text; + this.panel.reply.textContent = text || ''; + this.panel.reply.dataset.error = error ? 'true' : 'false'; + } + providerLabel() { if (this.provider === 'anthropic') return 'CLAUDE'; if (this.provider === 'ollama') return (this.model || 'LOCAL').toUpperCase().slice(0, 12); @@ -171,10 +188,12 @@ export class LocalVoiceController { } const reply = String(turn.text || '').trim() || (rounds ? 'Done.' : ''); this.pushHistory({ role: 'assistant', content: reply, toolCalls: [] }); - if (reply) await this.speak(reply); + this.showReply(reply); + if (reply && this.active) await this.speak(reply); this.setStatus(this.active ? 'listening' : 'idle', this.active ? `${this.providerLabel()} · ${reply.slice(0, 70)}` : undefined); } catch (error) { this.lastError = String(error?.message || error); + this.showReply(this.lastError, { error: true }); this.setStatus(this.active ? 'listening' : 'idle', `${this.providerLabel()} · ${this.lastError.slice(0, 80)}`); if (typed) console.warn('[local-voice]', this.lastError); } finally { @@ -296,3 +315,32 @@ export class LocalVoiceController { return { provider: this.provider, model: this.model, lang: this.lang, active: this.active, busy: this.busy, history: this.history.length, lastError: this.lastError }; } } + +/** + * Typed command box + last-reply line under the mic control. Voice is optional: + * the same turn loop runs from the keyboard (car passengers, quiet rooms, + * browsers without SpeechRecognition). + */ +function createLocalPanel(root, { onSubmit, placeholder }) { + if (!root || typeof document === 'undefined') return null; + root.querySelector('.gev-local-voice')?.remove(); + const panel = document.createElement('div'); + panel.className = 'gev-local-voice'; + panel.innerHTML = ` +
+ +
+ `; + const form = panel.querySelector('form'); + const input = panel.querySelector('input'); + input.placeholder = placeholder; + form.addEventListener('submit', (event) => { + event.preventDefault(); + const text = input.value; + input.value = ''; + onSubmit(text); + }); + input.addEventListener('keydown', (event) => event.stopPropagation()); // keep app hotkeys (Space, 1-7) out of the box + root.appendChild(panel); + return { root: panel, input, reply: panel.querySelector('.gev-local-voice-reply') }; +} diff --git a/style.css b/style.css index 3941027..11be37c 100644 --- a/style.css +++ b/style.css @@ -9520,3 +9520,39 @@ body.scene-playback-mode #key-setup { border-color: rgba(255, 170, 150, 0.65); outline: none; } + +/* Local voice provider: typed command box + last reply (src/voice/localVoice.js) */ +.gev-local-voice { + display: flex; + flex-direction: column; + gap: 4px; + margin-top: 6px; + width: 100%; +} +.gev-local-voice-input { + width: 100%; + box-sizing: border-box; + font: inherit; + font-size: 12px; + color: inherit; + background: rgba(0, 0, 0, 0.35); + border: 1px solid rgba(255, 255, 255, 0.25); + border-radius: 4px; + padding: 4px 6px; + outline: none; +} +.gev-local-voice-input:focus { + border-color: rgba(255, 255, 255, 0.6); +} +.gev-local-voice-reply { + font-size: 12px; + line-height: 1.35; + max-height: 9.5em; + overflow-y: auto; + white-space: pre-wrap; + opacity: 0.9; + padding: 2px 2px 0; +} +.gev-local-voice-reply[data-error='true'] { + color: #ff8a80; +}