diff --git a/apps/desktop/src/app/contrib/wiring.tsx b/apps/desktop/src/app/contrib/wiring.tsx index 8f1108cdc225d..02f95d9765a9d 100644 --- a/apps/desktop/src/app/contrib/wiring.tsx +++ b/apps/desktop/src/app/contrib/wiring.tsx @@ -66,7 +66,7 @@ import { setMessages } from '@/store/session' import { clearSessionTodos, setSessionTodos, todosForHydration } from '@/store/todos' -import { armWakeWord } from '@/store/wake-word' +import { armWakeWord, stopClientCapture } from '@/store/wake-word' import { isSecondaryWindow } from '@/store/windows' import { useSkinCommand } from '@/themes/use-skin-command' @@ -689,6 +689,10 @@ export function ContribWiring({ children }: { children: ReactNode }) { if (event.type === 'wake.detected') { const payload = event.payload as { profile?: null | string; start_new_session?: boolean } | undefined + // Free the Mac mic so voice conversation can open getUserMedia. + // Server already pauses the detector lease; this stops client PCM feed. + stopClientCapture() + // Audible confirmation that the wake registered, before voice capture // starts. Gated by the shared sound-mute toggle. playWakeSound() diff --git a/apps/desktop/src/lib/wake-client-capture.ts b/apps/desktop/src/lib/wake-client-capture.ts new file mode 100644 index 0000000000000..1bb7867bccab3 --- /dev/null +++ b/apps/desktop/src/lib/wake-client-capture.ts @@ -0,0 +1,178 @@ +/** + * Client-side mic capture for remote wake word. + * + * When the backend arms with `capture: "client"`, PortAudio runs on a headless + * VM with no mic. The desktop opens getUserMedia here, resamples to 16 kHz + * mono int16 frames, and pushes them via `wake.feed` so openWakeWord still + * runs server-side without requiring a server sound device. + */ + +const TARGET_RATE = 16_000 +const DEFAULT_FRAME = 1280 // 80 ms @ 16 kHz — matches tools/wake_word.py + +export type WakeFeedRequester = ( + method: string, + params?: Record +) => Promise + +export interface ClientWakeCaptureOptions { + /** Samples per frame at 16 kHz (from wake.start response). */ + frameLength?: number + request: WakeFeedRequester + onError?: (error: Error) => void +} + +export interface ClientWakeCaptureHandle { + stop: () => void + readonly active: boolean +} + +function downsampleTo16k(input: Float32Array, inputRate: number): Float32Array { + if (inputRate === TARGET_RATE) { + return input + } + if (inputRate <= 0) { + return new Float32Array(0) + } + const ratio = inputRate / TARGET_RATE + const outLen = Math.max(1, Math.floor(input.length / ratio)) + const out = new Float32Array(outLen) + for (let i = 0; i < outLen; i++) { + const start = Math.floor(i * ratio) + const end = Math.min(input.length, Math.floor((i + 1) * ratio)) + let sum = 0 + let count = 0 + for (let j = start; j < end; j++) { + sum += input[j] ?? 0 + count++ + } + out[i] = count > 0 ? sum / count : 0 + } + return out +} + +function floatToInt16LE(input: Float32Array): ArrayBuffer { + const buf = new ArrayBuffer(input.length * 2) + const view = new DataView(buf) + for (let i = 0; i < input.length; i++) { + const s = Math.max(-1, Math.min(1, input[i] ?? 0)) + view.setInt16(i * 2, s < 0 ? s * 0x8000 : s * 0x7fff, true) + } + return buf +} + +function bytesToBase64(buf: ArrayBuffer): string { + const bytes = new Uint8Array(buf) + let binary = '' + const chunk = 0x8000 + for (let i = 0; i < bytes.length; i += chunk) { + binary += String.fromCharCode(...bytes.subarray(i, i + chunk)) + } + return btoa(binary) +} + +/** + * Start streaming the default microphone to `wake.feed`. + * Returns a handle whose `stop()` ends tracks + audio graph. + */ +export async function startClientWakeCapture( + options: ClientWakeCaptureOptions +): Promise { + const frameLength = Math.max(160, Math.trunc(options.frameLength || DEFAULT_FRAME)) + const audioWindow = window as Window & { webkitAudioContext?: typeof AudioContext } + const AudioContextCtor = window.AudioContext || audioWindow.webkitAudioContext + if (!AudioContextCtor) { + throw new Error('AudioContext unavailable for client wake capture') + } + if (!navigator.mediaDevices?.getUserMedia) { + throw new Error('getUserMedia unavailable for client wake capture') + } + + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { + channelCount: 1, + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true + }, + video: false + }) + + const context = new AudioContextCtor() + const source = context.createMediaStreamSource(stream) + // ScriptProcessor is deprecated but widely available and simple for PCM export. + // Buffer size 4096 keeps callback rate reasonable on desktop. + const processor = context.createScriptProcessor(4096, 1, 1) + const mute = context.createGain() + mute.gain.value = 0 + + let pending = new Float32Array(0) + let stopped = false + let inflight = false + + const pushFrame = async (frame: Float32Array) => { + if (stopped || inflight) { + return + } + inflight = true + try { + const pcm = floatToInt16LE(frame) + await options.request('wake.feed', { + pcm: bytesToBase64(pcm), + sample_rate: TARGET_RATE + }) + } catch (error) { + options.onError?.(error instanceof Error ? error : new Error(String(error))) + } finally { + inflight = false + } + } + + processor.onaudioprocess = event => { + if (stopped) { + return + } + const input = event.inputBuffer.getChannelData(0) + const at16k = downsampleTo16k(input, context.sampleRate) + // Append to pending and emit full frames + const merged = new Float32Array(pending.length + at16k.length) + merged.set(pending, 0) + merged.set(at16k, pending.length) + let offset = 0 + while (offset + frameLength <= merged.length) { + const frame = merged.subarray(offset, offset + frameLength) + offset += frameLength + void pushFrame(new Float32Array(frame)) + } + pending = merged.subarray(offset) + } + + source.connect(processor) + processor.connect(mute) + mute.connect(context.destination) + + if (context.state === 'suspended') { + await context.resume().catch(() => undefined) + } + + return { + get active() { + return !stopped + }, + stop() { + if (stopped) { + return + } + stopped = true + try { + processor.disconnect() + source.disconnect() + mute.disconnect() + } catch { + // ignore + } + void context.close().catch(() => undefined) + stream.getTracks().forEach(t => t.stop()) + } + } +} diff --git a/apps/desktop/src/store/wake-word.ts b/apps/desktop/src/store/wake-word.ts index 8f214f8989692..89328ecce7701 100644 --- a/apps/desktop/src/store/wake-word.ts +++ b/apps/desktop/src/store/wake-word.ts @@ -1,6 +1,10 @@ import { atom } from 'nanostores' import { $gateway } from '@/store/gateway' +import { + startClientWakeCapture, + type ClientWakeCaptureHandle +} from '@/lib/wake-client-capture' // "Hey Hermes" wake-word listener state for the composer toggle. The gateway is // the single source of truth (the listener lives in the backend and is shared @@ -33,16 +37,62 @@ const INITIAL_WAKE_WORD_STATE: WakeWordState = { export const $wakeWord = atom(INITIAL_WAKE_WORD_STATE) +/** Active client mic stream for remote wake (capture: client). */ +let clientCapture: ClientWakeCaptureHandle | null = null + +/** Stop client-side PCM capture (also called on wake.detected before voice). */ +export function stopClientCapture(): void { + clientCapture?.stop() + clientCapture = null +} + +async function maybeStartClientCapture(result: WakeStartResponse | null | undefined): Promise { + stopClientCapture() + if (!result?.started) { + return + } + const mode = (result.capture || '').toLowerCase() + if (mode !== 'client' && mode !== 'remote' && mode !== 'external') { + return + } + try { + clientCapture = await startClientWakeCapture({ + frameLength: result.frame_length, + request: gatewayRequester + }) + } catch (error) { + const current = $wakeWord.get() + $wakeWord.set({ + ...current, + listening: false, + notice: + error instanceof Error + ? error.message + : 'Failed to open the client microphone for wake word', + pending: false + }) + // Best-effort: release server lease if client mic failed. + try { + await gatewayRequester('wake.stop', {}) + } catch { + // ignore + } + } +} + export interface WakeStatusResponse { /** Armed but the selected backend input delivers only silence. */ audio_silent?: boolean available?: boolean + /** local | client | auto — where PCM is captured. */ + capture?: string configured_surface?: string /** Config truth (wake_word.enabled) — drives post-voice re-arm. */ enabled?: boolean hint?: string input_device?: WakeInputDeviceStatus listening?: boolean + local_input_available?: boolean owned_by_caller?: boolean owner_surface?: string | null phrase?: string @@ -50,12 +100,15 @@ export interface WakeStatusResponse { } export interface WakeStartResponse { + capture?: string enabled_persisted?: boolean + frame_length?: number hint?: string owner_surface?: string | null phrase?: string provider?: string reason?: string + sample_rate?: number started?: boolean } @@ -152,10 +205,13 @@ export function applyWakeStartResult(result: WakeStartResponse | null | undefine pending: false, phrase: result.phrase?.trim() || current.phrase }) + void maybeStartClientCapture(result) return } + stopClientCapture() + $wakeWord.set({ ...current, // The backend probes requirements on start; an explicit "unavailable" @@ -174,6 +230,7 @@ export function applyWakeStartResult(result: WakeStartResponse | null | undefine export function applyWakeStopResult(result: WakeStopResponse | null | undefined): void { const current = $wakeWord.get() + stopClientCapture() $wakeWord.set({ ...current, enabled: result?.disabled_persisted ? false : current.enabled, @@ -199,7 +256,10 @@ export async function armWakeWord(request: WakeRequester = gatewayRequester): Pr return } - const result = await request('wake.start', { surface: 'gui' }) + const result = await request('wake.start', { + surface: 'gui', + client_capture: true + }) applyWakeStartResult(result) } catch { // Older backends / transient failures — keep whatever we last knew. @@ -229,7 +289,13 @@ export async function toggleWakeWord(request: WakeRequester = gatewayRequester): // persist: true — a deliberate click is consent, so the backend flips // wake_word.enabled in config.yaml (on/off) and the choice sticks for // future sessions. Auto-arm (armWakeWord) never passes it. - applyWakeStartResult(await request('wake.start', { persist: true, surface: 'gui' })) + applyWakeStartResult( + await request('wake.start', { + persist: true, + surface: 'gui', + client_capture: true + }) + ) } } catch (error) { const current = $wakeWord.get() @@ -274,10 +340,23 @@ export async function resumeWakeAfterVoice(request: WakeRequester = gatewayReque } if (status.listening) { + // Server lease is still armed (e.g. wake.resume after voice). + // Client PCM was stopped on wake.detected — reattach if needed. + const mode = (status.capture || '').toLowerCase() + if (mode === 'client' || mode === 'remote' || mode === 'external') { + void maybeStartClientCapture({ + started: true, + capture: 'client', + frame_length: 1280 + }) + } return } - const started = await request('wake.start', { surface: 'gui' }) + const started = await request('wake.start', { + surface: 'gui', + client_capture: true + }) applyWakeStartResult(started) if (started?.started) { @@ -298,5 +377,6 @@ export async function resumeWakeAfterVoice(request: WakeRequester = gatewayReque /** Test-only reset. */ export function resetWakeWordState(): void { + stopClientCapture() $wakeWord.set(INITIAL_WAKE_WORD_STATE) } diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index 4ff18177a4538..0ae798094afd4 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -1577,6 +1577,7 @@ DEFAULT_CONFIG = { "enabled": False, "surface": "auto", # eligible surface: "auto" (first claimant) | "cli" | "tui" | "gui" "input_device": None, # PortAudio input device index/name; null uses the process default + "capture": "auto", # auto | local | client — where PCM is captured (client = desktop streams mic via wake.feed) "provider": "openwakeword", # "openwakeword" (free, local) | "sherpa" (free, ANY phrase, no training) | "porcupine" (premium; needs PORCUPINE_ACCESS_KEY) "phrase": "hey hermes", # for "sherpa" this IS the detected phrase (any text works); for other engines it's a cosmetic label — detection is keyed by the model/keyword below "sensitivity": 0.6, # 0.0-1.0 detection threshold, consistent across engines (higher = stricter, fewer false triggers) diff --git a/tests/tools/test_wake_word.py b/tests/tools/test_wake_word.py index 21907073fd9b3..0ac450f5c5a17 100644 --- a/tests/tools/test_wake_word.py +++ b/tests/tools/test_wake_word.py @@ -171,12 +171,14 @@ def test_requirements_fresh_install_lazy_allowed(monkeypatch): def test_requirements_deps_present_but_no_audio_hint(monkeypatch): """Once deps ARE installed, a failing audio probe blocks with a mic hint (lazy installs can't fix a missing audio device).""" + _voice_loop_ready(monkeypatch) monkeypatch.setattr(ww, "_audio_available", lambda: False) + monkeypatch.setattr(ww, "_local_input_device_ready", lambda: False) monkeypatch.setattr("tools.lazy_deps.is_available", lambda f: True) monkeypatch.setattr("tools.lazy_deps._allow_lazy_installs", lambda: True) - r = ww.check_wake_word_requirements({"provider": "openwakeword"}) + r = ww.check_wake_word_requirements({"provider": "openwakeword", "capture": "local"}) assert r["available"] is False - assert "audio device" in r["hint"] + assert "audio device" in r["hint"] or "microphone" in r["hint"].lower() # ── openWakeWord engine (bundled model + base-model fetch) ─────────────── @@ -624,3 +626,85 @@ def test_machine_lock_is_released_when_owner_process_exits(tmp_path): if process.is_alive(): process.terminate() process.join(10) + + +# ── Client capture (remote desktop mic → wake.feed) ────────────────────── + + +def test_resolve_capture_mode_auto_and_prefer_client(monkeypatch): + monkeypatch.setattr(ww, "_local_input_device_ready", lambda: False) + # auto without prefer_client stays local (CLI/TUI/status semantics) + assert ww.resolve_capture_mode({"capture": "auto"}) == "local" + assert ww.resolve_capture_mode({"capture": "auto"}, prefer_client=True) == "client" + assert ww.resolve_capture_mode({"capture": "local"}, prefer_client=True) == "local" + assert ww.resolve_capture_mode({"capture": "client"}) == "client" + assert ww.resolve_capture_mode({"capture": "auto"}, force_local=True) == "local" + monkeypatch.setattr(ww, "_local_input_device_ready", lambda: True) + assert ww.resolve_capture_mode({"capture": "auto"}) == "local" + assert ww.resolve_capture_mode({"capture": "auto"}, prefer_client=True) == "client" + + +def test_requirements_client_capture_without_local_mic(monkeypatch): + monkeypatch.setattr(ww, "_audio_available", lambda: False) + monkeypatch.setattr(ww, "_local_input_device_ready", lambda: False) + monkeypatch.setattr(ww, "_stt_ready", lambda: True) + monkeypatch.setattr(ww, "_tts_ready", lambda: True) + + class _LD: + @staticmethod + def is_available(feature): + return True + + @staticmethod + def _allow_lazy_installs(): + return False + + @staticmethod + def feature_install_command(feature): + return "" + + monkeypatch.setattr(ww, "lazy_deps", _LD, raising=False) + import tools.lazy_deps as real_ld + monkeypatch.setattr("tools.lazy_deps.is_available", lambda f: True) + monkeypatch.setattr("tools.lazy_deps._allow_lazy_installs", lambda: False) + + reqs = ww.check_wake_word_requirements({"capture": "client", "provider": "openwakeword"}) + assert reqs["available"] is True + assert reqs["capture"] == "client" + + +def test_client_capture_feed_fires(monkeypatch, tmp_path): + import numpy as np + + monkeypatch.setattr(ww, "_build_engine", lambda cfg: _FakeEngine(fire=True)) + monkeypatch.setattr(ww, "_lock_path", lambda: tmp_path / "wake.lock") + # External mode must not import sounddevice + monkeypatch.setattr( + ww, + "_import_audio", + lambda: (_ for _ in ()).throw(OSError("no local mic")), + ) + owner = object() + fired = threading.Event() + + def _on_wake(): + fired.set() + + ww.start_listening(_on_wake, owner=owner, config={}, external_audio=True) + assert ww.is_listening() is True + info = ww.detector_frame_info() + fl = int(info["frame_length"]) + # Non-silent frame so silence flag does not dominate + pcm = (np.ones(fl, dtype=np.int16) * 5000).tobytes() + assert ww.feed_audio(owner=owner, pcm_int16=pcm) is True + assert fired.wait(2.0) + assert ww.stop_listening(owner=owner) is True + + +def test_feed_audio_rejects_wrong_owner(monkeypatch, tmp_path): + monkeypatch.setattr(ww, "_build_engine", lambda cfg: _FakeEngine(fire=False)) + monkeypatch.setattr(ww, "_lock_path", lambda: tmp_path / "wake.lock") + owner = object() + ww.start_listening(lambda: None, owner=owner, config={}, external_audio=True) + assert ww.feed_audio(owner=object(), pcm_int16=b"\x00\x00") is False + assert ww.stop_listening(owner=owner) is True diff --git a/tools/wake_word.py b/tools/wake_word.py index 0786985031070..b71c47cb0b8fe 100644 --- a/tools/wake_word.py +++ b/tools/wake_word.py @@ -76,6 +76,11 @@ _DEFAULTS: Dict[str, Any] = { "enabled": False, "surface": "auto", "input_device": None, + # Where PCM is captured: + # "local" — PortAudio on the backend host (historic default) + # "client" — desktop/TUI streams int16 frames via wake.feed + # "auto" — local when a device exists, else client capture + "capture": "auto", "provider": "openwakeword", "phrase": "hey hermes", "sensitivity": 0.6, @@ -244,6 +249,68 @@ def wake_phrase(cfg: Optional[Dict[str, Any]] = None) -> str: return str(_get(cfg, "phrase")) or "hey hermes" +def resolve_capture_mode( + cfg: Optional[Dict[str, Any]] = None, + *, + prefer_client: bool = False, + force_local: bool = False, +) -> str: + """Return ``local`` or ``client`` capture mode for this arm. + + ``prefer_client`` is set by remote desktop (Mac mic, headless backend). + ``force_local`` keeps CLI/TUI on the process mic. Config ``capture`` is + ``auto`` | ``local`` | ``client``. + """ + cfg = cfg if cfg is not None else load_wake_word_config() + if force_local: + return "local" + raw = str(_get(cfg, "capture") or "auto").strip().lower() + if raw in ("client", "remote", "external"): + return "client" + if raw == "local": + return "local" + # auto + if prefer_client: + return "client" + # Prefer local when a PortAudio input exists. Without an explicit client + # preference (desktop remote), stay on local so CLI/TUI/status still + # require a server mic instead of advertising a capture path only the + # desktop can feed. + if _local_input_device_ready(): + return "local" + return "local" + + +def _local_input_device_ready() -> bool: + """True when PortAudio is importable and at least one input device exists.""" + try: + sd, _ = _import_audio() + except (ImportError, OSError): + return False + try: + devices = sd.query_devices() + except Exception: + return False + if isinstance(devices, dict): + return int(devices.get("max_input_channels") or 0) > 0 + try: + for dev in devices: + channels = dev.get("max_input_channels") if isinstance(dev, dict) else None + if channels is None: + channels = getattr(dev, "max_input_channels", 0) + if int(channels or 0) > 0: + return True + except Exception: + return False + # Also accept a resolvable default input (some hosts list devices oddly). + try: + info = sd.query_devices(None, "input") + channels = info.get("max_input_channels") if isinstance(info, dict) else 0 + return int(channels or 0) > 0 + except Exception: + return False + + def wake_surface_enabled(surface: str, cfg: Optional[Dict[str, Any]] = None) -> bool: """Should ``surface`` (``cli`` / ``tui`` / ``gui``) host the listener? @@ -838,7 +905,7 @@ def check_wake_word_requirements(cfg: Optional[Dict[str, Any]] = None) -> Dict[s hint = lazy_deps.feature_install_command(feature) or "" elif not tflite_ok: hint = "The wake word needs the tflite runtime on this Mac: pip install ai-edge-litert" - elif deps_ok and not audio_ok: + elif deps_ok and not audio_ok and resolve_capture_mode(cfg) == "local": hint = "Microphone capture needs sounddevice + numpy and a working audio device." elif not stt_ok or not tts_ok: missing = " and ".join( @@ -847,12 +914,31 @@ def check_wake_word_requirements(cfg: Optional[Dict[str, Any]] = None) -> Dict[s hint = (f"Wake word needs {missing} configured — run `hermes tools` " f"(Voice section) or see the voice-mode docs.") + capture_mode = resolve_capture_mode(cfg) + local_input_ok = _local_input_device_ready() if deps_ok else False + # Client capture needs deps (engine) but not a server-side PortAudio device. + if capture_mode == "client": + mic_ok = deps_ok or (not deps_ok and lazy_ok) + if deps_ok and not hint: + # No server mic required; clear the local-device hint if that was set. + if hint.startswith("Microphone capture needs"): + hint = "" + else: + mic_ok = (deps_ok and audio_ok) or (not deps_ok and lazy_ok) + if deps_ok and not audio_ok and not hint: + hint = ( + "No local microphone on this backend. Remote desktop can stream " + "the client mic — set wake_word.capture: client or use a desktop " + "build with client-capture wake support." + ) + return { - "available": key_ok and stt_ok and tts_ok and tflite_ok - and ((deps_ok and audio_ok) or (not deps_ok and lazy_ok)), + "available": key_ok and stt_ok and tts_ok and tflite_ok and mic_ok, "provider": provider, "deps_available": deps_ok, "audio_available": audio_ok, + "local_input_available": local_input_ok, + "capture": capture_mode, "access_key_set": key_ok, "stt_available": stt_ok, "tts_available": tts_ok, @@ -875,18 +961,28 @@ class WakeWordDetector: def __init__(self, engine: _Engine, on_wake: Callable[[], None], cooldown: float = _FIRE_COOLDOWN_SECONDS, on_failure: Optional[Callable[["WakeWordDetector"], None]] = None, - input_device: int | str | None = None): + input_device: int | str | None = None, + external_audio: bool = False): self.engine = engine self.on_wake = on_wake self.cooldown = cooldown self.on_failure = on_failure self.input_device = input_device - self.input_device_details: Dict[str, Any] = {"selector": input_device} + self.external_audio = bool(external_audio) + self.input_device_details: Dict[str, Any] = ( + {"selector": "client", "name": "client capture", "hostapi": "remote"} + if self.external_audio + else {"selector": input_device} + ) self._thread: Optional[threading.Thread] = None self._stop = threading.Event() self._callback_inflight = threading.Event() self._last_fire = 0.0 self._lock = threading.Lock() + # Client-capture PCM queue (int16 mono frames). Local mode ignores this. + import queue as _queue + + self._audio_q: "_queue.Queue[Any]" = _queue.Queue(maxsize=64) # True when the stream is open but every frame is (near-)silence. # Surfaced via wake.status / /wake status so users can tell "armed" # from "deaf". @@ -898,8 +994,50 @@ class WakeWordDetector: t = self._thread return t is not None and t.is_alive() + def feed(self, pcm_int16) -> None: + """Enqueue one int16 mono frame (or raw bytes) for client capture. + + Frame length should match ``engine.frame_length`` (typically 1280 samples + at 16 kHz). Short frames are zero-padded; long frames are split. + """ + if not self.external_audio: + return + try: + import numpy as np + except Exception: + return + if isinstance(pcm_int16, (bytes, bytearray, memoryview)): + arr = np.frombuffer(pcm_int16, dtype=np.int16) + else: + arr = np.asarray(pcm_int16, dtype=np.int16).reshape(-1) + fl = int(self.engine.frame_length) + if fl <= 0: + return + # Split / pad into engine frames + offset = 0 + n = int(arr.shape[0]) + while offset < n: + chunk = arr[offset : offset + fl] + offset += fl + if chunk.shape[0] < fl: + pad = np.zeros(fl, dtype=np.int16) + pad[: chunk.shape[0]] = chunk + chunk = pad + try: + self._audio_q.put_nowait(chunk) + except Exception: + # Drop oldest on overflow so we stay real-time + try: + self._audio_q.get_nowait() + except Exception: + pass + try: + self._audio_q.put_nowait(chunk) + except Exception: + pass + def start(self) -> None: - """Open the mic and begin listening. Idempotent.""" + """Open the mic (or client feeder) and begin listening. Idempotent.""" with self._lock: if self._thread is not None and self._thread.is_alive(): return @@ -950,39 +1088,53 @@ class WakeWordDetector: def _run(self, ready: threading.Event, startup_errors: list[BaseException]) -> None: - try: - sd, _ = _import_audio() - except (ImportError, OSError) as e: - logger.error("wake word: audio libraries unavailable: %s", e) - startup_errors.append(e) - ready.set() - return - frame_length = self.engine.frame_length - self.input_device_details = _describe_input_device(sd, self.input_device) - logger.info( - "wake word: opening microphone device=%s selector=%r hostapi=%s " - "default_rate=%s requested_rate=%d", - self.input_device_details.get("name") or "system default", - self.input_device, - self.input_device_details.get("hostapi") or "unknown", - self.input_device_details.get("default_samplerate") or "unknown", - SAMPLE_RATE, - ) - try: - stream = sd.InputStream( - device=self.input_device, - samplerate=SAMPLE_RATE, - channels=1, - dtype="int16", - blocksize=frame_length, + stream = None + + if self.external_audio: + # Drain any stale frames from a previous arm. + try: + while True: + self._audio_q.get_nowait() + except Exception: + pass + logger.info( + "wake word: client-capture mode (frame=%d, rate=%d) — waiting for wake.feed", + frame_length, SAMPLE_RATE, ) - stream.start() - except Exception as e: - logger.error("wake word: failed to open microphone: %s", e) - startup_errors.append(e) - ready.set() - return + else: + try: + sd, _ = _import_audio() + except (ImportError, OSError) as e: + logger.error("wake word: audio libraries unavailable: %s", e) + startup_errors.append(e) + ready.set() + return + + self.input_device_details = _describe_input_device(sd, self.input_device) + logger.info( + "wake word: opening microphone device=%s selector=%r hostapi=%s " + "default_rate=%s requested_rate=%d", + self.input_device_details.get("name") or "system default", + self.input_device, + self.input_device_details.get("hostapi") or "unknown", + self.input_device_details.get("default_samplerate") or "unknown", + SAMPLE_RATE, + ) + try: + stream = sd.InputStream( + device=self.input_device, + samplerate=SAMPLE_RATE, + channels=1, + dtype="int16", + blocksize=frame_length, + ) + stream.start() + except Exception as e: + logger.error("wake word: failed to open microphone: %s", e) + startup_errors.append(e) + ready.set() + return # Drop any buffered audio/feature state so a resume right after a voice # turn can't immediately re-fire on audio captured before the pause (the @@ -992,7 +1144,8 @@ class WakeWordDetector: except Exception: pass - logger.info("wake word: listening (frame=%d, rate=%d)", frame_length, SAMPLE_RATE) + logger.info("wake word: listening (frame=%d, rate=%d, external=%s)", + frame_length, SAMPLE_RATE, self.external_audio) ready.set() failed = False # ~seconds of consecutive near-zero frames before we flag the stream @@ -1001,7 +1154,18 @@ class WakeWordDetector: try: while not self._stop.is_set(): try: - data, _overflow = stream.read(frame_length) + if self.external_audio: + try: + frame = self._audio_q.get(timeout=0.25) + except Exception: + # No client frames yet — count as silence for status. + self._silent_frames += 1 + if self._silent_frames == silent_alert_frames: + self.audio_silent = True + continue + data = frame + else: + data, _overflow = stream.read(frame_length) except Exception as e: logger.warning("wake word: stream read error: %s", e) failed = not self._stop.is_set() @@ -1045,11 +1209,12 @@ class WakeWordDetector: else: logger.debug("wake word: detection within cooldown — ignored") finally: - try: - stream.stop() - stream.close() - except Exception: - pass + if stream is not None: + try: + stream.stop() + stream.close() + except Exception: + pass logger.info("wake word: stream closed") if failed and self.on_failure is not None: self.on_failure(self) @@ -1136,6 +1301,7 @@ def start_listening( *, owner: object, config: Optional[Dict[str, Any]] = None, + external_audio: bool = False, ) -> WakeWordDetector: """Claim, build, and start the detector. Idempotent for the same owner. @@ -1163,6 +1329,7 @@ def start_listening( on_wake, on_failure=_detector_failed, input_device=_input_device(cfg), + external_audio=external_audio, ) _detector = detector _detector_owner = owner @@ -1265,3 +1432,31 @@ def get_last_match() -> Optional[tuple[str, str]]: if det is None: return None return getattr(det.engine, "last_match", None) + + +def feed_audio(*, owner: object, pcm_int16) -> bool: + """Push client-captured PCM into the armed detector (client capture mode). + + Returns True when the frame was accepted for ``owner``'s armed detector. + """ + with _detector_lock: + if _detector is None or _detector_owner is not owner: + return False + if not _detector.external_audio: + return False + det = _detector + det.feed(pcm_int16) + return True + + +def detector_frame_info() -> Dict[str, Any]: + """Sample rate + frame length for client capture streamers.""" + with _detector_lock: + det = _detector + if det is None: + return {"sample_rate": SAMPLE_RATE, "frame_length": 1280} + return { + "sample_rate": SAMPLE_RATE, + "frame_length": int(getattr(det.engine, "frame_length", 1280) or 1280), + "external_audio": bool(det.external_audio), + } diff --git a/tui_gateway/server.py b/tui_gateway/server.py index a36a539408b1f..bd8844a15136d 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -13119,8 +13119,10 @@ def _(rid, params: dict) -> dict: from tools.wake_word import ( WakeWordInUse, check_wake_word_requirements, + detector_frame_info, load_wake_word_config, owns_listener, + resolve_capture_mode, start_listening, wake_phrase, wake_surface_enabled, @@ -13129,16 +13131,25 @@ def _(rid, params: dict) -> dict: return _err(rid, 5026, f"wake module unavailable: {e}") cfg = load_wake_word_config() + # Desktop remote (gui) prefers client capture: Mac mic → wake.feed PCM, + # while the engine still runs on the backend. CLI/TUI stay local. + prefer_client = surface in ("gui", "desktop") or bool(params.get("client_capture")) + capture_mode = resolve_capture_mode(cfg, prefer_client=prefer_client) + external_audio = capture_mode == "client" # Requirements first: a gesture on an unarmed-able setup (no STT/TTS, no # mic, missing key) must refuse WITHOUT flipping wake_word.enabled — else # config says on while nothing can ever arm, and auto-arm paths churn. - reqs = check_wake_word_requirements(cfg) + # Temporarily stamp capture so the probe matches the arm mode. + probe_cfg = dict(cfg) + probe_cfg["capture"] = capture_mode + reqs = check_wake_word_requirements(probe_cfg) if not reqs["available"]: logger.warning("wake.start(%s): not available — %s", surface, reqs.get("hint")) return _ok(rid, { "started": False, "reason": "unavailable", "hint": reqs.get("hint") or "", + "capture": capture_mode, }) enabled_persisted = False if persist and not cfg.get("enabled"): @@ -13202,7 +13213,12 @@ def _(rid, params: dict) -> dict: reset_transport(token) try: - start_listening(_on_detect, owner=transport, config=cfg) + start_listening( + _on_detect, + owner=transport, + config=cfg, + external_audio=external_audio, + ) except WakeWordInUse: return _ok(rid, { "started": False, @@ -13216,13 +13232,20 @@ def _(rid, params: dict) -> dict: with _wake_lock: _wake_owner_transport = transport _wake_owner_surface = surface - logger.info("wake.start(%s): listening for %r (%s)", surface, reqs["phrase"], reqs["provider"]) + frame = detector_frame_info() + logger.info( + "wake.start(%s): listening for %r (%s) capture=%s frame=%s", + surface, reqs["phrase"], reqs["provider"], capture_mode, frame.get("frame_length"), + ) return _ok(rid, { "started": True, "phrase": reqs["phrase"], "provider": reqs["provider"], "owner_surface": surface, "enabled_persisted": enabled_persisted, + "capture": capture_mode, + "sample_rate": frame.get("sample_rate", 16000), + "frame_length": frame.get("frame_length", 1280), }) @@ -13308,6 +13331,7 @@ def _(rid, params: dict) -> dict: hint = f"Wake-word input device could not be resolved: {input_device['error']}" if silent and not hint: hint = silent_audio_hint(input_device) + capture = reqs.get("capture") or str(cfg.get("capture") or "auto") return _ok(rid, { "listening": listening, "owned_by_caller": owned_by_caller, @@ -13323,11 +13347,52 @@ def _(rid, params: dict) -> dict: "enabled": bool(cfg.get("enabled")), # Armed but deaf despite an open stream; see platform-specific hint. "audio_silent": silent, + "capture": capture, + "local_input_available": bool(reqs.get("local_input_available")), }) except Exception as e: return _err(rid, 5026, str(e)) +@method("wake.feed") +def _(rid, params: dict) -> dict: + """Push client-captured PCM into the armed wake detector. + + Params: + pcm: base64-encoded int16 mono little-endian samples (preferred), OR + pcm_b64: alias of pcm + Optional: + sample_rate: must be 16000 (ignored if missing; mismatched rates rejected) + + Used when ``wake.start`` returned ``capture: "client"`` so remote backends + without a microphone can still run openWakeWord on Mac/desktop audio. + """ + transport = current_transport() or _stdio_transport + raw_b64 = params.get("pcm") or params.get("pcm_b64") or "" + if not isinstance(raw_b64, str) or not raw_b64.strip(): + return _err(rid, 4001, "wake.feed requires base64 pcm") + try: + import base64 + pcm = base64.b64decode(raw_b64, validate=False) + except Exception as e: + return _err(rid, 4001, f"invalid base64 pcm: {e}") + if not pcm: + return _ok(rid, {"fed": False, "reason": "empty"}) + # Soft size cap (~0.5s of 16kHz int16 mono = 16000 bytes) + if len(pcm) > 64000: + return _err(rid, 4001, "pcm frame too large") + sr = params.get("sample_rate") + if sr is not None and int(sr) not in (0, 16000): + return _err(rid, 4001, "wake.feed only accepts 16 kHz PCM") + try: + from tools.wake_word import feed_audio + ok = feed_audio(owner=transport, pcm_int16=pcm) + except Exception as e: + logger.debug("wake.feed failed: %s", e) + return _err(rid, 5026, str(e)) + return _ok(rid, {"fed": bool(ok), "reason": None if ok else "not_owner"}) + + @method("voice.toggle") def _(rid, params: dict) -> dict: """CLI parity for the ``/voice`` slash command. diff --git a/website/docs/user-guide/features/wake-word.md b/website/docs/user-guide/features/wake-word.md index 19752893c4fdf..622532cd5f688 100644 --- a/website/docs/user-guide/features/wake-word.md +++ b/website/docs/user-guide/features/wake-word.md @@ -35,6 +35,43 @@ spoken command ends the conversation instead of being sent to the agent. Only a whole-utterance stop command matches, so a real request like "stop the docker container" still goes through normally. + + +## Remote desktop (client capture) + +When the desktop app connects to a **remote** Hermes backend (for example a +headless Docker host or a machine in another room), the backend often has **no +microphone**. Server-side PortAudio then fails with “Failed to open the +wake-word microphone.” + +Hermes supports **client capture** for that case: + +1. The desktop arms wake with `capture: client` (automatic for the GUI when the + backend has no local input device, or set explicitly below). +2. openWakeWord still runs **on the backend** (same engines, same models). +3. The desktop opens the **local Mac/PC microphone**, resamples to 16 kHz mono + int16, and streams short frames via the `wake.feed` RPC. +4. On detection the backend emits `wake.detected` as usual; the desktop starts + the normal voice pipeline on the client mic. + +```yaml +wake_word: + enabled: true + capture: auto # auto | local | client + # auto — local PortAudio unless the desktop arms with client_capture + # local — always open the backend mic (CLI/TUI default) + # client — always expect wake.feed PCM from the desktop (remote-friendly) +``` + +The desktop GUI always passes `client_capture: true` on `wake.start`, so remote +backends without a mic arm in client mode automatically. CLI and TUI keep local +capture unless you set `capture: client` explicitly. + +Privacy note: with client capture, wake PCM travels over the authenticated +desktop↔backend WebSocket (same channel as the rest of the session). Detection +still does not send audio to third-party wake APIs; the engine is local to the +backend process. + ## Engines | Engine | Cost | API key | Notes | @@ -82,6 +119,7 @@ wake_word: enabled: false surface: auto # eligible surface: "auto" | "cli" | "tui" | "gui" input_device: null # PortAudio input index or device-name substring; null = process default + capture: auto # auto | local | client — where PCM is captured (see Remote desktop) provider: openwakeword # "openwakeword" (free, local) | "sherpa" (free, any phrase) | "porcupine" phrase: "hey hermes" # cosmetic label only — detection is keyed by the model/keyword below sensitivity: 0.6 # 0.0-1.0 — higher = stricter (fewer false triggers), consistent across all engines