diff --git a/apps/desktop/src/app/chat/right-rail/preview-pane.test.tsx b/apps/desktop/src/app/chat/right-rail/preview-pane.test.tsx index 4696af124d355..3118861d14336 100644 --- a/apps/desktop/src/app/chat/right-rail/preview-pane.test.tsx +++ b/apps/desktop/src/app/chat/right-rail/preview-pane.test.tsx @@ -179,10 +179,9 @@ describe('PreviewPane console state', () => { ) }) - await waitFor( - () => expect(rendered.container.querySelector('iframe')).not.toBeNull(), - { container: rendered.container } - ) + await waitFor(() => expect(rendered.container.querySelector('iframe')).not.toBeNull(), { + container: rendered.container + }) expect(rendered.container.querySelector('iframe')?.getAttribute('src')).toBe('blob:pdf-preview-1') expect(readFileDataUrl).toHaveBeenCalledWith('/tmp/spec.pdf') const blob = createObjectURL.mock.calls[0]?.[0] @@ -328,10 +327,9 @@ describe('PreviewPane console state', () => { $connection.set({ baseUrl: 'http://macmini', mode: 'remote', profile: 'macmini' } as never) }) - await waitFor( - () => expect(rendered.container.querySelector('iframe')).not.toBeNull(), - { container: rendered.container } - ) + await waitFor(() => expect(rendered.container.querySelector('iframe')).not.toBeNull(), { + container: rendered.container + }) expect(api).toHaveBeenCalledWith({ path: `/api/fs/read-data-url?path=${encodeURIComponent(filePath)}`, profile: 'macmini' diff --git a/apps/desktop/src/app/session/hooks/use-prompt-actions/slash.ts b/apps/desktop/src/app/session/hooks/use-prompt-actions/slash.ts index c9f1836e65e32..d88d36548ff33 100644 --- a/apps/desktop/src/app/session/hooks/use-prompt-actions/slash.ts +++ b/apps/desktop/src/app/session/hooks/use-prompt-actions/slash.ts @@ -663,6 +663,7 @@ export function useSlashCommand(deps: SlashCommandDeps) { client_capture: true, surface: 'gui' }) + applyWakeStatus(current) return current diff --git a/apps/desktop/src/lib/wake-client-capture.ts b/apps/desktop/src/lib/wake-client-capture.ts index 3681bffe35c6f..cabcee9038628 100644 --- a/apps/desktop/src/lib/wake-client-capture.ts +++ b/apps/desktop/src/lib/wake-client-capture.ts @@ -10,10 +10,7 @@ const TARGET_RATE = 16_000 const DEFAULT_FRAME = 1280 // 80 ms @ 16 kHz — matches tools/wake_word.py -export type WakeFeedRequester = ( - method: string, - params?: Record -) => Promise +export type WakeFeedRequester = (method: string, params?: Record) => Promise export interface ClientWakeCaptureOptions { /** Samples per frame at 16 kHz (from wake.start response). */ @@ -31,33 +28,41 @@ function downsampleTo16k(input: Float32Array, inputRate: number): Float32Array { if (inputRate === TARGET_RATE) { return input } + if (inputRate <= 0) { return new Float32Array(0) } + const ratio = inputRate / TARGET_RATE const outLen = Math.max(1, Math.floor(input.length / ratio)) const out = new Float32Array(outLen) + for (let i = 0; i < outLen; i++) { const start = Math.floor(i * ratio) const end = Math.min(input.length, Math.floor((i + 1) * ratio)) let sum = 0 let count = 0 + for (let j = start; j < end; j++) { sum += input[j] ?? 0 count++ } + out[i] = count > 0 ? sum / count : 0 } + return out } function floatToInt16LE(input: Float32Array): ArrayBuffer { const buf = new ArrayBuffer(input.length * 2) const view = new DataView(buf) + for (let i = 0; i < input.length; i++) { const s = Math.max(-1, Math.min(1, input[i] ?? 0)) view.setInt16(i * 2, s < 0 ? s * 0x8000 : s * 0x7fff, true) } + return buf } @@ -65,9 +70,11 @@ function bytesToBase64(buf: ArrayBuffer): string { const bytes = new Uint8Array(buf) let binary = '' const chunk = 0x8000 + for (let i = 0; i < bytes.length; i += chunk) { binary += String.fromCharCode(...bytes.subarray(i, i + chunk)) } + return btoa(binary) } @@ -75,15 +82,15 @@ function bytesToBase64(buf: ArrayBuffer): string { * Start streaming the default microphone to `wake.feed`. * Returns a handle whose `stop()` ends tracks + audio graph. */ -export async function startClientWakeCapture( - options: ClientWakeCaptureOptions -): Promise { +export async function startClientWakeCapture(options: ClientWakeCaptureOptions): Promise { const frameLength = Math.max(160, Math.trunc(options.frameLength || DEFAULT_FRAME)) const audioWindow = window as Window & { webkitAudioContext?: typeof AudioContext } const AudioContextCtor = window.AudioContext || audioWindow.webkitAudioContext + if (!AudioContextCtor) { throw new Error('AudioContext unavailable for client wake capture') } + if (!navigator.mediaDevices?.getUserMedia) { throw new Error('getUserMedia unavailable for client wake capture') } @@ -123,13 +130,17 @@ export async function startClientWakeCapture( if (draining) { return } + draining = true + try { while (!stopped && queue.length > 0) { const batch = queue.splice(0, MAX_FRAMES_PER_FEED) + if (batch.length === 0) { break } + try { const merged = new Float32Array(batch.length * frameLength) batch.forEach((frame, i) => merged.set(frame, i * frameLength)) @@ -145,6 +156,7 @@ export async function startClientWakeCapture( } } finally { draining = false + if (!stopped && queue.length > 0) { void drainQueue() } @@ -155,10 +167,13 @@ export async function startClientWakeCapture( if (stopped) { return } + queue.push(frame) + while (queue.length > MAX_QUEUED_FRAMES) { queue.shift() } + void drainQueue() } @@ -166,6 +181,7 @@ export async function startClientWakeCapture( if (stopped) { return } + const input = event.inputBuffer.getChannelData(0) const at16k = downsampleTo16k(input, context.sampleRate) // Append to pending and emit full frames @@ -173,11 +189,13 @@ export async function startClientWakeCapture( merged.set(pending, 0) merged.set(at16k, pending.length) let offset = 0 + while (offset + frameLength <= merged.length) { const frame = merged.subarray(offset, offset + frameLength) offset += frameLength enqueueFrame(new Float32Array(frame)) } + pending = merged.subarray(offset) } @@ -197,8 +215,10 @@ export async function startClientWakeCapture( if (stopped) { return } + stopped = true queue.length = 0 + try { processor.disconnect() source.disconnect() @@ -206,6 +226,7 @@ export async function startClientWakeCapture( } catch { // ignore } + void context.close().catch(() => undefined) stream.getTracks().forEach(t => t.stop()) } diff --git a/apps/desktop/src/store/preview.ts b/apps/desktop/src/store/preview.ts index 7dd238eba5c17..5a0212b32c92d 100644 --- a/apps/desktop/src/store/preview.ts +++ b/apps/desktop/src/store/preview.ts @@ -117,6 +117,7 @@ function isPdfFileTarget(target: PreviewTarget): boolean { * obsolete raw-binary path after Desktop itself has been upgraded. */ export function decodePreviewTabs(raw: string): PreviewTab[] { const parsed = JSON.parse(raw) as unknown + const tabs = (Array.isArray(parsed) ? parsed.filter(isPreviewTab) : []).map(tab => isPdfFileTarget(tab.target) && tab.target.previewKind === 'binary' ? { ...tab, target: { ...tab.target, previewKind: 'pdf' as const } } diff --git a/apps/desktop/src/store/updates.test.ts b/apps/desktop/src/store/updates.test.ts index 161f7480f10ae..bb6f775f58884 100644 --- a/apps/desktop/src/store/updates.test.ts +++ b/apps/desktop/src/store/updates.test.ts @@ -772,6 +772,7 @@ describe('applyBackendUpdate recovery', () => { it('restores the fixed action deadline after reconnecting', async () => { updateHermesSpy.mockResolvedValue({ action_id: 'a'.repeat(32), ok: true, name: 'hermes-update', pid: 1 }) + const running = { exit_code: null, lines: ['still running'], @@ -783,6 +784,7 @@ describe('applyBackendUpdate recovery', () => { for (let attempt = 0; attempt < 119; attempt += 1) { getActionStatusSpy.mockResolvedValueOnce(running) } + getActionStatusSpy.mockRejectedValueOnce(new Error('ECONNRESET')).mockResolvedValue(running) const promise = applyBackendUpdate() diff --git a/apps/desktop/src/store/updates.ts b/apps/desktop/src/store/updates.ts index 1c99356030ccf..8b95b2eaded7e 100644 --- a/apps/desktop/src/store/updates.ts +++ b/apps/desktop/src/store/updates.ts @@ -569,9 +569,11 @@ async function runBackendUpdate(): Promise { try { const previousStatus = $backendUpdateStatus.get() const requestedTargetSha = previousStatus?.commits?.at(0)?.sha + const previousVersion = previousStatus?.targetSha?.startsWith('backend:') ? previousStatus.targetSha.slice('backend:'.length) : undefined + const started = await updateHermes() if (!started.ok) { diff --git a/apps/desktop/src/store/wake-word.ts b/apps/desktop/src/store/wake-word.ts index 744ff85f7e53b..13ed1f098aefe 100644 --- a/apps/desktop/src/store/wake-word.ts +++ b/apps/desktop/src/store/wake-word.ts @@ -1,9 +1,6 @@ import { atom } from 'nanostores' -import { - type ClientWakeCaptureHandle, - startClientWakeCapture -} from '@/lib/wake-client-capture' +import { type ClientWakeCaptureHandle, startClientWakeCapture } from '@/lib/wake-client-capture' import { $gateway } from '@/store/gateway' // "Hey Hermes" wake-word listener state for the composer toggle. The gateway is @@ -48,13 +45,17 @@ export function stopClientCapture(): void { async function maybeStartClientCapture(result: WakeStartResponse | null | undefined): Promise { stopClientCapture() + if (!result?.started) { return } + const mode = (result.capture || '').toLowerCase() + if (mode !== 'client' && mode !== 'remote' && mode !== 'external') { return } + try { clientCapture = await startClientWakeCapture({ frameLength: result.frame_length, @@ -65,12 +66,10 @@ async function maybeStartClientCapture(result: WakeStartResponse | null | undefi $wakeWord.set({ ...current, listening: false, - notice: - error instanceof Error - ? error.message - : 'Failed to open the client microphone for wake word', + notice: error instanceof Error ? error.message : 'Failed to open the client microphone for wake word', pending: false }) + // Best-effort: release server lease if client mic failed. try { await gatewayRequester('wake.stop', {}) @@ -255,12 +254,14 @@ export async function armWakeWord(request: WakeRequester = gatewayRequester): Pr client_capture: true, surface: 'gui' }) + applyWakeStatus(status) if (!status?.available || status.listening) { // Armed already (e.g. another surface/restart) — reattach feeder if client. if (status?.listening) { const mode = (status.capture || '').toLowerCase() + if (mode === 'client' || mode === 'remote' || mode === 'external') { void maybeStartClientCapture({ started: true, @@ -269,6 +270,7 @@ export async function armWakeWord(request: WakeRequester = gatewayRequester): Pr }) } } + return } @@ -276,6 +278,7 @@ export async function armWakeWord(request: WakeRequester = gatewayRequester): Pr surface: 'gui', client_capture: true }) + applyWakeStartResult(result) } catch { // Older backends / transient failures — keep whatever we last knew. @@ -350,6 +353,7 @@ export async function resumeWakeAfterVoice(request: WakeRequester = gatewayReque client_capture: true, surface: 'gui' }) + applyWakeStatus(status) // Config says off (or the feature can't run) — off is the correct rest @@ -362,6 +366,7 @@ export async function resumeWakeAfterVoice(request: WakeRequester = gatewayReque // Server lease is still armed (e.g. wake.resume after voice). // Client PCM was stopped on wake.detected — reattach if needed. const mode = (status.capture || '').toLowerCase() + if (mode === 'client' || mode === 'remote' || mode === 'external') { void maybeStartClientCapture({ started: true, @@ -369,6 +374,7 @@ export async function resumeWakeAfterVoice(request: WakeRequester = gatewayReque frame_length: status.frame_length ?? 1280 }) } + return } @@ -376,6 +382,7 @@ export async function resumeWakeAfterVoice(request: WakeRequester = gatewayReque surface: 'gui', client_capture: true }) + applyWakeStartResult(started) if (started?.started) {