From 7b006ea6e8668bacecd9807a4db82c034d8371d6 Mon Sep 17 00:00:00 2001 From: kshitij Date: Fri, 7 Aug 2026 18:17:44 +0530 Subject: [PATCH] feat(stt): idle unload for local whisper model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The local faster-whisper model singleton (_local_model) is loaded once and never released — the 'base' model holds ~370 MB of RAM/VRAM for the entire lifetime of the process, even when no voice messages arrive for hours or days. On long-running gateway processes (especially with local LLMs competing for the same GPU) this is wasteful. Add a config-driven idle unload: after stt.local.unload_after_idle_seconds (default 0 = never) of no transcription activity, a lightweight daemon thread sets _local_model = None so the Python GC can reclaim the ctranslate2 objects. The next voice message reloads the model transparently (the existing lazy-load path handles it). The watcher: - Checks every 30s whether idle time exceeds the configured threshold - Acquires _local_model_lock before unloading (prevents races with concurrent transcriptions that are mid-load) - Exits immediately if the model is already None (unloaded by another path, e.g. the CUDA fallback eviction) - Is restarted by each transcription with the current config value, so changing stt.local.unload_after_idle_seconds in config.yaml takes effect on the next voice message without a process restart Default is 0 (never unload) — zero behavior change for existing users. Recommended value for gateway processes: 300 (5 minutes). 15 tests: config resolution (garbage/negative/None fallbacks), unload safety (already-None, lock acquisition), touch timestamp, watcher lifecycle (unload after timeout, no unload within timeout, exits when model already None, stopped on new start). Existing STT test suite unchanged. --- cli-config.yaml.example | 3 +- hermes_cli/config_defaults.py | 1 + tests/tools/test_stt_idle_unload.py | 189 +++++++++++++++++++++++ tools/transcription_tools.py | 84 ++++++++++ website/docs/user-guide/configuration.md | 3 +- 5 files changed, 278 insertions(+), 2 deletions(-) create mode 100644 tests/tools/test_stt_idle_unload.py diff --git a/cli-config.yaml.example b/cli-config.yaml.example index 53a53b3685d79..176a24fbf67fd 100644 --- a/cli-config.yaml.example +++ b/cli-config.yaml.example @@ -1210,7 +1210,8 @@ stt: # # Set false to restore raw behavior (e.g. transcribing music/ambient audio). # vad_min_silence_ms: 500 # min silence (ms) that splits speech chunks when vad is on # no_speech_prob_threshold: 0.6 # drop a segment only if no_speech_prob > this... - # logprob_threshold: -1.0 # ...AND avg_logprob < this (both must hit — quiet real speech survives) + # logprob_threshold: -1.0 # ...AND avg_logprob < this (both must hit — quiet real speech survives) + # unload_after_idle_seconds: 0 # 0=never unload (default); 300=frees RAM/VRAM after 5min with no voice messages language: "en" # GLOBAL language hint for every STT provider (per-provider language wins). Set "" for auto-detect. # groq: # model: "whisper-large-v3-turbo" diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index 13554ff68e679..d8b0ee363a70e 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -1537,6 +1537,7 @@ DEFAULT_CONFIG = { "vad_min_silence_ms": 500, # min silence (ms) that splits speech chunks when vad is on "no_speech_prob_threshold": 0.6, # drop a segment only if no_speech_prob is ABOVE this... "logprob_threshold": -1.0, # ...AND its avg_logprob is BELOW this (both must hit) + "unload_after_idle_seconds": 0, # 0=never (default); 300=frees RAM/VRAM after 5min idle }, "groq": { "model": "whisper-large-v3-turbo", # whisper-large-v3, whisper-large-v3-turbo, distil-whisper-large-v3-en diff --git a/tests/tools/test_stt_idle_unload.py b/tests/tools/test_stt_idle_unload.py new file mode 100644 index 0000000000000..70c61e30387fa --- /dev/null +++ b/tests/tools/test_stt_idle_unload.py @@ -0,0 +1,189 @@ +"""Tests for the local whisper model idle-unload mechanism. + +The local faster-whisper model singleton (``_local_model``) is loaded once +and never released — hundreds of MB of RAM/VRAM sit idle between voice +messages on long-running gateway processes. These tests verify the +config-driven idle unload: after ``unload_after_idle_seconds`` of no +transcription activity, a daemon thread sets ``_local_model = None`` so the +memory can be reclaimed. The next transcription reloads the model +transparently. + +Contract under test: + +1. Default is 0 (never unload) — no thread is started. +2. After a successful transcription with a non-zero timeout, the watcher + thread is started and eventually unloads the model. +3. ``_touch_transcription_time`` resets the idle timer. +4. ``_unload_local_model`` is safe to call when the model is already None. +5. Config resolution handles garbage values gracefully. +""" + +import time +from unittest.mock import MagicMock, patch + +import pytest + +from tools.transcription_tools import ( + _get_idle_unload_seconds, + _touch_transcription_time, + _unload_local_model, + _start_idle_unload_watcher, + _IDLE_UNLOAD_CHECK_INTERVAL, +) +import tools.transcription_tools as tt + + +# ============================================================================ +# Config resolution +# ============================================================================ + + +class TestGetIdleUnloadSeconds: + def test_default_is_zero(self): + assert _get_idle_unload_seconds({}) == 0 + + def test_explicit_value(self): + assert _get_idle_unload_seconds({"unload_after_idle_seconds": 300}) == 300 + + def test_zero_means_never(self): + assert _get_idle_unload_seconds({"unload_after_idle_seconds": 0}) == 0 + + def test_negative_clamped_to_zero(self): + assert _get_idle_unload_seconds({"unload_after_idle_seconds": -5}) == 0 + + def test_garbage_falls_back_to_zero(self): + assert _get_idle_unload_seconds({"unload_after_idle_seconds": "never"}) == 0 + + def test_none_falls_back_to_zero(self): + assert _get_idle_unload_seconds({"unload_after_idle_seconds": None}) == 0 + + +# ============================================================================ +# _unload_local_model +# ============================================================================ + + +class TestUnloadLocalModel: + def test_unloads_when_model_present(self): + mock_model = MagicMock(name="whisper_model") + with patch.object(tt, "_local_model", mock_model), \ + patch.object(tt, "_local_model_name", "base"): + _unload_local_model() + assert tt._local_model is None + assert tt._local_model_name is None + + def test_safe_when_already_none(self): + with patch.object(tt, "_local_model", None), \ + patch.object(tt, "_local_model_name", None): + # Must not raise + _unload_local_model() + assert tt._local_model is None + + def test_acquires_model_lock(self): + """The unload must hold _local_model_lock to prevent races with + concurrent transcriptions that are mid-load.""" + mock_model = MagicMock(name="whisper_model") + with patch.object(tt, "_local_model", mock_model), \ + patch.object(tt, "_local_model_name", "base"), \ + patch.object(tt, "_local_model_lock") as mock_lock: + _unload_local_model() + mock_lock.__enter__.assert_called() + mock_lock.__exit__.assert_called() + + +# ============================================================================ +# _touch_transcription_time +# ============================================================================ + + +class TestTouchTranscriptionTime: + def test_sets_timestamp(self): + original = tt._last_transcription_time + try: + tt._last_transcription_time = 0.0 + _touch_transcription_time() + assert tt._last_transcription_time > 0 + finally: + tt._last_transcription_time = original + + +# ============================================================================ +# Watcher thread (with mocked time) +# ============================================================================ + + +class TestIdleUnloadWatcher: + def test_default_zero_does_not_start_thread(self): + """When unload_after_idle_seconds is 0, no watcher should be started.""" + with patch.object(tt, "_local_model", MagicMock()), \ + patch.object(tt, "_start_idle_unload_watcher") as mock_start: + # Simulate _transcribe_local's behavior: only start if > 0 + idle_timeout = _get_idle_unload_seconds({"unload_after_idle_seconds": 0}) + if idle_timeout > 0: + _start_idle_unload_watcher(idle_timeout) + mock_start.assert_not_called() + + def test_watcher_unloads_after_timeout(self): + """The watcher unloads the model after the configured idle period.""" + mock_model = MagicMock(name="whisper_model") + # Use a very short timeout and patch the check interval to 0.01s + # so the test runs in < 1 second. + with patch.object(tt, "_local_model", mock_model), \ + patch.object(tt, "_local_model_name", "base"), \ + patch.object(tt, "_IDLE_UNLOAD_CHECK_INTERVAL", 0.01), \ + patch.object(tt, "_last_transcription_time", time.monotonic() - 100): + _start_idle_unload_watcher(timeout_seconds=1) + # Wait for the watcher to fire + for _ in range(100): + if tt._local_model is None: + break + time.sleep(0.02) + assert tt._local_model is None + + def test_watcher_does_not_unload_within_timeout(self): + """The watcher does NOT unload when the model was recently used.""" + mock_model = MagicMock(name="whisper_model") + original_model = tt._local_model + original_name = tt._local_model_name + original_interval = tt._IDLE_UNLOAD_CHECK_INTERVAL + original_ts = tt._last_transcription_time + try: + tt._local_model = mock_model + tt._local_model_name = "base" + tt._IDLE_UNLOAD_CHECK_INTERVAL = 0.01 + tt._last_transcription_time = time.monotonic() + _start_idle_unload_watcher(timeout_seconds=100) + # Give it a few check cycles — model must survive + time.sleep(0.1) + assert tt._local_model is not None + finally: + tt._local_model = original_model + tt._local_model_name = original_name + tt._IDLE_UNLOAD_CHECK_INTERVAL = original_interval + tt._last_transcription_time = original_ts + + def test_watcher_exits_when_model_already_none(self): + """If the model was unloaded by another path, the watcher exits.""" + with patch.object(tt, "_local_model", None), \ + patch.object(tt, "_IDLE_UNLOAD_CHECK_INTERVAL", 0.01), \ + patch.object(tt, "_last_transcription_time", 0.0): + _start_idle_unload_watcher(timeout_seconds=1) + time.sleep(0.05) + # No crash, no hang — watcher detected _local_model is None and exited + + def test_watcher_stopped_on_new_start(self): + """Starting a new watcher stops the previous one.""" + mock_model = MagicMock(name="whisper_model") + with patch.object(tt, "_local_model", mock_model), \ + patch.object(tt, "_local_model_name", "base"), \ + patch.object(tt, "_IDLE_UNLOAD_CHECK_INTERVAL", 0.01), \ + patch.object(tt, "_last_transcription_time", time.monotonic()): + _start_idle_unload_watcher(timeout_seconds=100) + first_thread = tt._idle_unload_thread + assert first_thread is not None and first_thread.is_alive() + _start_idle_unload_watcher(timeout_seconds=200) + second_thread = tt._idle_unload_thread + assert second_thread is not first_thread + # First thread should have been stopped + time.sleep(0.05) + assert not first_thread.is_alive() diff --git a/tools/transcription_tools.py b/tools/transcription_tools.py index 7633a629436ac..f60fb1d7f1479 100644 --- a/tools/transcription_tools.py +++ b/tools/transcription_tools.py @@ -139,6 +139,19 @@ _local_model_name: Optional[str] = None # None` and download/load the whisper model twice (#24767). _local_model_lock = threading.Lock() +# --- Idle unload --------------------------------------------------------------- +# The model singleton above is loaded once and never released — hundreds of MB +# of RAM/VRAM sit idle between voice messages. On long-running gateway +# processes (especially with local LLMs competing for the same GPU) this is +# wasteful. A lightweight daemon thread checks _last_transcription_time and +# unloads the model after a configurable idle period. The next voice message +# reloads it transparently. +_last_transcription_time: float = 0.0 +_idle_unload_thread: Optional[threading.Thread] = None +_idle_unload_stop = threading.Event() + +_IDLE_UNLOAD_CHECK_INTERVAL = 30 # seconds between idle checks + # --------------------------------------------------------------------------- # Config helpers # --------------------------------------------------------------------------- @@ -1470,6 +1483,72 @@ def _should_force_faster_whisper_cpu() -> bool: return _sysctl_value("hw.optional.arm64") == "1" +def _get_idle_unload_seconds(local_cfg: Dict[str, Any]) -> int: + """Resolve the idle unload timeout from config. + + 0 = never unload (default). Negative values are treated as 0. + """ + try: + val = int(local_cfg.get("unload_after_idle_seconds", 0)) + except (TypeError, ValueError): + return 0 + return max(val, 0) + + +def _unload_local_model() -> None: + """Release the cached local whisper model and free its memory. + + Safe to call from any thread. The model lock prevents races with a + concurrent transcription that is mid-load. + """ + global _local_model, _local_model_name + with _local_model_lock: + if _local_model is not None: + logger.info( + "Unloading local whisper model '%s' after idle timeout", + _local_model_name or "unknown", + ) + _local_model = None + _local_model_name = None + + +def _start_idle_unload_watcher(timeout_seconds: int) -> None: + """Start (or replace) the background idle-unload thread. + + The thread checks every ``_IDLE_UNLOAD_CHECK_INTERVAL`` seconds whether + ``_last_transcription_time`` is older than ``timeout_seconds``. If so, it + unloads the model and exits. The next transcription restarts it. + """ + global _idle_unload_thread + # Stop any existing watcher — a new timeout may have been configured + _idle_unload_stop.set() + if _idle_unload_thread is not None and _idle_unload_thread.is_alive(): + _idle_unload_thread.join(timeout=5) + _idle_unload_stop.clear() + + def _watch(): + while not _idle_unload_stop.is_set(): + if _idle_unload_stop.wait(_IDLE_UNLOAD_CHECK_INTERVAL): + break + if _local_model is None: + break + idle_for = time.monotonic() - _last_transcription_time + if idle_for >= timeout_seconds: + _unload_local_model() + break + + _idle_unload_thread = threading.Thread( + target=_watch, name="hermes-stt-idle-unload", daemon=True + ) + _idle_unload_thread.start() + + +def _touch_transcription_time() -> None: + """Record that a transcription just completed (resets the idle timer).""" + global _last_transcription_time + _last_transcription_time = time.monotonic() + + def _load_local_whisper_model(model_name: str, device: str = "auto", compute_type: str = "auto"): """Load faster-whisper with graceful CUDA → CPU fallback. @@ -1705,6 +1784,11 @@ def _transcribe_local(file_path: str, model_name: str) -> Dict[str, Any]: Path(file_path).name, model_name, info.language, info.duration, ) + _touch_transcription_time() + idle_timeout = _get_idle_unload_seconds(local_cfg) + if idle_timeout > 0: + _start_idle_unload_watcher(idle_timeout) + return {"success": True, "transcript": transcript, "provider": "local"} except Exception as e: diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md index 0798c766f1721..36934d1bb1331 100644 --- a/website/docs/user-guide/configuration.md +++ b/website/docs/user-guide/configuration.md @@ -1872,6 +1872,7 @@ stt: vad_min_silence_ms: 500 # min silence (ms) that splits speech chunks when vad is on no_speech_prob_threshold: 0.6 # drop a segment only when no_speech_prob > this... logprob_threshold: -1.0 # ...AND avg_logprob < this (both must hit — quiet real speech survives) + unload_after_idle_seconds: 0 # 0=never unload (default); 300=frees ~370MB RAM after 5min idle groq: language: "" # per-provider override of stt.language openai: @@ -1886,7 +1887,7 @@ Set `stt.echo_transcripts: false` when the gateway should transcribe voice notes Provider behavior: -- `local` uses `faster-whisper` running on your machine. Install it separately with `pip install faster-whisper`. Silence-hallucination hardening is on by default: a Silero VAD filter keeps silence/noise from ever reaching Whisper, cross-window conditioning is disabled, and segments the model itself flags as probably-not-speech *and* low-confidence are dropped. Set `stt.local.vad: false` to transcribe non-speech audio (music, ambient) with the raw behavior. +- `local` uses `faster-whisper` running on your machine. Install it separately with `pip install faster-whisper`. Silence-hallucination hardening is on by default: a Silero VAD filter keeps silence/noise from ever reaching Whisper, cross-window conditioning is disabled, and segments the model itself flags as probably-not-speech *and* low-confidence are dropped. Set `stt.local.vad: false` to transcribe non-speech audio (music, ambient) with the raw behavior. The model stays loaded in memory between voice messages for low-latency transcription; set `stt.local.unload_after_idle_seconds` (e.g. `300` for 5 minutes) to automatically release the model when idle — useful on gateway processes or machines with limited RAM/VRAM. - `groq` uses Groq's Whisper-compatible endpoint and reads `GROQ_API_KEY`. Pass `stt.groq.language` (or the global `HERMES_LOCAL_STT_LANGUAGE` env var) to skip auto-detection and reduce latency. - `openai` uses the OpenAI speech API and reads `VOICE_TOOLS_OPENAI_KEY`.