7b006ea6e8
The local faster-whisper model singleton (_local_model) is loaded once
and never released — the 'base' model holds ~370 MB of RAM/VRAM for
the entire lifetime of the process, even when no voice messages arrive
for hours or days. On long-running gateway processes (especially with
local LLMs competing for the same GPU) this is wasteful.
Add a config-driven idle unload: after stt.local.unload_after_idle_seconds
(default 0 = never) of no transcription activity, a lightweight daemon
thread sets _local_model = None so the Python GC can reclaim the
ctranslate2 objects. The next voice message reloads the model
transparently (the existing lazy-load path handles it).
The watcher:
- Checks every 30s whether idle time exceeds the configured threshold
- Acquires _local_model_lock before unloading (prevents races with
concurrent transcriptions that are mid-load)
- Exits immediately if the model is already None (unloaded by another
path, e.g. the CUDA fallback eviction)
- Is restarted by each transcription with the current config value,
so changing stt.local.unload_after_idle_seconds in config.yaml takes
effect on the next voice message without a process restart
Default is 0 (never unload) — zero behavior change for existing users.
Recommended value for gateway processes: 300 (5 minutes).
15 tests: config resolution (garbage/negative/None fallbacks), unload
safety (already-None, lock acquisition), touch timestamp, watcher
lifecycle (unload after timeout, no unload within timeout, exits when
model already None, stopped on new start). Existing STT test suite
unchanged.
|
||
|---|---|---|
| .. | ||
| computer_use | ||
| environments | ||
| neutts_samples | ||
| wakewords | ||
| __init__.py | ||
| ansi_strip.py | ||
| approval.py | ||
| async_delegation.py | ||
| audio_container.py | ||
| binary_extensions.py | ||
| blueprints.py | ||
| browser_camofox.py | ||
| browser_camofox_state.py | ||
| browser_cdp_tool.py | ||
| browser_dialog_tool.py | ||
| browser_supervisor.py | ||
| browser_tool.py | ||
| budget_config.py | ||
| checkpoint_manager.py | ||
| clarify_gateway.py | ||
| clarify_tool.py | ||
| close_terminal_tool.py | ||
| code_execution_tool.py | ||
| computer_use_tool.py | ||
| credential_files.py | ||
| cronjob_tools.py | ||
| daemon_pool.py | ||
| debug_helpers.py | ||
| delegate_tool.py | ||
| delegation_live_log.py | ||
| desktop_ui.py | ||
| discord_tool.py | ||
| env_passthrough.py | ||
| env_probe.py | ||
| fal_common.py | ||
| feishu_doc_tool.py | ||
| feishu_drive_tool.py | ||
| file_operations.py | ||
| file_state.py | ||
| file_tools.py | ||
| flux3_video_tool.py | ||
| focus_pane_tool.py | ||
| fuzzy_match.py | ||
| homeassistant_tool.py | ||
| hook_output_spill.py | ||
| image_generation_tool.py | ||
| image_source.py | ||
| interrupt.py | ||
| kanban_tools.py | ||
| lazy_deps.py | ||
| managed_tool_gateway.py | ||
| mcp_dashboard_oauth.py | ||
| mcp_oauth.py | ||
| mcp_oauth_manager.py | ||
| mcp_schema_cache.py | ||
| mcp_stdio_watchdog.py | ||
| mcp_tool.py | ||
| memory_tool.py | ||
| microsoft_graph_auth.py | ||
| microsoft_graph_client.py | ||
| neutts_synth.py | ||
| open_preview_tool.py | ||
| openrouter_client.py | ||
| osv_check.py | ||
| patch_parser.py | ||
| path_security.py | ||
| process_registry.py | ||
| project_tools.py | ||
| react_to_message_tool.py | ||
| read_extract.py | ||
| read_preview_tool.py | ||
| read_terminal_tool.py | ||
| registry.py | ||
| schema_sanitizer.py | ||
| send_message_tool.py | ||
| session_search_tool.py | ||
| skill_manager_tool.py | ||
| skill_provenance.py | ||
| skill_usage.py | ||
| skills_ast_audit.py | ||
| skills_guard.py | ||
| skills_hub.py | ||
| skills_sync.py | ||
| skills_sync_client.py | ||
| skills_tool.py | ||
| slash_confirm.py | ||
| terminal_hints.py | ||
| terminal_tool.py | ||
| thread_context.py | ||
| threat_patterns.py | ||
| tirith_security.py | ||
| todo_tool.py | ||
| tool_backend_helpers.py | ||
| tool_output_limits.py | ||
| tool_result_storage.py | ||
| tool_search.py | ||
| transcription_tools.py | ||
| tts_streaming.py | ||
| tts_text_normalize.py | ||
| tts_tool.py | ||
| url_safety.py | ||
| video_generation_tool.py | ||
| vision_tools.py | ||
| voice_mode.py | ||
| wake_word.py | ||
| web_tools.py | ||
| website_policy.py | ||
| working_diff.py | ||
| write_approval.py | ||
| x_search_tool.py | ||
| xai_http.py | ||
| xai_video_tools.py | ||
| yuanbao_tools.py | ||