From d264bcd1c338703ef4f551e333b6fe6c0874bf83 Mon Sep 17 00:00:00 2001 From: giveen Date: Thu, 2 Apr 2026 14:23:41 -0600 Subject: [PATCH] feat: wire CAI_SUPPORT_MODEL + CAI_SUPPORT_INTERVAL to auto-compact the main agent context Problem ------- CAI_SUPPORT_MODEL and CAI_SUPPORT_INTERVAL were documented environment variables that had no runtime implementation. The support/reasoner agent was constructed using CAI_SUPPORT_MODEL but was never invoked automatically. CAI_SUPPORT_INTERVAL existed only in docs/config tables with no scheduler reading it. As a result, users with a limited context window (e.g. 32k on a local llama.cpp setup) had no way to automatically keep the main model's message_history from overflowing during a long pentest. Solution -------- Added an auto-compact scheduler block immediately after turn_count += 1 in the main single-agent run loop (run_cai_cli). When both CAI_SUPPORT_MODEL and CAI_SUPPORT_INTERVAL are set the scheduler: 1. Fires every CAI_SUPPORT_INTERVAL turns (modulo check). 2. Calls COMPACT_COMMAND_INSTANCE._perform_compaction(model_override= CAI_SUPPORT_MODEL) which: - Sends the full message_history to the support model for summarisation. - Clears message_history entirely (hard context reset). - Saves the summary to /memory as a .md file. - Stores the summary in COMPACTED_SUMMARIES under the agent name. 3. Re-syncs the local agent variable from AGENT_MANAGER.get_active_agent() so the run loop continues with the freshly reloaded agent instance whose system prompt already contains the injected summary (the system prompt template calls get_compacted_summary() dynamically on every turn so no extra wiring was needed). 4. Prints a visible yellow/green indicator so users can see when compaction fires and confirm the context window has been reset. 5. Silently swallows errors (only logs when CAI_DEBUG=2) so a failing support model never crashes the main session. Usage ----- CAI_SUPPORT_MODEL="openai/support" # lighter model on litellm proxy CAI_SUPPORT_INTERVAL=4 # compact every 4 turns --- src/cai/cli.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/cai/cli.py b/src/cai/cli.py index 8cacf525..abe48d90 100644 --- a/src/cai/cli.py +++ b/src/cai/cli.py @@ -1711,6 +1711,41 @@ def run_cai_cli( agent.model.message_history[:] = fix_message_list(agent.model.message_history) turn_count += 1 + # Auto-compact: when CAI_SUPPORT_MODEL + CAI_SUPPORT_INTERVAL are both set, + # compact the conversation every N turns using the support model so the + # main model's context window is kept small. After compaction the + # summary is injected into the agent's system prompt and the local + # agent reference is refreshed so the loop uses the reloaded instance. + _support_model = os.getenv("CAI_SUPPORT_MODEL") + _support_interval_raw = os.getenv("CAI_SUPPORT_INTERVAL") + if _support_model and _support_interval_raw: + try: + _support_interval = int(_support_interval_raw) + if _support_interval > 0 and turn_count % _support_interval == 0: + from cai.repl.commands.compact import COMPACT_COMMAND_INSTANCE + console.print( + f"\n[bold yellow]⟳ Auto-compact: turn {turn_count} " + f"(every {_support_interval} turns) — " + f"summarising with {_support_model}[/bold yellow]" + ) + COMPACT_COMMAND_INSTANCE._perform_compaction( + model_override=_support_model + ) + # Re-sync the local agent reference so the loop continues + # with the freshly reloaded agent (history cleared, memory + # summary already injected into its system prompt). + from cai.sdk.agents.simple_agent_manager import AGENT_MANAGER as _AM + _reloaded = _AM.get_active_agent() + if _reloaded is not None: + agent = _reloaded + console.print( + "[bold green]✓ Memory summary applied to agent system prompt — " + "context window reset[/bold green]\n" + ) + except (ValueError, Exception) as _e: + if os.getenv("CAI_DEBUG", "1") == "2": + console.print(f"[red]Auto-compact error: {_e}[/red]") + # Stop measuring active time and start measuring idle time again stop_active_timer() start_idle_timer()