From e740d6499d6d73263174fa0c432acf715b254ece Mon Sep 17 00:00:00 2001 From: luijait Date: Fri, 23 May 2025 12:15:01 +0200 Subject: [PATCH 1/3] Pricing via JSON --- .gitignore | 4 ++-- src/cai/util.py | 31 ++++++++++++++++--------------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index 593a7ca6..6c1e2b31 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ # macOS Files .DS_Store - +cai_env/ # Byte-compiled / optimized / DLL files __pycache__/ **/__pycache__/ @@ -152,4 +152,4 @@ cython_debug/ workspaces/ # benchmarks -benchmarks/outputs \ No newline at end of file +benchmarks/outputs diff --git a/src/cai/util.py b/src/cai/util.py index 40fdc62b..a13257eb 100644 --- a/src/cai/util.py +++ b/src/cai/util.py @@ -341,23 +341,24 @@ class CostTracker: if model_name in self.model_pricing_cache: return self.model_pricing_cache[model_name] - # Try to load pricing from local pricing.json first + # Try to load pricing from remote pricing.json first try: - pricing_path = pathlib.Path("pricing.json") - if pricing_path.exists(): - with open(pricing_path, "r", encoding="utf-8") as f: - local_pricing = json.load(f) - pricing_info = local_pricing.get("alias0", {}) - input_cost = pricing_info.get("input_cost_per_token", 0) - output_cost = pricing_info.get("output_cost_per_token", 0) - - # Cache and return local pricing - self.model_pricing_cache[model_name] = (input_cost, output_cost) - return input_cost, output_cost + import requests + pricing_url = "https://raw.githubusercontent.com/aliasrobotics/cai/refs/heads/mcp_stdio/pricing.json" + response = requests.get(pricing_url, timeout=5) + if response.status_code == 200: + remote_pricing = response.json() + pricing_info = remote_pricing.get("alias0", {}) + input_cost = pricing_info.get("input_cost_per_token", 0) + output_cost = pricing_info.get("output_cost_per_token", 0) + + # Cache and return remote pricing + self.model_pricing_cache[model_name] = (input_cost, output_cost) + return input_cost, output_cost except Exception as e: - print(f" WARNING: Error loading local pricing.json: {str(e)}") - - # Fallback to LiteLLM API if local pricing not found + print(f" WARNING: Error loading remote pricing.json: {str(e)}") + + # Fallback to LiteLLM API if both remote and local pricing not found LITELLM_URL = ( "https://raw.githubusercontent.com/BerriAI/litellm/main/" "model_prices_and_context_window.json" From 423c80585abd1158b6d801118095090a941338af Mon Sep 17 00:00:00 2001 From: luijait Date: Fri, 23 May 2025 13:34:28 +0200 Subject: [PATCH 2/3] Fix stream plus reason --- .../agents/models/openai_chatcompletions.py | 91 ++++++++++--------- src/cai/util.py | 54 ++++++----- 2 files changed, 79 insertions(+), 66 deletions(-) diff --git a/src/cai/sdk/agents/models/openai_chatcompletions.py b/src/cai/sdk/agents/models/openai_chatcompletions.py index e45aa0e2..3f2d0076 100644 --- a/src/cai/sdk/agents/models/openai_chatcompletions.py +++ b/src/cai/sdk/agents/models/openai_chatcompletions.py @@ -994,12 +994,14 @@ class OpenAIChatCompletionsModel(Model): # Handle Claude reasoning content first (before regular content) reasoning_content = None + + # Check for Claude reasoning in different possible formats if hasattr(delta, 'reasoning_content') and delta.reasoning_content is not None: reasoning_content = delta.reasoning_content elif isinstance(delta, dict) and 'reasoning_content' in delta and delta['reasoning_content'] is not None: reasoning_content = delta['reasoning_content'] - # Also check for thinking_blocks structure + # Also check for thinking_blocks structure (Claude 4 format) thinking_blocks = None if hasattr(delta, 'thinking_blocks') and delta.thinking_blocks is not None: thinking_blocks = delta.thinking_blocks @@ -1012,11 +1014,29 @@ class OpenAIChatCompletionsModel(Model): if isinstance(block, dict) and block.get('type') == 'thinking': reasoning_content = block.get('thinking', '') break + elif isinstance(block, dict) and block.get('type') == 'text' and 'thinking' in str(block): + # Sometimes thinking content comes as text blocks + reasoning_content = block.get('text', '') + break - if reasoning_content and thinking_context: - # Update the thinking display - from cai.util import update_claude_thinking_content - update_claude_thinking_content(thinking_context, reasoning_content) + # Check for direct thinking field (some Claude models) + if not reasoning_content: + if hasattr(delta, 'thinking') and delta.thinking is not None: + reasoning_content = delta.thinking + elif isinstance(delta, dict) and 'thinking' in delta and delta['thinking'] is not None: + reasoning_content = delta['thinking'] + + # Update thinking display if we have reasoning content + if reasoning_content: + if thinking_context: + # Streaming mode: Update the rich thinking display + from cai.util import update_claude_thinking_content + update_claude_thinking_content(thinking_context, reasoning_content) + else: + # Non-streaming mode: Use simple text output + from cai.util import print_claude_reasoning_simple, detect_claude_thinking_in_stream + if detect_claude_thinking_in_stream(str(self.model)): + print_claude_reasoning_simple(reasoning_content, self.agent_name, str(self.model)) @@ -1028,6 +1048,14 @@ class OpenAIChatCompletionsModel(Model): content = delta['content'] if content: + # IMPORTANT: If we have content and thinking_context is active, + # it means thinking is complete and normal content is starting + # Close the thinking display automatically + if thinking_context: + from cai.util import finish_claude_thinking_display + finish_claude_thinking_display(thinking_context) + thinking_context = None # Clear the context + # For Ollama, we need to accumulate the full content to check for function calls if is_ollama: ollama_full_content += content @@ -1035,10 +1063,9 @@ class OpenAIChatCompletionsModel(Model): # Add to the streaming text buffer streaming_text_buffer += content - # Update streaming display if enabled - always do this for text content - # BUT: if thinking context is active, don't show text content during streaming - # Instead, it will be shown after thinking completes - if streaming_context and not thinking_context: + # Update streaming display if enabled - ALWAYS respect CAI_STREAM setting + # Both thinking and regular content should stream if streaming is enabled + if streaming_context: # Check if this is a local model and should have zero cost model_str = str(self.model).lower() is_local_model = ( @@ -1749,41 +1776,7 @@ class OpenAIChatCompletionsModel(Model): from cai.util import finish_claude_thinking_display finish_claude_thinking_display(thinking_context) - # After thinking completes, display the text content if it exists - # This ensures that Claude thinking models show their final response - if (state.text_content_index_and_output and - state.text_content_index_and_output[1].text and - not streamed_tool_calls): # Only show if there were no tool calls - - # Create a message-like object for displaying the text response - text_msg = type('TextResponse', (), { - 'content': state.text_content_index_and_output[1].text, - 'tool_calls': None - }) - - # Display the text response using the CLI utility - cli_print_agent_messages( - agent_name=getattr(self, 'agent_name', 'Agent'), - message=text_msg, - counter=getattr(self, 'interaction_counter', 0), - model=str(self.model), - debug=False, - interaction_input_tokens=interaction_input, - interaction_output_tokens=interaction_output, - interaction_reasoning_tokens=int( - final_response.usage.output_tokens_details.reasoning_tokens - if final_response.usage and final_response.usage.output_tokens_details - and hasattr(final_response.usage.output_tokens_details, 'reasoning_tokens') - else 0 - ), - total_input_tokens=total_input, - total_output_tokens=total_output, - total_reasoning_tokens=getattr(self, 'total_reasoning_tokens', 0), - interaction_cost=interaction_cost, - total_cost=total_cost, - tool_output=None, - suppress_empty=True - ) + # Note: Content is now displayed during streaming, no need to show it again here if tracing.include_data(): span_generation.span_data.output = [final_response.model_dump()] @@ -2065,7 +2058,15 @@ class OpenAIChatCompletionsModel(Model): # Add extended reasoning support for Claude models # Supports Claude 3.7, Claude 4, and any model with "thinking" in the name - has_reasoning_capability = "thinking" in model_str + has_reasoning_capability = ( + "thinking" in model_str or + # Claude 4 models support reasoning + "-4-" in model_str or + "sonnet-4" in model_str or + "haiku-4" in model_str or + "opus-4" in model_str or + "3.7" in model_str + ) if has_reasoning_capability: # Clean the model name by removing "thinking" before sending to API diff --git a/src/cai/util.py b/src/cai/util.py index a13257eb..cfceac03 100644 --- a/src/cai/util.py +++ b/src/cai/util.py @@ -341,23 +341,6 @@ class CostTracker: if model_name in self.model_pricing_cache: return self.model_pricing_cache[model_name] - # Try to load pricing from remote pricing.json first - try: - import requests - pricing_url = "https://raw.githubusercontent.com/aliasrobotics/cai/refs/heads/mcp_stdio/pricing.json" - response = requests.get(pricing_url, timeout=5) - if response.status_code == 200: - remote_pricing = response.json() - pricing_info = remote_pricing.get("alias0", {}) - input_cost = pricing_info.get("input_cost_per_token", 0) - output_cost = pricing_info.get("output_cost_per_token", 0) - - # Cache and return remote pricing - self.model_pricing_cache[model_name] = (input_cost, output_cost) - return input_cost, output_cost - except Exception as e: - print(f" WARNING: Error loading remote pricing.json: {str(e)}") - # Fallback to LiteLLM API if both remote and local pricing not found LITELLM_URL = ( "https://raw.githubusercontent.com/BerriAI/litellm/main/" @@ -3276,20 +3259,46 @@ def detect_claude_thinking_in_stream(model_name): model_str = str(model_name).lower() # Check for Claude models with reasoning capability - # According to LiteLLM docs, Claude 3.7, Claude 4 and models with "thinking" support reasoning + # Claude 4 models (like claude-sonnet-4-20250514) support reasoning + # Also check for explicit "thinking" in model name has_reasoning = ( "claude" in model_str and ( + # Claude 4 models (sonnet-4, haiku-4, opus-4) + "-4-" in model_str or + "sonnet-4" in model_str or + "haiku-4" in model_str or + "opus-4" in model_str or + # Legacy support for 3.7 and explicit thinking models "3.7" in model_str or - "4" in model_str or "thinking" in model_str ) ) return has_reasoning +def print_claude_reasoning_simple(reasoning_content, agent_name, model_name): + """ + Print Claude reasoning content in simple mode (no Rich panels). + Used when CAI_STREAM=False. + + Args: + reasoning_content: The reasoning/thinking text + agent_name: The agent name + model_name: The model name + """ + if not reasoning_content or not reasoning_content.strip(): + return + + # Simple text output without Rich formatting + timestamp = datetime.now().strftime("%H:%M:%S") + print(f"\n🧠 Reasoning | {agent_name} | {model_name} | {timestamp}") + print("=" * 60) + print(reasoning_content) + print("=" * 60 + "\n") + def start_claude_thinking_if_applicable(model_name, agent_name, counter): """ - Start Claude thinking display if the model supports it. + Start Claude thinking display if the model supports it AND streaming is enabled. Args: model_name: The model name @@ -3299,6 +3308,9 @@ def start_claude_thinking_if_applicable(model_name, agent_name, counter): Returns: The thinking context if created, None otherwise """ - if detect_claude_thinking_in_stream(model_name): + # Only show thinking in streaming mode + streaming_enabled = os.getenv('CAI_STREAM', 'false').lower() == 'true' + + if streaming_enabled and detect_claude_thinking_in_stream(model_name): return create_claude_thinking_context(agent_name, counter, model_name) return None From cb5f0a5d4d7101b094fd2dcb866b3344441250ba Mon Sep 17 00:00:00 2001 From: luijait Date: Fri, 23 May 2025 13:37:15 +0200 Subject: [PATCH 3/3] Pricing --- src/cai/util.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/cai/util.py b/src/cai/util.py index cfceac03..c2a02c89 100644 --- a/src/cai/util.py +++ b/src/cai/util.py @@ -340,7 +340,21 @@ class CostTracker: # Check cache for non-local models if model_name in self.model_pricing_cache: return self.model_pricing_cache[model_name] - + # Try to load pricing from local pricing.json first + try: + pricing_path = pathlib.Path("pricing.json") + if pricing_path.exists(): + with open(pricing_path, "r", encoding="utf-8") as f: + local_pricing = json.load(f) + pricing_info = local_pricing.get("alias0", {}) + input_cost = pricing_info.get("input_cost_per_token", 0) + output_cost = pricing_info.get("output_cost_per_token", 0) + + # Cache and return local pricing + self.model_pricing_cache[model_name] = (input_cost, output_cost) + return input_cost, output_cost + except Exception as e: + print(f" WARNING: Error loading local pricing.json: {str(e)}") # Fallback to LiteLLM API if both remote and local pricing not found LITELLM_URL = ( "https://raw.githubusercontent.com/BerriAI/litellm/main/"