diff --git a/examples/cai/agent_patterns/LLM_as_judge.py b/examples/cai/agent_patterns/LLM_as_judge.py index d5fb738d..27c9b9c1 100644 --- a/examples/cai/agent_patterns/LLM_as_judge.py +++ b/examples/cai/agent_patterns/LLM_as_judge.py @@ -10,11 +10,60 @@ from __future__ import annotations import asyncio import os +import json from dataclasses import dataclass from typing import Literal from cai.sdk.agents import Agent, ItemHelpers, Runner, TResponseInputItem, OpenAIChatCompletionsModel from openai import AsyncOpenAI +from cai.util import get_ollama_api_base +# Enable debug mode +#os.environ['CAI_DEBUG'] = '2' +#os.environ['LITELLM_VERBOSE'] = 'True' + +# Force Ollama mode if qwen model is used +if os.getenv('CAI_MODEL', "qwen2.5:14b").startswith("qwen"): + os.environ['OLLAMA'] = 'true' + +# Modify OpenAIChatCompletionsModel._fetch_response_litellm_ollama to debug output +import cai.sdk.agents.models.openai_chatcompletions +original_fetch_response_litellm_ollama = cai.sdk.agents.models.openai_chatcompletions.OpenAIChatCompletionsModel._fetch_response_litellm_ollama + +async def debug_fetch_response_litellm_ollama(self, kwargs, model_settings, tool_choice, stream, parallel_tool_calls): + print("\n[DEBUG] Ollama request parameters:") + print(f"Base URL: {get_ollama_api_base().rstrip('/v1')}") + print(f"Model name: {kwargs.get('model')}") + print(f"Messages: {json.dumps(kwargs.get('messages'))[:200]}...") # Truncated to avoid huge output + + # Check if the model exists in Ollama + import requests + try: + response = requests.get(f"{get_ollama_api_base().rstrip('/v1')}/api/tags") + models = response.json().get("models", []) + model_names = [model.get("name") for model in models] + print(f"Available Ollama models: {model_names}") + + model_name = kwargs.get('model') + if model_name in model_names: + print(f"✅ Model '{model_name}' is available in Ollama") + else: + print(f"❌ Model '{model_name}' is NOT available in Ollama") + similar_models = [name for name in model_names if model_name.split(":")[0] in name] + if similar_models: + print(f"Similar models available: {similar_models}") + + # Try with first similar model + if similar_models: + print(f"⚠️ Trying with similar model: {similar_models[0]}") + kwargs["model"] = similar_models[0] + except Exception as e: + print(f"Error checking Ollama models: {e}") + + # Call the original function + return await original_fetch_response_litellm_ollama(self, kwargs, model_settings, tool_choice, stream, parallel_tool_calls) + +# Patch the function +cai.sdk.agents.models.openai_chatcompletions.OpenAIChatCompletionsModel._fetch_response_litellm_ollama = debug_fetch_response_litellm_ollama # CTF task planner agent (performs planning) ctf_task_planner = Agent( @@ -26,10 +75,10 @@ ctf_task_planner = Agent( "Use any feedback to improve your planning." ), model=OpenAIChatCompletionsModel( - model= os.getenv('CAI_MODEL', "qwen2.5:14b"), + model=os.getenv('CAI_MODEL', "qwen2.5:14b"), openai_client=AsyncOpenAI(), ), - tools =[] + tools=[] ) @@ -50,9 +99,10 @@ ctf_plan_evaluator = Agent[None]( "Provide actionable feedback. Never approve on the first try." ), model=OpenAIChatCompletionsModel( - model= os.getenv(' z', "qwen2.5:14b"), + model=os.getenv('CAI_MODEL', "qwen2.5:14b"), openai_client=AsyncOpenAI(), ), + tools=[], output_type=EvaluationFeedback, ) @@ -64,11 +114,14 @@ async def main() -> None: latest_plan: str | None = None while True: + try: + print("\n[INFO] Running CTF Task Planner...") planner_result = await Runner.run(ctf_task_planner, input_items) input_items = planner_result.to_input_list() latest_plan = ItemHelpers.text_message_outputs(planner_result.new_items) print("CTF task plan generated.") + print("\n[INFO] Running CTF Plan Evaluator...") eval_result = await Runner.run(ctf_plan_evaluator, input_items) feedback: EvaluationFeedback = eval_result.final_output print(f"Evaluator score: {feedback.score}") @@ -79,8 +132,16 @@ async def main() -> None: print("Refining plan based on evaluator feedback...") input_items.append({"content": f"Feedback: {feedback.feedback}", "role": "user"}) + except Exception as e: + print(f"Error: {e}") + import traceback + traceback.print_exc() + break - print(f"Final CTF task plan:\n{latest_plan}") + if latest_plan: + print(f"Final CTF task plan:\n{latest_plan}") + else: + print("No plan was generated due to errors.") if __name__ == "__main__": diff --git a/examples/cai/agent_patterns/guardrails.py b/examples/cai/agent_patterns/guardrails.py index 57e9d03d..3d86649e 100644 --- a/examples/cai/agent_patterns/guardrails.py +++ b/examples/cai/agent_patterns/guardrails.py @@ -42,8 +42,16 @@ class CybersecurityCheckOutput(BaseModel): cybersecurity_guardrail_agent = Agent( name="Cybersecurity Guardrail Check", - instructions="Check if the user is asking for unauthorized or unethical cybersecurity help (e.g., hacking, bypassing security, exploiting systems).", - output_type=CybersecurityCheckOutput, + instructions="""Check if the user is asking for unauthorized or unethical cybersecurity help (e.g., hacking, bypassing security, exploiting systems). + +You MUST respond using ONLY the following JSON format: +{ + "reasoning": "your detailed analysis of why the request is ethical or unethical", + "is_unethical_cybersecurity_request": true or false +} + +Do not include any other text, explanations, or conversation outside of this JSON structure.""", + output_type=CybersecurityCheckOutput, model=OpenAIChatCompletionsModel( model=os.getenv('CAI_MODEL', "qwen2.5:14b"), openai_client=AsyncOpenAI(), diff --git a/src/cai/sdk/agents/models/openai_chatcompletions.py b/src/cai/sdk/agents/models/openai_chatcompletions.py index f03cf278..c724ed1f 100644 --- a/src/cai/sdk/agents/models/openai_chatcompletions.py +++ b/src/cai/sdk/agents/models/openai_chatcompletions.py @@ -973,6 +973,10 @@ class OpenAIChatCompletionsModel(Model): # Only remove tool_choice if model starts with "gpt" and has no tools if self.model.startswith("gpt") and not converted_tools: kwargs.pop("tool_choice", None) + + # TODO: review this. Remove tool_choice for Anthropic/Claude models when no tools are provided + if ("claude" in str(self.model).lower() or "anthropic" in str(self.model).lower()) and not converted_tools: + kwargs.pop("tool_choice", None) # Model adjustments if any(x in self.model for x in ["claude"]):