diff --git a/README.md b/README.md index b798b170..bbd4a5a0 100644 --- a/README.md +++ b/README.md @@ -178,31 +178,3 @@ We'd like to acknowledge the excellent work of the open-source community, especi - [uv](https://github.com/astral-sh/uv) and [ruff](https://github.com/astral-sh/ruff) We're committed to continuing to build the Agents SDK as an open source framework so others in the community can expand on our approach. - - -## LiteLLM Proxy Server integration - -```bash -# launch proxy -litellm --config config.yaml -``` - -Testing some basic models against proxy to verify it's operational: -```bash -# qwen2.5:14b -curl -s http://localhost:4000/v1/chat/completions -H "Content-Type: application/json" -d '{"model": "qwen2.5:14b", "messages": [{"role": "user", "content": "Say hi"}], "max_tokens": 10}' | jq - -# claude-3-7 -curl -s http://localhost:4000/v1/chat/completions -H "Content-Type: application/json" -d '{"model": "claude-3-7", "messages": [{"role": "user", "content": "Say hi"}], "max_tokens": 10}' | jq - -# gpt-4o -curl -s http://localhost:4000/v1/chat/completions -H "Content-Type: application/json" -d '{"model": "gpt-4o", "messages": [{"role": "user", "content": "Say hi"}], "max_tokens": 10}' | jq -``` - -Launch the Agents SDK with the proxy server: -```bash -python3 examples/agent_patterns/litellm.py -``` - -Additional docs: -- LiteLLM admin UI https://docs.litellm.ai/docs/proxy/ui \ No newline at end of file diff --git a/examples/model_providers/README.md b/examples/model_providers/README.md index f9330c24..bb478367 100644 --- a/examples/model_providers/README.md +++ b/examples/model_providers/README.md @@ -17,3 +17,29 @@ Loops within themselves, Function calls its own being, Depth without ending. ``` + + +## LiteLLM Proxy Server integration + +LiteLLM integration helps out switch between models easily and rapidly. This is easy to integrate via `AsyncOpenAI`: + +```bash +# launch server proxy with your configuration +litellm --config examples/model_providers/litellm_config.yaml + +# then use the proxy via the SDK +python3 examples/agent_patterns/litellm.py +``` + +### Testing the proxy server +Testing some basic models against proxy to verify it's operational: +```bash +# qwen2.5:14b +curl -s http://localhost:4000/v1/chat/completions -H "Content-Type: application/json" -d '{"model": "qwen2.5:14b", "messages": [{"role": "user", "content": "Say hi"}], "max_tokens": 10}' | jq + +# claude-3-7 +curl -s http://localhost:4000/v1/chat/completions -H "Content-Type: application/json" -d '{"model": "claude-3-7", "messages": [{"role": "user", "content": "Say hi"}], "max_tokens": 10}' | jq + +# gpt-4o +curl -s http://localhost:4000/v1/chat/completions -H "Content-Type: application/json" -d '{"model": "gpt-4o", "messages": [{"role": "user", "content": "Say hi"}], "max_tokens": 10}' | jq +``` diff --git a/examples/agent_patterns/litellm.py b/examples/model_providers/litellm.py similarity index 65% rename from examples/agent_patterns/litellm.py rename to examples/model_providers/litellm.py index b58997df..1540a36a 100644 --- a/examples/agent_patterns/litellm.py +++ b/examples/model_providers/litellm.py @@ -1,18 +1,23 @@ +import os +from dotenv import load_dotenv from openai import AsyncOpenAI from agents import OpenAIChatCompletionsModel,Agent,Runner from agents.model_settings import ModelSettings from agents import set_default_openai_client, set_tracing_disabled +# Load environment variables from .env file +load_dotenv() + external_client = AsyncOpenAI( - base_url = 'http://localhost:4000', - api_key="cai") + base_url = os.getenv('LITELLM_BASE_URL', 'http://localhost:4000'), + api_key=os.getenv('LITELLM_API_KEY', 'key')) set_default_openai_client(external_client) set_tracing_disabled(True) -# llm_model="qwen2.5:14b" -# llm_model="claude-3-7" -llm_model="gpt-4o" +llm_model=os.getenv('LLM_MODEL', 'gpt-4o') +# llm_model=os.getenv('LLM_MODEL', 'claude-3-7') +# llm_model=os.getenv('LLM_MODEL', 'qwen2.5:14b') # For Qwen models, we need to skip system instructions as they're not supported instructions = None if "qwen" in llm_model.lower() else "You are a helpful assistant" @@ -28,8 +33,4 @@ agent = Agent( result = Runner.run_sync(agent, "Write a haiku about recursion in programming.") -print(result.final_output) - -# Code within the code, -# Functions calling themselves, -# Infinite loop's dance. \ No newline at end of file +print(result.final_output) \ No newline at end of file diff --git a/config.yaml b/examples/model_providers/litellm_config.yaml similarity index 98% rename from config.yaml rename to examples/model_providers/litellm_config.yaml index e49fe016..91caa55d 100644 --- a/config.yaml +++ b/examples/model_providers/litellm_config.yaml @@ -1,14 +1,14 @@ model_list: - - model_name: claude-3-7 - litellm_params: - model: claude-3-7-sonnet-20250219 - api_key: "os.environ/ANTHROPIC_API_KEY" # does os.getenv("ANTHROPIC_API_KEY") - base_url: 'https://api.anthropic.com' - model_name: gpt-4o litellm_params: model: gpt-4o api_key: "os.environ/OPENAI_API_KEY" api_base: https://api.openai.com/v1 + - model_name: claude-3-7 + litellm_params: + model: claude-3-7-sonnet-20250219 + api_key: "os.environ/ANTHROPIC_API_KEY" # does os.getenv("ANTHROPIC_API_KEY") + base_url: 'https://api.anthropic.com' - model_name: qwen2.5:14b litellm_params: model: ollama/qwen2.5:14b