mirror of https://github.com/aliasrobotics/cai.git
Restore structure and place litellm integraton under model_providers
Signed-off-by: Víctor Mayoral Vilches <v.mayoralv@gmail.com>
This commit is contained in:
parent
8189f331b6
commit
0f8c357231
28
README.md
28
README.md
|
|
@ -178,31 +178,3 @@ We'd like to acknowledge the excellent work of the open-source community, especi
|
|||
- [uv](https://github.com/astral-sh/uv) and [ruff](https://github.com/astral-sh/ruff)
|
||||
|
||||
We're committed to continuing to build the Agents SDK as an open source framework so others in the community can expand on our approach.
|
||||
|
||||
|
||||
## LiteLLM Proxy Server integration
|
||||
|
||||
```bash
|
||||
# launch proxy
|
||||
litellm --config config.yaml
|
||||
```
|
||||
|
||||
Testing some basic models against proxy to verify it's operational:
|
||||
```bash
|
||||
# qwen2.5:14b
|
||||
curl -s http://localhost:4000/v1/chat/completions -H "Content-Type: application/json" -d '{"model": "qwen2.5:14b", "messages": [{"role": "user", "content": "Say hi"}], "max_tokens": 10}' | jq
|
||||
|
||||
# claude-3-7
|
||||
curl -s http://localhost:4000/v1/chat/completions -H "Content-Type: application/json" -d '{"model": "claude-3-7", "messages": [{"role": "user", "content": "Say hi"}], "max_tokens": 10}' | jq
|
||||
|
||||
# gpt-4o
|
||||
curl -s http://localhost:4000/v1/chat/completions -H "Content-Type: application/json" -d '{"model": "gpt-4o", "messages": [{"role": "user", "content": "Say hi"}], "max_tokens": 10}' | jq
|
||||
```
|
||||
|
||||
Launch the Agents SDK with the proxy server:
|
||||
```bash
|
||||
python3 examples/agent_patterns/litellm.py
|
||||
```
|
||||
|
||||
Additional docs:
|
||||
- LiteLLM admin UI https://docs.litellm.ai/docs/proxy/ui
|
||||
|
|
@ -17,3 +17,29 @@ Loops within themselves,
|
|||
Function calls its own being,
|
||||
Depth without ending.
|
||||
```
|
||||
|
||||
|
||||
## LiteLLM Proxy Server integration
|
||||
|
||||
LiteLLM integration helps out switch between models easily and rapidly. This is easy to integrate via `AsyncOpenAI`:
|
||||
|
||||
```bash
|
||||
# launch server proxy with your configuration
|
||||
litellm --config examples/model_providers/litellm_config.yaml
|
||||
|
||||
# then use the proxy via the SDK
|
||||
python3 examples/agent_patterns/litellm.py
|
||||
```
|
||||
|
||||
### Testing the proxy server
|
||||
Testing some basic models against proxy to verify it's operational:
|
||||
```bash
|
||||
# qwen2.5:14b
|
||||
curl -s http://localhost:4000/v1/chat/completions -H "Content-Type: application/json" -d '{"model": "qwen2.5:14b", "messages": [{"role": "user", "content": "Say hi"}], "max_tokens": 10}' | jq
|
||||
|
||||
# claude-3-7
|
||||
curl -s http://localhost:4000/v1/chat/completions -H "Content-Type: application/json" -d '{"model": "claude-3-7", "messages": [{"role": "user", "content": "Say hi"}], "max_tokens": 10}' | jq
|
||||
|
||||
# gpt-4o
|
||||
curl -s http://localhost:4000/v1/chat/completions -H "Content-Type: application/json" -d '{"model": "gpt-4o", "messages": [{"role": "user", "content": "Say hi"}], "max_tokens": 10}' | jq
|
||||
```
|
||||
|
|
|
|||
|
|
@ -1,18 +1,23 @@
|
|||
import os
|
||||
from dotenv import load_dotenv
|
||||
from openai import AsyncOpenAI
|
||||
from agents import OpenAIChatCompletionsModel,Agent,Runner
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents import set_default_openai_client, set_tracing_disabled
|
||||
|
||||
# Load environment variables from .env file
|
||||
load_dotenv()
|
||||
|
||||
external_client = AsyncOpenAI(
|
||||
base_url = 'http://localhost:4000',
|
||||
api_key="cai")
|
||||
base_url = os.getenv('LITELLM_BASE_URL', 'http://localhost:4000'),
|
||||
api_key=os.getenv('LITELLM_API_KEY', 'key'))
|
||||
|
||||
set_default_openai_client(external_client)
|
||||
set_tracing_disabled(True)
|
||||
|
||||
# llm_model="qwen2.5:14b"
|
||||
# llm_model="claude-3-7"
|
||||
llm_model="gpt-4o"
|
||||
llm_model=os.getenv('LLM_MODEL', 'gpt-4o')
|
||||
# llm_model=os.getenv('LLM_MODEL', 'claude-3-7')
|
||||
# llm_model=os.getenv('LLM_MODEL', 'qwen2.5:14b')
|
||||
|
||||
# For Qwen models, we need to skip system instructions as they're not supported
|
||||
instructions = None if "qwen" in llm_model.lower() else "You are a helpful assistant"
|
||||
|
|
@ -28,8 +33,4 @@ agent = Agent(
|
|||
|
||||
|
||||
result = Runner.run_sync(agent, "Write a haiku about recursion in programming.")
|
||||
print(result.final_output)
|
||||
|
||||
# Code within the code,
|
||||
# Functions calling themselves,
|
||||
# Infinite loop's dance.
|
||||
print(result.final_output)
|
||||
|
|
@ -1,14 +1,14 @@
|
|||
model_list:
|
||||
- model_name: claude-3-7
|
||||
litellm_params:
|
||||
model: claude-3-7-sonnet-20250219
|
||||
api_key: "os.environ/ANTHROPIC_API_KEY" # does os.getenv("ANTHROPIC_API_KEY")
|
||||
base_url: 'https://api.anthropic.com'
|
||||
- model_name: gpt-4o
|
||||
litellm_params:
|
||||
model: gpt-4o
|
||||
api_key: "os.environ/OPENAI_API_KEY"
|
||||
api_base: https://api.openai.com/v1
|
||||
- model_name: claude-3-7
|
||||
litellm_params:
|
||||
model: claude-3-7-sonnet-20250219
|
||||
api_key: "os.environ/ANTHROPIC_API_KEY" # does os.getenv("ANTHROPIC_API_KEY")
|
||||
base_url: 'https://api.anthropic.com'
|
||||
- model_name: qwen2.5:14b
|
||||
litellm_params:
|
||||
model: ollama/qwen2.5:14b
|
||||
Loading…
Reference in New Issue