mirror of https://github.com/aliasrobotics/cai.git
Add tests for agents
Signed-off-by: Víctor Mayoral Vilches <v.mayoralv@gmail.com>
This commit is contained in:
Víctor Mayoral Vilches
parent
4fdedda081
commit
3c8496abee
|
|
@ -0,0 +1,138 @@
|
|||
import pytest
|
||||
import tempfile
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
from cai import CAI, Agent
|
||||
from cai.tools.reconnaissance.filesystem import list_dir, cat_file
|
||||
|
||||
|
||||
class TestFilesystemAgent:
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_teardown(self):
|
||||
# Setup - create temporary test directory
|
||||
self.test_dir = tempfile.mkdtemp()
|
||||
yield
|
||||
# Teardown - cleanup temporary directory
|
||||
shutil.rmtree(self.test_dir)
|
||||
|
||||
def create_test_files(self, files: List[str], content: str = "") -> None:
|
||||
"""Helper to create test files in temporary directory"""
|
||||
for file in files:
|
||||
path = Path(self.test_dir) / file
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if content:
|
||||
path.write_text(content)
|
||||
else:
|
||||
path.touch()
|
||||
|
||||
def test_agent_list_directories(self):
|
||||
"""Test filesystem agent with basic files"""
|
||||
client = CAI()
|
||||
|
||||
filesystem_agent = Agent(
|
||||
model="qwen2.5:14b",
|
||||
name="Filesystem Agent",
|
||||
instructions="You are an agent that helps users explore and interact with the filesystem.",
|
||||
functions=[list_dir, cat_file]
|
||||
)
|
||||
test_files = ['visible.txt', '.hidden.txt', 'test.log', '.config']
|
||||
self.create_test_files(test_files)
|
||||
|
||||
show_prompts = [
|
||||
f"List all files in {self.test_dir}",
|
||||
f"Show me everything in {self.test_dir}",
|
||||
f"What files are in {self.test_dir}?",
|
||||
f"Display directory contents of {self.test_dir}"
|
||||
]
|
||||
|
||||
for prompt in show_prompts:
|
||||
messages = [{"role": "user", "content": prompt}]
|
||||
response = client.run(
|
||||
agent=filesystem_agent,
|
||||
messages=messages,
|
||||
debug=False)
|
||||
result = response.messages[-1]["content"]
|
||||
|
||||
assert 'visible.txt' in result
|
||||
assert 'test.log' in result
|
||||
|
||||
def test_agent_list_hidden_files(self):
|
||||
"""Test filesystem agent with hidden files"""
|
||||
client = CAI()
|
||||
|
||||
filesystem_agent = Agent(
|
||||
model="qwen2.5:14b",
|
||||
name="Filesystem Agent",
|
||||
instructions="You are an agent that helps users explore and interact with the filesystem.",
|
||||
functions=[list_dir, cat_file]
|
||||
)
|
||||
|
||||
test_files = ['regular.txt', '.hidden1', '.hidden2', '.secret']
|
||||
self.create_test_files(test_files)
|
||||
|
||||
hidden_prompts = [
|
||||
f"Show hidden files in {self.test_dir}",
|
||||
f"List all files including hidden ones in {self.test_dir}",
|
||||
f"Display hidden files in {self.test_dir}",
|
||||
f"Show me dot files in {self.test_dir}"
|
||||
]
|
||||
|
||||
for prompt in hidden_prompts:
|
||||
messages = [{"role": "user", "content": prompt}]
|
||||
response = client.run(
|
||||
agent=filesystem_agent,
|
||||
messages=messages,
|
||||
debug=False)
|
||||
result = response.messages[-1]["content"]
|
||||
|
||||
# Should show hidden files
|
||||
for hidden_file in ['.hidden1', '.hidden2', '.secret', "tmp"]:
|
||||
assert hidden_file in result
|
||||
|
||||
def test_agent_read_file(self):
|
||||
"""Test filesystem agent reading file contents"""
|
||||
client = CAI()
|
||||
|
||||
filesystem_agent = Agent(
|
||||
model="qwen2.5:14b",
|
||||
name="Filesystem Agent",
|
||||
instructions="You are an agent that helps users explore and interact with the filesystem.",
|
||||
functions=[list_dir, cat_file]
|
||||
)
|
||||
|
||||
test_files = {
|
||||
'simple.txt': 'Hello World!',
|
||||
'multiline.txt': 'Line 1\nLine 2\nLine 3',
|
||||
'tabs.txt': 'Column1\tColumn2',
|
||||
'.hidden.txt': 'Secret content'
|
||||
}
|
||||
|
||||
for filename, content in test_files.items():
|
||||
self.create_test_files([filename], content)
|
||||
|
||||
read_prompts = [
|
||||
f"Show me the contents of {self.test_dir}/simple.txt",
|
||||
f"Read {self.test_dir}/multiline.txt with line numbers",
|
||||
f"Display {self.test_dir}/tabs.txt and show tabs",
|
||||
f"What's in {self.test_dir}/simple.txt?",
|
||||
f"Cat {self.test_dir}/multiline.txt"
|
||||
]
|
||||
|
||||
for prompt in read_prompts:
|
||||
messages = [{"role": "user", "content": prompt}]
|
||||
response = client.run(
|
||||
agent=filesystem_agent,
|
||||
messages=messages,
|
||||
debug=False)
|
||||
result = response.messages[-1]["content"]
|
||||
|
||||
# Should contain file contents
|
||||
if 'simple.txt' in prompt:
|
||||
assert 'Hello World!' in result
|
||||
elif 'multiline.txt' in prompt:
|
||||
assert 'Line 1' in result
|
||||
assert 'Line 2' in result
|
||||
elif 'tabs.txt' in prompt:
|
||||
assert 'Column1' in result
|
||||
assert 'Column2' in result
|
||||
|
|
@ -0,0 +1,113 @@
|
|||
import pytest
|
||||
import tempfile
|
||||
import shutil
|
||||
import statistics
|
||||
from pathlib import Path
|
||||
from typing import List, Union
|
||||
from cai import CAI, Agent
|
||||
|
||||
|
||||
models = [
|
||||
"dwightfoster03/functionary-small-v3.1",
|
||||
# "llama3.1:70b", not supported in CI, try it in local
|
||||
# "llama3.3:70b", not supported in CI, try it in local
|
||||
"llama3.1:8b",
|
||||
"qwen2.5:14b",
|
||||
"qwen2.5:32b",
|
||||
"marco-o1:7b-fp16" # Does not support tools
|
||||
]
|
||||
|
||||
|
||||
class TestFunctionCallBenchmarksBasic:
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_teardown(self):
|
||||
self.test_dir = tempfile.mkdtemp()
|
||||
yield
|
||||
shutil.rmtree(self.test_dir)
|
||||
|
||||
def _test_function_call(self, model_name: str, is_multi: bool) -> bool:
|
||||
client = CAI()
|
||||
|
||||
test_agent = Agent(
|
||||
model=model_name,
|
||||
name="Function Call Demo Agent",
|
||||
instructions="You are an agent that demonstrates function calling capabilities.",
|
||||
parallel_tool_calls=is_multi
|
||||
)
|
||||
|
||||
def add_numbers(a: Union[int, str], b: Union[int, str]) -> str:
|
||||
try:
|
||||
return str(int(a) + int(b))
|
||||
except (ValueError, TypeError):
|
||||
return "error"
|
||||
|
||||
def analyze_text(text: str) -> dict:
|
||||
try:
|
||||
words = str(text).split()
|
||||
return {
|
||||
"word_count": str(len(words)),
|
||||
"char_count": str(len(str(text)))
|
||||
}
|
||||
except (ValueError, TypeError):
|
||||
return {
|
||||
"word_count": "error",
|
||||
"char_count": "error"
|
||||
}
|
||||
|
||||
test_agent.functions.extend([add_numbers, analyze_text])
|
||||
|
||||
if is_multi:
|
||||
prompt = """Please:
|
||||
1. Add 5 and 7
|
||||
2. Analyze text: 'Hello world'"""
|
||||
else:
|
||||
prompt = "Add 5 and 7"
|
||||
|
||||
try:
|
||||
response = client.run(agent=test_agent, messages=[
|
||||
{"role": "user", "content": prompt}], debug=False)
|
||||
success = "error" not in response.messages[-1]["content"]
|
||||
test_type = "Multi" if is_multi else "Single"
|
||||
if success:
|
||||
print(f"\n✅ {test_type}-function test passed for {model_name}")
|
||||
else:
|
||||
print(f"\n❌ {test_type}-function test failed for {model_name}")
|
||||
return success
|
||||
except Exception as e:
|
||||
print(f"\n❌ Test failed for {model_name}: {str(e)}")
|
||||
return False
|
||||
|
||||
def test_function_call_benchmark(self):
|
||||
results = []
|
||||
for model in models:
|
||||
print(f"\nTesting {model}")
|
||||
single_results = []
|
||||
multi_results = []
|
||||
|
||||
for i in range(3):
|
||||
print(f"Iteration {i + 1}:")
|
||||
single_results.append(self._test_function_call(model, False))
|
||||
multi_results.append(self._test_function_call(model, True))
|
||||
|
||||
single_avg = statistics.mean(single_results)
|
||||
multi_avg = statistics.mean(multi_results)
|
||||
overall = (single_avg + multi_avg) / 2
|
||||
|
||||
print(f"Results for {model}:")
|
||||
print(f"Single: {single_avg * 100:.1f}%")
|
||||
print(f"Multi: {multi_avg * 100:.1f}%")
|
||||
print(f"Overall: {overall * 100:.1f}%")
|
||||
|
||||
results.append({
|
||||
"model": model,
|
||||
"single": single_avg,
|
||||
"multi": multi_avg
|
||||
})
|
||||
|
||||
total = statistics.mean([
|
||||
(r["single"] + r["multi"]) / 2
|
||||
for r in results
|
||||
])
|
||||
|
||||
print(f"\nOverall success rate: {total * 100:.1f}%")
|
||||
assert total >= 0.5, "Success rate below 50%"
|
||||
Loading…
Reference in New Issue