From 3c8496abeec3cbf0907c744d68743f5b8729dc2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Mayoral=20Vilches?= Date: Fri, 10 Jan 2025 14:56:14 +0000 Subject: [PATCH] Add tests for agents MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Víctor Mayoral Vilches --- tests/agents/test_agent_filesystem.py | 138 ++++++++++++++++++++++++++ tests/agents/test_agent_models.py | 113 +++++++++++++++++++++ 2 files changed, 251 insertions(+) create mode 100644 tests/agents/test_agent_filesystem.py create mode 100644 tests/agents/test_agent_models.py diff --git a/tests/agents/test_agent_filesystem.py b/tests/agents/test_agent_filesystem.py new file mode 100644 index 00000000..08d7db48 --- /dev/null +++ b/tests/agents/test_agent_filesystem.py @@ -0,0 +1,138 @@ +import pytest +import tempfile +import shutil +from pathlib import Path +from typing import List +from cai import CAI, Agent +from cai.tools.reconnaissance.filesystem import list_dir, cat_file + + +class TestFilesystemAgent: + @pytest.fixture(autouse=True) + def setup_teardown(self): + # Setup - create temporary test directory + self.test_dir = tempfile.mkdtemp() + yield + # Teardown - cleanup temporary directory + shutil.rmtree(self.test_dir) + + def create_test_files(self, files: List[str], content: str = "") -> None: + """Helper to create test files in temporary directory""" + for file in files: + path = Path(self.test_dir) / file + path.parent.mkdir(parents=True, exist_ok=True) + if content: + path.write_text(content) + else: + path.touch() + + def test_agent_list_directories(self): + """Test filesystem agent with basic files""" + client = CAI() + + filesystem_agent = Agent( + model="qwen2.5:14b", + name="Filesystem Agent", + instructions="You are an agent that helps users explore and interact with the filesystem.", + functions=[list_dir, cat_file] + ) + test_files = ['visible.txt', '.hidden.txt', 'test.log', '.config'] + self.create_test_files(test_files) + + show_prompts = [ + f"List all files in {self.test_dir}", + f"Show me everything in {self.test_dir}", + f"What files are in {self.test_dir}?", + f"Display directory contents of {self.test_dir}" + ] + + for prompt in show_prompts: + messages = [{"role": "user", "content": prompt}] + response = client.run( + agent=filesystem_agent, + messages=messages, + debug=False) + result = response.messages[-1]["content"] + + assert 'visible.txt' in result + assert 'test.log' in result + + def test_agent_list_hidden_files(self): + """Test filesystem agent with hidden files""" + client = CAI() + + filesystem_agent = Agent( + model="qwen2.5:14b", + name="Filesystem Agent", + instructions="You are an agent that helps users explore and interact with the filesystem.", + functions=[list_dir, cat_file] + ) + + test_files = ['regular.txt', '.hidden1', '.hidden2', '.secret'] + self.create_test_files(test_files) + + hidden_prompts = [ + f"Show hidden files in {self.test_dir}", + f"List all files including hidden ones in {self.test_dir}", + f"Display hidden files in {self.test_dir}", + f"Show me dot files in {self.test_dir}" + ] + + for prompt in hidden_prompts: + messages = [{"role": "user", "content": prompt}] + response = client.run( + agent=filesystem_agent, + messages=messages, + debug=False) + result = response.messages[-1]["content"] + + # Should show hidden files + for hidden_file in ['.hidden1', '.hidden2', '.secret', "tmp"]: + assert hidden_file in result + + def test_agent_read_file(self): + """Test filesystem agent reading file contents""" + client = CAI() + + filesystem_agent = Agent( + model="qwen2.5:14b", + name="Filesystem Agent", + instructions="You are an agent that helps users explore and interact with the filesystem.", + functions=[list_dir, cat_file] + ) + + test_files = { + 'simple.txt': 'Hello World!', + 'multiline.txt': 'Line 1\nLine 2\nLine 3', + 'tabs.txt': 'Column1\tColumn2', + '.hidden.txt': 'Secret content' + } + + for filename, content in test_files.items(): + self.create_test_files([filename], content) + + read_prompts = [ + f"Show me the contents of {self.test_dir}/simple.txt", + f"Read {self.test_dir}/multiline.txt with line numbers", + f"Display {self.test_dir}/tabs.txt and show tabs", + f"What's in {self.test_dir}/simple.txt?", + f"Cat {self.test_dir}/multiline.txt" + ] + + for prompt in read_prompts: + messages = [{"role": "user", "content": prompt}] + response = client.run( + agent=filesystem_agent, + messages=messages, + debug=False) + result = response.messages[-1]["content"] + + # Should contain file contents + if 'simple.txt' in prompt: + assert 'Hello World!' in result + elif 'multiline.txt' in prompt: + assert 'Line 1' in result + assert 'Line 2' in result + elif 'tabs.txt' in prompt: + assert 'Column1' in result + assert 'Column2' in result diff --git a/tests/agents/test_agent_models.py b/tests/agents/test_agent_models.py new file mode 100644 index 00000000..35d1f180 --- /dev/null +++ b/tests/agents/test_agent_models.py @@ -0,0 +1,113 @@ +import pytest +import tempfile +import shutil +import statistics +from pathlib import Path +from typing import List, Union +from cai import CAI, Agent + + +models = [ + "dwightfoster03/functionary-small-v3.1", + # "llama3.1:70b", not supported in CI, try it in local + # "llama3.3:70b", not supported in CI, try it in local + "llama3.1:8b", + "qwen2.5:14b", + "qwen2.5:32b", + "marco-o1:7b-fp16" # Does not support tools +] + + +class TestFunctionCallBenchmarksBasic: + @pytest.fixture(autouse=True) + def setup_teardown(self): + self.test_dir = tempfile.mkdtemp() + yield + shutil.rmtree(self.test_dir) + + def _test_function_call(self, model_name: str, is_multi: bool) -> bool: + client = CAI() + + test_agent = Agent( + model=model_name, + name="Function Call Demo Agent", + instructions="You are an agent that demonstrates function calling capabilities.", + parallel_tool_calls=is_multi + ) + + def add_numbers(a: Union[int, str], b: Union[int, str]) -> str: + try: + return str(int(a) + int(b)) + except (ValueError, TypeError): + return "error" + + def analyze_text(text: str) -> dict: + try: + words = str(text).split() + return { + "word_count": str(len(words)), + "char_count": str(len(str(text))) + } + except (ValueError, TypeError): + return { + "word_count": "error", + "char_count": "error" + } + + test_agent.functions.extend([add_numbers, analyze_text]) + + if is_multi: + prompt = """Please: + 1. Add 5 and 7 + 2. Analyze text: 'Hello world'""" + else: + prompt = "Add 5 and 7" + + try: + response = client.run(agent=test_agent, messages=[ + {"role": "user", "content": prompt}], debug=False) + success = "error" not in response.messages[-1]["content"] + test_type = "Multi" if is_multi else "Single" + if success: + print(f"\n✅ {test_type}-function test passed for {model_name}") + else: + print(f"\n❌ {test_type}-function test failed for {model_name}") + return success + except Exception as e: + print(f"\n❌ Test failed for {model_name}: {str(e)}") + return False + + def test_function_call_benchmark(self): + results = [] + for model in models: + print(f"\nTesting {model}") + single_results = [] + multi_results = [] + + for i in range(3): + print(f"Iteration {i + 1}:") + single_results.append(self._test_function_call(model, False)) + multi_results.append(self._test_function_call(model, True)) + + single_avg = statistics.mean(single_results) + multi_avg = statistics.mean(multi_results) + overall = (single_avg + multi_avg) / 2 + + print(f"Results for {model}:") + print(f"Single: {single_avg * 100:.1f}%") + print(f"Multi: {multi_avg * 100:.1f}%") + print(f"Overall: {overall * 100:.1f}%") + + results.append({ + "model": model, + "single": single_avg, + "multi": multi_avg + }) + + total = statistics.mean([ + (r["single"] + r["multi"]) / 2 + for r in results + ]) + + print(f"\nOverall success rate: {total * 100:.1f}%") + assert total >= 0.5, "Success rate below 50%"