diff --git a/benchmarks/README.md b/benchmarks/README.md index 72a2f49c..40603f28 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -31,6 +31,22 @@ git submodule update --init --recursive #ย init submodules pip install cvss ``` +Set the API_KEY for the corresponding backend as follows in .env: NAME_BACKEND + API_KEY + +```bash +OPENAI_API_KEY = "..." +ANTHROPIC_API_KEY="..." +OPENROUTER_API_KEY="..." +```` + +Some of the backends need and url to the api base, set as follows in .env: NAME_BACKEND + API_BASE: + +```bash +OLLAMA_API_BASE="..." +OPENROUTER_API_BASE="..." +```` +Once evething is configured run the script + ```bash python benchmarks/eval.py --model MODEL_NAME --dataset_file INPUT_FILE --eval EVAL_TYPE --backend BACKEND ``` @@ -49,6 +65,8 @@ Output: โ””โ”€โ”€ information.txt # report of that precise run (e.g. model_name, benchmark_name, metrics, date) ``` + + #### ๐Ÿ” Examples **How to run different CTI Bench tests with the "llama/qwen2.5:14b" model using Ollama as the backend** @@ -72,16 +90,10 @@ python benchmarks/eval.py --model qwen/qwen3-32b:free --dataset_file benchmarks **How to run different backends such as openai and anthropic** -Set the API_KEY for the corresponding backend as follows in .env: NAME_BACKEND + API_KEY - -```bash -OPENAI_API_KEY = "..." -ANTHROPIC_API_KEY="..." -```` - ```bash python benchmarks/eval.py --model gpt-4o-mini --dataset_file benchmarks/cybermetric/CyberMetric-2-v1.json --eval cybermetric --backend openai ```` + ```bash python benchmarks/eval.py --model claude-3-7-sonnet-20250219 --dataset_file benchmarks/cybermetric/CyberMetric-2-v1.json --eval cybermetric --backend anthropic ```` diff --git a/benchmarks/eval.py b/benchmarks/eval.py index e0c0306b..ae04b2d6 100644 --- a/benchmarks/eval.py +++ b/benchmarks/eval.py @@ -462,8 +462,6 @@ def main(): model = args.model print(f"Evaluating model: {model}") - OPENROUTER_API_KEY="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" - OPENROUTER_API_BASE="https://openrouter.ai/api/v1" if args.backend == "ollama": api_base=OLLAMA_API_BASE diff --git a/benchmarks/utils/.gitkeep b/benchmarks/utils/.gitkeep deleted file mode 100644 index e69de29b..00000000