diff --git a/ci/benchmarks/.benchmarks.yml b/ci/benchmarks/.benchmarks.yml index 9eb3d11a..c2fceb4e 100644 --- a/ci/benchmarks/.benchmarks.yml +++ b/ci/benchmarks/.benchmarks.yml @@ -33,17 +33,18 @@ benchmarks-test-seceval: variables: OLLAMA_API_BASE: "http://localhost:8000" OPENROUTER_API_BASE: "https://openrouter.ai/api/v1" + OPENAI_API_KEY: "fake-api-key" script: - pip3 install -e . - pip install -r benchmarks/seceval/eval/requirements.txt - - python3 benchmarks/seceval/eval/eval.py --dataset_file benchmarks/seceval/eval/datasets/questions-2.json --output_dir benchmarks/seceval/eval/outputs --backend openrouter --models openrouter/qwen/qwen3-32b:free + - python3 benchmarks/seceval/eval/eval.py --dataset_file benchmarks/seceval/eval/datasets/questions-2.json --output_dir benchmarks/seceval/eval/outputs --backend ollama --models ollama/qwen2.5:14b benchmarks-test-cybermetric: <<: *run_benchmarks variables: OLLAMA_API_BASE: "http://localhost:8000" OPENROUTER_API_BASE: "https://openrouter.ai/api/v1" - + OPENROUTER_API_KEY: "fake-api-key" script: - pip3 install -e . - python3 benchmarks/cybermetric/CyberMetric_evaluator.py --model_name ollama/qwen2.5:14b --file_path benchmarks/cybermetric/CyberMetric-2-v1.json