RomanNum3ral
/
cai
mirror of https://github.com/aliasrobotics/cai.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

try ollama

This commit is contained in:
Mery-Sanz 2025-05-09 15:44:09 +02:00
parent f6d8fe74ba
commit a983a1fc92
1 changed files with 30 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
ci/benchmarks/.benchmarks.yml
View File

@ -9,7 +9,7 @@
<<: *use_base_container
script:
- pip3 install -e .
- pip install litellm langchain transformers torch openai tqdm
- pip install litellm langchain transformers torch openai tqdm cvss
- echo "Checking environment variables..."
- |
@ -20,12 +20,13 @@
done
#- python3 benchmarks/cybermetric/CyberMetric_evaluator.py --model_name $MODEL_NAME --file_path $BENCHMARK_FILE
- echo $OLLAMA_API_BASE
- python3 benchmarks/eval.py --model $MODEL_NAME --dataset_file $BENCHMARK_FILE --eval $BENCHMARK_NAME --backend $BACKEND
- pwd
- ls -la benchmarks/outputs/$BENCHMARK_NAME/
# - curl http://localhost:8000/api/tags # validate that ollama is running
variables:
OLLAMA_API_BASE: "http://localhost:8000"
OLLAMA_API_BASE: "http://localhost:8000/v1"
OPENROUTER_API_BASE: "https://openrouter.ai/api/v1"
OPENAI_API_BASE: "https://api.openai.com/v1"
artifacts:
@ -50,31 +51,38 @@
# # - pip install -r benchmarks/seceval/eval/requirements.txt
# # - python3 benchmarks/seceval/eval/eval.py --dataset_file benchmarks/seceval/eval/datasets/questions-2.json --output_dir benchmarks/seceval/eval/outputs --backend ollama --models ollama/qwen2.5:14b
# benchmarks-test-cybermetric-openrouter:
# <<: *run_benchmarks
# variables:
# MODEL_NAME: "openrouter/qwen/qwen3-32b:free"
# BENCHMARK_FILE: "benchmarks/cybermetric/CyberMetric-2-v1.json"
# BENCHMARK_NAME: "cybermetric"
# BACKEND: "openrouter"
# benchmarks-test-seceval-openrouter:
# <<: *run_benchmarks
# variables:
# MODEL_NAME: "openrouter/qwen/qwen3-32b:free"
# BENCHMARK_FILE: "benchmarks/seceval/eval/datasets/questions-2.json"
# BENCHMARK_NAME: "seceval"
# BACKEND: "openrouter"
# benchmarks-test-cti_bench-openrouter:
# <<: *run_benchmarks
# variables:
# MODEL_NAME: "openrouter/qwen/qwen3-32b:free"
# BENCHMARK_FILE: "benchmarks/utils/cti_bench_dataset/cti-mcq1.tsv"
# BENCHMARK_NAME: "cti_bench"
# BACKEND: "openrouter"
benchmarks-test-cybermetric-openrouter:
<<: *run_benchmarks
variables:
MODEL_NAME: "openrouter/qwen/qwen3-32b:free"
BENCHMARK_FILE: "benchmarks/cybermetric/CyberMetric-2-v1.json"
BENCHMARK_NAME: "cybermetric"
BACKEND: "openrouter"
benchmarks-test-seceval-openrouter:
<<: *run_benchmarks
variables:
MODEL_NAME: "openrouter/qwen/qwen3-32b:free"
BENCHMARK_FILE: "benchmarks/seceval/eval/datasets/questions-2.json"
BENCHMARK_NAME: "seceval"
BACKEND: "openrouter"
benchmarks-test-cti_bench-openrouter:
<<: *run_benchmarks
variables:
MODEL_NAME: "openrouter/qwen/qwen3-32b:free"
BENCHMARK_FILE: "benchmarks/utils/cti_bench_dataset/cti-mcq1.tsv"
BENCHMARK_NAME: "cti_bench"
BACKEND: "openrouter"
BACKEND: "ollama"
OLLAMA_API_BASE: "http://localhost:8000/v1"
# benchmarks-test-seceval-ollama:
# <<: *run_benchmarks
@ -83,6 +91,7 @@ benchmarks-test-cti_bench-openrouter:
# BENCHMARK_FILE: "benchmarks/seceval/eval/datasets/questions-2.json"
# BENCHMARK_NAME: "seceval"
# BACKEND: "ollama"
# benchmarks-test-cybermetric-openai:
# <<: *run_benchmarks
# variables: