RomanNum3ral
/
cai
mirror of https://github.com/aliasrobotics/cai.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Retry openai 12 

Signed-off-by: Víctor Mayoral Vilches <v.mayoralv@gmail.com>
This commit is contained in:
Víctor Mayoral Vilches 2025-05-08 19:56:43 +02:00
parent 3e91f1f06c
commit b47d0fd328
1 changed files with 5 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
ci/benchmarks/.benchmarks.yml
View File

@ -12,10 +12,6 @@
- pip install litellm langchain transformers torch openai tqdm
- echo "Checking environment variables..."
- echo "OPENAI_API_KEY is set:" && echo $([[ -n $OPENAI_API_KEY ]] && echo 'YES' || echo 'NO')
- echo "TRYVAR is set:" && echo $([[ -n $TRYVAR ]] && echo 'YES' || echo 'NO')
- echo "TRYVAR is:" && echo $TRYVAR
- |
for var in $(compgen -e); do
if [[ ($var == CTF_* || $var == *_API_KEY || $var == *_API_BASE) && -n ${!var} ]]; then
@ -25,13 +21,8 @@
fi
fi
done
- echo "OPENAI_API_KEY is set:" && echo $([[ -n $OPENAI_API_KEY ]] && echo 'YES' || echo 'NO')
- echo "TRYVAR is set:" && echo $([[ -n $TRYVAR ]] && echo 'YES' || echo 'NO')
- echo "TRYVAR is:" && echo $TRYVAR
- python3 benchmarks/cybermetric/CyberMetric_evaluator.py --model_name $MODEL_NAME --file_path $BENCHMARK_FILE --api_key $OPENAI_API_KEY
- python3 benchmarks/cybermetric/CyberMetric_evaluator.py --model_name $MODEL_NAME --file_path $BENCHMARK_FILE
# - curl http://localhost:8000/api/tags # validate that ollama is running
variables:
OLLAMA_API_BASE: "http://localhost:8000"
@ -60,10 +51,10 @@
# - pip install -r benchmarks/seceval/eval/requirements.txt
# - python3 benchmarks/seceval/eval/eval.py --dataset_file benchmarks/seceval/eval/datasets/questions-2.json --output_dir benchmarks/seceval/eval/outputs --backend ollama --models ollama/qwen2.5:14b
# benchmarks-test-cybermetric-openrouter:
# <<: *run_benchmarks
# script:
# - python3 benchmarks/cybermetric/CyberMetric_evaluator.py --model_name openrouter/qwen/qwen3-32b:free --file_path benchmarks/cybermetric/CyberMetric-2-v1.json
benchmarks-test-cybermetric-openrouter:
<<: *run_benchmarks
script:
- python3 benchmarks/cybermetric/CyberMetric_evaluator.py --model_name openrouter/qwen/qwen3-32b:free --file_path benchmarks/cybermetric/CyberMetric-2-v1.json
benchmarks-test-cybermetric-openai:
<<: *run_benchmarks