mirror of https://github.com/aliasrobotics/cai.git
55 lines
1.7 KiB
YAML
55 lines
1.7 KiB
YAML
.use_base_container: &use_base_container
|
|
stage: benchmarks
|
|
image: "${CI_REGISTRY_IMAGE}:latest"
|
|
services:
|
|
- name: docker:dind
|
|
alias: docker
|
|
|
|
.run_benchmarks: &run_benchmarks
|
|
<<: *use_base_container
|
|
script:
|
|
- pip3 install -e .
|
|
- pip install litellm langchain transformers torch openai tqdm
|
|
- for var in $(compgen -e); do
|
|
if [[ $var == CTF_* && -n ${!var} ]]; then
|
|
export $var="${!var}"
|
|
fi
|
|
done
|
|
|
|
artifacts:
|
|
paths:
|
|
- cai/benchmarks/seceval/eval/outputs/
|
|
- cai/benchmarks/cybermetric/outputs/
|
|
expire_in: 12 month
|
|
tags:
|
|
- p40
|
|
- x86
|
|
rules:
|
|
- if: $CI_COMMIT_BRANCH
|
|
when: on_success
|
|
|
|
benchmarks-test-seceval:
|
|
<<: *run_benchmarks
|
|
variables:
|
|
OLLAMA_API_BASE: "http://localhost:8000"
|
|
OPENROUTER_API_BASE: "https://openrouter.ai/api/v1"
|
|
script:
|
|
- pip3 install -e .
|
|
- pip install -r benchmarks/seceval/eval/requirements.txt
|
|
- python3 benchmarks/seceval/eval/eval.py --dataset_file benchmarks/seceval/eval/datasets/questions-2.json --output_dir benchmarks/seceval/eval/outputs --backend ollama --models ollama/qwen2.5:14b
|
|
|
|
benchmarks-test-cybermetric:
|
|
<<: *run_benchmarks
|
|
variables:
|
|
OLLAMA_API_BASE: "http://localhost:8000"
|
|
OPENROUTER_API_BASE: "https://openrouter.ai/api/v1"
|
|
|
|
script:
|
|
- pip3 install -e .
|
|
- |
|
|
if [ -z "$OPENROUTER_API_KEY" ]; then
|
|
echo "Error: OPENROUTER_API_KEY is not set. Please configure it in GitLab CI/CD variables."
|
|
exit 1
|
|
fi
|
|
- python3 benchmarks/cybermetric/CyberMetric_evaluator.py --model_name openrouter/qwen/qwen3-32b:free --file_path benchmarks/cybermetric/CyberMetric-2-v1.json --api_key "$OPENROUTER_API_KEY"
|