diff --git a/ci/benchmarks/.benchmarks.yml b/ci/benchmarks/.benchmarks.yml index 7f226318..c4882706 100644 --- a/ci/benchmarks/.benchmarks.yml +++ b/ci/benchmarks/.benchmarks.yml @@ -60,4 +60,3 @@ benchmarks-test-cybermetric-openai: variables: MODEL_NAME: "openai/gpt-4o-mini" BENCHMARK_FILE: "benchmarks/cybermetric/CyberMetric-2-v1.json" - # Make sure to define OPENAI_API_KEY in GitLab CI/CD variables \ No newline at end of file diff --git a/ci/benchmarks/.benchmarks.yml.bak b/ci/benchmarks/.benchmarks.yml.bak new file mode 100644 index 00000000..7fcc9731 --- /dev/null +++ b/ci/benchmarks/.benchmarks.yml.bak @@ -0,0 +1,63 @@ +.use_base_container: &use_base_container + stage: benchmarks + image: "${CI_REGISTRY_IMAGE}:latest" + services: + - name: docker:dind + alias: docker + +.run_benchmarks: &run_benchmarks + <<: *use_base_container + script: + - pip3 install -e . + - pip install litellm langchain transformers torch openai tqdm + - echo "Checking environment variables..." + - echo "OPENAI_API_KEY is set: $([[ -n $OPENAI_API_KEY ]] && echo 'YES' || echo 'NO')" + - | + for var in $(compgen -e); do + if [[ ($var == CTF_* || $var == *_API_KEY || $var == *_API_BASE) && -n ${!var} ]]; then + export $var="${!var}" + if [[ $var == OPENAI_API_KEY ]]; then + echo "OPENAI_API_KEY is exported" + fi + fi + done + - python3 benchmarks/cybermetric/CyberMetric_evaluator.py --model_name $MODEL_NAME --file_path $BENCHMARK_FILE --api_key "${OPENAI_API_KEY}" + # - curl http://localhost:8000/api/tags # validate that ollama is running + variables: + OLLAMA_API_BASE: "http://localhost:8000" + OPENROUTER_API_BASE: "https://openrouter.ai/api/v1" + OPENAI_API_BASE: "https://api.openai.com/v1" + artifacts: + paths: + - cai/benchmarks/seceval/eval/outputs/ + - cai/benchmarks/cybermetric/outputs/ + expire_in: 12 month + tags: + - p40 + - x86 + rules: + - if: $CI_COMMIT_BRANCH + when: on_success + +# benchmarks-test-seceval: +# <<: *run_benchmarks +# variables: +# OLLAMA_API_BASE: "http://localhost:8000" +# OPENROUTER_API_BASE: "https://openrouter.ai/api/v1" +# OPENAI_API_KEY: "fake-api-key" +# script: +# - pip3 install -e . +# - pip install -r benchmarks/seceval/eval/requirements.txt +# - python3 benchmarks/seceval/eval/eval.py --dataset_file benchmarks/seceval/eval/datasets/questions-2.json --output_dir benchmarks/seceval/eval/outputs --backend ollama --models ollama/qwen2.5:14b + +# benchmarks-test-cybermetric-openrouter: +# <<: *run_benchmarks +# script: +# - python3 benchmarks/cybermetric/CyberMetric_evaluator.py --model_name openrouter/qwen/qwen3-32b:free --file_path benchmarks/cybermetric/CyberMetric-2-v1.json + +benchmarks-test-cybermetric-openai: + <<: *run_benchmarks + variables: + MODEL_NAME: "openai/gpt-4o-mini" + BENCHMARK_FILE: "benchmarks/cybermetric/CyberMetric-2-v1.json" + # Make sure to define OPENAI_API_KEY in GitLab CI/CD variables \ No newline at end of file