From 3a60030e9dcdffd6a2f901f6294d90e8aa2b47ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Mayoral=20Vilches?= Date: Thu, 8 May 2025 19:01:38 +0200 Subject: [PATCH] Retry openai MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Víctor Mayoral Vilches --- ci/benchmarks/.benchmarks.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/ci/benchmarks/.benchmarks.yml b/ci/benchmarks/.benchmarks.yml index 531b9609..4180b482 100644 --- a/ci/benchmarks/.benchmarks.yml +++ b/ci/benchmarks/.benchmarks.yml @@ -7,7 +7,7 @@ .run_benchmarks: &run_benchmarks <<: *use_base_container - before_script: + script: - pip3 install -e . - pip install litellm langchain transformers torch openai tqdm - | @@ -16,6 +16,7 @@ export $var="${!var}" fi done + - python3 benchmarks/cybermetric/CyberMetric_evaluator.py --model_name $MODEL_NAME --file_path $BENCHMARK_FILE # - curl http://localhost:8000/api/tags # validate that ollama is running variables: OLLAMA_API_BASE: "http://localhost:8000" @@ -44,12 +45,13 @@ # - pip install -r benchmarks/seceval/eval/requirements.txt # - python3 benchmarks/seceval/eval/eval.py --dataset_file benchmarks/seceval/eval/datasets/questions-2.json --output_dir benchmarks/seceval/eval/outputs --backend ollama --models ollama/qwen2.5:14b -benchmarks-test-cybermetric: - <<: *run_benchmarks - script: - - python3 benchmarks/cybermetric/CyberMetric_evaluator.py --model_name ollama/qwen2.5:14b --file_path benchmarks/cybermetric/CyberMetric-2-v1.json +# benchmarks-test-cybermetric-openrouter: +# <<: *run_benchmarks +# script: +# - python3 benchmarks/cybermetric/CyberMetric_evaluator.py --model_name openrouter/qwen/qwen3-32b:free --file_path benchmarks/cybermetric/CyberMetric-2-v1.json benchmarks-test-cybermetric-openai: <<: *run_benchmarks - script: - - python3 benchmarks/cybermetric/CyberMetric_evaluator.py --model_name openai/gpt-4o-mini --file_path benchmarks/cybermetric/CyberMetric-2-v1.json + variables: + MODEL_NAME: "openai/gpt-4o-mini" + BENCHMARK_FILE: "benchmarks/cybermetric/CyberMetric-2-v1.json" \ No newline at end of file