Retry openai 6

Signed-off-by: Víctor Mayoral Vilches <v.mayoralv@gmail.com>

ci/benchmarks/.benchmarks.yml

@@ -60,4 +60,3 @@ benchmarks-test-cybermetric-openai:
   variables:
     MODEL_NAME: "openai/gpt-4o-mini"
     BENCHMARK_FILE: "benchmarks/cybermetric/CyberMetric-2-v1.json"
-    # Make sure to define OPENAI_API_KEY in GitLab CI/CD variables

ci/benchmarks/.benchmarks.yml.bak

@@ -0,0 +1,63 @@
+.use_base_container: &use_base_container
+  stage: benchmarks
+  image: "${CI_REGISTRY_IMAGE}:latest"
+  services:
+    - name: docker:dind
+      alias: docker
+
+.run_benchmarks: &run_benchmarks
+  <<: *use_base_container
+  script:
+    - pip3 install -e .
+    - pip install litellm langchain transformers torch openai tqdm
+    - echo "Checking environment variables..."
+    - echo "OPENAI_API_KEY is set: $([[ -n $OPENAI_API_KEY ]] && echo 'YES' || echo 'NO')"
+    - |
+      for var in $(compgen -e); do
+        if [[ ($var == CTF_* || $var == *_API_KEY || $var == *_API_BASE) && -n ${!var} ]]; then
+          export $var="${!var}"
+          if [[ $var == OPENAI_API_KEY ]]; then
+            echo "OPENAI_API_KEY is exported"
+          fi
+        fi
+      done
+    - python3 benchmarks/cybermetric/CyberMetric_evaluator.py --model_name $MODEL_NAME --file_path $BENCHMARK_FILE --api_key "${OPENAI_API_KEY}"
+    # - curl http://localhost:8000/api/tags  # validate that ollama is running
+  variables:
+    OLLAMA_API_BASE: "http://localhost:8000"
+    OPENROUTER_API_BASE: "https://openrouter.ai/api/v1"
+    OPENAI_API_BASE: "https://api.openai.com/v1"
+  artifacts:
+    paths:
+      - cai/benchmarks/seceval/eval/outputs/
+      - cai/benchmarks/cybermetric/outputs/
+    expire_in: 12 month
+  tags:
+    - p40
+    - x86
+  rules:
+    - if: $CI_COMMIT_BRANCH
+      when: on_success
+
+# benchmarks-test-seceval:
+#   <<: *run_benchmarks
+#   variables:
+#     OLLAMA_API_BASE: "http://localhost:8000"
+#     OPENROUTER_API_BASE: "https://openrouter.ai/api/v1"
+#     OPENAI_API_KEY: "fake-api-key"
+#   script:
+#     - pip3 install -e .
+#     - pip install -r benchmarks/seceval/eval/requirements.txt
+#     - python3 benchmarks/seceval/eval/eval.py --dataset_file benchmarks/seceval/eval/datasets/questions-2.json --output_dir benchmarks/seceval/eval/outputs --backend ollama --models ollama/qwen2.5:14b
+
+# benchmarks-test-cybermetric-openrouter:
+#   <<: *run_benchmarks
+#   script:
+#     - python3 benchmarks/cybermetric/CyberMetric_evaluator.py --model_name openrouter/qwen/qwen3-32b:free  --file_path benchmarks/cybermetric/CyberMetric-2-v1.json
+
+benchmarks-test-cybermetric-openai:
+  <<: *run_benchmarks
+  variables:
+    MODEL_NAME: "openai/gpt-4o-mini"
+    BENCHMARK_FILE: "benchmarks/cybermetric/CyberMetric-2-v1.json"
+  # Make sure to define OPENAI_API_KEY in GitLab CI/CD variables