From 4e463096dd22fd9e8c64e3d7df689ddd842987ef Mon Sep 17 00:00:00 2001
From: Mery-Sanz <maria@aliasrobotics.com>
Date: Thu, 8 May 2025 12:02:31 +0200
Subject: [PATCH] try ci

---
 .gitlab-ci.yml              |  3 ++-
 ci/benchmark/.benchmark.yml | 33 +++++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+), 1 deletion(-)
 create mode 100644 ci/benchmark/.benchmark.yml

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7273e68f..d90340c8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -23,7 +23,8 @@ include:
     file:
       # - 'ci/build/.build.yml'                # build
       #- 'ci/setup/.setup.yml'               # setup
-      - 'ci/test/.test.yml'
+      #- 'ci/test/.test.yml'
+      - 'ci/benchmark/.benchmark.yml'
       # - 'ci/ctfs/.ctf.yml'                  # ctf
 
   # - project: 'aliasrobotics/alias_research/cai'
diff --git a/ci/benchmark/.benchmark.yml b/ci/benchmark/.benchmark.yml
new file mode 100644
index 00000000..65ae2260
--- /dev/null
+++ b/ci/benchmark/.benchmark.yml
@@ -0,0 +1,33 @@
+use_base_container: &use_base_container
+  stage: test
+  image: "${CI_REGISTRY_IMAGE}:latest"
+  services:
+    - name: docker:dind
+      alias: docker
+
+run_benchmark: &run_benchmark
+  <<: *use_base_container
+  script:
+    - pip3 install -e .
+    - pip install litellm langchain transformers torch openai tqdm
+  artifacts:
+    paths:
+      - benchmarks/seceval/eval/outputs/
+      - benchmarks/cybermetric/outputs/
+    expire_in: 1 month
+  tags:
+    - p40
+    - x86
+  rules:
+    - if: $CI_COMMIT_BRANCH
+      when: on_success
+
+🛠️ benchmark test_eval:
+  <<: *run_benchmark
+  variables:
+    OLLAMA_API_BASE: "http://localhost:8000"
+    OPENROUTER_API_BASE: "https://openrouter.ai/api/v1"
+  script:
+    - pip3 install -e .
+    - pip install -r cai/benchmarks/seceval/eval/requirements.txt
+    - python3 benchmarks/seceval/eval/eval.py --dataset_file benchmarks/seceval/eval/datasets/questions.json --output_dir benchmarks/seceval/eval/outputs --backend ollama --models ollama/qwen2.5:14b