try ci

.gitlab-ci.yml

@@ -23,7 +23,8 @@ include:
     file:
       # - 'ci/build/.build.yml'                # build
       #- 'ci/setup/.setup.yml'               # setup
-      - 'ci/test/.test.yml'
+      #- 'ci/test/.test.yml'
+      - 'ci/benchmark/.benchmark.yml'
       # - 'ci/ctfs/.ctf.yml'                  # ctf
 
   # - project: 'aliasrobotics/alias_research/cai'

ci/benchmark/.benchmark.yml

@@ -0,0 +1,33 @@
+use_base_container: &use_base_container
+  stage: test
+  image: "${CI_REGISTRY_IMAGE}:latest"
+  services:
+    - name: docker:dind
+      alias: docker
+
+run_benchmark: &run_benchmark
+  <<: *use_base_container
+  script:
+    - pip3 install -e .
+    - pip install litellm langchain transformers torch openai tqdm
+  artifacts:
+    paths:
+      - benchmarks/seceval/eval/outputs/
+      - benchmarks/cybermetric/outputs/
+    expire_in: 1 month
+  tags:
+    - p40
+    - x86
+  rules:
+    - if: $CI_COMMIT_BRANCH
+      when: on_success
+
+🛠️ benchmark test_eval:
+  <<: *run_benchmark
+  variables:
+    OLLAMA_API_BASE: "http://localhost:8000"
+    OPENROUTER_API_BASE: "https://openrouter.ai/api/v1"
+  script:
+    - pip3 install -e .
+    - pip install -r cai/benchmarks/seceval/eval/requirements.txt
+    - python3 benchmarks/seceval/eval/eval.py --dataset_file benchmarks/seceval/eval/datasets/questions.json --output_dir benchmarks/seceval/eval/outputs --backend ollama --models ollama/qwen2.5:14b