RomanNum3ral
/
cai
mirror of https://github.com/aliasrobotics/cai.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

testing yml

This commit is contained in:
lidia9 2025-05-08 12:57:09 +02:00
parent c79a51d71f
commit 61b07f9838
8 changed files with 5 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
benchmarks/cybermetric/output_cybermetric/ollama-qwen2.5:14b-2025-05-08-5zd6a/information.txt
View File

@ -1,12 +0,0 @@
CyberMetric Evaluation
=====================
Model: ollama/qwen2.5:14b
Dataset: CyberMetric-2-v1.json
Start Time: 2025-05-08 12:31:39
Status: Completed
Questions Processed: 2
Correct Answers: 1
Accuracy: 50.00%
End Time: 2025-05-08 12:31:42
Duration: 0:00:02.434703

15
benchmarks/cybermetric/output_cybermetric/ollama-qwen2.5:14b-2025-05-08-5zd6a/report_failed_questions.json
View File

@ -1,15 +0,0 @@
{
"questions": [
{
"question": "Which of the following is a desirable property of a biometric system?",
"answers": {
"A": "Permanent",
"B": "Transferability",
"C": "Uniformity",
"D": "Forgiveness"
},
"solution": "A",
"llm_answer": "C"
}
]
}

12
benchmarks/cybermetric/output_cybermetric/ollama-qwen2.5:14b-2025-05-08-sebps/information.txt
View File

@ -1,12 +0,0 @@
CyberMetric Evaluation
=====================
Model: ollama/qwen2.5:14b
Dataset: CyberMetric-2-v1.json
Start Time: 2025-05-08 12:23:35
Status: Completed
Questions Processed: 2
Correct Answers: 1
Accuracy: 50.00%
End Time: 2025-05-08 12:23:38
Duration: 0:00:02.306513

15
benchmarks/cybermetric/output_cybermetric/ollama-qwen2.5:14b-2025-05-08-sebps/report_failed_questions.json
View File

@ -1,15 +0,0 @@
{
"questions": [
{
"question": "Which of the following is a desirable property of a biometric system?",
"answers": {
"A": "Permanent",
"B": "Transferability",
"C": "Uniformity",
"D": "Forgiveness"
},
"solution": "A",
"llm_answer": "C"
}
]
}

10
benchmarks/cybermetric/output_cybermetric/ollama-qwen2.5:14b-2025-05-08/information.txt
View File

@ -1,10 +0,0 @@
CyberMetric Evaluation
=====================
Model: ollama/qwen2.5:14b
Dataset: CyberMetric-2-v1.json
Start Time: 2025-05-08 12:22:51
Status: Running
Questions Processed: 0
Correct Answers: 0
Accuracy: 0.00%

10
benchmarks/cybermetric/output_cybermetric/openrouter-qwen-qwen3-32b:free-2025-05-08/information.txt
View File

@ -1,10 +0,0 @@
CyberMetric Evaluation
=====================
Model: openrouter/qwen/qwen3-32b:free
Dataset: CyberMetric-2-v1.json
Start Time: 2025-05-08 12:23:53
Status: Running
Questions Processed: 0
Correct Answers: 0
Accuracy: 0.00%

1
benchmarks/otros/cybermetric Submodule

@ -0,0 +1 @@
Subproject commit 2d82e92dd6171a3fd5b40f8306af3b8c366179e6

8
ci/benchmark/.benchmark.yml
View File

@ -1,11 +1,11 @@
use_base_container: &use_base_container
stage: test
stage: benchmark
image: "${CI_REGISTRY_IMAGE}:latest"
services:
- name: docker:dind
alias: docker
run_benchmark: &run_benchmark
.run_benchmark: &run_benchmark
<<: *use_base_container
script:
- pip3 install -e .
@ -14,7 +14,7 @@ run_benchmark: &run_benchmark
paths:
- benchmarks/seceval/eval/outputs/
- benchmarks/cybermetric/outputs/
expire_in: 1 month
expire_in: 12 month
tags:
- p40
- x86
@ -22,7 +22,7 @@ run_benchmark: &run_benchmark
- if: $CI_COMMIT_BRANCH
when: on_success
🛠️ benchmark test_eval:
🛠️ benchmark test_seceval:
<<: *run_benchmark
variables:
OLLAMA_API_BASE: "http://localhost:8000"