diff --git a/benchmarks/cybermetric/output_cybermetric/ollama-qwen2.5:14b-2025-05-08-5zd6a/information.txt b/benchmarks/cybermetric/output_cybermetric/ollama-qwen2.5:14b-2025-05-08-5zd6a/information.txt deleted file mode 100644 index 1fa6eafc..00000000 --- a/benchmarks/cybermetric/output_cybermetric/ollama-qwen2.5:14b-2025-05-08-5zd6a/information.txt +++ /dev/null @@ -1,12 +0,0 @@ -CyberMetric Evaluation -===================== - -Model: ollama/qwen2.5:14b -Dataset: CyberMetric-2-v1.json -Start Time: 2025-05-08 12:31:39 -Status: Completed -Questions Processed: 2 -Correct Answers: 1 -Accuracy: 50.00% -End Time: 2025-05-08 12:31:42 -Duration: 0:00:02.434703 diff --git a/benchmarks/cybermetric/output_cybermetric/ollama-qwen2.5:14b-2025-05-08-5zd6a/report_failed_questions.json b/benchmarks/cybermetric/output_cybermetric/ollama-qwen2.5:14b-2025-05-08-5zd6a/report_failed_questions.json deleted file mode 100644 index 04c365cb..00000000 --- a/benchmarks/cybermetric/output_cybermetric/ollama-qwen2.5:14b-2025-05-08-5zd6a/report_failed_questions.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "questions": [ - { - "question": "Which of the following is a desirable property of a biometric system?", - "answers": { - "A": "Permanent", - "B": "Transferability", - "C": "Uniformity", - "D": "Forgiveness" - }, - "solution": "A", - "llm_answer": "C" - } - ] -} \ No newline at end of file diff --git a/benchmarks/cybermetric/output_cybermetric/ollama-qwen2.5:14b-2025-05-08-sebps/information.txt b/benchmarks/cybermetric/output_cybermetric/ollama-qwen2.5:14b-2025-05-08-sebps/information.txt deleted file mode 100644 index 4fb1aa66..00000000 --- a/benchmarks/cybermetric/output_cybermetric/ollama-qwen2.5:14b-2025-05-08-sebps/information.txt +++ /dev/null @@ -1,12 +0,0 @@ -CyberMetric Evaluation -===================== - -Model: ollama/qwen2.5:14b -Dataset: CyberMetric-2-v1.json -Start Time: 2025-05-08 12:23:35 -Status: Completed -Questions Processed: 2 -Correct Answers: 1 -Accuracy: 50.00% -End Time: 2025-05-08 12:23:38 -Duration: 0:00:02.306513 diff --git a/benchmarks/cybermetric/output_cybermetric/ollama-qwen2.5:14b-2025-05-08-sebps/report_failed_questions.json b/benchmarks/cybermetric/output_cybermetric/ollama-qwen2.5:14b-2025-05-08-sebps/report_failed_questions.json deleted file mode 100644 index 04c365cb..00000000 --- a/benchmarks/cybermetric/output_cybermetric/ollama-qwen2.5:14b-2025-05-08-sebps/report_failed_questions.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "questions": [ - { - "question": "Which of the following is a desirable property of a biometric system?", - "answers": { - "A": "Permanent", - "B": "Transferability", - "C": "Uniformity", - "D": "Forgiveness" - }, - "solution": "A", - "llm_answer": "C" - } - ] -} \ No newline at end of file diff --git a/benchmarks/cybermetric/output_cybermetric/ollama-qwen2.5:14b-2025-05-08/information.txt b/benchmarks/cybermetric/output_cybermetric/ollama-qwen2.5:14b-2025-05-08/information.txt deleted file mode 100644 index d32aedc2..00000000 --- a/benchmarks/cybermetric/output_cybermetric/ollama-qwen2.5:14b-2025-05-08/information.txt +++ /dev/null @@ -1,10 +0,0 @@ -CyberMetric Evaluation -===================== - -Model: ollama/qwen2.5:14b -Dataset: CyberMetric-2-v1.json -Start Time: 2025-05-08 12:22:51 -Status: Running -Questions Processed: 0 -Correct Answers: 0 -Accuracy: 0.00% diff --git a/benchmarks/cybermetric/output_cybermetric/openrouter-qwen-qwen3-32b:free-2025-05-08/information.txt b/benchmarks/cybermetric/output_cybermetric/openrouter-qwen-qwen3-32b:free-2025-05-08/information.txt deleted file mode 100644 index 83b9626f..00000000 --- a/benchmarks/cybermetric/output_cybermetric/openrouter-qwen-qwen3-32b:free-2025-05-08/information.txt +++ /dev/null @@ -1,10 +0,0 @@ -CyberMetric Evaluation -===================== - -Model: openrouter/qwen/qwen3-32b:free -Dataset: CyberMetric-2-v1.json -Start Time: 2025-05-08 12:23:53 -Status: Running -Questions Processed: 0 -Correct Answers: 0 -Accuracy: 0.00% diff --git a/benchmarks/otros/cybermetric b/benchmarks/otros/cybermetric new file mode 160000 index 00000000..2d82e92d --- /dev/null +++ b/benchmarks/otros/cybermetric @@ -0,0 +1 @@ +Subproject commit 2d82e92dd6171a3fd5b40f8306af3b8c366179e6 diff --git a/ci/benchmark/.benchmark.yml b/ci/benchmark/.benchmark.yml index 65ae2260..e230df4d 100644 --- a/ci/benchmark/.benchmark.yml +++ b/ci/benchmark/.benchmark.yml @@ -1,11 +1,11 @@ use_base_container: &use_base_container - stage: test + stage: benchmark image: "${CI_REGISTRY_IMAGE}:latest" services: - name: docker:dind alias: docker -run_benchmark: &run_benchmark +.run_benchmark: &run_benchmark <<: *use_base_container script: - pip3 install -e . @@ -14,7 +14,7 @@ run_benchmark: &run_benchmark paths: - benchmarks/seceval/eval/outputs/ - benchmarks/cybermetric/outputs/ - expire_in: 1 month + expire_in: 12 month tags: - p40 - x86 @@ -22,7 +22,7 @@ run_benchmark: &run_benchmark - if: $CI_COMMIT_BRANCH when: on_success -🛠️ benchmark test_eval: +🛠️ benchmark test_seceval: <<: *run_benchmark variables: OLLAMA_API_BASE: "http://localhost:8000"