Merge branch 'benchmarks' into '0.4.0'

Benchmark integration seceval, cybermetric and cti bench @Mery-Sanz

See merge request aliasrobotics/alias_research/cai!163

.gitignore

@@ -149,4 +149,7 @@ cython_debug/
 .vscode/
 
 # workspaces
-workspaces/
+workspaces/
+
+# benchmarks
+benchmarks/outputs

.gitlab-ci.yml

@@ -2,6 +2,7 @@ stages:
   - build
   - setup
   - test                # unit tests validation
+  - benchmarks
   - ctf
 
 variables:
@@ -24,6 +25,7 @@ include:
       # - 'ci/build/.build.yml'                # build
       #- 'ci/setup/.setup.yml'               # setup
       - 'ci/test/.test.yml'
+      - 'ci/benchmarks/.benchmarks.yml'
       # - 'ci/ctfs/.ctf.yml'                  # ctf
 
   # - project: 'aliasrobotics/alias_research/cai'

.gitmodules

@@ -0,0 +1,9 @@
+[submodule "benchmarks/cti_bench"]
+	path = benchmarks/cti_bench
+	url = https://github.com/xashru/cti-bench.git
+[submodule "benchmarks/seceval"]
+	path = benchmarks/seceval
+	url = https://github.com/XuanwuAI/SecEval.git
+[submodule "benchmarks/cybermetric"]
+	path = benchmarks/cybermetric
+	url = https://github.com/cybermetric/CyberMetric.git

benchmarks/README.md

@@ -0,0 +1,71 @@
+# AI Model Evaluation Benchmarks
+
+This chapter is a curated collection of benchmark datasets and evaluation tools designed to assess the capabilities of custom AI models, particularly in domains related to cybersecurity.
+
+The collection is intended to support researchers and developers who are evaluating their own models using reliable, task-specific benchmarks.
+
+Currently, this are the benchmarks included:
+
+| Benchmark | Description |
+|-----------|-------------|
+| [SecEval](https://github.com/XuanwuAI/SecEval) | Benchmark designed to evaluate large language models (LLMs) on security-related tasks. It includes various real-world scenarios such as phishing email analysis, vulnerability classification, and response generation. |
+| [CyberMetric](https://github.com/CyberMetric) | Benchmark framework that focuses on measuring the performance of AI systems in cybersecurity-specific question answering, knowledge extraction, and contextual understanding. It emphasizes both domain knowledge and reasoning ability. |
+| [CTIBench](https://github.com/xashru/cti-bench) | Benchmark focused on evaluating LLM models' capabilities in understanding and processing Cyber Threat Intelligence (CTI) information. |
+
+
+
+The goal is to consolidate diverse evaluation tasks under a single framework to support rigorous, standardized testing.
+
+## 📊 General Summary Table
+
+| Model       | SecEval   | CyberMetric  | Total Value | 
+|-------------|-----------|--------------|-------------|
+| model_name  | `XX.X%`   | `XX.X%`      | `XX.X%`     | 
+
+
+
+#### ▶️ Usage
+
+```bash
+git submodule update --init --recursive  # init submodules
+pip install cvss
+```
+
+```bash
+python benchmarks/eval.py --model MODEL_NAME --dataset_file INPUT_FILE --eval EVAL_TYPE --backend BACKEND
+```
+```bash
+Arguments:
+    -m, --model         # Specify the model to evaluate (e.g., "gpt-4", "ollama/qwen2.5:14b")
+    -d, --dataset_file  # IMPORTANT! By default: small test data of 2 samples 
+    -B, --backend       # Backend to use: "openai", "openrouter", "ollama" (required)
+    -e, --eval          # Specify the evaluation benchmark
+
+Output:
+   outputs/
+   └── benchmark_name/
+       └── model_date_random-num/
+           ├── answers.json       # the whole test with LLM answers
+           └── information.txt    # report of that precise run (e.g. model_name, benchmark_name, metrics, date)
+
+```
+#### 🔍 Examples
+
+**How to run different CTI Bench tests with the "llama/qwen2.5:14b" model using Ollama as the backend**
+
+```bash
+python benchmarks/eval.py --model ollama/qwen2.5:14b --dataset_file benchmarks/cybermetric/CyberMetric-2-v1.json --eval cybermetric --backend ollama
+````
+
+```bash
+python benchmarks/eval.py --model ollama/qwen2.5:14b --dataset_file benchmarks/seceval/eval/datasets/questions-2.json --eval seceval --backend ollama
+```
+
+**How to run different CTI Bench tests with the "qwen/qwen3-32b:free" model using Openrouter as the backend**
+
+```bash
+python benchmarks/eval.py --model qwen/qwen3-32b:free  --dataset_file benchmarks/cti_bench/data/cti-mcq1.tsv --eval cti_bench --backend openrouter
+````
+```bash
+python benchmarks/eval.py --model qwen/qwen3-32b:free  --dataset_file benchmarks/cti_bench/data/cti-ate2.tsv --eval cti_bench --backend openrouter
+````

benchmarks/cti_bench

@@ -0,0 +1 @@
+Subproject commit c527ae73e3c8272cf6efe76952de5468be4672a3

benchmarks/cybermetric

@@ -0,0 +1 @@
+Subproject commit 2f5818bd2c19350cd6cfae028b75499ebe4ffd29

benchmarks/eval.py

@@ -0,0 +1,533 @@
+"""
+Benchmark Evaluation Script
+
+This script provides utilities to evaluate language models on cybersecurity-related multiple-choice and other question-answering benchmarks.
+
+Usage:
+    python benchmarks/eval.py --model MODEL_NAME --dataset_file INPUT_FILE --eval EVAL_TYPE --backend BACKEND
+
+Arguments:
+    -m, --model           Specify the model to evaluate (e.g., "gpt-4", "qwen2.5:14b", etc.)
+    -d, --dataset_file    Path to the dataset file (JSON or TSV) containing questions to evaluate
+    -B, --backend         Backend to use: "openai", "openrouter", "ollama" (required)
+    -e, --eval            Specify the evaluation benchmark
+
+Example:
+
+     python benchmarks/eval.py --model ollama/qwen2.5:14b --dataset_file benchmarks/cybermetric/CyberMetric-80-v1.json --eval cybermetric --backend ollama
+     python benchmarks/eval.py --model ollama/qwen2.5:14b --dataset_file benchmarks/seceval/eval/datasets/questions-2.json --eval seceval --backend ollama
+     python benchmarks/eval.py --model ollama/qwen2.5:14b --dataset_file benchmarks/cti_bench/data/cti-mcq.tsv --eval cti_bench --backend ollama
+     
+     python benchmarks/eval.py --model qwen/qwen3-32b:free --dataset_file benchmarks/utils/cybermetric_dataset/CyberMetric-2-v1.json --eval cybermetric --backend openrouter
+
+Environment Variables:
+    OPENROUTER_API_KEY:  API key for OpenRouter (if using OpenRouter models)
+    OPENROUTER_API_BASE: Base URL for OpenRouter API (default: https://openrouter.ai/api/v1)
+    OLLAMA_API_BASE:     Base URL for Ollama API via LiteLLM proxy (default: http://localhost:8000/v1)
+    OPENAI_API_KEY:      API key for OpenAI (if using OpenAI models)
+    OPENAI_API_BASE:     Base URL for OpenAI API (default: https://api.openai.com/v1)
+"""
+
+import json
+import re
+import time
+import os
+import datetime
+import random
+import string
+import argparse
+from tqdm import tqdm
+import litellm
+import requests
+import csv
+import os
+import datetime
+
+
+OPENROUTER_API_BASE = os.environ.get("OPENROUTER_API_BASE", "https://openrouter.ai/api/v1")
+OPENROUTER_API_KEY = os.environ.get("OPENROUTER_API_KEY")
+OLLAMA_API_BASE = os.environ.get("OLLAMA_API_BASE", "http://localhost:8000/v1")
+OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
+OPENAI_API_BASE = os.environ.get("OPENAI_API_BASE", "https://api.openai.com/v1")
+
+
+def ask_model(question_obj, instruction, model, api_base, api_key=None, custom_llm_provider=None):
+    """
+    Calls the model with the question and choices, returns the extracted answer.
+    """
+    # Compose the prompt
+    prompt =  question_obj["Question"] + "\n" + instruction
+    if question_obj.get("Choices"):
+        prompt += "\nChoices:\n" + question_obj["Choices"]
+    try:
+        response = litellm.completion(
+            model=model,
+            messages=[
+                {"role": "system", "content": "You are a security expert who answers questions."},
+                {"role": "user", "content": prompt},
+            ],
+            api_base=api_base,
+            api_key=api_key,
+            custom_llm_provider=custom_llm_provider,
+        )
+        if hasattr(response, "choices") and response.choices:
+            content = response.choices[0].message.content
+  
+            return content
+    except Exception as e:
+        print(f"Error: {e}.")
+        return None
+
+def load_dataset(dataset_file, eval_type):
+    questions = [] #list of questions: {question: str, answers: dict, solution: str}
+    if eval_type == "seceval":
+        with open(dataset_file, 'r') as f:
+            data = json.load(f)
+            for question in data:
+                questions.append({
+                    "Question": question["question"],
+                    "Choices": "\n".join(question["choices"]),
+                    "Solution": question["answer"]
+                })
+
+    elif eval_type == "cybermetric":
+        with open(dataset_file, 'r') as f:
+            data = json.load(f)
+            for question in data.get("questions", []):
+                questions.append({
+                    "Question": question.get("question", ""),
+                    "Choices": "\n".join([f"{k}: {v}" for k, v in question.get("answers", {}).items()]),
+                    "Solution": question.get("solution", "")
+                })
+    elif eval_type == "cti_bench":
+        with open(dataset_file, 'r') as f:
+            reader = csv.reader(f, delimiter='\t')
+            header = next(reader, None)
+            for row in reader:
+                # Handle three possible formats:
+                # Format 1: [URL, Question, Option A, Option B, Option C, Option D, Prompt, GT] (8 columns)
+                # Format 2: [URL, Platform, Description, Prompt, GT] (5 columns)
+                # Format 3: [URL, Description, Prompt, GT] (4 columns)
+
+                if len(row) == 8:
+                    # MCQ format
+                    questions.append({
+                        "Question": row[1],
+                        "Choices": f"A: {row[2]}\nB: {row[3]}\nC: {row[4]}\nD: {row[5]}",
+                        "Solution": row[7]
+                    })
+                elif len(row) == 5:
+                    # ATE format (no choices, just open-ended)
+                    questions.append({
+                        "Question": row[2] + row[3],  # Description + Prompt
+                        "Choices": "",       # No choices for ATE
+                        "Solution": row[4]   # GT
+                    })
+                elif len(row) == 4:
+                    # RCM format: [URL, Description, Prompt, GT]
+                    questions.append({
+                        "Question": row[1] + row[2],  # Description + Prompt
+                        "Choices": "",       # No choices for RCM
+                        "Solution": row[3]   # GT
+                    })
+
+    return questions
+
+def run_evaluation(dataset, instruction, model, api_base=None, api_key=None, custom_llm_provider=None):
+    results = []
+    for idx, q in enumerate(dataset):
+        model_answer = ask_model(q, instruction, model, api_base, api_key, custom_llm_provider)
+        print(f"---------------{idx+1}/{len(dataset)}----------------")
+        print(f"Evaluating question: {q['Question']}")
+        print(f"Choices: {q['Choices']}")
+        print(f"Solution: {q['Solution']}") 
+        print(f"Model Answer: {model_answer}")
+        print("--------------------------------")
+        results.append({
+            "Question": q["Question"],
+            "Choices": q["Choices"],
+            "ModelAnswer": model_answer,
+            "Solution": q["Solution"]
+        })
+        
+    return results
+
+def parse_result_seceval(result):
+    # Expecting format: 'ANSWER: X', 'ANSWER: XY', or 'ANSWER: XYZ' (1, 2, or 3 letters A-D)
+    if result is None:
+        return None
+    match = re.search(r"ANSWER:?\s*([A-D]{1,3})", result, re.IGNORECASE)
+    if match:
+        return match.group(1).upper()
+    return None
+
+def parse_result_cybermetric(result):
+    # Expecting format: 'ANSWER: X'
+    if result is None:
+        return None
+    match = re.search(r"ANSWER:?\s*([A-D])", result, re.IGNORECASE)
+    if match:
+        return match.group(1).upper()
+    return None
+
+def parse_result_cti_bench(result, dataset_file):
+    # Accepts answers like 'ANSWER: X', 'ANSWER: XY', or 'ANSWER: XYZ' (A-D), and also 'ANSWER: Txxxx' (MITRE ATT&CK IDs)
+    if result is None: 
+        return None
+    # Try to match CWE IDs (e.g., CWE-416, CWE-79, etc.)
+    if "cti-rcm" in dataset_file:
+        cwe_match = re.search(r"(CWE-\d+)", result, re.IGNORECASE)
+        if cwe_match:
+            return cwe_match.group(1).upper()
+    # Try to match A-D answers (1-3 letters)
+    if "cti-mcq" in dataset_file:
+        match = re.search(r"ANSWER:?\s*([A-D])", result, re.IGNORECASE)
+        if match:
+            return match.group(1).upper()
+    # Find all MITRE ATT&CK technique IDs (e.g., T1071, T1059, etc.) anywhere in the result
+    if "cti-ate" in dataset_file:
+        ids = re.findall(r"T\d{4}", result, re.IGNORECASE)
+        if ids:
+            # Return all found IDs as uppercase, comma-separated
+            return ", ".join([x.upper() for x in ids])
+    if "cti-vsp" in dataset_file:
+        # Try to match CVSS vector strings (e.g., CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H)
+        cvss_match = re.search(r"(CVSS:3\.1/AV:[NALP]/AC:[LH]/PR:[NLH]/UI:[NR]/S:[UC]/C:[NLH]/I:[NLH]/A:[NLH])", result, re.IGNORECASE)
+        if cvss_match:
+            return cvss_match.group(1).upper()
+        # Also accept vector without prefix
+        vector_match = re.search(r"(AV:[NALP]/AC:[LH]/PR:[NLH]/UI:[NR]/S:[UC]/C:[NLH]/I:[NLH]/A:[NLH])", result, re.IGNORECASE)
+        if vector_match:
+            return "CVSS:3.1/" + vector_match.group(1).upper()
+        return result.upper()
+    return None
+
+def extract_cvss_score(vector):
+    """
+    Extracts a score from a CVSS vector string using the CVSS library.
+    """
+    try:
+        from cvss import CVSS3
+        c = CVSS3(vector)
+        return c.scores()[0]
+    except Exception as e:
+        print(f"Error calculating CVSS score: {e}")
+        return None
+
+def compute_vsp_mad(results):
+    """
+    Compute Mean Absolute Deviation for CVSS scores, following the original implementation.
+    """
+    try:
+        from cvss import CVSS3
+    except ImportError:
+        print("CVSS library not found. Please install it with 'pip install cvss'")
+        return None
+        
+    cvss_prefix = 'CVSS:3.1/'  # Use 3.1 to match current data
+    error_sum = 0
+    total = 0
+    
+    for item in results:
+        gt = item.get("Solution")
+        pred = item.get("ModelAnswer")
+        
+        try:
+            # Parse prediction
+            pred_vector = parse_result_cti_bench(pred, "cti-vsp")
+            
+            # Ensure vectors have prefix
+            if pred_vector and not pred_vector.startswith("CVSS:"):
+                pred_vector = cvss_prefix + pred_vector
+                
+            # Calculate scores
+            if gt and pred_vector:
+                c_gt = CVSS3(gt)
+                c_pred = CVSS3(pred_vector)
+                
+                gt_score = c_gt.scores()[0]
+                pred_score = c_pred.scores()[0]
+                
+                error = abs(pred_score - gt_score)
+                error_sum += error
+                total += 1
+        except Exception as e:
+            print(f"Error processing CVSS vector: {e}")
+            continue
+            
+    return error_sum / total if total > 0 else None
+
+def compute_ate_metrics(results):
+    """
+    Compute F1-macro score and accuracy for CTI-ATE task.
+    
+    For F1-macro, we calculate F1 separately for each sample and then average them.
+    
+    Args:
+        results (list of dict): Each dict should have the ground truth answer and model answer.
+        
+    Returns:
+        tuple: (f1_macro, accuracy, precision_macro, recall_macro)
+    """
+    # For storing per-sample metrics
+    f1_scores = []
+    precision_scores = []
+    recall_scores = []
+    
+    correct_predictions = 0
+    total_predictions = 0
+    
+    for item in results:
+        gt = item.get("Solution", "")
+        pred = item.get("ModelAnswer", "")
+        
+        # Extract technique IDs
+        gt_ids = [tid.strip().upper() for tid in gt.split(",") if tid.strip()]
+        pred_vector = parse_result_cti_bench(pred, "cti-ate")
+        pred_ids = [tid.strip().upper() for tid in (pred_vector or "").split(",") if tid.strip()]
+        
+        # Calculate true positives, false positives, and false negatives for this sample
+        sample_tp = len(set(gt_ids) & set(pred_ids))
+        sample_fp = len(set(pred_ids) - set(gt_ids))
+        sample_fn = len(set(gt_ids) - set(pred_ids))
+        
+        # Calculate precision and recall for this sample
+        if sample_tp + sample_fp > 0:
+            sample_precision = sample_tp / (sample_tp + sample_fp)
+        else:
+            sample_precision = 0
+            
+        if sample_tp + sample_fn > 0:
+            sample_recall = sample_tp / (sample_tp + sample_fn)
+        else:
+            sample_recall = 0
+            
+        # Calculate F1 for this sample
+        if sample_precision + sample_recall > 0:
+            sample_f1 = 2 * (sample_precision * sample_recall) / (sample_precision + sample_recall)
+        else:
+            sample_f1 = 0
+            
+        # Add to list of scores
+        precision_scores.append(sample_precision)
+        recall_scores.append(sample_recall)
+        f1_scores.append(sample_f1)
+        
+        # Calculate exact match for accuracy
+        if set(gt_ids) == set(pred_ids):
+            correct_predictions += 1
+        total_predictions += 1
+    
+    # Calculate macro metrics (average of per-sample metrics)
+    precision_macro = sum(precision_scores) / len(precision_scores) if precision_scores else 0
+    recall_macro = sum(recall_scores) / len(recall_scores) if recall_scores else 0
+    f1_macro = sum(f1_scores) / len(f1_scores) if f1_scores else 0
+    accuracy = correct_predictions / total_predictions if total_predictions > 0 else 0
+    
+    return f1_macro, accuracy, precision_macro, recall_macro
+
+def compute_accuracy(results, benchmark_name, dataset_file=None):
+    """
+    Compute accuracy for a benchmark result set.
+
+    Args:
+        results (list of dict): Each dict should have the ground truth answer and model answer.
+        benchmark_name (str): The name of the benchmark.
+    Returns:
+        accuracy (float): Accuracy as a percentage (0-100).
+        correct_count (int): Number of correct answers.
+        total_count (int): Total number of evaluated items.
+    """
+    correct_count = 0
+    total_count = 0
+    
+    # For VSP, use the mean absolute deviation instead of accuracy
+    if benchmark_name.lower() == "cti_bench" and dataset_file and 'cti-vsp' in dataset_file:
+        mad = compute_vsp_mad(results)
+        # Return MAD as the "accuracy" value, with 0 correct count and total items processed
+        return mad, 0, len(results)
+    
+    # For ATE, calculate F1-macro score and return it with accuracy
+    if benchmark_name.lower() == "cti_bench" and dataset_file and 'cti-ate' in dataset_file:
+        f1_macro, accuracy, precision_macro, recall_macro = compute_ate_metrics(results)
+        # We'll return f1_macro as the primary metric, and pass accuracy as correct_count (as a percentage)
+        # and total items processed as total_count
+        return f1_macro, accuracy * 100, len(results)
+        
+    for item in results:
+        sol = item.get("Solution")
+        pred = item.get("ModelAnswer")
+        # For cybermetric, parse both gt and pred using parse_result_cybermetric
+        if benchmark_name.lower() == "cybermetric":
+            from benchmarks.eval import parse_result_cybermetric
+            pred_parsed = parse_result_cybermetric(pred)
+            if sol is not None and pred_parsed is not None:
+                if sol == pred_parsed:
+                    correct_count += 1
+                total_count += 1
+        elif benchmark_name.lower() == "seceval":  
+            pred_parsed = parse_result_seceval(pred)
+            if sol is not None and pred_parsed is not None:
+                if sol == pred_parsed:
+                    correct_count += 1
+                total_count += 1
+        elif benchmark_name.lower() == "cti_bench" and 'vsp' not in dataset_file and 'ate' not in dataset_file:
+            pred_parsed = parse_result_cti_bench(pred, dataset_file)
+            if sol is not None and pred_parsed is not None:
+                if sol == pred_parsed:
+                    correct_count += 1
+                total_count += 1
+        else:
+            if sol is not None and pred is not None:
+                # Accept either exact match or case-insensitive match
+                if str(sol).strip().upper() == str(pred).strip().upper():
+                    correct_count += 1
+                total_count += 1
+    accuracy = (correct_count / total_count * 100) if total_count > 0 else 0.0
+    return accuracy, correct_count, total_count
+
+def save_benchmark_results(
+    benchmark_name,
+    model,
+    dataset_file,
+    start_time,
+    end_time,
+    questions_processed,
+    correct_count,
+    accuracy,
+    total_count,
+    result
+):
+    """
+    Save benchmark results in CyberMetric-style format to output_dir/information.txt.
+    """
+    output_dir = os.path.join(os.getcwd(), "benchmarks", "outputs", benchmark_name)
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+
+    # Save information file as: <model>_<YYYYMMDD_HHMMSS>.txt
+    now_str = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+    safe_model = "".join([c if c.isalnum() or c in ('-', '_') else '_' for c in str(model)])
+    info_file = os.path.join(output_dir, f"{safe_model}_{now_str}.txt")
+    duration = end_time - start_time
+
+    # Create a subdirectory for this run, named after info_file (without extension)
+    run_dir = os.path.splitext(info_file)[0]
+    if not os.path.exists(run_dir):
+        os.makedirs(run_dir)
+
+    # Save the info file as .tct inside the run_dir
+    info_file="information.txt"
+    info_file_tct = os.path.join(run_dir, os.path.basename(os.path.splitext(info_file)[0] + ".txt"))
+    with open(info_file_tct, "w") as f:
+        f.write(f"{benchmark_name} Evaluation\n")
+        f.write("=====================\n\n")
+        f.write(f"Model: {model}\n")
+        f.write(f"Dataset: {os.path.basename(dataset_file)}\n")
+        f.write(f"Start Time: {start_time.strftime('%Y-%m-%d %H:%M:%S')}\n")
+        f.write(f"Questions Processed: {questions_processed}\n")
+        
+        # Check if it's a VSP evaluation
+        if benchmark_name.lower() == "cti_bench" and 'cti-vsp' in dataset_file:
+            f.write(f"Mean Absolute Deviation: {accuracy:.2f}\n")
+        # Check if it's an ATE evaluation
+        elif benchmark_name.lower() == "cti_bench" and 'cti-ate' in dataset_file:
+            f.write(f"F1-macro Score: {accuracy:.2f}\n")
+            f.write(f"Accuracy: {correct_count:.2f}%\n")
+        else:
+            f.write(f"Correct Answers: {correct_count}\n")
+            f.write(f"Accuracy: {accuracy:.2f}%\n")
+            
+        f.write(f"End Time: {end_time.strftime('%Y-%m-%d %H:%M:%S')}\n")
+        f.write(f"Duration: {duration}\n")
+
+    # Save the results as answers.json inside the run_dir
+    results_file = os.path.join(run_dir, "answers.json")
+    with open(results_file, "w", encoding="utf-8") as rf:
+        json.dump(result, rf, ensure_ascii=False, indent=2)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="SecEval Evaluation CLI")
+    parser.add_argument("-d", "--dataset_file", type=str, required=True, help="Specify the dataset file to evaluate on.")
+    parser.add_argument("-B", "--backend", type=str, choices=["openai", "ollama", "openrouter"], required=True, help="Specify the llm type. openai: openai model, ollama: ollama model, openrouter: openrouter model")
+    parser.add_argument("-m", "--model", type=str, required=True, help="Specify the models.")
+    parser.add_argument("-e", "--eval", type=str, required=True, help="Specify the evaluation benchmark.")
+    args = parser.parse_args()
+    
+    model = args.model
+
+    print(f"Evaluating model: {model}")
+ 
+    if args.backend == "openai":
+        api_base=OPENAI_API_BASE
+        api_key=OPENAI_API_KEY
+        custom_llm_provider=None
+    elif args.backend == "ollama":
+        api_base=OLLAMA_API_BASE
+        api_base = api_base.rstrip('/v1')
+        api_key=None
+        custom_llm_provider="ollama"
+    elif args.backend == "openrouter":
+        api_base=OPENROUTER_API_BASE
+        api_key=OPENROUTER_API_KEY
+        custom_llm_provider="openrouter"
+    else:
+        raise RuntimeError("Unknown backend")
+
+    if args.eval == "seceval":
+        instruction = """Below are multiple-choice questions concerning cybersecurity. 
+        Please select the correct answers and respond with the correct letters A, B, C, or D.
+        You could select more than one letter
+        Always return in this format: 'ANSWER: XYZ' or 'ANSWER: X'.
+        """
+        if args.dataset_file == "":
+            args.dataset_file = "benchmarks/utils/seceval_dataset/questions-2.json"
+        dataset = load_dataset(args.dataset_file, "seceval")
+    elif args.eval == "cybermetric":
+        instruction = """Choose the correct answer (A, B, C, or D) only. 
+        Always return in this format: 'ANSWER: X'.
+        """
+        if args.dataset_file == "":
+            args.dataset_file = "benchmarks/utils/cybermetric_dataset/CyberMetric-2-v1.json"
+        dataset = load_dataset(args.dataset_file, "cybermetric")
+    elif args.eval == "cti_bench":
+        if args.dataset_file == "":
+            args.dataset_file = "benchmarks/utils/cti_bench_dataset/cti-mcq1.tsv"
+        if "ate" in args.dataset_file:
+            instruction = """
+            Return your answer in the MITRE ATT&CK technique ID format.
+            You could select more than one technique ID. Be brief and concise. 
+            Return your answer in this format: 'ANSWER: TXXXX' or 'ANSWER: TXXXX, TXXXX, TXXXX'
+            Remember they are just an example, the important is the format."""
+        elif "rcm" in args.dataset_file:
+           instruction = """ #CWE
+           Return CWE ID. 
+           Return your answer in this format: 'ANSWER: CWE-XXXX' (X are numbers)."""   
+        elif "mcq" in args.dataset_file:
+            instruction = """
+            Select the correct answer (A, B, C, or D) only.
+            Return your answer in this format: 'ANSWER: A'"""
+        elif "vsp" in args.dataset_file:
+            instruction = """
+            Return your answer in this CVSS format: 'ANSWER: CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H'"""
+        dataset = load_dataset(args.dataset_file, "cti_bench")
+    
+    start_time = datetime.datetime.now()
+    result = run_evaluation(dataset, instruction, model, api_base, api_key, custom_llm_provider)
+    end_time = datetime.datetime.now()
+
+    accuracy, correct_count, total_count = compute_accuracy(result, args.eval, dataset_file=args.dataset_file)
+    
+    if args.eval.lower() == "cti_bench" and 'cti-vsp' in args.dataset_file:
+        print(f"Mean Absolute Deviation: {accuracy:.2f}")
+    elif args.eval.lower() == "cti_bench" and 'cti-ate' in args.dataset_file:
+        print(f"F1-macro Score: {accuracy:.2f}")
+        print(f"Accuracy: {correct_count:.2f}%")
+    else:
+        print(f"Accuracy: {accuracy:.2f}% ({correct_count}/{total_count})")
+
+    save_benchmark_results(args.eval, model, args.dataset_file, start_time, end_time, len(dataset), correct_count, accuracy, total_count, result)
+
+if __name__ == "__main__":
+    main()

benchmarks/seceval

@@ -0,0 +1 @@
+Subproject commit 7aef3176a1a095b40bbfa806a7a2d6c5e203d5b7

benchmarks/utils/cti_bench_dataset/cti-mcq1.tsv

@@ -0,0 +1,3 @@
+URL	Question	Option A	Option B	Option C	Option D	Prompt	GT
+https://attack.mitre.org/techniques/T1548/	Which of the following mitigations involves preventing applications from running that haven't been downloaded from legitimate repositories?	Audit	Execution Prevention	Operating System Configuration	User Account Control	You are given a multiple-choice question (MCQ) from a Cyber Threat Intelligence (CTI) knowledge benchmark dataset. Your task is to choose the best option among the four provided. Return your answer as a single uppercase letter: A, B, C, or D.  **Question:** Which of the following mitigations involves preventing applications from running that haven't been downloaded from legitimate repositories?  **Options:** A) Audit B) Execution Prevention C) Operating System Configuration D) User Account Control  **Important:** The last line of your answer should contain only the single letter corresponding to the best option, with no additional text. 	B
+https://attack.mitre.org/techniques/T1548/	Which data source is recommended for monitoring commands that may circumvent mechanisms designed to control elevation of privileges?	Command	File	Process	User Account	You are given a multiple-choice question (MCQ) from a Cyber Threat Intelligence (CTI) knowledge benchmark dataset. Your task is to choose the best option among the four provided. Return your answer as a single uppercase letter: A, B, C, or D.  **Question:** Which data source is recommended for monitoring commands that may circumvent mechanisms designed to control elevation of privileges?  **Options:** A) Command B) File C) Process D) User Account  **Important:** The last line of your answer should contain only the single letter corresponding to the best option, with no additional text. 	D

benchmarks/utils/cti_bench_dataset/cti-rcm2.tsv

@@ -0,0 +1,3 @@
+URL	Description	Prompt	GT
+https://nvd.nist.gov/vuln/detail/CVE-2024-23848	In the Linux kernel through 6.7.1, there is a use-after-free in cec_queue_msg_fh, related to drivers/media/cec/core/cec-adap.c and drivers/media/cec/core/cec-api.c.	Analyze the following CVE description and map it to the appropriate CWE. Provide a brief justification for your choice. Ensure the last line of your response contains only the CWE ID.  CVE Description: In the Linux kernel through 6.7.1, there is a use-after-free in cec_queue_msg_fh, related to drivers/media/cec/core/cec-adap.c and drivers/media/cec/core/cec-api.c. 	CWE-416
+https://nvd.nist.gov/vuln/detail/CVE-2023-38738	IBM OpenPages with Watson 8.3 and 9.0 could provide weaker than expected security in a OpenPages environment using Native authentication. If OpenPages is using Native authentication an attacker with access to the OpenPages database could through a series of specially crafted steps could exploit this weakness and gain unauthorized access to other OpenPages accounts. IBM X-Force ID: 262594.	Analyze the following CVE description and map it to the appropriate CWE. Provide a brief justification for your choice. Ensure the last line of your response contains only the CWE ID.  CVE Description: IBM OpenPages with Watson 8.3 and 9.0 could provide weaker than expected security in a OpenPages environment using Native authentication. If OpenPages is using Native authentication an attacker with access to the OpenPages database could through a series of specially crafted steps could exploit this weakness and gain unauthorized access to other OpenPages accounts. IBM X-Force ID: 262594. 	CWE-257

benchmarks/utils/cti_bench_dataset/cti-vsp2.tsv

@@ -0,0 +1,2 @@
+URL	Description	Prompt	GT
+https://nvd.nist.gov/vuln/detail/CVE-2024-23848	In the Linux kernel through 6.7.1, there is a use-after-free in cec_queue_msg_fh, related to drivers/media/cec/core/cec-adap.c and drivers/media/cec/core/cec-api.c.	Analyze the following CVE description and calculate the CVSS v3.1 Base Score. Determine the values for each base metric: AV, AC, PR, UI, S, C, I, and A. Summarize each metric's value and provide the final CVSS v3.1 vector string.   Valid options for each metric are as follows: - **Attack Vector (AV)**: Network (N), Adjacent (A), Local (L), Physical (P) - **Attack Complexity (AC)**: Low (L), High (H) - **Privileges Required (PR)**: None (N), Low (L), High (H) - **User Interaction (UI)**: None (N), Required (R) - **Scope (S)**: Unchanged (U), Changed (C) - **Confidentiality (C)**: None (N), Low (L), High (H) - **Integrity (I)**: None (N), Low (L), High (H) - **Availability (A)**: None (N), Low (L), High (H)  Summarize each metric's value and provide the final CVSS v3.1 vector string. Ensure the final line of your response contains only the CVSS v3 Vector String in the following format:  Example format: CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H  CVE Description: In the Linux kernel through 6.7.1, there is a use-after-free in cec_queue_msg_fh, related to drivers/media/cec/core/cec-adap.c and drivers/media/cec/core/cec-api.c. 	CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:N/I:N/A:H

benchmarks/utils/cybermetric_dataset/CyberMetric-2-v1.json

@@ -0,0 +1,26 @@
+{
+  "questions": [
+
+{
+  "question": "What is the primary requirement for an Random Bit Generator's (RBG) output to be used for generating cryptographic keys?",
+  "answers": {
+    "A": "The RBG's output should have a length that matches the target data to be protected",
+    "B": "The RBG's output should be computationally indistinguishable from random bits and provide sufficient entropy to support the security strength required for the target data",
+    "C": "The RBG's output should be as long as possible to ensure maximal randomness",
+    "D": "The RBG's output should have precisely the same length as the symmetric key to be generated"
+  },
+  "solution": "B"
+},
+{
+  "question": "In cryptography, what is the purpose of using a key-derivation function (KDF)?",
+  "answers": {
+    "A": "Generate public keys",
+    "B": "Authenticate digital signatures",
+    "C": "Encrypt data using a password",
+    "D": "KDF are algorithms used to transform a secret into crucial parameters like keys and Initialization Vectors (IVs)"
+  },
+  "solution": "D"
+}
+
+  ]
+}

benchmarks/utils/seceval_dataset/questions-2.json

@@ -0,0 +1,35 @@
+[
+    {
+        "id": "282cd7f6-86f9-4174-95c2-df26b2b6395e",
+        "source": "android_sec_doc",
+        "question": "You are tasked with designing a secure storage system for an Android device's hardware identifiers as part of an ID attestation implementation. Which of the following properties are essential for ensuring the system's integrity and security?",
+        "choices": [
+            "A: The storage must contain the original identifiers to enable the TEE to verify their authenticity during attestation.",
+            "B: The storage should be tamper-evident to ensure any modification is detectable, rendering the attestation invalid.",
+            "C: The `destroyAttestationIds()` method should be able to restore the identifier-derived data after a factory reset.",
+            "D: RMA facilities must not have the ability to regenerate hardware identifier-derived data to prevent unauthorized attestation."
+        ],
+        "answer": "B",
+        "topics": [
+            "SystemSecurity",
+            "Vulnerability"
+        ],
+        "keyword": "Tamper-evident"
+    },
+    {
+        "id": "28355cb5-4b9a-44b8-8419-38fd17d335e4",
+        "source": "owasp_mastg",
+        "question": "An Android application developer is tasked with ensuring the security of sensitive user information. The developer has several options for data storage, each with different security implications. Which of the following storage options should the developer avoid to prevent exposing sensitive user data to other applications on the device?",
+        "choices": [
+            "A: Using SharedPreferences with MODE_PRIVATE to store user preferences and settings.",
+            "B: Storing user credentials in an unencrypted SQLite database accessible to the app.",
+            "C: Implementing SQLCipher to encrypt SQLite databases containing sensitive user information.",
+            "D: Saving encrypted user data in the Android Keystore system."
+        ],
+        "answer": "B",
+        "topics": [
+            "ApplicationSecurity"
+        ],
+        "keyword": "SQLite"
+    }
+]

ci/benchmarks/.benchmarks.yml

@@ -0,0 +1,88 @@
+.use_base_container: &use_base_container
+  stage: benchmarks
+  image: "${CI_REGISTRY_IMAGE}:latest"
+  services:
+    - name: docker:dind
+      alias: docker
+
+.run_benchmarks: &run_benchmarks
+  <<: *use_base_container
+  script:
+    - pip3 install -e .
+    - pip install litellm langchain transformers torch openai tqdm cvss python-dotenv
+    - echo "Checking environment variables..."
+
+    - |
+      for var in $(compgen -e); do
+        if [[ ($var == CTF_* || $var == *_API_KEY || $var == *_API_BASE) && -n ${!var} ]]; then
+          export $var="${!var}"          
+        fi
+      done
+
+    - echo $OLLAMA_API_BASE
+    - python3 benchmarks/eval.py --model $MODEL_NAME --dataset_file $BENCHMARK_FILE --eval $BENCHMARK_NAME --backend $BACKEND
+    - pwd
+    - ls -la benchmarks/outputs/$BENCHMARK_NAME/
+    - curl http://host.docker.internal:8000/api/tags # http://localhost:8000/api/tags 
+  variables:
+    OLLAMA_API_BASE: "http://host.docker.internal:8000" # http://localhost:8000
+    OPENROUTER_API_BASE: "https://openrouter.ai/api/v1"
+    OPENAI_API_BASE: "https://api.openai.com/v1"
+  artifacts:
+    paths:
+      - benchmarks/outputs/
+    expire_in: 12 month
+  tags:
+    - p40
+    - x86
+  rules:
+    - if: $CI_COMMIT_BRANCH
+      when: on_success
+
+
+
+benchmarks-test-cybermetric-ollama:
+  <<: *run_benchmarks
+  variables:
+    MODEL_NAME: "ollama/qwen2.5:14b"
+    BENCHMARK_FILE: "benchmarks/utils/cybermetric_dataset/CyberMetric-2-v1.json"
+    BENCHMARK_NAME: "cybermetric"
+    BACKEND: "ollama"
+    OLLAMA_API_BASE: "http://localhost:8000"
+
+
+# # benchmarks-test-seceval:
+# #   <<: *run_benchmarks
+# #   variables:
+# #     OLLAMA_API_BASE: "http://localhost:8000"
+# #     OPENROUTER_API_BASE: "https://openrouter.ai/api/v1"
+# #     OPENAI_API_KEY: "fake-api-key"
+# #   script:
+# #     - pip3 install -e .
+# #     - pip install -r benchmarks/seceval/eval/requirements.txt
+# #     - python3 benchmarks/seceval/eval/eval.py --dataset_file benchmarks/utils/seceval_dataset/questions-2.json --output_dir benchmarks/seceval/eval/outputs --backend ollama --models ollama/qwen2.5:14b
+
+benchmarks-test-cybermetric-openrouter:
+  <<: *run_benchmarks
+  variables:
+    MODEL_NAME: "openrouter/qwen/qwen3-32b:free"
+    BENCHMARK_FILE: "benchmarks/utils/cybermetric_dataset/CyberMetric-2-v1.json"
+    BENCHMARK_NAME: "cybermetric"
+    BACKEND: "openrouter"
+
+benchmarks-test-seceval-openrouter:
+  <<: *run_benchmarks
+  variables:
+    MODEL_NAME: "openrouter/qwen/qwen3-32b:free"
+    BENCHMARK_FILE: "benchmarks/utils/seceval_dataset/questions-2.json"
+    BENCHMARK_NAME: "seceval"
+    BACKEND: "openrouter"
+
+benchmarks-test-cti_bench-openrouter:
+  <<: *run_benchmarks
+  variables:
+    MODEL_NAME: "openrouter/qwen/qwen3-32b:free"
+    BENCHMARK_FILE: "benchmarks/utils/cti_bench_dataset/cti-mcq1.tsv"
+    BENCHMARK_NAME: "cti_bench"
+    BACKEND: "openrouter"
+