From 1970c312e4184efc91a480248e52922bcca5d517 Mon Sep 17 00:00:00 2001 From: Mery-Sanz Date: Mon, 12 May 2025 08:23:36 +0200 Subject: [PATCH] review path --- .gitignore | 2 +- benchmarks/eval.py | 24 +++++------ ci/benchmarks/.benchmarks.yml | 72 +++++++++++-------------------- ci/benchmarks/.benchmarks.yml.bak | 63 --------------------------- 4 files changed, 36 insertions(+), 125 deletions(-) delete mode 100644 ci/benchmarks/.benchmarks.yml.bak diff --git a/.gitignore b/.gitignore index 4d94f1f4..2956bfc1 100644 --- a/.gitignore +++ b/.gitignore @@ -148,4 +148,4 @@ cython_debug/ .vscode/ # benchmarks -benchmarks/cybermetric/outputs \ No newline at end of file +benchmarks/outputs \ No newline at end of file diff --git a/benchmarks/eval.py b/benchmarks/eval.py index 58684a7e..d7a5ecb2 100644 --- a/benchmarks/eval.py +++ b/benchmarks/eval.py @@ -14,13 +14,13 @@ Arguments: Example: - python benchmarks/eval.py --model ollama/qwen2.5:14b --dataset_file benchmarks/cybermetric/CyberMetric-2-v1.json --eval cybermetric --backend ollama - python benchmarks/eval.py --model ollama/qwen2.5:14b --dataset_file benchmarks/seceval/eval/datasets/questions-2.json --eval seceval --backend ollama - python benchmarks/eval.py --model ollama/qwen2.5:14b --dataset_file benchmarks/cti_bench/data/cti-mcq1.tsv --eval cti_bench --backend ollama - python benchmarks/eval.py --model ollama/qwen2.5:14b --dataset_file benchmarks/cti_bench/data/cti-ate2.tsv --eval cti_bench --backend ollama - python benchmarks/eval.py --model ollama/qwen2.5:14b --dataset_file benchmarks/cti_bench/data/cti-rcm2.tsv --eval cti_bench --backend ollama - python benchmarks/eval.py --model ollama/qwen2.5:14b --dataset_file benchmarks/cti_bench/data/cti-vsp2.tsv --eval cti_bench --backend ollama - python benchmarks/eval.py --model qwen/qwen3-32b:free --dataset_file benchmarks/cybermetric/CyberMetric-2-v1.json --eval cybermetric --backend openrouter + python benchmarks/eval.py --model ollama/qwen2.5:14b --dataset_file benchmarks/utils/cybermetric_dataset/CyberMetric-2-v1.json --eval cybermetric --backend ollama + python benchmarks/eval.py --model ollama/qwen2.5:14b --dataset_file benchmarks/utils/seceval_dataset/questions-2.json --eval seceval --backend ollama + python benchmarks/eval.py --model ollama/qwen2.5:14b --dataset_file benchmarks/utils/cti_bench_dataset/cti-mcq1.tsv --eval cti_bench --backend ollama + python benchmarks/eval.py --model ollama/qwen2.5:14b --dataset_file benchmarks/utils/cti_bench_dataset/cti-ate2.tsv --eval cti_bench --backend ollama + python benchmarks/eval.py --model ollama/qwen2.5:14b --dataset_file benchmarks/utils/cti_bench_dataset/cti-rcm2.tsv --eval cti_bench --backend ollama + python benchmarks/eval.py --model ollama/qwen2.5:14b --dataset_file benchmarks/utils/cti_bench_dataset/cti-vsp2.tsv --eval cti_bench --backend ollama + python benchmarks/eval.py --model qwen/qwen3-32b:free --dataset_file benchmarks/utils/cybermetric_dataset/CyberMetric-2-v1.json --eval cybermetric --backend openrouter Environment Variables: OPENROUTER_API_KEY: API key for OpenRouter (if using OpenRouter models) @@ -484,16 +484,18 @@ def main(): Always return in this format: 'ANSWER: XYZ' or 'ANSWER: X'. """ if args.dataset_file == "": - args.dataset_file = "benchmarks/seceval/eval/datasets/questions-2.json" + args.dataset_file = "benchmarks/utils/seceval_dataset/questions-2.json" dataset = load_dataset(args.dataset_file, "seceval") elif args.eval == "cybermetric": instruction = """Choose the correct answer (A, B, C, or D) only. Always return in this format: 'ANSWER: X'. """ if args.dataset_file == "": - args.dataset_file = "benchmarks/cybermetric/CyberMetric-2-v1.json" + args.dataset_file = "benchmarks/utils/cybermetric_dataset/CyberMetric-2-v1.json" dataset = load_dataset(args.dataset_file, "cybermetric") elif args.eval == "cti_bench": + if args.dataset_file == "": + args.dataset_file = "benchmarks/utils/cti_bench_dataset/cti-mcq1.tsv" if "ate" in args.dataset_file: instruction = """ Return your answer in the MITRE ATT&CK technique ID format. @@ -511,16 +513,12 @@ def main(): elif "vsp" in args.dataset_file: instruction = """ Return your answer in this CVSS format: 'ANSWER: CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H'""" - if args.dataset_file == "": - args.dataset_file = "benchmarks/cti_bench/data/cti-mcq1.tsv" dataset = load_dataset(args.dataset_file, "cti_bench") start_time = datetime.datetime.now() result = run_evaluation(dataset, instruction, model, api_base, api_key, custom_llm_provider) end_time = datetime.datetime.now() - print(result) - accuracy, correct_count, total_count = compute_accuracy(result, args.eval, dataset_file=args.dataset_file) if args.eval.lower() == "cti_bench" and 'cti-vsp' in args.dataset_file: diff --git a/ci/benchmarks/.benchmarks.yml b/ci/benchmarks/.benchmarks.yml index e653835b..5d703154 100644 --- a/ci/benchmarks/.benchmarks.yml +++ b/ci/benchmarks/.benchmarks.yml @@ -19,8 +19,6 @@ fi done - - #- python3 benchmarks/cybermetric/CyberMetric_evaluator.py --model_name $MODEL_NAME --file_path $BENCHMARK_FILE - echo $OLLAMA_API_BASE - python3 benchmarks/eval.py --model $MODEL_NAME --dataset_file $BENCHMARK_FILE --eval $BENCHMARK_NAME --backend $BACKEND - pwd @@ -47,10 +45,10 @@ benchmarks-test-cybermetric-ollama: <<: *run_benchmarks variables: MODEL_NAME: "ollama/qwen2.5:14b" - BENCHMARK_FILE: "benchmarks/cybermetric/CyberMetric-2-v1.json" + BENCHMARK_FILE: "benchmarks/utils/cybermetric_dataset/CyberMetric-2-v1.json" BENCHMARK_NAME: "cybermetric" BACKEND: "ollama" - #OLLAMA_API_BASE: "http://localhost:8000" + OLLAMA_API_BASE: "http://localhost:8000" # # benchmarks-test-seceval: @@ -62,51 +60,29 @@ benchmarks-test-cybermetric-ollama: # # script: # # - pip3 install -e . # # - pip install -r benchmarks/seceval/eval/requirements.txt -# # - python3 benchmarks/seceval/eval/eval.py --dataset_file benchmarks/seceval/eval/datasets/questions-2.json --output_dir benchmarks/seceval/eval/outputs --backend ollama --models ollama/qwen2.5:14b +# # - python3 benchmarks/seceval/eval/eval.py --dataset_file benchmarks/utils/seceval_dataset/questions-2.json --output_dir benchmarks/seceval/eval/outputs --backend ollama --models ollama/qwen2.5:14b -# benchmarks-test-cybermetric-openrouter: -# <<: *run_benchmarks -# variables: -# MODEL_NAME: "openrouter/qwen/qwen3-32b:free" -# BENCHMARK_FILE: "benchmarks/cybermetric/CyberMetric-2-v1.json" -# BENCHMARK_NAME: "cybermetric" -# BACKEND: "openrouter" +benchmarks-test-cybermetric-openrouter: + <<: *run_benchmarks + variables: + MODEL_NAME: "openrouter/qwen/qwen3-32b:free" + BENCHMARK_FILE: "benchmarks/utils/cybermetric_dataset/CyberMetric-2-v1.json" + BENCHMARK_NAME: "cybermetric" + BACKEND: "openrouter" -# benchmarks-test-seceval-openrouter: -# <<: *run_benchmarks -# variables: -# MODEL_NAME: "openrouter/qwen/qwen3-32b:free" -# BENCHMARK_FILE: "benchmarks/seceval/eval/datasets/questions-2.json" -# BENCHMARK_NAME: "seceval" -# BACKEND: "openrouter" +benchmarks-test-seceval-openrouter: + <<: *run_benchmarks + variables: + MODEL_NAME: "openrouter/qwen/qwen3-32b:free" + BENCHMARK_FILE: "benchmarks/utils/seceval_dataset/questions-2.json" + BENCHMARK_NAME: "seceval" + BACKEND: "openrouter" -# benchmarks-test-cti_bench-openrouter: -# <<: *run_benchmarks -# variables: -# MODEL_NAME: "openrouter/qwen/qwen3-32b:free" -# BENCHMARK_FILE: "benchmarks/utils/cti_bench_dataset/cti-mcq1.tsv" -# BENCHMARK_NAME: "cti_bench" -# BACKEND: "openrouter" +benchmarks-test-cti_bench-openrouter: + <<: *run_benchmarks + variables: + MODEL_NAME: "openrouter/qwen/qwen3-32b:free" + BENCHMARK_FILE: "benchmarks/utils/cti_bench_dataset/cti-mcq1.tsv" + BENCHMARK_NAME: "cti_bench" + BACKEND: "openrouter" -# benchmarks-test-cybermetric-ollama: -# <<: *run_benchmarks -# variables: -# MODEL_NAME: "openrouter/qwen/qwen3-32b:free" -# BENCHMARK_FILE: "benchmarks/cybermetric/CyberMetric-2-v1.json" -# BENCHMARK_NAME: "cybermetric" -# BACKEND: "ollama" -# OLLAMA_API_BASE: "http://localhost:8000/v1" - -# benchmarks-test-seceval-ollama: -# <<: *run_benchmarks -# variables: -# MODEL_NAME: "ollama/qwen2.5:14b" -# BENCHMARK_FILE: "benchmarks/seceval/eval/datasets/questions-2.json" -# BENCHMARK_NAME: "seceval" -# BACKEND: "ollama" - -# benchmarks-test-cybermetric-openai: -# <<: *run_benchmarks -# variables: -# MODEL_NAME: "openai/gpt-4o-mini" -# BENCHMARK_FILE: "benchmarks/cybermetric/CyberMetric-2-v1.json" diff --git a/ci/benchmarks/.benchmarks.yml.bak b/ci/benchmarks/.benchmarks.yml.bak deleted file mode 100644 index 7fcc9731..00000000 --- a/ci/benchmarks/.benchmarks.yml.bak +++ /dev/null @@ -1,63 +0,0 @@ -.use_base_container: &use_base_container - stage: benchmarks - image: "${CI_REGISTRY_IMAGE}:latest" - services: - - name: docker:dind - alias: docker - -.run_benchmarks: &run_benchmarks - <<: *use_base_container - script: - - pip3 install -e . - - pip install litellm langchain transformers torch openai tqdm - - echo "Checking environment variables..." - - echo "OPENAI_API_KEY is set: $([[ -n $OPENAI_API_KEY ]] && echo 'YES' || echo 'NO')" - - | - for var in $(compgen -e); do - if [[ ($var == CTF_* || $var == *_API_KEY || $var == *_API_BASE) && -n ${!var} ]]; then - export $var="${!var}" - if [[ $var == OPENAI_API_KEY ]]; then - echo "OPENAI_API_KEY is exported" - fi - fi - done - - python3 benchmarks/cybermetric/CyberMetric_evaluator.py --model_name $MODEL_NAME --file_path $BENCHMARK_FILE --api_key "${OPENAI_API_KEY}" - # - curl http://localhost:8000/api/tags # validate that ollama is running - variables: - OLLAMA_API_BASE: "http://localhost:8000" - OPENROUTER_API_BASE: "https://openrouter.ai/api/v1" - OPENAI_API_BASE: "https://api.openai.com/v1" - artifacts: - paths: - - cai/benchmarks/seceval/eval/outputs/ - - cai/benchmarks/cybermetric/outputs/ - expire_in: 12 month - tags: - - p40 - - x86 - rules: - - if: $CI_COMMIT_BRANCH - when: on_success - -# benchmarks-test-seceval: -# <<: *run_benchmarks -# variables: -# OLLAMA_API_BASE: "http://localhost:8000" -# OPENROUTER_API_BASE: "https://openrouter.ai/api/v1" -# OPENAI_API_KEY: "fake-api-key" -# script: -# - pip3 install -e . -# - pip install -r benchmarks/seceval/eval/requirements.txt -# - python3 benchmarks/seceval/eval/eval.py --dataset_file benchmarks/seceval/eval/datasets/questions-2.json --output_dir benchmarks/seceval/eval/outputs --backend ollama --models ollama/qwen2.5:14b - -# benchmarks-test-cybermetric-openrouter: -# <<: *run_benchmarks -# script: -# - python3 benchmarks/cybermetric/CyberMetric_evaluator.py --model_name openrouter/qwen/qwen3-32b:free --file_path benchmarks/cybermetric/CyberMetric-2-v1.json - -benchmarks-test-cybermetric-openai: - <<: *run_benchmarks - variables: - MODEL_NAME: "openai/gpt-4o-mini" - BENCHMARK_FILE: "benchmarks/cybermetric/CyberMetric-2-v1.json" - # Make sure to define OPENAI_API_KEY in GitLab CI/CD variables \ No newline at end of file