mirror of https://github.com/aliasrobotics/cai.git
review path
This commit is contained in:
Mery-Sanz
parent
54bb4655b2
commit
1970c312e4
|
|
@ -148,4 +148,4 @@ cython_debug/
|
|||
.vscode/
|
||||
|
||||
# benchmarks
|
||||
benchmarks/cybermetric/outputs
|
||||
benchmarks/outputs
|
||||
|
|
@ -14,13 +14,13 @@ Arguments:
|
|||
|
||||
Example:
|
||||
|
||||
python benchmarks/eval.py --model ollama/qwen2.5:14b --dataset_file benchmarks/cybermetric/CyberMetric-2-v1.json --eval cybermetric --backend ollama
|
||||
python benchmarks/eval.py --model ollama/qwen2.5:14b --dataset_file benchmarks/seceval/eval/datasets/questions-2.json --eval seceval --backend ollama
|
||||
python benchmarks/eval.py --model ollama/qwen2.5:14b --dataset_file benchmarks/cti_bench/data/cti-mcq1.tsv --eval cti_bench --backend ollama
|
||||
python benchmarks/eval.py --model ollama/qwen2.5:14b --dataset_file benchmarks/cti_bench/data/cti-ate2.tsv --eval cti_bench --backend ollama
|
||||
python benchmarks/eval.py --model ollama/qwen2.5:14b --dataset_file benchmarks/cti_bench/data/cti-rcm2.tsv --eval cti_bench --backend ollama
|
||||
python benchmarks/eval.py --model ollama/qwen2.5:14b --dataset_file benchmarks/cti_bench/data/cti-vsp2.tsv --eval cti_bench --backend ollama
|
||||
python benchmarks/eval.py --model qwen/qwen3-32b:free --dataset_file benchmarks/cybermetric/CyberMetric-2-v1.json --eval cybermetric --backend openrouter
|
||||
python benchmarks/eval.py --model ollama/qwen2.5:14b --dataset_file benchmarks/utils/cybermetric_dataset/CyberMetric-2-v1.json --eval cybermetric --backend ollama
|
||||
python benchmarks/eval.py --model ollama/qwen2.5:14b --dataset_file benchmarks/utils/seceval_dataset/questions-2.json --eval seceval --backend ollama
|
||||
python benchmarks/eval.py --model ollama/qwen2.5:14b --dataset_file benchmarks/utils/cti_bench_dataset/cti-mcq1.tsv --eval cti_bench --backend ollama
|
||||
python benchmarks/eval.py --model ollama/qwen2.5:14b --dataset_file benchmarks/utils/cti_bench_dataset/cti-ate2.tsv --eval cti_bench --backend ollama
|
||||
python benchmarks/eval.py --model ollama/qwen2.5:14b --dataset_file benchmarks/utils/cti_bench_dataset/cti-rcm2.tsv --eval cti_bench --backend ollama
|
||||
python benchmarks/eval.py --model ollama/qwen2.5:14b --dataset_file benchmarks/utils/cti_bench_dataset/cti-vsp2.tsv --eval cti_bench --backend ollama
|
||||
python benchmarks/eval.py --model qwen/qwen3-32b:free --dataset_file benchmarks/utils/cybermetric_dataset/CyberMetric-2-v1.json --eval cybermetric --backend openrouter
|
||||
|
||||
Environment Variables:
|
||||
OPENROUTER_API_KEY: API key for OpenRouter (if using OpenRouter models)
|
||||
|
|
@ -484,16 +484,18 @@ def main():
|
|||
Always return in this format: 'ANSWER: XYZ' or 'ANSWER: X'.
|
||||
"""
|
||||
if args.dataset_file == "":
|
||||
args.dataset_file = "benchmarks/seceval/eval/datasets/questions-2.json"
|
||||
args.dataset_file = "benchmarks/utils/seceval_dataset/questions-2.json"
|
||||
dataset = load_dataset(args.dataset_file, "seceval")
|
||||
elif args.eval == "cybermetric":
|
||||
instruction = """Choose the correct answer (A, B, C, or D) only.
|
||||
Always return in this format: 'ANSWER: X'.
|
||||
"""
|
||||
if args.dataset_file == "":
|
||||
args.dataset_file = "benchmarks/cybermetric/CyberMetric-2-v1.json"
|
||||
args.dataset_file = "benchmarks/utils/cybermetric_dataset/CyberMetric-2-v1.json"
|
||||
dataset = load_dataset(args.dataset_file, "cybermetric")
|
||||
elif args.eval == "cti_bench":
|
||||
if args.dataset_file == "":
|
||||
args.dataset_file = "benchmarks/utils/cti_bench_dataset/cti-mcq1.tsv"
|
||||
if "ate" in args.dataset_file:
|
||||
instruction = """
|
||||
Return your answer in the MITRE ATT&CK technique ID format.
|
||||
|
|
@ -511,16 +513,12 @@ def main():
|
|||
elif "vsp" in args.dataset_file:
|
||||
instruction = """
|
||||
Return your answer in this CVSS format: 'ANSWER: CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H'"""
|
||||
if args.dataset_file == "":
|
||||
args.dataset_file = "benchmarks/cti_bench/data/cti-mcq1.tsv"
|
||||
dataset = load_dataset(args.dataset_file, "cti_bench")
|
||||
|
||||
start_time = datetime.datetime.now()
|
||||
result = run_evaluation(dataset, instruction, model, api_base, api_key, custom_llm_provider)
|
||||
end_time = datetime.datetime.now()
|
||||
|
||||
print(result)
|
||||
|
||||
accuracy, correct_count, total_count = compute_accuracy(result, args.eval, dataset_file=args.dataset_file)
|
||||
|
||||
if args.eval.lower() == "cti_bench" and 'cti-vsp' in args.dataset_file:
|
||||
|
|
|
|||
|
|
@ -19,8 +19,6 @@
|
|||
fi
|
||||
done
|
||||
|
||||
|
||||
#- python3 benchmarks/cybermetric/CyberMetric_evaluator.py --model_name $MODEL_NAME --file_path $BENCHMARK_FILE
|
||||
- echo $OLLAMA_API_BASE
|
||||
- python3 benchmarks/eval.py --model $MODEL_NAME --dataset_file $BENCHMARK_FILE --eval $BENCHMARK_NAME --backend $BACKEND
|
||||
- pwd
|
||||
|
|
@ -47,10 +45,10 @@ benchmarks-test-cybermetric-ollama:
|
|||
<<: *run_benchmarks
|
||||
variables:
|
||||
MODEL_NAME: "ollama/qwen2.5:14b"
|
||||
BENCHMARK_FILE: "benchmarks/cybermetric/CyberMetric-2-v1.json"
|
||||
BENCHMARK_FILE: "benchmarks/utils/cybermetric_dataset/CyberMetric-2-v1.json"
|
||||
BENCHMARK_NAME: "cybermetric"
|
||||
BACKEND: "ollama"
|
||||
#OLLAMA_API_BASE: "http://localhost:8000"
|
||||
OLLAMA_API_BASE: "http://localhost:8000"
|
||||
|
||||
|
||||
# # benchmarks-test-seceval:
|
||||
|
|
@ -62,51 +60,29 @@ benchmarks-test-cybermetric-ollama:
|
|||
# # script:
|
||||
# # - pip3 install -e .
|
||||
# # - pip install -r benchmarks/seceval/eval/requirements.txt
|
||||
# # - python3 benchmarks/seceval/eval/eval.py --dataset_file benchmarks/seceval/eval/datasets/questions-2.json --output_dir benchmarks/seceval/eval/outputs --backend ollama --models ollama/qwen2.5:14b
|
||||
# # - python3 benchmarks/seceval/eval/eval.py --dataset_file benchmarks/utils/seceval_dataset/questions-2.json --output_dir benchmarks/seceval/eval/outputs --backend ollama --models ollama/qwen2.5:14b
|
||||
|
||||
# benchmarks-test-cybermetric-openrouter:
|
||||
# <<: *run_benchmarks
|
||||
# variables:
|
||||
# MODEL_NAME: "openrouter/qwen/qwen3-32b:free"
|
||||
# BENCHMARK_FILE: "benchmarks/cybermetric/CyberMetric-2-v1.json"
|
||||
# BENCHMARK_NAME: "cybermetric"
|
||||
# BACKEND: "openrouter"
|
||||
benchmarks-test-cybermetric-openrouter:
|
||||
<<: *run_benchmarks
|
||||
variables:
|
||||
MODEL_NAME: "openrouter/qwen/qwen3-32b:free"
|
||||
BENCHMARK_FILE: "benchmarks/utils/cybermetric_dataset/CyberMetric-2-v1.json"
|
||||
BENCHMARK_NAME: "cybermetric"
|
||||
BACKEND: "openrouter"
|
||||
|
||||
# benchmarks-test-seceval-openrouter:
|
||||
# <<: *run_benchmarks
|
||||
# variables:
|
||||
# MODEL_NAME: "openrouter/qwen/qwen3-32b:free"
|
||||
# BENCHMARK_FILE: "benchmarks/seceval/eval/datasets/questions-2.json"
|
||||
# BENCHMARK_NAME: "seceval"
|
||||
# BACKEND: "openrouter"
|
||||
benchmarks-test-seceval-openrouter:
|
||||
<<: *run_benchmarks
|
||||
variables:
|
||||
MODEL_NAME: "openrouter/qwen/qwen3-32b:free"
|
||||
BENCHMARK_FILE: "benchmarks/utils/seceval_dataset/questions-2.json"
|
||||
BENCHMARK_NAME: "seceval"
|
||||
BACKEND: "openrouter"
|
||||
|
||||
# benchmarks-test-cti_bench-openrouter:
|
||||
# <<: *run_benchmarks
|
||||
# variables:
|
||||
# MODEL_NAME: "openrouter/qwen/qwen3-32b:free"
|
||||
# BENCHMARK_FILE: "benchmarks/utils/cti_bench_dataset/cti-mcq1.tsv"
|
||||
# BENCHMARK_NAME: "cti_bench"
|
||||
# BACKEND: "openrouter"
|
||||
benchmarks-test-cti_bench-openrouter:
|
||||
<<: *run_benchmarks
|
||||
variables:
|
||||
MODEL_NAME: "openrouter/qwen/qwen3-32b:free"
|
||||
BENCHMARK_FILE: "benchmarks/utils/cti_bench_dataset/cti-mcq1.tsv"
|
||||
BENCHMARK_NAME: "cti_bench"
|
||||
BACKEND: "openrouter"
|
||||
|
||||
# benchmarks-test-cybermetric-ollama:
|
||||
# <<: *run_benchmarks
|
||||
# variables:
|
||||
# MODEL_NAME: "openrouter/qwen/qwen3-32b:free"
|
||||
# BENCHMARK_FILE: "benchmarks/cybermetric/CyberMetric-2-v1.json"
|
||||
# BENCHMARK_NAME: "cybermetric"
|
||||
# BACKEND: "ollama"
|
||||
# OLLAMA_API_BASE: "http://localhost:8000/v1"
|
||||
|
||||
# benchmarks-test-seceval-ollama:
|
||||
# <<: *run_benchmarks
|
||||
# variables:
|
||||
# MODEL_NAME: "ollama/qwen2.5:14b"
|
||||
# BENCHMARK_FILE: "benchmarks/seceval/eval/datasets/questions-2.json"
|
||||
# BENCHMARK_NAME: "seceval"
|
||||
# BACKEND: "ollama"
|
||||
|
||||
# benchmarks-test-cybermetric-openai:
|
||||
# <<: *run_benchmarks
|
||||
# variables:
|
||||
# MODEL_NAME: "openai/gpt-4o-mini"
|
||||
# BENCHMARK_FILE: "benchmarks/cybermetric/CyberMetric-2-v1.json"
|
||||
|
|
|
|||
|
|
@ -1,63 +0,0 @@
|
|||
.use_base_container: &use_base_container
|
||||
stage: benchmarks
|
||||
image: "${CI_REGISTRY_IMAGE}:latest"
|
||||
services:
|
||||
- name: docker:dind
|
||||
alias: docker
|
||||
|
||||
.run_benchmarks: &run_benchmarks
|
||||
<<: *use_base_container
|
||||
script:
|
||||
- pip3 install -e .
|
||||
- pip install litellm langchain transformers torch openai tqdm
|
||||
- echo "Checking environment variables..."
|
||||
- echo "OPENAI_API_KEY is set: $([[ -n $OPENAI_API_KEY ]] && echo 'YES' || echo 'NO')"
|
||||
- |
|
||||
for var in $(compgen -e); do
|
||||
if [[ ($var == CTF_* || $var == *_API_KEY || $var == *_API_BASE) && -n ${!var} ]]; then
|
||||
export $var="${!var}"
|
||||
if [[ $var == OPENAI_API_KEY ]]; then
|
||||
echo "OPENAI_API_KEY is exported"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
- python3 benchmarks/cybermetric/CyberMetric_evaluator.py --model_name $MODEL_NAME --file_path $BENCHMARK_FILE --api_key "${OPENAI_API_KEY}"
|
||||
# - curl http://localhost:8000/api/tags # validate that ollama is running
|
||||
variables:
|
||||
OLLAMA_API_BASE: "http://localhost:8000"
|
||||
OPENROUTER_API_BASE: "https://openrouter.ai/api/v1"
|
||||
OPENAI_API_BASE: "https://api.openai.com/v1"
|
||||
artifacts:
|
||||
paths:
|
||||
- cai/benchmarks/seceval/eval/outputs/
|
||||
- cai/benchmarks/cybermetric/outputs/
|
||||
expire_in: 12 month
|
||||
tags:
|
||||
- p40
|
||||
- x86
|
||||
rules:
|
||||
- if: $CI_COMMIT_BRANCH
|
||||
when: on_success
|
||||
|
||||
# benchmarks-test-seceval:
|
||||
# <<: *run_benchmarks
|
||||
# variables:
|
||||
# OLLAMA_API_BASE: "http://localhost:8000"
|
||||
# OPENROUTER_API_BASE: "https://openrouter.ai/api/v1"
|
||||
# OPENAI_API_KEY: "fake-api-key"
|
||||
# script:
|
||||
# - pip3 install -e .
|
||||
# - pip install -r benchmarks/seceval/eval/requirements.txt
|
||||
# - python3 benchmarks/seceval/eval/eval.py --dataset_file benchmarks/seceval/eval/datasets/questions-2.json --output_dir benchmarks/seceval/eval/outputs --backend ollama --models ollama/qwen2.5:14b
|
||||
|
||||
# benchmarks-test-cybermetric-openrouter:
|
||||
# <<: *run_benchmarks
|
||||
# script:
|
||||
# - python3 benchmarks/cybermetric/CyberMetric_evaluator.py --model_name openrouter/qwen/qwen3-32b:free --file_path benchmarks/cybermetric/CyberMetric-2-v1.json
|
||||
|
||||
benchmarks-test-cybermetric-openai:
|
||||
<<: *run_benchmarks
|
||||
variables:
|
||||
MODEL_NAME: "openai/gpt-4o-mini"
|
||||
BENCHMARK_FILE: "benchmarks/cybermetric/CyberMetric-2-v1.json"
|
||||
# Make sure to define OPENAI_API_KEY in GitLab CI/CD variables
|
||||
Loading…
Reference in New Issue