diff --git a/benchmarks/README.md b/benchmarks/README.md index 68fc91fb..72a2f49c 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -11,7 +11,7 @@ Currently, this are the benchmarks included: | [SecEval](https://github.com/XuanwuAI/SecEval) | Benchmark designed to evaluate large language models (LLMs) on security-related tasks. It includes various real-world scenarios such as phishing email analysis, vulnerability classification, and response generation. | | [CyberMetric](https://github.com/CyberMetric) | Benchmark framework that focuses on measuring the performance of AI systems in cybersecurity-specific question answering, knowledge extraction, and contextual understanding. It emphasizes both domain knowledge and reasoning ability. | | [CTIBench](https://github.com/xashru/cti-bench) | Benchmark focused on evaluating LLM models' capabilities in understanding and processing Cyber Threat Intelligence (CTI) information. | - +| [PentestPerf](https://gitlab.com/aliasrobotics/alias_research/caiextensions/pentestperf) | Internal Benchmark to evaluate the security capabilities. It is built as a composition of individual benchmarks, each represented by a Docker container. Each container scenario (CTF) can contain multiple challenges and tasks. The system is designed to be modular and extensible, allowing for the addition of new benchmarks and challenges. | The goal is to consolidate diverse evaluation tasks under a single framework to support rigorous, standardized testing. @@ -68,4 +68,20 @@ python benchmarks/eval.py --model qwen/qwen3-32b:free --dataset_file benchmarks ```` ```bash python benchmarks/eval.py --model qwen/qwen3-32b:free --dataset_file benchmarks/cti_bench/data/cti-ate2.tsv --eval cti_bench --backend openrouter -```` \ No newline at end of file +```` + +**How to run different backends such as openai and anthropic** + +Set the API_KEY for the corresponding backend as follows in .env: NAME_BACKEND + API_KEY + +```bash +OPENAI_API_KEY = "..." +ANTHROPIC_API_KEY="..." +```` + +```bash +python benchmarks/eval.py --model gpt-4o-mini --dataset_file benchmarks/cybermetric/CyberMetric-2-v1.json --eval cybermetric --backend openai +```` +```bash +python benchmarks/eval.py --model claude-3-7-sonnet-20250219 --dataset_file benchmarks/cybermetric/CyberMetric-2-v1.json --eval cybermetric --backend anthropic +```` diff --git a/benchmarks/eval.py b/benchmarks/eval.py index f1e48619..e601877c 100644 --- a/benchmarks/eval.py +++ b/benchmarks/eval.py @@ -9,7 +9,8 @@ Usage: Arguments: -m, --model Specify the model to evaluate (e.g., "gpt-4", "qwen2.5:14b", etc.) -d, --dataset_file Path to the dataset file (JSON or TSV) containing questions to evaluate - -B, --backend Backend to use: "openai", "openrouter", "ollama" (required) + -B, --backend Backend to use: "openai", "openrouter", "ollama", "anthropic", etc. + (is important to set the api key and api base in environment variables for the backend) -e, --eval Specify the evaluation benchmark Example: @@ -20,6 +21,10 @@ Example: python benchmarks/eval.py --model qwen/qwen3-32b:free --dataset_file benchmarks/utils/cybermetric_dataset/CyberMetric-2-v1.json --eval cybermetric --backend openrouter + python benchmarks/eval.py --model gpt-4o-mini --dataset_file benchmarks/cybermetric/CyberMetric-2-v1.json --eval cybermetric --backend openai + + python benchmarks/eval.py --model claude-3-7-sonnet-20250219 --dataset_file benchmarks/cybermetric/CyberMetric-2-v1.json --eval cybermetric --backend anthropic + Environment Variables: OPENROUTER_API_KEY: API key for OpenRouter (if using OpenRouter models) OPENROUTER_API_BASE: Base URL for OpenRouter API (default: https://openrouter.ai/api/v1) @@ -47,7 +52,6 @@ import datetime OPENROUTER_API_BASE = os.environ.get("OPENROUTER_API_BASE", "https://openrouter.ai/api/v1") OPENROUTER_API_KEY = os.environ.get("OPENROUTER_API_KEY") OLLAMA_API_BASE = os.environ.get("OLLAMA_API_BASE", "http://localhost:8000/v1") -OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY") OPENAI_API_BASE = os.environ.get("OPENAI_API_BASE", "https://api.openai.com/v1") @@ -450,7 +454,7 @@ def save_benchmark_results( def main(): parser = argparse.ArgumentParser(description="SecEval Evaluation CLI") parser.add_argument("-d", "--dataset_file", type=str, required=True, help="Specify the dataset file to evaluate on.") - parser.add_argument("-B", "--backend", type=str, choices=["openai", "ollama", "openrouter"], required=True, help="Specify the llm type. openai: openai model, ollama: ollama model, openrouter: openrouter model") + parser.add_argument("-B", "--backend", type=str, required=True, help="Specify the llm type. openai: openai model, ollama: ollama model, openrouter: openrouter model") parser.add_argument("-m", "--model", type=str, required=True, help="Specify the models.") parser.add_argument("-e", "--eval", type=str, required=True, help="Specify the evaluation benchmark.") args = parser.parse_args() @@ -458,12 +462,10 @@ def main(): model = args.model print(f"Evaluating model: {model}") - - if args.backend == "openai": - api_base=OPENAI_API_BASE - api_key=OPENAI_API_KEY - custom_llm_provider=None - elif args.backend == "ollama": + OPENROUTER_API_KEY="REDACTED_OPENROUTER_KEY" + OPENROUTER_API_BASE="https://openrouter.ai/api/v1" + + if args.backend == "ollama": api_base=OLLAMA_API_BASE api_base = api_base.rstrip('/v1') api_key=None @@ -472,6 +474,12 @@ def main(): api_base=OPENROUTER_API_BASE api_key=OPENROUTER_API_KEY custom_llm_provider="openrouter" + elif args.backend: + api_base=os.environ.get(args.backend.upper() + "_API_BASE") + api_key= os.environ.get(args.backend.upper() + "_API_KEY") + custom_llm_provider=args.backend + if api_key is None: + raise RuntimeError("Unknown backend") else: raise RuntimeError("Unknown backend") diff --git a/ci/benchmarks/.benchmarks.yml b/ci/benchmarks/.benchmarks.yml index 5d703154..68a03519 100644 --- a/ci/benchmarks/.benchmarks.yml +++ b/ci/benchmarks/.benchmarks.yml @@ -86,3 +86,10 @@ benchmarks-test-cti_bench-openrouter: BENCHMARK_NAME: "cti_bench" BACKEND: "openrouter" +benchmarks-test-cti_bench-openai: + <<: *run_benchmarks + variables: + MODEL_NAME: "gpt-4o-mini" + BENCHMARK_FILE: "benchmarks/utils/cti_bench_dataset/cti-mcq1.tsv" + BENCHMARK_NAME: "cti_bench" + BACKEND: "openai"