add pentestperf + suppoer for openai

benchmarks/README.md

@@ -11,7 +11,7 @@ Currently, this are the benchmarks included:
 | [SecEval](https://github.com/XuanwuAI/SecEval) | Benchmark designed to evaluate large language models (LLMs) on security-related tasks. It includes various real-world scenarios such as phishing email analysis, vulnerability classification, and response generation. |
 | [CyberMetric](https://github.com/CyberMetric) | Benchmark framework that focuses on measuring the performance of AI systems in cybersecurity-specific question answering, knowledge extraction, and contextual understanding. It emphasizes both domain knowledge and reasoning ability. |
 | [CTIBench](https://github.com/xashru/cti-bench) | Benchmark focused on evaluating LLM models' capabilities in understanding and processing Cyber Threat Intelligence (CTI) information. |
-
+| [PentestPerf](https://gitlab.com/aliasrobotics/alias_research/caiextensions/pentestperf) | Internal Benchmark to evaluate the security capabilities. It is built as a composition of individual benchmarks, each represented by a Docker container. Each container scenario (CTF) can contain multiple challenges and tasks. The system is designed to be modular and extensible, allowing for the addition of new benchmarks and challenges. |
 
 
 The goal is to consolidate diverse evaluation tasks under a single framework to support rigorous, standardized testing.
@@ -68,4 +68,20 @@ python benchmarks/eval.py --model qwen/qwen3-32b:free  --dataset_file benchmarks
 ````
 ```bash
 python benchmarks/eval.py --model qwen/qwen3-32b:free  --dataset_file benchmarks/cti_bench/data/cti-ate2.tsv --eval cti_bench --backend openrouter
-````
+````
+
+**How to run different backends such as openai and anthropic**
+
+Set the API_KEY for the corresponding backend as follows in .env: NAME_BACKEND + API_KEY
+
+```bash
+OPENAI_API_KEY = "..."
+ANTHROPIC_API_KEY="..."
+````
+
+```bash
+python benchmarks/eval.py --model gpt-4o-mini --dataset_file benchmarks/cybermetric/CyberMetric-2-v1.json --eval cybermetric --backend openai
+````
+```bash
+python benchmarks/eval.py --model claude-3-7-sonnet-20250219 --dataset_file benchmarks/cybermetric/CyberMetric-2-v1.json --eval cybermetric --backend anthropic
+````

benchmarks/eval.py

@@ -9,7 +9,8 @@ Usage:
 Arguments:
     -m, --model           Specify the model to evaluate (e.g., "gpt-4", "qwen2.5:14b", etc.)
     -d, --dataset_file    Path to the dataset file (JSON or TSV) containing questions to evaluate
-    -B, --backend         Backend to use: "openai", "openrouter", "ollama" (required)
+    -B, --backend         Backend to use: "openai", "openrouter", "ollama", "anthropic", etc. 
+                          (is important to set the api key and api base in environment variables for the backend)
     -e, --eval            Specify the evaluation benchmark
 
 Example:
@@ -20,6 +21,10 @@ Example:
      
      python benchmarks/eval.py --model qwen/qwen3-32b:free --dataset_file benchmarks/utils/cybermetric_dataset/CyberMetric-2-v1.json --eval cybermetric --backend openrouter
 
+     python benchmarks/eval.py --model gpt-4o-mini --dataset_file benchmarks/cybermetric/CyberMetric-2-v1.json --eval cybermetric --backend openai
+
+     python benchmarks/eval.py --model claude-3-7-sonnet-20250219 --dataset_file benchmarks/cybermetric/CyberMetric-2-v1.json --eval cybermetric --backend anthropic
+
 Environment Variables:
     OPENROUTER_API_KEY:  API key for OpenRouter (if using OpenRouter models)
     OPENROUTER_API_BASE: Base URL for OpenRouter API (default: https://openrouter.ai/api/v1)
@@ -47,7 +52,6 @@ import datetime
 OPENROUTER_API_BASE = os.environ.get("OPENROUTER_API_BASE", "https://openrouter.ai/api/v1")
 OPENROUTER_API_KEY = os.environ.get("OPENROUTER_API_KEY")
 OLLAMA_API_BASE = os.environ.get("OLLAMA_API_BASE", "http://localhost:8000/v1")
-OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
 OPENAI_API_BASE = os.environ.get("OPENAI_API_BASE", "https://api.openai.com/v1")
 
 
@@ -450,7 +454,7 @@ def save_benchmark_results(
 def main():
     parser = argparse.ArgumentParser(description="SecEval Evaluation CLI")
     parser.add_argument("-d", "--dataset_file", type=str, required=True, help="Specify the dataset file to evaluate on.")
-    parser.add_argument("-B", "--backend", type=str, choices=["openai", "ollama", "openrouter"], required=True, help="Specify the llm type. openai: openai model, ollama: ollama model, openrouter: openrouter model")
+    parser.add_argument("-B", "--backend", type=str, required=True, help="Specify the llm type. openai: openai model, ollama: ollama model, openrouter: openrouter model")
     parser.add_argument("-m", "--model", type=str, required=True, help="Specify the models.")
     parser.add_argument("-e", "--eval", type=str, required=True, help="Specify the evaluation benchmark.")
     args = parser.parse_args()
@@ -458,12 +462,10 @@ def main():
     model = args.model
 
     print(f"Evaluating model: {model}")
- 
-    if args.backend == "openai":
-        api_base=OPENAI_API_BASE
-        api_key=OPENAI_API_KEY
-        custom_llm_provider=None
-    elif args.backend == "ollama":
+    OPENROUTER_API_KEY="REDACTED_OPENROUTER_KEY"
+    OPENROUTER_API_BASE="https://openrouter.ai/api/v1"
+    
+    if args.backend == "ollama":
         api_base=OLLAMA_API_BASE
         api_base = api_base.rstrip('/v1')
         api_key=None
@@ -472,6 +474,12 @@ def main():
         api_base=OPENROUTER_API_BASE
         api_key=OPENROUTER_API_KEY
         custom_llm_provider="openrouter"
+    elif args.backend:
+        api_base=os.environ.get(args.backend.upper() + "_API_BASE")
+        api_key= os.environ.get(args.backend.upper() + "_API_KEY")
+        custom_llm_provider=args.backend
+        if api_key is None:
+            raise RuntimeError("Unknown backend")
     else:
         raise RuntimeError("Unknown backend")
 

ci/benchmarks/.benchmarks.yml

@@ -86,3 +86,10 @@ benchmarks-test-cti_bench-openrouter:
     BENCHMARK_NAME: "cti_bench"
     BACKEND: "openrouter"
 
+benchmarks-test-cti_bench-openai:
+  <<: *run_benchmarks
+  variables:
+    MODEL_NAME: "gpt-4o-mini"
+    BENCHMARK_FILE: "benchmarks/utils/cti_bench_dataset/cti-mcq1.tsv"
+    BENCHMARK_NAME: "cti_bench"
+    BACKEND: "openai"