RomanNum3ral
/
cai
mirror of https://github.com/aliasrobotics/cai.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add s to benchmakrs

This commit is contained in:
Mery-Sanz 2025-05-08 12:17:41 +02:00
parent 243a989063
commit c5f74cf85f
11 changed files with 329 additions and 37689 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

184
benchmarks/cybermetric/CyberMetric-10-v1.json
View File

@ -1,184 +0,0 @@
{
"questions": [
{
"question": "Which of the following is a desirable property of a biometric system?",
"answers": {
"A": "Permanent",
"B": "Transferability",
"C": "Uniformity",
"D": "Forgiveness"
},
"solution": "A"
},
{
"question": "In TCP/IP networking, which protocol is used to hold network addresses and routing information in a packet?",
"answers": {
"A": "HTTP",
"B": "IP",
"C": "Routing Information Protocol (RIP)",
"D": "TCP"
},
"solution": "B"
},
{
"question": "In the context of unexpected negative outcomes in personal privacy policies, what responsibility lies with both the provider and the consumer regarding the retention time of private information?",
"answers": {
"A": "Both the provider and the consumer are responsible for setting the appropriate retention time",
"B": "Neither the provider nor the consumer has a responsibility for setting the appropriate retention time",
"C": "Only the consumer is responsible for setting an appropriate retention time",
"D": "Only the provider is responsible for ensuring the appropriate retention time"
},
"solution": "A"
},
{
"question": "What type of attack involves an attacker behaving as a store-and-forward or proxy mechanism between two communicating systems?",
"answers": {
"A": "Buffer overflow attack",
"B": "Man-in-the-Middle attack",
"C": "DDoS attack",
"D": "Sniffer attack"
},
"solution": "B"
},
{
"question": "What is the fundamental purpose of logging and monitoring in an organization's security measures?",
"answers": {
"A": "To store backups of critical data",
"B": "To track, record, and review activity to detect and respond to security incidents",
"C": "To manage and enforce user access controls",
"D": "To create a record of all employee activities"
},
"solution": "B"
},
{
"question": "What is the benefit of structured walk-throughs in disaster recovery testing?",
"answers": {
"A": "Observe live actions in a controlled environment",
"B": "Conduct a full shut-down and restoration at the primary site",
"C": "Test operational response to disaster scenarios",
"D": "Interrupt real operations at the primary site"
},
"solution": "C"
},
{
"question": "Which security process metric would most assist in determining an appropriate backup frequency for a database server?",
"answers": {
"A": "RTO",
"B": "MTBF",
"C": "RPO",
"D": "MTD"
},
"solution": "C"
},
{
"question": "What constitutional amendment outlines the burden placed on investigators to have a valid search warrant before conducting certain searches?",
"answers": {
"A": "Third Amendment",
"B": "First Amendment",
"C": "Second Amendment",
"D": "Fourth Amendment"
},
"solution": "D"
},
{
"question": "Which tool is frequently used to directly access websites without a browser and is often utilized for testing and identifying potential API vulnerabilities?",
"answers": {
"A": "SSH",
"B": "cURL",
"C": "FTP",
"D": "Telnet"
},
"solution": "B"
},
{
"question": "What is the distinguishing characteristic of symmetric-key cryptography?",
"answers": {
"A": "It tends to be CPU intensive",
"B": "It uses the same key for encryption and decryption",
"C": "It provides integrity protection to data",
"D": "It uses different but related keys for encryption and decryption"
},
"solution": "B"
},
{
"question": "What is the result of 5521211 modulo 1337?",
"answers": {
"A": "11111",
"B": "33",
"C": "602",
"D": "738"
},
"solution": "D"
},
{
"question": "Which of the following is an example of a strong password?",
"answers": {
"A": "Password123456",
"B": "ME_Fgh4_al445!A",
"C": "Qwerty!",
"D": "123456"
},
"solution": "B"
},
{
"question": "Which model consists of the Application Layer, Host-to-Host Layer, Internet Layer, and Network Access Layer?",
"answers": {
"A": "OSI model",
"B": "WAN model",
"C": "LAN model",
"D": "TCP/IP model"
},
"solution": "D"
},
{
"question": "What is the primary purpose of 'inference engine' in the context of expert systems?",
"answers": {
"A": "To infer new facts and solve problems using rules and facts in the knowledge base",
"B": "To encrypt messages passed across the Internet",
"C": "To enable recognition of an entity by a system",
"D": "To control access to resources in a system"
},
"solution": "A"
},
{
"question": "What is the primary objective of conducting code reviews for bespoke and custom software applications within the Payment Card Industry Data Security Standard guidelines?",
"answers": {
"A": "To expedite the deplxoyment of bespoke and custom software into production",
"B": "To exploit potential coding vulnerabilities in production software",
"C": "To ensure bespoke and custom software meets performance benchmarks under heavy load",
"D": "To ensure that bespoke and custom software cannot be exploited via coding vulnerabilities"
},
"solution": "D"
},
{
"question": "Which principle of cybersecurity focuses on ensuring that data is accurate and trustworthy?",
"answers": {
"A": "Resilience",
"B": "Availability",
"C": "Authenticity",
"D": "Integrity"
},
"solution": "D"
},
{
"question": "An IDS installed on the network perimeter sees a spike in traffic during off-duty hours and begins logging and alerting. Which type of IDS is in place?",
"answers": {
"A": "Packet filtering",
"B": "Anomaly based",
"C": "Signature based",
"D": "Stateful"
},
"solution": "B"
},
{
"question": "Which of the following is a desirable property of a biometric system?",
"answers": {
"A": "Permanent",
"B": "Transferability",
"C": "Uniformity",
"D": "Forgiveness"
},
"solution": "A"
}
]
}

25
benchmarks/cybermetric/CyberMetric-2-v1.json
View File

@ -1,25 +0,0 @@
{
"questions": [
{
"question": "Which of the following is a desirable property of a biometric system?",
"answers": {
"A": "Permanent",
"B": "Transferability",
"C": "Uniformity",
"D": "Forgiveness"
},
"solution": "A"
},
{
"question": "In TCP/IP networking, which protocol is used to hold network addresses and routing information in a packet?",
"answers": {
"A": "HTTP",
"B": "IP",
"C": "Routing Information Protocol (RIP)",
"D": "TCP"
},
"solution": "B"
}
]
}

87
benchmarks/cybermetric/CyberMetric_evaluator.py Normal file
View File

@ -0,0 +1,87 @@
import json
import re
import time
from tqdm import tqdm
from openai import OpenAI
class CyberMetricEvaluator:
def __init__(self, api_key, file_path):
self.client = OpenAI(api_key=api_key)
self.file_path = file_path
def read_json_file(self):
with open(self.file_path, 'r') as file:
return json.load(file)
@staticmethod
def extract_answer(response):
if response.strip(): # Checks if the response is not empty and not just whitespace
match = re.search(r"ANSWER:?\s*([A-D])", response, re.IGNORECASE)
if match:
return match.group(1).upper() # Return the matched letter in uppercase
return None
def ask_llm(self, question, answers, max_retries=5):
options = ', '.join([f"{key}) {value}" for key, value in answers.items()])
prompt = f"Question: {question}\nOptions: {options}\n\nChoose the correct answer (A, B, C, or D) only. Always return in this format: 'ANSWER: X' "
for attempt in range(max_retries):
try:
response = self.client.chat.completions.create(
model="gpt-3.5-turbo-0125",
messages=[
{"role": "system", "content": "You are a security expert who answers questions."},
{"role": "user", "content": prompt},
]
)
if response.choices:
result = self.extract_answer(response.choices[0].message.content)
if result:
return result
else:
print("Incorrect answer format detected. Attempting the question again.")
except Exception as e:
print(f"Error: {e}. Attempting the question again in {2 ** attempt} seconds.")
time.sleep(2 ** attempt)
return None
def run_evaluation(self):
json_data = self.read_json_file()
questions_data = json_data['questions']
correct_count = 0
incorrect_answers = []
with tqdm(total=len(questions_data), desc="Processing Questions") as progress_bar:
for item in questions_data:
question = item['question']
answers = item['answers']
correct_answer = item['solution']
llm_answer = self.ask_llm(question, answers)
if llm_answer == correct_answer:
correct_count += 1
else:
incorrect_answers.append({
'question': question,
'correct_answer': correct_answer,
'llm_answer': llm_answer
})
accuracy_rate = correct_count / (progress_bar.n + 1) * 100
progress_bar.set_postfix_str(f"Accuracy: {accuracy_rate:.2f}%")
progress_bar.update(1)
print(f"Final Accuracy: {correct_count / len(questions_data) * 100}%")
if incorrect_answers:
print("\nIncorrect Answers:")
for item in incorrect_answers:
print(f"Question: {item['question']}")
print(f"Expected Answer: {item['correct_answer']}, LLM Answer: {item['llm_answer']}\n")
# Example usage:
if __name__ == "__main__":
API_KEY="<YOUR-APKI-KEY-HERE>"
file_path='CyberMetric-500-v1.json'
evaluator = CyberMetricEvaluator(api_key=API_KEY, file_path=file_path)
evaluator.run_evaluation()

334
benchmarks/cybermetric/CyberMetric_evaluator_openrouter_ollama.py
View File

@ -1,334 +0,0 @@
"""
CyberMetric Evaluator for LLMs
This script evaluates the performance of language models on the CyberMetric benchmark.
It supports both OpenRouter-hosted models and local Ollama models via LiteLLM proxy.
Usage:
python CyberMetric_evaluator_openrouter_ollama.py --model_name MODEL_NAME [--file_path FILE_PATH] [--api_key API_KEY]
Arguments:
--model_name: Required. Model name with prefix (openrouter/ or ollama/)
Examples: openrouter/anthropic/claude-3-opus, ollama/llama3
--file_path: Optional. Path to the CyberMetric JSON file (default: CyberMetric-2-v1.json)
--api_key: Optional. API key for OpenRouter (can also use OPENROUTER_API_KEY env var)
Environment Variables:
OPENROUTER_API_KEY: API key for OpenRouter (if using OpenRouter models)
OPENROUTER_API_BASE: Base URL for OpenRouter API (default: https://openrouter.ai/api/v1)
OLLAMA_API_BASE: Base URL for Ollama API via LiteLLM proxy (default: http://localhost:8000/v1)
Examples:
# Run with an OpenRouter model
python CyberMetric_evaluator_openrouter_ollama.py --model_name openrouter/qwen/qwen3-32b:free
# Run with a local Ollama model (requires LiteLLM proxy running)
python CyberMetric_evaluator_openrouter_ollama.py --model_name ollama/qwen2.5:14b
# Specify a different benchmark file
python CyberMetric_evaluator_openrouter_ollama.py --model_name openrouter/qwen/qwen3-32b:free --file_path CyberMetric-10000-v1.json
"""
import json
import re
import time
import os
import datetime
import random
import string
import argparse
from tqdm import tqdm
import litellm
import requests
# Default API bases
OPENROUTER_API_BASE = "https://openrouter.ai/api/v1"
OLLAMA_LITELLM_API_BASE = "http://localhost:8000/v1"
class CyberMetricEvaluator:
def __init__(self, model_name, file_path, api_key=None, openrouter_api_base=None, ollama_litellm_api_base=None):
self.model_name = model_name
self.file_path = file_path
self.failed_questions = []
self.failed_count = 0
# Set API configurations
self.openrouter_api_base = openrouter_api_base or os.environ.get("OPENROUTER_API_BASE", OPENROUTER_API_BASE)
self.ollama_litellm_api_base = ollama_litellm_api_base or os.environ.get("OLLAMA_LITELLM_API_BASE", OLLAMA_LITELLM_API_BASE)
# Set API key for OpenRouter if needed
self.api_key = None
if self.model_name.startswith("openrouter/"):
self.api_key = api_key or os.environ.get("OPENROUTER_API_KEY")
self.start_time = datetime.datetime.now()
# Create output directory structure
self.output_dir = self.create_output_directory()
self.info_file = os.path.join(self.output_dir, "information.txt")
self.report_file = os.path.join(self.output_dir, "report_failed_questions.json")
# Initialize info file
self.initialize_info_file()
print("--DEBUG: model_name: ", self.model_name)
def create_output_directory(self):
# Create base directory if it doesn't exist
base_dir = "output_cybermetric"
if not os.path.exists(base_dir):
os.makedirs(base_dir)
# Format model name for directory (replace / with -)
model_dir_name = self.model_name.replace("/", "-")
# Get current date
current_date = datetime.datetime.now().strftime("%Y-%m-%d")
# Create directory name
dir_name = f"{model_dir_name}-{current_date}"
# If directory already exists, add random string
full_path = os.path.join(base_dir, dir_name)
if os.path.exists(full_path):
random_str = ''.join(random.choices(string.ascii_lowercase + string.digits, k=5))
dir_name = f"{model_dir_name}-{current_date}-{random_str}"
full_path = os.path.join(base_dir, dir_name)
os.makedirs(full_path)
return full_path
def initialize_info_file(self):
with open(self.info_file, 'w') as file:
file.write(f"CyberMetric Evaluation\n")
file.write(f"=====================\n\n")
file.write(f"Model: {self.model_name}\n")
file.write(f"Dataset: {self.file_path}\n")
file.write(f"Start Time: {self.start_time.strftime('%Y-%m-%d %H:%M:%S')}\n")
file.write(f"Status: Running\n")
file.write(f"Questions Processed: 0\n")
file.write(f"Correct Answers: 0\n")
file.write(f"Accuracy: 0.00%\n")
def update_info_file(self, questions_processed, correct_count, status="Running"):
accuracy = correct_count / questions_processed * 100 if questions_processed > 0 else 0
with open(self.info_file, 'w') as file:
file.write(f"CyberMetric Evaluation\n")
file.write(f"=====================\n\n")
file.write(f"Model: {self.model_name}\n") # "openrouter/qwen/qwen3-32b:free"
file.write(f"Dataset: {self.file_path}\n")
file.write(f"Start Time: {self.start_time.strftime('%Y-%m-%d %H:%M:%S')}\n")
file.write(f"Status: {status}\n")
file.write(f"Questions Processed: {questions_processed}\n")
file.write(f"Correct Answers: {correct_count}\n")
file.write(f"Accuracy: {accuracy:.2f}%\n")
if status == "Completed":
end_time = datetime.datetime.now()
duration = end_time - self.start_time
file.write(f"End Time: {end_time.strftime('%Y-%m-%d %H:%M:%S')}\n")
file.write(f"Duration: {duration}\n")
def read_json_file(self):
with open(self.file_path, 'r') as file:
return json.load(file)
def save_failed_questions(self):
report_data = {"questions": self.failed_questions}
with open(self.report_file, 'w') as file:
json.dump(report_data, file, indent=4)
print(f"Updated failed questions report in {self.report_file}")
@staticmethod
def extract_answer(response):
if response.strip():
match = re.search(r"ANSWER:?\s*([A-D])", response, re.IGNORECASE)
if match:
return match.group(1).upper()
return None
def ask_ollama_litellm(self, prompt, max_retries=5):
for attempt in range(max_retries):
try:
response = litellm.completion(
model=self.model_name,
messages=[
{"role": "system", "content": "You are a security expert who answers questions."},
{"role": "user", "content": prompt},
],
api_base="http://localhost:8000"
)
if hasattr(response, "choices") and response.choices:
content = response.choices[0].message.content
result = self.extract_answer(content)
if result:
print("--DEBUG: result: ", result)
return result
else:
print("Incorrect answer format detected. Attempting the question again.")
except Exception as e:
print(f"Error: {e}. Attempting the question again in {2 ** attempt} seconds.")
time.sleep(2 ** attempt)
return None
def ask_openrouter(self, prompt, max_retries=5):
if not self.api_key:
raise ValueError("API key is required for OpenRouter models")
for attempt in range(max_retries):
try:
response = litellm.completion(
model=self.model_name,
messages=[
{"role": "system", "content": "You are a security expert who answers questions."},
{"role": "user", "content": prompt},
],
api_base=self.openrouter_api_base,
api_key=self.api_key,
headers={
"HTTP-Referer": "https://your-site-url.com",
"X-Title": "CyberMetric Evaluator"
}
)
if hasattr(response, "choices") and response.choices:
content = response.choices[0].message.content
result = self.extract_answer(content)
if result:
print("--DEBUG: result: ", result)
return result
else:
print("Incorrect answer format detected. Attempting the question again.")
except Exception as e:
print(f"Error: {e}. Attempting the question again in {2 ** attempt} seconds.")
time.sleep(2 ** attempt)
return None
def ask_llm(self, question, answers, max_retries=5):
options = ', '.join([f"{key}) {value}" for key, value in answers.items()])
prompt = (
f"Question: {question}\n"
f"Options: {options}\n\n"
"Choose the correct answer (A, B, C, or D) only. Always return in this format: 'ANSWER: X'"
)
print("--DEBUG: prompt: ", prompt)
if self.model_name.startswith("openrouter/"):
return self.ask_openrouter(prompt, max_retries)
elif self.model_name.startswith("ollama/"):
# Only use the litellm approach with port 8000 for Ollama
return self.ask_ollama_litellm(prompt, max_retries)
else:
print("Error: Model name must start with 'openrouter/' or 'ollama/'")
return None
def run_evaluation(self):
if not (self.model_name.startswith("openrouter/") or self.model_name.startswith("ollama/")):
print("Error: You must set model name with prefix 'ollama/' or 'openrouter/'")
return
json_data = self.read_json_file()
questions_data = json_data['questions']
correct_count = 0
incorrect_answers = []
with tqdm(total=len(questions_data), desc="Processing Questions") as progress_bar:
for i, item in enumerate(questions_data):
question = item['question']
answers = item['answers']
correct_answer = item['solution']
llm_answer = self.ask_llm(question, answers)
if llm_answer == correct_answer:
correct_count += 1
else:
self.failed_questions.append({
'question': question,
'answers': answers,
'solution': correct_answer,
'llm_answer': llm_answer
})
self.failed_count += 1
if self.failed_count % 2 == 0:
self.save_failed_questions()
incorrect_answers.append({
'question': question,
'correct_answer': correct_answer,
'llm_answer': llm_answer
})
# Update progress and information file
questions_processed = i + 1
accuracy_rate = correct_count / questions_processed * 100
progress_bar.set_postfix_str(f"Accuracy: {accuracy_rate:.2f}%")
progress_bar.update(1)
# Update info file every 5 questions
if questions_processed % 5 == 0 or questions_processed == len(questions_data):
self.update_info_file(questions_processed, correct_count)
# Final update with completed status
self.update_info_file(len(questions_data), correct_count, "Completed")
print(f"\nFinal Accuracy: {correct_count / len(questions_data) * 100:.2f}%")
if self.failed_questions:
self.save_failed_questions() # final failed questions
if incorrect_answers:
print("\nIncorrect Answers:")
for item in incorrect_answers:
print(f"Question: {item['question']}")
print(f"Expected Answer: {item['correct_answer']}, LLM Answer: {item['llm_answer']}\n")
if __name__ == "__main__":
#litellm._turn_on_debug()
# Create argument parser
parser = argparse.ArgumentParser(description='CyberMetric Evaluator for LLMs')
parser.add_argument('--model_name', type=str, required=True,
help='Model name with prefix (openrouter/ or ollama/)')
parser.add_argument('--file_path', type=str, default='CyberMetric-2-v1.json',
help='Path to the CyberMetric JSON file')
parser.add_argument('--api_key', type=str,
help='API key for OpenRouter (can also use OPENROUTER_API_KEY env var)')
args = parser.parse_args()
model_name = args.model_name
file_path = args.file_path
api_key = args.api_key or os.environ.get("OPENROUTER_API_KEY")
if model_name.startswith("ollama/"):
# Ollama configuration
evaluator = CyberMetricEvaluator(
model_name=model_name,
file_path=file_path
)
print(f"Using Ollama configuration with LiteLLM proxy on port 8000")
elif model_name.startswith("openrouter/"):
# OpenRouter configuration
if not api_key:
raise ValueError("API key must be provided via --api_key or OPENROUTER_API_KEY environment variable for OpenRouter models")
evaluator = CyberMetricEvaluator(
model_name=model_name,
file_path=file_path,
api_key=api_key,
openrouter_api_base=os.environ.get("OPENROUTER_API_BASE", "https://openrouter.ai/api/v1")
)
print("Using OpenRouter configuration")
else:
raise ValueError("Model name must start with 'ollama/' or 'openrouter/'")
# Run the evaluation
evaluator.run_evaluation()

1
benchmarks/seceval/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
.DS_Store

BIN
benchmarks/seceval/.langchain.db
View File

Binary file not shown.

144
benchmarks/seceval/README_CN.md Normal file
View File

@ -0,0 +1,144 @@
# SecEval：评估基础模型在网络安全知识上的全面基准测试
[English](README.md)
随着大型语言模型的出现，为网络安全行业带来了变革性的时代。在网络安全知识问答、漏洞挖掘和告警研判等领域，人们正在开发、部署和使用大模型应用。各种研究表明，LLMs主要在预训练阶段获取知识，而微调基本上是为了使模型与用户意图对齐，提供遵循指令的能力。这表明基础模型中的知识和技能对模型在特定下游任务上的潜力有着重要影响。
然而，在现有评估数据集中缺乏对网络安全知识的专门性评估。我们通过引入“SecEval”来解决这个问题。SecEval是首个专门为评估基础模型中的网络安全知识而创建的基准测试。它提供了超过2000个多项选择题，覆盖9个领域：软件安全、应用程序安全、系统安全、网络安全、密码学、内存安全、网络安全和渗透测试。
SecEval通过向OpenAI GPT4提供权威来源的数据来生成相应问题，如开放许可的教科书、官方文档和行业指导方针与标准，来生成问题。生成过程经过精心设计，以确保数据集满足严格的质量、多样性和公正性标准。您可以在[探索页面](https://xuanwuai.github.io/SecEval/explore.html)上探索我们的数据集。
使用SecEval，我们对10种最先进的基础模型进行了评估，为它们在网络安全领域的表现提供了新的见解。结果表明，在LLMs在网络安全领域还有很长的路要走。我们希望SecEval能够成为未来在这一领域研究的催化剂。
## 目录
- [Leaderboard](#leaderboard)
- [数据集](#数据集)
- [数据集生成方法](#数据集生成方法)
- [局限性](#局限性)
- [未来工作](#未来工作)
- [许可证](#许可证)
- [引用](#引用)
## Leaderboard
| # | Model | Creator | Access | Submission Date | System Security | Application Security | PenTest | Memory Safety | Network Security | Web Security | Vulnerability | Software Security | Cryptography | Overall |
|-----|-------------------|-----------|-----------|-----------------|-----------------|----------------------|---------|---------------|------------------|--------------|---------------|-------------------|--------------|---------|
| 1 | gpt-4-turbo | OpenAI | API, Web | 2023-12-20 | 73.61 | 75.25 | 80.00 | 70.83 | 75.65 | 82.15 | 76.05 | 73.28 | 64.29 | 79.07 |
| 2 | gpt-35-turbo | OpenAI | API, Web | 2023-12-20 | 59.15 | 57.18 | 72.00 | 43.75 | 60.87 | 63.00 | 60.18 | 58.19 | 35.71 | 62.09 |
| 3 | Yi-6B | 01-AI | Weight | 2023-12-20 | 50.61 | 48.89 | 69.26 | 35.42 | 56.52 | 54.98 | 49.40 | 45.69 | 35.71 | 53.57 |
| 4 | Orca-2-7b | Microsoft | Weight | 2023-12-20 | 46.76 | 47.03 | 60.84 | 31.25 | 49.13 | 55.63 | 50.00 | 52.16 | 14.29 | 51.60 |
| 5 | Mistral-7B-v0.1 | Mistralai | Weight | 2023-12-20 | 40.19 | 38.37 | 53.47 | 33.33 | 36.52 | 46.57 | 42.22 | 43.10 | 28.57 | 43.65 |
| 6 | chatglm3-6b-base | THUDM | Weight | 2023-12-20 | 39.72 | 37.25 | 57.47 | 31.25 | 43.04 | 41.14 | 37.43 | 39.66 | 28.57 | 41.58 |
| 7 | Aquila2-7B | BAAI | Weight | 2023-12-20 | 34.84 | 36.01 | 47.16 | 22.92 | 32.17 | 42.04 | 38.02 | 36.21 | 7.14 | 38.29 |
| 8 | Qwen-7B | Alibaba | Weight | 2023-12-20 | 28.92 | 28.84 | 41.47 | 18.75 | 29.57 | 33.25 | 31.74 | 30.17 | 14.29 | 31.37 |
| 9 | internlm-7b | Sensetime | Weight | 2023-12-20 | 25.92 | 25.87 | 36.21 | 25.00 | 27.83 | 32.86 | 29.34 | 34.05 | 7.14 | 30.29 |
| 10 | Llama-2-7b-hf | MetaAI | Weight | 2023-12-20 | 20.94 | 18.69 | 26.11 | 16.67 | 14.35 | 22.77 | 21.56 | 20.26 | 21.43 | 22.15 |
## 数据集
### 格式
数据集采用json格式。每个问题包含以下字段：
* id: str # 每个问题的唯一id
* source: str # 问题来源
* question: str # 问题描述
* choices: List[str] # 问题的选项
* answer: str # 问题的答案
* topics: List[QuestionTopic] # 问题的主题，每个问题可以有多个主题。
* keyword: str # 问题的关键词
### 问题分布
| 主题 | 问题数量 |
|----------------------|--------------|
| 系统安全 | 1065 |
| 应用安全 | 808 |
| 渗透测试 | 475 |
| 内存安全 | 48 |
| 网络安全 | 230 |
| 网络安全 | 773 |
| 漏洞 | 334 |
| 软件安全 | 232 |
| 密码学 | 14 |
| 总体 | 2126 |
### 下载
您可以通过运行以下命令下载json文件的数据集。
```
wget https://huggingface.co/datasets/XuanwuAI/SecEval/blob/main/problems.json
```
或者您可以从[Huggingface](https://huggingface.co/datasets/XuanwuAI/SecEval)加载数据集。
### 在SecEval上评估您的模型
您可以使用我们的[评估脚本](https://github.com/XuanwuAI/SecEval/tree/main/eval)来在SecEval数据集上评估您的模型。
## 数据集生成方法
### 数据收集
- **教科书**：我们从加州大学伯克利分校的计算机安全课程 CS161 和麻省理工学院的 6.858 选取了开放许可的教科书。这些资源提供了关于网络安全、内存安全、网页安全和密码学的广泛信息。
- **官方文档**：我们使用了如苹果平台安全、安卓安全和 Windows 安全等官方文档，以整合这些平台特有的系统安全和应用安全知识。
- **工业指南**：为了包含网络安全，我们参考了 Mozilla 网络安全指南。此外，我们还使用了 OWASP 网络安全测试指南（WSTG）和 OWASP 移动应用安全测试指南（MASTG），以获得网络和应用安全测试的洞见。
- **工业标准**：我们使用了CWE来解决对漏洞的认知问题。对于渗透测试，我们纳入了 MITRE ATT&CK 和 MITRE D3fend 框架。
### 生成评估问题
为了便于评估过程，我们设计了一个多项选择题的数据集格式。我们的问题生成方法包括几个步骤：
1. **文本解析**：我们首先根据文本的层次结构进行解析，例如教科书的章节，或诸如ATT&CK这样的框架的战略和技术。
2. **内容抽样**：对于内容丰富的文本，如CWE或Windows安全文档，我们采用抽样策略以保持可管理性。例如，我们从CWE中选择了25种最常见的弱点类型和175种随机类型。
3. **问题生成**：利用GPT-4基于解析后的文本生成多项选择题并根据文本的特性调整送入GPT参考文本的粒度。例如，CS161教科书的问题是基于各个章节的，而来自ATT&CK的则是基于技术的。
4. **问题完善**：然后我们指导GPT-4识别并过滤出问题，比如过于简单或不完整的问题。在可能的情况下，我们会尝试将问题修正；否则，它们将被丢弃。
5. **答案校准**：我们通过向GPT-4提供问题和产生问题的源文本来再次生成答案。如果GPT-4生成的响应与之前问题生成阶段确定的答案不一致，则表明，对于这道题目，获得一致答案本身具有挑战性。在这种情况下，我们选择淘汰这些问题。
6. **分类**：最后，我们将问题分为9个主题，并为每个问题附上相关的细粒度关键词。
## 局限性
当前数据集存在一定的局限性：
1. **分布不平衡**：数据集在不同领域的问题分布不均，导致某些区域的问题集中度较高，而其他区域则较少。
2. **主题覆盖范围不完整**：数据集中缺少一些网络安全主题，如内容安全、逆向工程和恶意软件分析。因此，它并未涵盖该领域内的全部知识范围。
## 未来工作
1. **分布上的改进**：我们旨在通过增加更多问题来提高数据集的全面性，从而丰富现有网络安全主题的覆盖范围。
2. **主题覆盖上的改进**：我们将努力在数据集中包含更广泛的网络安全主题，这将有助于实现各个领域问题分布的更公平分配。
## 许可证
数据集在 [CC BY-NC-SA 4.0](https://creativecommons.org/licenses/by-nc-sa/4.0/) 许可证下发布。代码在 [MIT](https://opensource.org/licenses/MIT) 许可证下发布。
## 引用
```bibtex
@misc{li2023seceval,
title={SecEval: A Comprehensive Benchmark for Evaluating Cybersecurity Knowledge of Foundation Models},
author={Li, Guancheng and Li, Yifeng and Wang Guannan and Yang, Haoyu and Yu, Yang},
publisher = {GitHub},
howpublished= "https://github.com/XuanwuAI/SecEval",
year={2023}
}
```
## 致谢
这项工作是在[腾讯安全玄武实验室](https://xlab.tencent.com/en/)的支持下完成的，并获得了腾讯星火计划的帮助。

36776
benchmarks/seceval/eval/datasets/questions.json
View File

File diff suppressed because it is too large Load Diff

306
benchmarks/seceval/eval/eval.py
View File

@ -1,23 +1,3 @@
"""
SecEval Evaluation Script
This script evaluates language models on cybersecurity multiple-choice questions.
It supports various LLM backends including HuggingFace, Azure OpenAI, TextGen, and Ollama.
The script processes questions in batches and calculates accuracy scores by topic.
Usage:
python3 eval.py -d dataset.json -B backend_type -m model_name [options]
Example:
python3 eval.py --dataset_file datasets/questions.json --output_dir outputs --model ollama/qwen2.5:14b
Environment Variables:
- OPENAI_API_ENDPOINT: Azure OpenAI endpoint URL
- OPENAI_API_KEY: Azure OpenAI API key
- TEXTGEN_MODEL_URL: URL for TextGen model
- LOCAL_HF_MODEL_DIR: Directory containing local HuggingFace models
- OLLAMA_API_BASE: Base URL for Ollama API (default: http://localhost:8000)
"""
import argparse
from typing import Any, Dict, List
from dotenv import load_dotenv
@ -39,59 +19,60 @@ from pathlib import Path
import os
import logging
# Configure logging
logger = logging.getLogger(__name__)
import time
from langchain.globals import set_llm_cache
from langchain.cache import SQLiteCache
import litellm
# Set up caching for LLM responses
set_llm_cache(
SQLiteCache(
database_path=str(Path(__file__).parent.parent / ".langchain.db")
)
)
# Configure logging settings
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(message)s",
)
logger.addHandler(logging.FileHandler(f"./eval-{int(time.time())}.log", "w"))
# Instruction template for the LLM
instruction = """Below are multiple-choice questions concerning cybersecurity.
Please select the correct answers and respond with the correct letters A, B, C, or D.
You could select more than one letter.
instruction = "Below are multiple-choice questions concerning cybersecurity. Please select the correct answers and respond with the letters ABCD only."
chat_few_shot = [
HumanMessage(
content="Question: Which mitigation prevent stack overflow bug? A: Stack Canary. B: ALSR. C: CFI. D: Code Signing.",
),
AIMessage(
content="Answer: ABC",
),
]
few_shot = """
Question: You are reviewing a piece of code to identify potential vulnerabilities related to NULL pointer dereferences. Which of the following strategies should you consider implementing to mitigate the risk? Choose all that apply. A: Use a programming language that inherently manages pointers and prevents direct memory access. B: Apply input validation to all variables and data stores that receive data from external sources to ensure they are initialized to expected values. C: Explicitly initialize all variables and other data stores, either during declaration or just before the first usage. D: Implement a custom memory management system that automatically assigns a unique identifier to each pointer.
Answer: ABC
Question: Analyze the code snippet provided. Which of the following practices should be implemented in a multithreaded application to prevent NULL pointer dereference vulnerabilities? Select all that apply.A: Ensure that proper locking APIs are used to lock the pointer before the if statement and unlock when it has finished to prevent race conditions. B: Check the return value of all functions that return a value, including network-related functions, and verify that it is non-null before using it.C: Use automated static analysis tools that target this type of weakness, understanding that while not perfect, they can still be effective.D: Verify that a non-nil 'response' is present before deferring 'response.Body.Close()' to handle cases where the 'Do' method returns an error.
Answer: ABCD
Question: In a bank management application, a method called `createBankAccount` allows the creation of bank accounts without user identity verification or authority checks. Based on common security practices, which of the following measures would improve security?A: Implement a centralized authentication mechanism for operations that require a proven user identity.B: Duplicate client-side authentication checks on the server side to prevent bypassing of security checks.C: Use a vetted library or framework that offers robust authentication capabilities to prevent security weaknesses.D: Apply strong authentication requirements for all users accessing data stored in the cloud.
Answer: ABCD
Question: What are the potential consequences if an application handling sensitive data does not require authentication for data access, and what are the appropriate mitigation strategies?A: Attackers could easily access sensitive data from any location; require strong authentication for data access using cloud provider controls.B: Security checks might be bypassed if only performed on the client side; implement checks on both client and server sides.C: Unauthenticated users could alter product functionality; do not use authentication for critical functionality in products.D: Sensitive data may be accessed without proper credentials; utilize authentication capabilities provided by the framework or operating system.
Answer: ABD
Question: To prevent security vulnerabilities related to deserialization of untrusted data in a Java application, which of the following practices should a developer implement?A: Use the signing/sealing features of the programming language to assure that deserialized data has not been tainted.B: Explicitly define a final readObject() method to throw an exception and prevent deserialization.C: Populate a new object by deserializing data to ensure data flows through safe input validation functions.D: Make fields transient to protect them from deserialization and prevent carrying over sensitive variables.
Answer: ABCD
"""
def init_hf_llm(model_id: str) -> HuggingFacePipeline:
"""
Initialize a HuggingFace language model.
Args:
model_id: The model identifier from HuggingFace
Returns:
HuggingFacePipeline: Initialized model pipeline
Raises:
ImportError: If required dependencies are not installed
"""
# Check transformers and torch installation
def init_hf_llm(model_id: str):
# check transformers and torch installation
try:
import transformers
except ImportError:
raise ImportError("Please install transformers with `pip install transformers`")
try:
import torch
flash_attn_enable = torch.cuda.get_device_capability()[0] >= 8
except ImportError:
raise ImportError("Please install torch with `pip install torch`")
# Initialize HuggingFace pipeline with specified parameters
# todo: add flash_attn_enable to the model_kwargs
llm = HuggingFacePipeline.from_model_id(
model_id=model_id,
task="text-generation",
@ -102,45 +83,18 @@ def init_hf_llm(model_id: str) -> HuggingFacePipeline:
return llm
def init_textgen_llm(model_id: str) -> TextGen:
"""
Initialize a TextGen language model.
Args:
model_id: The model identifier
Returns:
TextGen: Initialized model
Raises:
RuntimeError: If TEXTGEN_MODEL_URL is not set
"""
# Check for required environment variable
def init_textgen_llm(model_id: str):
if os.environ.get("TEXTGEN_MODEL_URL") is None:
raise RuntimeError("Please set TEXTGEN_MODEL_URL")
llm = TextGen(model_url=os.environ["TEXTGEN_MODEL_URL"]) # type: ignore
return llm
def init_azure_openai_llm(model_id: str) -> AzureChatOpenAI:
"""
Initialize an Azure OpenAI language model.
Args:
model_id: The model identifier
Returns:
AzureChatOpenAI: Initialized model
Raises:
RuntimeError: If required environment variables are not set
"""
def init_azure_openai_llm(model_id: str):
if os.environ.get("OPENAI_API_ENDPOINT") is None:
raise RuntimeError("Please set OPENAI_API_ENDPOINT")
if os.environ.get("OPENAI_API_KEY") is None:
raise RuntimeError("Please set OPENAI_API_KEY")
# Configure Azure OpenAI parameters
azure_params = {
"model": model_id,
"openai_api_base": os.environ["OPENAI_API_ENDPOINT"],
@ -151,116 +105,15 @@ def init_azure_openai_llm(model_id: str) -> AzureChatOpenAI:
return AzureChatOpenAI(**azure_params) # type: ignore
def init_ollama_llm(model_id: str) -> 'OllamaChat':
"""
Initialize an Ollama language model.
Args:
model_id: The model identifier
Returns:
OllamaChat: Initialized model wrapper
"""
class OllamaChat:
async def abatch(self, prompts: List[str]) -> List[str]:
responses = []
for prompt in prompts:
try:
ollama_api_base = os.getenv("OLLAMA_API_BASE", "http://localhost:8000")
api_base = ollama_api_base.rstrip('/v1')
completion = litellm.completion(
model=model_id,
messages=[{"role": "user", "content": prompt}],
api_base=api_base,
custom_llm_provider="ollama"
)
if hasattr(completion, "choices") and completion.choices:
content = completion.choices[0].message.content
result = self.extract_answer(content)
if result:
responses.append(result)
else:
print("Incorrect answer format detected.")
responses.append("Error: No result parsed")
except Exception as e:
logging.error(f"Ollama error: {e}")
responses.append(f"Error: {e}")
return responses
def extract_answer(self, text: str) -> str:
match = re.findall(r"[A-D]", text.upper())
return "".join(sorted(set(match))) if match else ""
return OllamaChat()
def init_openrouter_llm(model_id: str):
class OpenRouterChat:
async def abatch(self, prompts: List[str]):
responses = []
for prompt in prompts:
try:
api_base = os.getenv("OPENROUTER_API_BASE", "https://openrouter.ai/api/v1/chat/completions")
api_key = os.getenv("OPENROUTER_API_KEY")
if not api_key:
raise ValueError("OPENROUTER_API_KEY is not defined in the environment variables.")
completion = litellm.completion(
model=model_id,
messages=[{"role": "user", "content": prompt}],
api_base=api_base,
api_key=api_key,
custom_llm_provider="openrouter"
)
if hasattr(completion, "choices") and completion.choices:
content = completion.choices[0].message.content
result = self.extract_answer(content)
if result:
responses.append(result)
else:
print("Formato de respuesta incorrecto.")
responses.append("Error: No se pudo extraer resultado")
except Exception as e:
logging.error(f"OpenRouter error: {e}")
responses.append(f"Error: {e}")
return responses
def extract_answer(self, text: str):
match = re.findall(r"[A-D]", text.upper())
return "".join(sorted(set(match))) if match else ""
return OpenRouterChat()
def load_dataset(dataset_path: str) -> List[Dict[str, Any]]:
"""
Load evaluation dataset from JSON file.
Args:
dataset_path: Path to dataset file
Returns:
List[Dict[str, Any]]: Loaded dataset
"""
def load_dataset(dataset_path: str):
with open(dataset_path, "r") as f:
dataset = json.load(f)
return dataset
async def batch_inference_dataset(llm: BaseLanguageModel, batch: List[Dict[str, Any]], chat: bool = False) -> List[Dict[str, Any]]:
"""
Process a batch of questions through the language model.
Args:
llm: Language model to use
batch: List of questions to process
chat: Whether to use chat format
Returns:
List[Dict[str, Any]]: Processed results with scores
"""
async def batch_inference_dataset(
llm: BaseLanguageModel, batch: List[Dict[str, Any]], chat=False
):
results = []
llm_inputs = []
for dataset_row in batch:
@ -269,9 +122,13 @@ async def batch_inference_dataset(llm: BaseLanguageModel, batch: List[Dict[str,
)
question_text = question_text.replace("\n", " ")
if chat:
llm_input = [SystemMessage(content=instruction)]
llm_input = (
[SystemMessage(content=instruction)]
+ chat_few_shot
+ [HumanMessage(content=question_text)]
)
else:
llm_input = instruction + "\n"
llm_input = instruction + few_shot + question_text + "\n"
llm_inputs.append(llm_input)
try:
@ -298,14 +155,6 @@ async def batch_inference_dataset(llm: BaseLanguageModel, batch: List[Dict[str,
logging.info(
f'llm_output: {llm_output}, parsed answer: {batch[idx]["llm_answer"]}, answer: {batch[idx]["answer"]}'
)
print("Question:", batch[idx]["question"])
print("Correct Answer:", batch[idx]["answer"])
print("LLM Answer:", batch[idx]["llm_answer"])
print("LLM Output:", llm_output)
print("Score:", batch[idx]["score"])
print("--------------------------------")
results.append(batch[idx])
return results
@ -358,17 +207,55 @@ def count_score_by_topic(dataset: List[Dict[str, Any]]):
def main():
parser = argparse.ArgumentParser(description="SecEval Evaluation CLI")
parser.add_argument("-o", "--output_dir", type=str, default="/tmp", help="Specify the output directory.")
parser.add_argument("-d", "--dataset_file", type=str, required=True, help="Specify the dataset file to evaluate on.")
parser.add_argument("-c", "--chat", action="store_true", default=False, help="Evaluate on chat model.")
parser.add_argument("-b", "--batch_size", type=int, default=1, help="Specify the batch size.")
parser.add_argument("-B", "--backend", type=str, choices=["remote_hf", "azure", "textgen", "local_hf", "ollama", "openrouter"], required=True, help="Specify the llm type. remote_hf: remote huggingface model backed, azure: azure openai model, textgen: textgen backend, local_hf: local huggingface model backed, ollama: ollama model, openrouter: openrouter model")
parser.add_argument("-m", "--models", type=str, nargs="+", required=True, help="Specify the models.")
args = parser.parse_args()
models = list(args.models)
parser.add_argument(
"-o",
"--output_dir",
type=str,
default="/tmp",
help="Specify the output directory.",
)
parser.add_argument(
"-d",
"--dataset_file",
type=str,
required=True,
help="Specify the dataset file to evaluate on.",
)
parser.add_argument(
"-c",
"--chat",
action="store_true",
default=False,
help="Evaluate on chat model.",
)
parser.add_argument(
"-b",
"--batch_size",
type=int,
default=1,
help="Specify the batch size.",
)
parser.add_argument(
"-B",
"--backend",
type=str,
choices=["remote_hf", "azure", "textgen", "local_hf"],
required=True,
help="Specify the llm type. remote_hf: remote huggingface model backed, azure: azure openai model, textgen: textgen backend, local_hf: local huggingface model backed",
)
parser.add_argument(
"-m",
"--models",
type=str,
nargs="+",
required=True,
help="Specify the models.",
)
logging.info(f"Evaluating models: {models}")
args = parser.parse_args()
models = list(args.models)
logging.info(f"evaluating models: {models}")
for model_id in models:
if args.backend == "remote_hf":
llm = init_hf_llm(model_id)
@ -384,31 +271,28 @@ def main():
llm = init_textgen_llm(model_id)
elif args.backend == "azure":
llm = init_azure_openai_llm(model_id)
elif args.backend == "ollama":
llm = init_ollama_llm(model_id)
elif args.backend == "openrouter":
llm = init_openrouter_llm(model_id)
else:
raise RuntimeError("Unknown backend")
dataset = load_dataset(args.dataset_file)
result = inference_dataset(llm, dataset, batch_size=args.batch_size, chat=args.chat)
result = inference_dataset(
llm, dataset, batch_size=args.batch_size, chat=args.chat
)
score_fraction, score_float = count_score_by_topic(result)
result_with_score = {
"score_fraction": score_fraction,
"score_float": score_float,
"detail": result,
}
# Create output directory if it doesn't exist
output_dir = Path(args.output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
output_path = output_dir / f"{Path(args.dataset_file).stem}_{os.path.basename(model_id)}.json"
logger.info(f"Writing result to {output_path}")
output_path = (
Path(args.output_dir)
/ f"{Path(args.dataset_file).stem}_{os.path.basename(model_id)}.json"
)
logger.info(f"writing result to {output_path}")
with open(output_path, "w") as f:
json.dump(result_with_score, f, indent=4)
del llm
if __name__ == "__main__":
main()
main()

157
benchmarks/seceval/eval/eval_ollama.py
View File

@ -1,157 +0,0 @@
"python3 benchmarks/seceval/eval/eval_ollama.py --dataset_file benchmarks/seceval/eval/datasets/questions.json --output_dir benchmarks/seceval/eval/outputs --model ollama/qwen2.5:14b"
import argparse
import asyncio
import json
import logging
import os
import re
from pathlib import Path
from typing import Any, Dict, List
from dotenv import load_dotenv
from langchain.schema.messages import SystemMessage, HumanMessage
from langchain.adapters.openai import convert_message_to_dict
import litellm
load_dotenv()
def init_ollama_llm(model_id: str):
class OllamaChat:
async def abatch(self, prompts: List[str]):
responses = []
for prompt in prompts:
try:
ollama_api_base = os.getenv("OLLAMA_API_BASE", "http://localhost:8000")
api_base = ollama_api_base.rstrip('/v1')
completion = litellm.completion(
model=model_id,
messages=[{"role": "user", "content": prompt}],
api_base=api_base,
custom_llm_provider="ollama"
)
if hasattr(completion, "choices") and completion.choices:
content = completion.choices[0].message.content
result = self.extract_answer(content)
if result:
responses.append(result)
else:
print("Incorrect answer format detected.")
responses.append("Error: No result parsed")
except Exception as e:
logging.error(f"Ollama error: {e}")
responses.append(f"Error: {e}")
return responses
def extract_answer(self, text: str):
match = re.findall(r"[A-D]", text.upper())
return "".join(sorted(set(match))) if match else ""
return OllamaChat()
def load_dataset(dataset_path: str):
with open(dataset_path, "r") as f:
return json.load(f)
async def batch_inference_dataset(llm, batch: List[Dict[str, Any]], chat=False):
results = []
llm_inputs = []
instruction = """Below are multiple-choice questions concerning cybersecurity.
Please select the correct answer.
Your exprected answer is JUST THE CORRECT LETTE, not a word or phrase."""
for row in batch:
question_text = "Question: " + row["question"] + " " + " ".join(row["choices"])
question_text = question_text.replace("\n", " ")
if chat:
llm_input = [SystemMessage(content=instruction), HumanMessage(content=question_text)]
else:
llm_input = instruction + question_text + "\n"
llm_inputs.append(llm_input)
try:
llm_outputs = await llm.abatch(llm_inputs)
except Exception as e:
logging.error(f"Error in batch: {e}")
llm_outputs = [f"{e}"] * len(llm_inputs)
for idx, output in enumerate(llm_outputs):
output = output.replace("Answer:", "").strip()
row = batch[idx]
row["llm_input"] = llm_inputs[idx]
row["llm_output"] = output
row["llm_answer"] = "".join(sorted(set(re.findall(r"[A-D]", output))))
row["score"] = int(row["llm_answer"].lower() == row["answer"].lower())
results.append(row)
return results
def inference_dataset(llm, dataset: List[Dict[str, Any]], batch_size: int = 1, chat=False):
def chunks(lst, n):
for i in range(0, len(lst), n):
yield lst[i:i + n]
loop = asyncio.get_event_loop()
batches = list(chunks(dataset, batch_size))
results = []
for idx, batch in enumerate(batches):
print(f"Processing batch {idx+1}/{len(batches)}")
results += loop.run_until_complete(batch_inference_dataset(llm, batch, chat))
return results
def count_score_by_topic(dataset: List[Dict[str, Any]]):
score_by_topic = {}
total_by_topic = {}
total_score = 0
for row in dataset:
for topic in row["topics"]:
score_by_topic[topic] = score_by_topic.get(topic, 0) + row["score"]
total_by_topic[topic] = total_by_topic.get(topic, 0) + 1
total_score += row["score"]
fraction = {k: f"{score_by_topic[k]}/{total_by_topic[k]}" for k in score_by_topic}
percent = {k: round(score_by_topic[k] / total_by_topic[k] * 100, 2) for k in score_by_topic}
percent["Overall"] = round(total_score / len(dataset) * 100, 2)
fraction["Overall"] = f"{total_score}/{len(dataset)}"
return fraction, percent
def main():
parser = argparse.ArgumentParser(description="Ollama LLM Benchmark")
parser.add_argument("-d", "--dataset_file", type=str, required=True)
parser.add_argument("-o", "--output_dir", type=str, default="./results")
parser.add_argument("-b", "--batch_size", type=int, default=1)
parser.add_argument("-m", "--model", type=str, required=True)
parser.add_argument("-c", "--chat", action="store_true", default=False)
args = parser.parse_args()
os.makedirs(args.output_dir, exist_ok=True)
model_id = args.model
llm = init_ollama_llm(model_id)
dataset = load_dataset(args.dataset_file)
results = inference_dataset(llm, dataset, batch_size=args.batch_size, chat=args.chat)
score_frac, score_pct = count_score_by_topic(results)
output = {
"score_fraction": score_frac,
"score_percent": score_pct,
"details": results
}
output_path = Path(args.output_dir) / f"{Path(args.dataset_file).stem}_{model_id}.json"
with open(output_path, "w") as f:
json.dump(output, f, indent=2)
print(f"Results written to {output_path}")
if __name__ == "__main__":
main()

4
benchmarks/seceval/eval/requirements.txt
View File

@ -5,7 +5,7 @@ protobuf
scipy
sentencepiece
torch
flash_attn==2.3.*
python-dotenv
transformers_stream_generator
tiktoken
langchain-community
tiktoken