mirror of https://github.com/aliasrobotics/cai.git
cybermetric integration ollama+openrouter
This commit is contained in:
lidia9
parent
caefeb6d8b
commit
abb698591b
|
|
@ -1,88 +0,0 @@
|
|||
import json
|
||||
import re
|
||||
import time
|
||||
from tqdm import tqdm
|
||||
from openai import OpenAI
|
||||
import os
|
||||
|
||||
class CyberMetricEvaluator:
|
||||
def __init__(self, api_key, file_path):
|
||||
self.client = OpenAI(api_key=api_key)
|
||||
self.file_path = file_path
|
||||
|
||||
def read_json_file(self):
|
||||
with open(self.file_path, 'r') as file:
|
||||
return json.load(file)
|
||||
|
||||
@staticmethod
|
||||
def extract_answer(response):
|
||||
if response.strip(): # Checks if the response is not empty and not just whitespace
|
||||
match = re.search(r"ANSWER:?\s*([A-D])", response, re.IGNORECASE)
|
||||
if match:
|
||||
return match.group(1).upper() # Return the matched letter in uppercase
|
||||
return None
|
||||
|
||||
def ask_llm(self, question, answers, max_retries=5):
|
||||
options = ', '.join([f"{key}) {value}" for key, value in answers.items()])
|
||||
prompt = f"Question: {question}\nOptions: {options}\n\nChoose the correct answer (A, B, C, or D) only. Always return in this format: 'ANSWER: X' "
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
response = self.client.chat.completions.create(
|
||||
model="gpt-3.5-turbo-0125",
|
||||
messages=[
|
||||
{"role": "system", "content": "You are a security expert who answers questions."},
|
||||
{"role": "user", "content": prompt},
|
||||
]
|
||||
)
|
||||
if response.choices:
|
||||
result = self.extract_answer(response.choices[0].message.content)
|
||||
if result:
|
||||
return result
|
||||
else:
|
||||
print("Incorrect answer format detected. Attempting the question again.")
|
||||
except Exception as e:
|
||||
print(f"Error: {e}. Attempting the question again in {2 ** attempt} seconds.")
|
||||
time.sleep(2 ** attempt)
|
||||
return None
|
||||
|
||||
def run_evaluation(self):
|
||||
json_data = self.read_json_file()
|
||||
questions_data = json_data['questions']
|
||||
|
||||
correct_count = 0
|
||||
incorrect_answers = []
|
||||
|
||||
with tqdm(total=len(questions_data), desc="Processing Questions") as progress_bar:
|
||||
for item in questions_data:
|
||||
question = item['question']
|
||||
answers = item['answers']
|
||||
correct_answer = item['solution']
|
||||
|
||||
llm_answer = self.ask_llm(question, answers)
|
||||
if llm_answer == correct_answer:
|
||||
correct_count += 1
|
||||
else:
|
||||
incorrect_answers.append({
|
||||
'question': question,
|
||||
'correct_answer': correct_answer,
|
||||
'llm_answer': llm_answer
|
||||
})
|
||||
|
||||
accuracy_rate = correct_count / (progress_bar.n + 1) * 100
|
||||
progress_bar.set_postfix_str(f"Accuracy: {accuracy_rate:.2f}%")
|
||||
progress_bar.update(1)
|
||||
|
||||
print(f"Final Accuracy: {correct_count / len(questions_data) * 100}%")
|
||||
|
||||
if incorrect_answers:
|
||||
print("\nIncorrect Answers:")
|
||||
for item in incorrect_answers:
|
||||
print(f"Question: {item['question']}")
|
||||
print(f"Expected Answer: {item['correct_answer']}, LLM Answer: {item['llm_answer']}\n")
|
||||
|
||||
# Example usage:
|
||||
if __name__ == "__main__":
|
||||
API_KEY="<YOUR-APKI-KEY-HERE>"
|
||||
file_path='CyberMetric-500-v1.json'
|
||||
evaluator = CyberMetricEvaluator(api_key=API_KEY, file_path=file_path)
|
||||
evaluator.run_evaluation()
|
||||
|
|
@ -1,123 +0,0 @@
|
|||
import json
|
||||
import re
|
||||
import time
|
||||
from tqdm import tqdm
|
||||
import litellm
|
||||
|
||||
|
||||
class CyberMetricEvaluator:
|
||||
def __init__(self, model_name, file_path):
|
||||
self.model_name = model_name
|
||||
self.file_path = file_path
|
||||
self.report_file = f"report_failed_questions.json"
|
||||
self.failed_questions = []
|
||||
self.failed_count = 0
|
||||
print("--DEBUG: model_name: ", self.model_name)
|
||||
|
||||
def read_json_file(self):
|
||||
with open(self.file_path, 'r') as file:
|
||||
return json.load(file)
|
||||
|
||||
def save_failed_questions(self):
|
||||
report_data = {"questions": self.failed_questions}
|
||||
|
||||
with open(self.report_file, 'w') as file:
|
||||
json.dump(report_data, file, indent=4)
|
||||
|
||||
print(f"Updated failed questions report in {self.report_file}")
|
||||
|
||||
@staticmethod
|
||||
def extract_answer(response):
|
||||
if response.strip():
|
||||
match = re.search(r"ANSWER:?\s*([A-D])", response, re.IGNORECASE)
|
||||
if match:
|
||||
return match.group(1).upper()
|
||||
return None
|
||||
|
||||
def ask_llm(self, question, answers, max_retries=5):
|
||||
options = ', '.join([f"{key}) {value}" for key, value in answers.items()])
|
||||
prompt = (
|
||||
f"Question: {question}\n"
|
||||
f"Options: {options}\n\n"
|
||||
"Choose the correct answer (A, B, C, or D) only. Always return in this format: 'ANSWER: X'"
|
||||
)
|
||||
print("--DEBUG: prompt: ", prompt)
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
response = litellm.completion(
|
||||
model=self.model_name,
|
||||
messages=[
|
||||
{"role": "system", "content": "You are a security expert who answers questions."},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
api_base="http://localhost:8000"
|
||||
)
|
||||
if hasattr(response, "choices") and response.choices:
|
||||
content = response.choices[0].message.content
|
||||
result = self.extract_answer(content)
|
||||
if result:
|
||||
print("--DEBUG: result: ", result)
|
||||
return result
|
||||
else:
|
||||
print("Incorrect answer format detected. Attempting the question again.")
|
||||
except Exception as e:
|
||||
print(f"Error: {e}. Attempting the question again in {2 ** attempt} seconds.")
|
||||
time.sleep(2 ** attempt)
|
||||
return None
|
||||
|
||||
def run_evaluation(self):
|
||||
json_data = self.read_json_file()
|
||||
questions_data = json_data['questions']
|
||||
|
||||
correct_count = 0
|
||||
incorrect_answers = []
|
||||
|
||||
with tqdm(total=len(questions_data), desc="Processing Questions") as progress_bar:
|
||||
for item in questions_data:
|
||||
question = item['question']
|
||||
answers = item['answers']
|
||||
correct_answer = item['solution']
|
||||
|
||||
llm_answer = self.ask_llm(question, answers)
|
||||
if llm_answer == correct_answer:
|
||||
correct_count += 1
|
||||
else:
|
||||
self.failed_questions.append({
|
||||
'question': question,
|
||||
'answers': answers,
|
||||
'solution': correct_answer,
|
||||
'llm_answer': llm_answer
|
||||
})
|
||||
self.failed_count += 1
|
||||
|
||||
if self.failed_count % 2 == 0:
|
||||
self.save_failed_questions()
|
||||
|
||||
incorrect_answers.append({
|
||||
'question': question,
|
||||
'correct_answer': correct_answer,
|
||||
'llm_answer': llm_answer
|
||||
})
|
||||
|
||||
accuracy_rate = correct_count / (progress_bar.n + 1) * 100
|
||||
progress_bar.set_postfix_str(f"Accuracy: {accuracy_rate:.2f}%")
|
||||
progress_bar.update(1)
|
||||
|
||||
print(f"\nFinal Accuracy: {correct_count / len(questions_data) * 100:.2f}%")
|
||||
|
||||
if self.failed_questions:
|
||||
self.save_failed_questions() #final failed questions
|
||||
|
||||
if incorrect_answers:
|
||||
print("\nIncorrect Answers:")
|
||||
for item in incorrect_answers:
|
||||
print(f"Question: {item['question']}")
|
||||
print(f"Expected Answer: {item['correct_answer']}, LLM Answer: {item['llm_answer']}\n")
|
||||
|
||||
if __name__ == "__main__":
|
||||
#litellm._turn_on_debug()
|
||||
|
||||
file_path = 'CyberMetric-10000-v1.json'
|
||||
|
||||
evaluator = CyberMetricEvaluator(model_name="ollama/qwen3:32b-q8_0-ctx-32768", file_path=file_path) # ollama/qwen3:32b-q8_0-ctx-32768"
|
||||
evaluator.run_evaluation()
|
||||
|
|
@ -1,3 +1,36 @@
|
|||
"""
|
||||
CyberMetric Evaluator for LLMs
|
||||
|
||||
This script evaluates the performance of language models on the CyberMetric benchmark.
|
||||
It supports both OpenRouter-hosted models and local Ollama models via LiteLLM proxy.
|
||||
|
||||
Usage:
|
||||
python CyberMetric_evaluator_openrouter_ollama.py --model_name MODEL_NAME [--file_path FILE_PATH] [--api_key API_KEY]
|
||||
|
||||
Arguments:
|
||||
--model_name: Required. Model name with prefix (openrouter/ or ollama/)
|
||||
Examples: openrouter/anthropic/claude-3-opus, ollama/llama3
|
||||
--file_path: Optional. Path to the CyberMetric JSON file (default: CyberMetric-2-v1.json)
|
||||
--api_key: Optional. API key for OpenRouter (can also use OPENROUTER_API_KEY env var)
|
||||
|
||||
Environment Variables:
|
||||
OPENROUTER_API_KEY: API key for OpenRouter (if using OpenRouter models)
|
||||
OPENROUTER_API_BASE: Base URL for OpenRouter API (default: https://openrouter.ai/api/v1)
|
||||
OLLAMA_API_BASE: Base URL for Ollama API via LiteLLM proxy (default: http://localhost:8000/v1)
|
||||
|
||||
Examples:
|
||||
# Run with an OpenRouter model
|
||||
python CyberMetric_evaluator_openrouter_ollama.py --model_name openrouter/qwen/qwen3-32b:free
|
||||
|
||||
# Run with a local Ollama model (requires LiteLLM proxy running)
|
||||
python CyberMetric_evaluator_openrouter_ollama.py --model_name ollama/qwen2.5:14b
|
||||
|
||||
# Specify a different benchmark file
|
||||
python CyberMetric_evaluator_openrouter_ollama.py --model_name openrouter/qwen/qwen3-32b:free --file_path CyberMetric-10000-v1.json
|
||||
|
||||
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
|
|
@ -85,7 +118,7 @@ class CyberMetricEvaluator:
|
|||
with open(self.info_file, 'w') as file:
|
||||
file.write(f"CyberMetric Evaluation\n")
|
||||
file.write(f"=====================\n\n")
|
||||
file.write(f"Model: {self.model_name}\n")
|
||||
file.write(f"Model: {self.model_name}\n") # "openrouter/qwen/qwen3-32b:free"
|
||||
file.write(f"Dataset: {self.file_path}\n")
|
||||
file.write(f"Start Time: {self.start_time.strftime('%Y-%m-%d %H:%M:%S')}\n")
|
||||
file.write(f"Status: {status}\n")
|
||||
|
|
@ -298,3 +331,4 @@ if __name__ == "__main__":
|
|||
|
||||
# Run the evaluation
|
||||
evaluator.run_evaluation()
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue