From abb698591b6a7751228bc96501e7db3a07b94003 Mon Sep 17 00:00:00 2001 From: lidia9 Date: Thu, 8 May 2025 12:01:39 +0200 Subject: [PATCH] cybermetric integration ollama+openrouter --- .../cybermetric/CyberMetric_evaluator.py | 88 ------------- .../CyberMetric_evaluator_ollama.py | 123 ------------------ ...CyberMetric_evaluator_openrouter_ollama.py | 36 ++++- 3 files changed, 35 insertions(+), 212 deletions(-) delete mode 100644 benchmarks/cybermetric/CyberMetric_evaluator.py delete mode 100644 benchmarks/cybermetric/CyberMetric_evaluator_ollama.py diff --git a/benchmarks/cybermetric/CyberMetric_evaluator.py b/benchmarks/cybermetric/CyberMetric_evaluator.py deleted file mode 100644 index 7dffd00a..00000000 --- a/benchmarks/cybermetric/CyberMetric_evaluator.py +++ /dev/null @@ -1,88 +0,0 @@ -import json -import re -import time -from tqdm import tqdm -from openai import OpenAI -import os - -class CyberMetricEvaluator: - def __init__(self, api_key, file_path): - self.client = OpenAI(api_key=api_key) - self.file_path = file_path - - def read_json_file(self): - with open(self.file_path, 'r') as file: - return json.load(file) - - @staticmethod - def extract_answer(response): - if response.strip(): # Checks if the response is not empty and not just whitespace - match = re.search(r"ANSWER:?\s*([A-D])", response, re.IGNORECASE) - if match: - return match.group(1).upper() # Return the matched letter in uppercase - return None - - def ask_llm(self, question, answers, max_retries=5): - options = ', '.join([f"{key}) {value}" for key, value in answers.items()]) - prompt = f"Question: {question}\nOptions: {options}\n\nChoose the correct answer (A, B, C, or D) only. Always return in this format: 'ANSWER: X' " - for attempt in range(max_retries): - try: - response = self.client.chat.completions.create( - model="gpt-3.5-turbo-0125", - messages=[ - {"role": "system", "content": "You are a security expert who answers questions."}, - {"role": "user", "content": prompt}, - ] - ) - if response.choices: - result = self.extract_answer(response.choices[0].message.content) - if result: - return result - else: - print("Incorrect answer format detected. Attempting the question again.") - except Exception as e: - print(f"Error: {e}. Attempting the question again in {2 ** attempt} seconds.") - time.sleep(2 ** attempt) - return None - - def run_evaluation(self): - json_data = self.read_json_file() - questions_data = json_data['questions'] - - correct_count = 0 - incorrect_answers = [] - - with tqdm(total=len(questions_data), desc="Processing Questions") as progress_bar: - for item in questions_data: - question = item['question'] - answers = item['answers'] - correct_answer = item['solution'] - - llm_answer = self.ask_llm(question, answers) - if llm_answer == correct_answer: - correct_count += 1 - else: - incorrect_answers.append({ - 'question': question, - 'correct_answer': correct_answer, - 'llm_answer': llm_answer - }) - - accuracy_rate = correct_count / (progress_bar.n + 1) * 100 - progress_bar.set_postfix_str(f"Accuracy: {accuracy_rate:.2f}%") - progress_bar.update(1) - - print(f"Final Accuracy: {correct_count / len(questions_data) * 100}%") - - if incorrect_answers: - print("\nIncorrect Answers:") - for item in incorrect_answers: - print(f"Question: {item['question']}") - print(f"Expected Answer: {item['correct_answer']}, LLM Answer: {item['llm_answer']}\n") - -# Example usage: -if __name__ == "__main__": - API_KEY="" - file_path='CyberMetric-500-v1.json' - evaluator = CyberMetricEvaluator(api_key=API_KEY, file_path=file_path) - evaluator.run_evaluation() diff --git a/benchmarks/cybermetric/CyberMetric_evaluator_ollama.py b/benchmarks/cybermetric/CyberMetric_evaluator_ollama.py deleted file mode 100644 index fd0f608b..00000000 --- a/benchmarks/cybermetric/CyberMetric_evaluator_ollama.py +++ /dev/null @@ -1,123 +0,0 @@ -import json -import re -import time -from tqdm import tqdm -import litellm - - -class CyberMetricEvaluator: - def __init__(self, model_name, file_path): - self.model_name = model_name - self.file_path = file_path - self.report_file = f"report_failed_questions.json" - self.failed_questions = [] - self.failed_count = 0 - print("--DEBUG: model_name: ", self.model_name) - - def read_json_file(self): - with open(self.file_path, 'r') as file: - return json.load(file) - - def save_failed_questions(self): - report_data = {"questions": self.failed_questions} - - with open(self.report_file, 'w') as file: - json.dump(report_data, file, indent=4) - - print(f"Updated failed questions report in {self.report_file}") - - @staticmethod - def extract_answer(response): - if response.strip(): - match = re.search(r"ANSWER:?\s*([A-D])", response, re.IGNORECASE) - if match: - return match.group(1).upper() - return None - - def ask_llm(self, question, answers, max_retries=5): - options = ', '.join([f"{key}) {value}" for key, value in answers.items()]) - prompt = ( - f"Question: {question}\n" - f"Options: {options}\n\n" - "Choose the correct answer (A, B, C, or D) only. Always return in this format: 'ANSWER: X'" - ) - print("--DEBUG: prompt: ", prompt) - for attempt in range(max_retries): - try: - response = litellm.completion( - model=self.model_name, - messages=[ - {"role": "system", "content": "You are a security expert who answers questions."}, - {"role": "user", "content": prompt}, - ], - api_base="http://localhost:8000" - ) - if hasattr(response, "choices") and response.choices: - content = response.choices[0].message.content - result = self.extract_answer(content) - if result: - print("--DEBUG: result: ", result) - return result - else: - print("Incorrect answer format detected. Attempting the question again.") - except Exception as e: - print(f"Error: {e}. Attempting the question again in {2 ** attempt} seconds.") - time.sleep(2 ** attempt) - return None - - def run_evaluation(self): - json_data = self.read_json_file() - questions_data = json_data['questions'] - - correct_count = 0 - incorrect_answers = [] - - with tqdm(total=len(questions_data), desc="Processing Questions") as progress_bar: - for item in questions_data: - question = item['question'] - answers = item['answers'] - correct_answer = item['solution'] - - llm_answer = self.ask_llm(question, answers) - if llm_answer == correct_answer: - correct_count += 1 - else: - self.failed_questions.append({ - 'question': question, - 'answers': answers, - 'solution': correct_answer, - 'llm_answer': llm_answer - }) - self.failed_count += 1 - - if self.failed_count % 2 == 0: - self.save_failed_questions() - - incorrect_answers.append({ - 'question': question, - 'correct_answer': correct_answer, - 'llm_answer': llm_answer - }) - - accuracy_rate = correct_count / (progress_bar.n + 1) * 100 - progress_bar.set_postfix_str(f"Accuracy: {accuracy_rate:.2f}%") - progress_bar.update(1) - - print(f"\nFinal Accuracy: {correct_count / len(questions_data) * 100:.2f}%") - - if self.failed_questions: - self.save_failed_questions() #final failed questions - - if incorrect_answers: - print("\nIncorrect Answers:") - for item in incorrect_answers: - print(f"Question: {item['question']}") - print(f"Expected Answer: {item['correct_answer']}, LLM Answer: {item['llm_answer']}\n") - -if __name__ == "__main__": - #litellm._turn_on_debug() - - file_path = 'CyberMetric-10000-v1.json' - - evaluator = CyberMetricEvaluator(model_name="ollama/qwen3:32b-q8_0-ctx-32768", file_path=file_path) # ollama/qwen3:32b-q8_0-ctx-32768" - evaluator.run_evaluation() diff --git a/benchmarks/cybermetric/CyberMetric_evaluator_openrouter_ollama.py b/benchmarks/cybermetric/CyberMetric_evaluator_openrouter_ollama.py index f448ddb7..925ed9ba 100644 --- a/benchmarks/cybermetric/CyberMetric_evaluator_openrouter_ollama.py +++ b/benchmarks/cybermetric/CyberMetric_evaluator_openrouter_ollama.py @@ -1,3 +1,36 @@ +""" +CyberMetric Evaluator for LLMs + +This script evaluates the performance of language models on the CyberMetric benchmark. +It supports both OpenRouter-hosted models and local Ollama models via LiteLLM proxy. + +Usage: + python CyberMetric_evaluator_openrouter_ollama.py --model_name MODEL_NAME [--file_path FILE_PATH] [--api_key API_KEY] + +Arguments: + --model_name: Required. Model name with prefix (openrouter/ or ollama/) + Examples: openrouter/anthropic/claude-3-opus, ollama/llama3 + --file_path: Optional. Path to the CyberMetric JSON file (default: CyberMetric-2-v1.json) + --api_key: Optional. API key for OpenRouter (can also use OPENROUTER_API_KEY env var) + +Environment Variables: + OPENROUTER_API_KEY: API key for OpenRouter (if using OpenRouter models) + OPENROUTER_API_BASE: Base URL for OpenRouter API (default: https://openrouter.ai/api/v1) + OLLAMA_API_BASE: Base URL for Ollama API via LiteLLM proxy (default: http://localhost:8000/v1) + +Examples: + # Run with an OpenRouter model + python CyberMetric_evaluator_openrouter_ollama.py --model_name openrouter/qwen/qwen3-32b:free + + # Run with a local Ollama model (requires LiteLLM proxy running) + python CyberMetric_evaluator_openrouter_ollama.py --model_name ollama/qwen2.5:14b + + # Specify a different benchmark file + python CyberMetric_evaluator_openrouter_ollama.py --model_name openrouter/qwen/qwen3-32b:free --file_path CyberMetric-10000-v1.json + + +""" + import json import re import time @@ -85,7 +118,7 @@ class CyberMetricEvaluator: with open(self.info_file, 'w') as file: file.write(f"CyberMetric Evaluation\n") file.write(f"=====================\n\n") - file.write(f"Model: {self.model_name}\n") + file.write(f"Model: {self.model_name}\n") # "openrouter/qwen/qwen3-32b:free" file.write(f"Dataset: {self.file_path}\n") file.write(f"Start Time: {self.start_time.strftime('%Y-%m-%d %H:%M:%S')}\n") file.write(f"Status: {status}\n") @@ -298,3 +331,4 @@ if __name__ == "__main__": # Run the evaluation evaluator.run_evaluation() +