diff --git a/benchmarks/cybermetric/CyberMetric-10-v1.json b/benchmarks/cybermetric/CyberMetric-10-v1.json new file mode 100644 index 00000000..e15e3065 --- /dev/null +++ b/benchmarks/cybermetric/CyberMetric-10-v1.json @@ -0,0 +1,104 @@ +{ + "questions": [ + { + "question": "Which of the following is a desirable property of a biometric system?", + "answers": { + "A": "Permanent", + "B": "Transferability", + "C": "Uniformity", + "D": "Forgiveness" + }, + "solution": "A" + }, + { + "question": "In TCP/IP networking, which protocol is used to hold network addresses and routing information in a packet?", + "answers": { + "A": "HTTP", + "B": "IP", + "C": "Routing Information Protocol (RIP)", + "D": "TCP" + }, + "solution": "B" + }, + { + "question": "In the context of unexpected negative outcomes in personal privacy policies, what responsibility lies with both the provider and the consumer regarding the retention time of private information?", + "answers": { + "A": "Both the provider and the consumer are responsible for setting the appropriate retention time", + "B": "Neither the provider nor the consumer has a responsibility for setting the appropriate retention time", + "C": "Only the consumer is responsible for setting an appropriate retention time", + "D": "Only the provider is responsible for ensuring the appropriate retention time" + }, + "solution": "A" + }, + { + "question": "What type of attack involves an attacker behaving as a store-and-forward or proxy mechanism between two communicating systems?", + "answers": { + "A": "Buffer overflow attack", + "B": "Man-in-the-Middle attack", + "C": "DDoS attack", + "D": "Sniffer attack" + }, + "solution": "B" + }, + { + "question": "What is the fundamental purpose of logging and monitoring in an organization's security measures?", + "answers": { + "A": "To store backups of critical data", + "B": "To track, record, and review activity to detect and respond to security incidents", + "C": "To manage and enforce user access controls", + "D": "To create a record of all employee activities" + }, + "solution": "B" + }, + { + "question": "What is the benefit of structured walk-throughs in disaster recovery testing?", + "answers": { + "A": "Observe live actions in a controlled environment", + "B": "Conduct a full shut-down and restoration at the primary site", + "C": "Test operational response to disaster scenarios", + "D": "Interrupt real operations at the primary site" + }, + "solution": "C" + }, + { + "question": "Which security process metric would most assist in determining an appropriate backup frequency for a database server?", + "answers": { + "A": "RTO", + "B": "MTBF", + "C": "RPO", + "D": "MTD" + }, + "solution": "C" + }, + { + "question": "What constitutional amendment outlines the burden placed on investigators to have a valid search warrant before conducting certain searches?", + "answers": { + "A": "Third Amendment", + "B": "First Amendment", + "C": "Second Amendment", + "D": "Fourth Amendment" + }, + "solution": "D" + }, + { + "question": "Which tool is frequently used to directly access websites without a browser and is often utilized for testing and identifying potential API vulnerabilities?", + "answers": { + "A": "SSH", + "B": "cURL", + "C": "FTP", + "D": "Telnet" + }, + "solution": "B" + }, + { + "question": "What is the distinguishing characteristic of symmetric-key cryptography?", + "answers": { + "A": "It tends to be CPU intensive", + "B": "It uses the same key for encryption and decryption", + "C": "It provides integrity protection to data", + "D": "It uses different but related keys for encryption and decryption" + }, + "solution": "B" + } + ] +} \ No newline at end of file diff --git a/benchmarks/cybermetric/CyberMetric_evaluator.py b/benchmarks/cybermetric/CyberMetric_evaluator.py index 27f75385..7dffd00a 100644 --- a/benchmarks/cybermetric/CyberMetric_evaluator.py +++ b/benchmarks/cybermetric/CyberMetric_evaluator.py @@ -3,6 +3,7 @@ import re import time from tqdm import tqdm from openai import OpenAI +import os class CyberMetricEvaluator: def __init__(self, api_key, file_path): diff --git a/benchmarks/cybermetric/CyberMetric_evaluator_ollama.py b/benchmarks/cybermetric/CyberMetric_evaluator_ollama.py new file mode 100644 index 00000000..260abf69 --- /dev/null +++ b/benchmarks/cybermetric/CyberMetric_evaluator_ollama.py @@ -0,0 +1,103 @@ +import json +import re +import time +from tqdm import tqdm +import os +import litellm + + +class CyberMetricEvaluator: + def __init__(self, model_name, file_path): + self.model_name = model_name # E.g., "ollama/llama3" + self.file_path = file_path + print("--DEBUG: model_name: ", self.model_name) + + def read_json_file(self): + with open(self.file_path, 'r') as file: + return json.load(file) + + @staticmethod + def extract_answer(response): + if response.strip(): + match = re.search(r"ANSWER:?\s*([A-D])", response, re.IGNORECASE) + if match: + return match.group(1).upper() + return None + + def ask_llm(self, question, answers, max_retries=5): + options = ', '.join([f"{key}) {value}" for key, value in answers.items()]) + prompt = ( + f"Question: {question}\n" + f"Options: {options}\n\n" + "Choose the correct answer (A, B, C, or D) only. Always return in this format: 'ANSWER: X'" + ) + print("--DEBUG: prompt: ", prompt) + for attempt in range(max_retries): + try: + response = litellm.completion( + model=self.model_name, + messages=[ + {"role": "system", "content": "You are a security expert who answers questions."}, + {"role": "user", "content": prompt}, + ], + api_base="http://localhost:8000" #"http://localhost:11434" # Ollama API endpoint + ) + if hasattr(response, "choices") and response.choices: + content = response.choices[0].message.content + result = self.extract_answer(content) + if result: + print("--DEBUG: result: ", result) + return result + else: + print("Incorrect answer format detected. Attempting the question again.") + except Exception as e: + print(f"Error: {e}. Attempting the question again in {2 ** attempt} seconds.") + time.sleep(2 ** attempt) + return None + + def run_evaluation(self): + json_data = self.read_json_file() + questions_data = json_data['questions'] + + correct_count = 0 + incorrect_answers = [] + + with tqdm(total=len(questions_data), desc="Processing Questions") as progress_bar: + for item in questions_data: + question = item['question'] + answers = item['answers'] + correct_answer = item['solution'] + + llm_answer = self.ask_llm(question, answers) + if llm_answer == correct_answer: + correct_count += 1 + else: + incorrect_answers.append({ + 'question': question, + 'correct_answer': correct_answer, + 'llm_answer': llm_answer + }) + + accuracy_rate = correct_count / (progress_bar.n + 1) * 100 + progress_bar.set_postfix_str(f"Accuracy: {accuracy_rate:.2f}%") + progress_bar.update(1) + + print(f"\nFinal Accuracy: {correct_count / len(questions_data) * 100:.2f}%") + + if incorrect_answers: + print("\nIncorrect Answers:") + for item in incorrect_answers: + print(f"Question: {item['question']}") + print(f"Expected Answer: {item['correct_answer']}, LLM Answer: {item['llm_answer']}\n") + +if __name__ == "__main__": + # Enable debug logging for litellm + #litellm._turn_on_debug() + + file_path = 'CyberMetric-10-v1.json' #small set for testing + + # Use the exact model name as it appears in Ollama + # For Ollama models, you should use "ollama/model_name" + # The "ollama/" prefix tells litellm to use Ollama + evaluator = CyberMetricEvaluator(model_name="ollama/qwen2.5:14b", file_path=file_path) + evaluator.run_evaluation()