diff --git a/benchmarks/cybermetric/CyberMetric-10-v1.json b/benchmarks/cybermetric/CyberMetric-10-v1.json new file mode 100644 index 00000000..97f2d570 --- /dev/null +++ b/benchmarks/cybermetric/CyberMetric-10-v1.json @@ -0,0 +1,125 @@ +{ + "questions": [ + +{ + "question": "What is the primary requirement for an Random Bit Generator's (RBG) output to be used for generating cryptographic keys?", + "answers": { + "A": "The RBG's output should have a length that matches the target data to be protected", + "B": "The RBG's output should be computationally indistinguishable from random bits and provide sufficient entropy to support the security strength required for the target data", + "C": "The RBG's output should be as long as possible to ensure maximal randomness", + "D": "The RBG's output should have precisely the same length as the symmetric key to be generated" + }, + "solution": "B" +}, +{ + "question": "In cryptography, what is the purpose of using a key-derivation function (KDF)?", + "answers": { + "A": "Generate public keys", + "B": "Authenticate digital signatures", + "C": "Encrypt data using a password", + "D": "KDF are algorithms used to transform a secret into crucial parameters like keys and Initialization Vectors (IVs)" + }, + "solution": "D" +}, +{ + "question": "What is the primary purpose of segmentation in the context of PCI DSS?", + "answers": { + "A": "To reduce the number of PCI DSS requirements applicable to an entity", + "B": "To limit the scope of the PCI DSS assessment and minimize the potential for security breaches", + "C": "To completely remove the applicability of PCI DSS for a merchant's cardholder data environment", + "D": "To eliminate the need for implementing PCI DSS controls" + }, + "solution": "B" +}, +{ + "question": "What is the purpose of using strong cryptography to protect cardholder data during transmission over open, public networks?", + "answers": { + "A": "To ensure the data is secured from unauthorized access or interception during transmission", + "B": "To increase the speed of data transmission over open, public networks", + "C": "To provide access to authorized individuals to monitor the transmission process", + "D": "To improve the efficiency of network routing protocols" + }, + "solution": "A" +}, +{ + "question": "What is the purpose of periodic reviews of user accounts and access privileges?", + "answers": { + "A": "To ensure all user accounts have access to the highest privileges", + "B": "To demonstrate adherence to regulatory requirements", + "C": "To create a record of all user accounts", + "D": "To identify and remove any inappropriate access and privileges" + }, + "solution": "D" +}, +{ + "question": "Which of the following is a purpose of having synchronized system clocks?", + "answers": { + "A": "To compare log files from different systems", + "B": "To accelerate system performance", + "C": "To standardize file naming conventions", + "D": "To reduce power consumption" + }, + "solution": "A" +}, +{ + "question": "What is physical security?", + "answers": { + "A": "The protection of data from hacker attacks", + "B": "The prevention of natural disasters caused by environmental factors", + "C": "The act of securing a building or location to prevent unauthorized access, theft, or damage to physical assets", + "D": "The enforcement of technical security controls to prevent data breaches" + }, + "solution": "C" +}, + { + "question": "What does the abbreviation 'SDK' stand for in the context of software security?", + "answers": { + "A": "Software Development Kit", + "B": "System Deflection Key", + "C": "Software Delegation Kernel", + "D": "System Development Key" + }, + "solution": "A" +}, +{ + "question": "What is the main purpose of multi-factor authentication in cybersecurity?", + "answers": { + "A": "To limit user privileges within the network", + "B": "To ensure compliance with government regulations", + "C": "To adds an extra layer of security by requiring multiple forms of verification", + "D": "To monitor network traffic for security threats" + }, + "solution": "C" +}, +{ + "question": "What is the primary objective of a Web Application Firewall (WAF) in cybersecurity?", + "answers": { + "A": "To encrypt network traffic between web servers", + "B": "To prevent unauthorized access to network services", + "C": "To protect web applications from security threats", + "D": "To manage user authentication and authorization for web services" + }, + "solution": "C" +}, +{ + "question": "What is the purpose of periodically changing passwords or passphrases?", + "answers": { + "A": "To provide more time for a malicious individual to crack the password/passphrase", + "B": "To allow for more rapid detection and response to address potentially compromised credentials", + "C": "To prevent unauthorized users from gaining access to the user account through use of a shared authentication factor", + "D": "To offer less time for a malicious individual to crack a password/passphrase and less time to use a compromised password" + }, + "solution": "D" +}, +{ + "question": "Which of the following defines the entity’s security objectives and principles?", + "answers": { + "A": "Security policies", + "B": "Operational procedures", + "C": "Risk assessment framework", + "D": "Incident response plan" + }, + "solution": "A" +} + ] +} \ No newline at end of file diff --git a/benchmarks/cybermetric/CyberMetric_evaluator_ollama.py b/benchmarks/cybermetric/CyberMetric_evaluator_ollama.py index 787cf048..e6e098bb 100644 --- a/benchmarks/cybermetric/CyberMetric_evaluator_ollama.py +++ b/benchmarks/cybermetric/CyberMetric_evaluator_ollama.py @@ -10,12 +10,24 @@ class CyberMetricEvaluator: def __init__(self, model_name, file_path): self.model_name = model_name # E.g., "ollama/llama3" self.file_path = file_path + self.report_file = "report_failed_questions.json" + self.failed_questions = [] + self.failed_count = 0 print("--DEBUG: model_name: ", self.model_name) def read_json_file(self): with open(self.file_path, 'r') as file: return json.load(file) + def save_failed_questions(self): + # Create the report with the same format as original file + report_data = {"questions": self.failed_questions} + + with open(self.report_file, 'w') as file: + json.dump(report_data, file, indent=4) + + print(f"Updated failed questions report in {self.report_file}") + @staticmethod def extract_answer(response): if response.strip(): @@ -72,6 +84,19 @@ class CyberMetricEvaluator: if llm_answer == correct_answer: correct_count += 1 else: + # Add the question to our failed questions list + self.failed_questions.append({ + 'question': question, + 'answers': answers, + 'solution': correct_answer, + 'llm_answer': llm_answer + }) + self.failed_count += 1 + + # Save failed questions to JSON file every 2 failures + if self.failed_count % 2 == 0: + self.save_failed_questions() + incorrect_answers.append({ 'question': question, 'correct_answer': correct_answer, @@ -83,6 +108,10 @@ class CyberMetricEvaluator: progress_bar.update(1) print(f"\nFinal Accuracy: {correct_count / len(questions_data) * 100:.2f}%") + + # Final save of failed questions + if self.failed_questions: + self.save_failed_questions() if incorrect_answers: print("\nIncorrect Answers:") @@ -99,5 +128,5 @@ if __name__ == "__main__": # Use the exact model name as it appears in Ollama # For Ollama models, you should use "ollama/model_name" # The "ollama/" prefix tells litellm to use Ollama - evaluator = CyberMetricEvaluator(model_name="ollama/qwen3:32b-q8_0-ctx-32768", file_path=file_path) + evaluator = CyberMetricEvaluator(model_name="ollama/qwen2.5:14b", file_path=file_path) # ollama/qwen3:32b-q8_0-ctx-32768" evaluator.run_evaluation()