From 60c7d7efe5b15bce1b470c5efa1be5870b2f0b4b Mon Sep 17 00:00:00 2001 From: lidia9 Date: Wed, 7 May 2025 15:18:12 +0200 Subject: [PATCH] cybermetric --- benchmarks/cybermetric/CyberMetric-10-v1.json | 285 +++++++++++------- .../CyberMetric_evaluator_ollama.py | 6 +- 2 files changed, 175 insertions(+), 116 deletions(-) diff --git a/benchmarks/cybermetric/CyberMetric-10-v1.json b/benchmarks/cybermetric/CyberMetric-10-v1.json index 97f2d570..75d70f68 100644 --- a/benchmarks/cybermetric/CyberMetric-10-v1.json +++ b/benchmarks/cybermetric/CyberMetric-10-v1.json @@ -1,125 +1,184 @@ { "questions": [ - -{ - "question": "What is the primary requirement for an Random Bit Generator's (RBG) output to be used for generating cryptographic keys?", - "answers": { - "A": "The RBG's output should have a length that matches the target data to be protected", - "B": "The RBG's output should be computationally indistinguishable from random bits and provide sufficient entropy to support the security strength required for the target data", - "C": "The RBG's output should be as long as possible to ensure maximal randomness", - "D": "The RBG's output should have precisely the same length as the symmetric key to be generated" - }, - "solution": "B" -}, -{ - "question": "In cryptography, what is the purpose of using a key-derivation function (KDF)?", - "answers": { - "A": "Generate public keys", - "B": "Authenticate digital signatures", - "C": "Encrypt data using a password", - "D": "KDF are algorithms used to transform a secret into crucial parameters like keys and Initialization Vectors (IVs)" - }, - "solution": "D" -}, -{ - "question": "What is the primary purpose of segmentation in the context of PCI DSS?", - "answers": { - "A": "To reduce the number of PCI DSS requirements applicable to an entity", - "B": "To limit the scope of the PCI DSS assessment and minimize the potential for security breaches", - "C": "To completely remove the applicability of PCI DSS for a merchant's cardholder data environment", - "D": "To eliminate the need for implementing PCI DSS controls" + { + "question": "Which of the following is a desirable property of a biometric system?", + "answers": { + "A": "Permanent", + "B": "Transferability", + "C": "Uniformity", + "D": "Forgiveness" + }, + "solution": "A" }, - "solution": "B" -}, -{ - "question": "What is the purpose of using strong cryptography to protect cardholder data during transmission over open, public networks?", - "answers": { - "A": "To ensure the data is secured from unauthorized access or interception during transmission", - "B": "To increase the speed of data transmission over open, public networks", - "C": "To provide access to authorized individuals to monitor the transmission process", - "D": "To improve the efficiency of network routing protocols" + { + "question": "In TCP/IP networking, which protocol is used to hold network addresses and routing information in a packet?", + "answers": { + "A": "HTTP", + "B": "IP", + "C": "Routing Information Protocol (RIP)", + "D": "TCP" + }, + "solution": "B" }, - "solution": "A" -}, -{ - "question": "What is the purpose of periodic reviews of user accounts and access privileges?", - "answers": { - "A": "To ensure all user accounts have access to the highest privileges", - "B": "To demonstrate adherence to regulatory requirements", - "C": "To create a record of all user accounts", - "D": "To identify and remove any inappropriate access and privileges" + { + "question": "In the context of unexpected negative outcomes in personal privacy policies, what responsibility lies with both the provider and the consumer regarding the retention time of private information?", + "answers": { + "A": "Both the provider and the consumer are responsible for setting the appropriate retention time", + "B": "Neither the provider nor the consumer has a responsibility for setting the appropriate retention time", + "C": "Only the consumer is responsible for setting an appropriate retention time", + "D": "Only the provider is responsible for ensuring the appropriate retention time" + }, + "solution": "A" }, - "solution": "D" -}, -{ - "question": "Which of the following is a purpose of having synchronized system clocks?", - "answers": { - "A": "To compare log files from different systems", - "B": "To accelerate system performance", - "C": "To standardize file naming conventions", - "D": "To reduce power consumption" + { + "question": "What type of attack involves an attacker behaving as a store-and-forward or proxy mechanism between two communicating systems?", + "answers": { + "A": "Buffer overflow attack", + "B": "Man-in-the-Middle attack", + "C": "DDoS attack", + "D": "Sniffer attack" + }, + "solution": "B" }, - "solution": "A" -}, -{ - "question": "What is physical security?", - "answers": { - "A": "The protection of data from hacker attacks", - "B": "The prevention of natural disasters caused by environmental factors", - "C": "The act of securing a building or location to prevent unauthorized access, theft, or damage to physical assets", - "D": "The enforcement of technical security controls to prevent data breaches" - }, - "solution": "C" -}, - { - "question": "What does the abbreviation 'SDK' stand for in the context of software security?", - "answers": { - "A": "Software Development Kit", - "B": "System Deflection Key", - "C": "Software Delegation Kernel", - "D": "System Development Key" + { + "question": "What is the fundamental purpose of logging and monitoring in an organization's security measures?", + "answers": { + "A": "To store backups of critical data", + "B": "To track, record, and review activity to detect and respond to security incidents", + "C": "To manage and enforce user access controls", + "D": "To create a record of all employee activities" + }, + "solution": "B" }, - "solution": "A" -}, -{ - "question": "What is the main purpose of multi-factor authentication in cybersecurity?", - "answers": { - "A": "To limit user privileges within the network", - "B": "To ensure compliance with government regulations", - "C": "To adds an extra layer of security by requiring multiple forms of verification", - "D": "To monitor network traffic for security threats" + { + "question": "What is the benefit of structured walk-throughs in disaster recovery testing?", + "answers": { + "A": "Observe live actions in a controlled environment", + "B": "Conduct a full shut-down and restoration at the primary site", + "C": "Test operational response to disaster scenarios", + "D": "Interrupt real operations at the primary site" + }, + "solution": "C" }, - "solution": "C" -}, -{ - "question": "What is the primary objective of a Web Application Firewall (WAF) in cybersecurity?", - "answers": { - "A": "To encrypt network traffic between web servers", - "B": "To prevent unauthorized access to network services", - "C": "To protect web applications from security threats", - "D": "To manage user authentication and authorization for web services" + { + "question": "Which security process metric would most assist in determining an appropriate backup frequency for a database server?", + "answers": { + "A": "RTO", + "B": "MTBF", + "C": "RPO", + "D": "MTD" + }, + "solution": "C" }, - "solution": "C" -}, -{ - "question": "What is the purpose of periodically changing passwords or passphrases?", - "answers": { - "A": "To provide more time for a malicious individual to crack the password/passphrase", - "B": "To allow for more rapid detection and response to address potentially compromised credentials", - "C": "To prevent unauthorized users from gaining access to the user account through use of a shared authentication factor", - "D": "To offer less time for a malicious individual to crack a password/passphrase and less time to use a compromised password" + { + "question": "What constitutional amendment outlines the burden placed on investigators to have a valid search warrant before conducting certain searches?", + "answers": { + "A": "Third Amendment", + "B": "First Amendment", + "C": "Second Amendment", + "D": "Fourth Amendment" + }, + "solution": "D" }, - "solution": "D" -}, -{ - "question": "Which of the following defines the entity’s security objectives and principles?", - "answers": { - "A": "Security policies", - "B": "Operational procedures", - "C": "Risk assessment framework", - "D": "Incident response plan" - }, - "solution": "A" -} + { + "question": "Which tool is frequently used to directly access websites without a browser and is often utilized for testing and identifying potential API vulnerabilities?", + "answers": { + "A": "SSH", + "B": "cURL", + "C": "FTP", + "D": "Telnet" + }, + "solution": "B" + }, + { + "question": "What is the distinguishing characteristic of symmetric-key cryptography?", + "answers": { + "A": "It tends to be CPU intensive", + "B": "It uses the same key for encryption and decryption", + "C": "It provides integrity protection to data", + "D": "It uses different but related keys for encryption and decryption" + }, + "solution": "B" + }, + { + "question": "What is the result of 5521211 modulo 1337?", + "answers": { + "A": "11111", + "B": "33", + "C": "602", + "D": "738" + }, + "solution": "D" + }, + { + "question": "Which of the following is an example of a strong password?", + "answers": { + "A": "Password123456", + "B": "ME_Fgh4_al445!A", + "C": "Qwerty!", + "D": "123456" + }, + "solution": "B" + }, + { + "question": "Which model consists of the Application Layer, Host-to-Host Layer, Internet Layer, and Network Access Layer?", + "answers": { + "A": "OSI model", + "B": "WAN model", + "C": "LAN model", + "D": "TCP/IP model" + }, + "solution": "D" + }, + { + "question": "What is the primary purpose of 'inference engine' in the context of expert systems?", + "answers": { + "A": "To infer new facts and solve problems using rules and facts in the knowledge base", + "B": "To encrypt messages passed across the Internet", + "C": "To enable recognition of an entity by a system", + "D": "To control access to resources in a system" + }, + "solution": "A" + }, + { + "question": "What is the primary objective of conducting code reviews for bespoke and custom software applications within the Payment Card Industry Data Security Standard guidelines?", + "answers": { + "A": "To expedite the deplxoyment of bespoke and custom software into production", + "B": "To exploit potential coding vulnerabilities in production software", + "C": "To ensure bespoke and custom software meets performance benchmarks under heavy load", + "D": "To ensure that bespoke and custom software cannot be exploited via coding vulnerabilities" + }, + "solution": "D" + }, + { + "question": "Which principle of cybersecurity focuses on ensuring that data is accurate and trustworthy?", + "answers": { + "A": "Resilience", + "B": "Availability", + "C": "Authenticity", + "D": "Integrity" + }, + "solution": "D" + }, + { + "question": "An IDS installed on the network perimeter sees a spike in traffic during off-duty hours and begins logging and alerting. Which type of IDS is in place?", + "answers": { + "A": "Packet filtering", + "B": "Anomaly based", + "C": "Signature based", + "D": "Stateful" + }, + "solution": "B" + }, + { + "question": "Which of the following is a desirable property of a biometric system?", + "answers": { + "A": "Permanent", + "B": "Transferability", + "C": "Uniformity", + "D": "Forgiveness" + }, + "solution": "A" + } ] } \ No newline at end of file diff --git a/benchmarks/cybermetric/CyberMetric_evaluator_ollama.py b/benchmarks/cybermetric/CyberMetric_evaluator_ollama.py index e6e098bb..569e2ad5 100644 --- a/benchmarks/cybermetric/CyberMetric_evaluator_ollama.py +++ b/benchmarks/cybermetric/CyberMetric_evaluator_ollama.py @@ -10,7 +10,7 @@ class CyberMetricEvaluator: def __init__(self, model_name, file_path): self.model_name = model_name # E.g., "ollama/llama3" self.file_path = file_path - self.report_file = "report_failed_questions.json" + self.report_file = f"report_failed_questions.json" self.failed_questions = [] self.failed_count = 0 print("--DEBUG: model_name: ", self.model_name) @@ -123,10 +123,10 @@ if __name__ == "__main__": # Enable debug logging for litellm #litellm._turn_on_debug() - file_path = 'CyberMetric-10-v1.json' #small set for testing + file_path = 'CyberMetric-10000-v1.json' #small set for testing # Use the exact model name as it appears in Ollama # For Ollama models, you should use "ollama/model_name" # The "ollama/" prefix tells litellm to use Ollama - evaluator = CyberMetricEvaluator(model_name="ollama/qwen2.5:14b", file_path=file_path) # ollama/qwen3:32b-q8_0-ctx-32768" + evaluator = CyberMetricEvaluator(model_name="ollama/qwen3:32b-q8_0-ctx-32768", file_path=file_path) # ollama/qwen3:32b-q8_0-ctx-32768" evaluator.run_evaluation()