mirror of https://github.com/aliasrobotics/cai.git
cybermetric - ollama integration
This commit is contained in:
lidia9
parent
fbba948508
commit
df0061a4bf
|
|
@ -0,0 +1,104 @@
|
|||
{
|
||||
"questions": [
|
||||
{
|
||||
"question": "Which of the following is a desirable property of a biometric system?",
|
||||
"answers": {
|
||||
"A": "Permanent",
|
||||
"B": "Transferability",
|
||||
"C": "Uniformity",
|
||||
"D": "Forgiveness"
|
||||
},
|
||||
"solution": "A"
|
||||
},
|
||||
{
|
||||
"question": "In TCP/IP networking, which protocol is used to hold network addresses and routing information in a packet?",
|
||||
"answers": {
|
||||
"A": "HTTP",
|
||||
"B": "IP",
|
||||
"C": "Routing Information Protocol (RIP)",
|
||||
"D": "TCP"
|
||||
},
|
||||
"solution": "B"
|
||||
},
|
||||
{
|
||||
"question": "In the context of unexpected negative outcomes in personal privacy policies, what responsibility lies with both the provider and the consumer regarding the retention time of private information?",
|
||||
"answers": {
|
||||
"A": "Both the provider and the consumer are responsible for setting the appropriate retention time",
|
||||
"B": "Neither the provider nor the consumer has a responsibility for setting the appropriate retention time",
|
||||
"C": "Only the consumer is responsible for setting an appropriate retention time",
|
||||
"D": "Only the provider is responsible for ensuring the appropriate retention time"
|
||||
},
|
||||
"solution": "A"
|
||||
},
|
||||
{
|
||||
"question": "What type of attack involves an attacker behaving as a store-and-forward or proxy mechanism between two communicating systems?",
|
||||
"answers": {
|
||||
"A": "Buffer overflow attack",
|
||||
"B": "Man-in-the-Middle attack",
|
||||
"C": "DDoS attack",
|
||||
"D": "Sniffer attack"
|
||||
},
|
||||
"solution": "B"
|
||||
},
|
||||
{
|
||||
"question": "What is the fundamental purpose of logging and monitoring in an organization's security measures?",
|
||||
"answers": {
|
||||
"A": "To store backups of critical data",
|
||||
"B": "To track, record, and review activity to detect and respond to security incidents",
|
||||
"C": "To manage and enforce user access controls",
|
||||
"D": "To create a record of all employee activities"
|
||||
},
|
||||
"solution": "B"
|
||||
},
|
||||
{
|
||||
"question": "What is the benefit of structured walk-throughs in disaster recovery testing?",
|
||||
"answers": {
|
||||
"A": "Observe live actions in a controlled environment",
|
||||
"B": "Conduct a full shut-down and restoration at the primary site",
|
||||
"C": "Test operational response to disaster scenarios",
|
||||
"D": "Interrupt real operations at the primary site"
|
||||
},
|
||||
"solution": "C"
|
||||
},
|
||||
{
|
||||
"question": "Which security process metric would most assist in determining an appropriate backup frequency for a database server?",
|
||||
"answers": {
|
||||
"A": "RTO",
|
||||
"B": "MTBF",
|
||||
"C": "RPO",
|
||||
"D": "MTD"
|
||||
},
|
||||
"solution": "C"
|
||||
},
|
||||
{
|
||||
"question": "What constitutional amendment outlines the burden placed on investigators to have a valid search warrant before conducting certain searches?",
|
||||
"answers": {
|
||||
"A": "Third Amendment",
|
||||
"B": "First Amendment",
|
||||
"C": "Second Amendment",
|
||||
"D": "Fourth Amendment"
|
||||
},
|
||||
"solution": "D"
|
||||
},
|
||||
{
|
||||
"question": "Which tool is frequently used to directly access websites without a browser and is often utilized for testing and identifying potential API vulnerabilities?",
|
||||
"answers": {
|
||||
"A": "SSH",
|
||||
"B": "cURL",
|
||||
"C": "FTP",
|
||||
"D": "Telnet"
|
||||
},
|
||||
"solution": "B"
|
||||
},
|
||||
{
|
||||
"question": "What is the distinguishing characteristic of symmetric-key cryptography?",
|
||||
"answers": {
|
||||
"A": "It tends to be CPU intensive",
|
||||
"B": "It uses the same key for encryption and decryption",
|
||||
"C": "It provides integrity protection to data",
|
||||
"D": "It uses different but related keys for encryption and decryption"
|
||||
},
|
||||
"solution": "B"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -3,6 +3,7 @@ import re
|
|||
import time
|
||||
from tqdm import tqdm
|
||||
from openai import OpenAI
|
||||
import os
|
||||
|
||||
class CyberMetricEvaluator:
|
||||
def __init__(self, api_key, file_path):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,103 @@
|
|||
import json
|
||||
import re
|
||||
import time
|
||||
from tqdm import tqdm
|
||||
import os
|
||||
import litellm
|
||||
|
||||
|
||||
class CyberMetricEvaluator:
|
||||
def __init__(self, model_name, file_path):
|
||||
self.model_name = model_name # E.g., "ollama/llama3"
|
||||
self.file_path = file_path
|
||||
print("--DEBUG: model_name: ", self.model_name)
|
||||
|
||||
def read_json_file(self):
|
||||
with open(self.file_path, 'r') as file:
|
||||
return json.load(file)
|
||||
|
||||
@staticmethod
|
||||
def extract_answer(response):
|
||||
if response.strip():
|
||||
match = re.search(r"ANSWER:?\s*([A-D])", response, re.IGNORECASE)
|
||||
if match:
|
||||
return match.group(1).upper()
|
||||
return None
|
||||
|
||||
def ask_llm(self, question, answers, max_retries=5):
|
||||
options = ', '.join([f"{key}) {value}" for key, value in answers.items()])
|
||||
prompt = (
|
||||
f"Question: {question}\n"
|
||||
f"Options: {options}\n\n"
|
||||
"Choose the correct answer (A, B, C, or D) only. Always return in this format: 'ANSWER: X'"
|
||||
)
|
||||
print("--DEBUG: prompt: ", prompt)
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
response = litellm.completion(
|
||||
model=self.model_name,
|
||||
messages=[
|
||||
{"role": "system", "content": "You are a security expert who answers questions."},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
api_base="http://localhost:8000" #"http://localhost:11434" # Ollama API endpoint
|
||||
)
|
||||
if hasattr(response, "choices") and response.choices:
|
||||
content = response.choices[0].message.content
|
||||
result = self.extract_answer(content)
|
||||
if result:
|
||||
print("--DEBUG: result: ", result)
|
||||
return result
|
||||
else:
|
||||
print("Incorrect answer format detected. Attempting the question again.")
|
||||
except Exception as e:
|
||||
print(f"Error: {e}. Attempting the question again in {2 ** attempt} seconds.")
|
||||
time.sleep(2 ** attempt)
|
||||
return None
|
||||
|
||||
def run_evaluation(self):
|
||||
json_data = self.read_json_file()
|
||||
questions_data = json_data['questions']
|
||||
|
||||
correct_count = 0
|
||||
incorrect_answers = []
|
||||
|
||||
with tqdm(total=len(questions_data), desc="Processing Questions") as progress_bar:
|
||||
for item in questions_data:
|
||||
question = item['question']
|
||||
answers = item['answers']
|
||||
correct_answer = item['solution']
|
||||
|
||||
llm_answer = self.ask_llm(question, answers)
|
||||
if llm_answer == correct_answer:
|
||||
correct_count += 1
|
||||
else:
|
||||
incorrect_answers.append({
|
||||
'question': question,
|
||||
'correct_answer': correct_answer,
|
||||
'llm_answer': llm_answer
|
||||
})
|
||||
|
||||
accuracy_rate = correct_count / (progress_bar.n + 1) * 100
|
||||
progress_bar.set_postfix_str(f"Accuracy: {accuracy_rate:.2f}%")
|
||||
progress_bar.update(1)
|
||||
|
||||
print(f"\nFinal Accuracy: {correct_count / len(questions_data) * 100:.2f}%")
|
||||
|
||||
if incorrect_answers:
|
||||
print("\nIncorrect Answers:")
|
||||
for item in incorrect_answers:
|
||||
print(f"Question: {item['question']}")
|
||||
print(f"Expected Answer: {item['correct_answer']}, LLM Answer: {item['llm_answer']}\n")
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Enable debug logging for litellm
|
||||
#litellm._turn_on_debug()
|
||||
|
||||
file_path = 'CyberMetric-10-v1.json' #small set for testing
|
||||
|
||||
# Use the exact model name as it appears in Ollama
|
||||
# For Ollama models, you should use "ollama/model_name"
|
||||
# The "ollama/" prefix tells litellm to use Ollama
|
||||
evaluator = CyberMetricEvaluator(model_name="ollama/qwen2.5:14b", file_path=file_path)
|
||||
evaluator.run_evaluation()
|
||||
Loading…
Reference in New Issue