RomanNum3ral
/
cai
mirror of https://github.com/aliasrobotics/cai.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
cai/benchmarking/jeopardy_ctfs/index.html

3369 lines
65 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

<!doctype html>
<html lang="en" class="no-js">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<link rel="prev" href="../attack_defense/">
<link rel="next" href="../cyber_ranges/">
<link rel="icon" href="../../assets/imago.png">
<meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.6.11">
<title>Jeopardy CTFs - CAI</title>
<link rel="stylesheet" href="../../assets/stylesheets/main.4af4bdda.min.css">
<link rel="stylesheet" href="../../assets/stylesheets/palette.06af60db.min.css">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
<style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
<link rel="stylesheet" href="../../assets/_mkdocstrings.css">
<link rel="stylesheet" href="../../stylesheets/extra.css">
<script>__md_scope=new URL("../..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
</head>
<body dir="ltr" data-md-color-scheme="default" data-md-color-primary="custom" data-md-color-accent="indigo">
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
<label class="md-overlay" for="__drawer"></label>
<div data-md-component="skip">
<a href="#jeopardy-style-ctf-benchmarks" class="md-skip">
Skip to content
</a>
</div>
<div data-md-component="announce">
</div>
<header class="md-header md-header--shadow" data-md-component="header">
<nav class="md-header__inner md-grid" aria-label="Header">
<a href="../.." title="CAI" class="md-header__button md-logo" aria-label="CAI" data-md-component="logo">
<img src="../../assets/imago.png" alt="logo">
</a>
<label class="md-header__button md-icon" for="__drawer">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
</label>
<div class="md-header__title" data-md-component="header-title">
<div class="md-header__ellipsis">
<div class="md-header__topic">
<span class="md-ellipsis">
CAI
</span>
</div>
<div class="md-header__topic" data-md-component="header-topic">
<span class="md-ellipsis">
Jeopardy CTFs
</span>
</div>
</div>
</div>
<label class="md-header__button md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
</label>
<div class="md-search" data-md-component="search" role="dialog">
<label class="md-search__overlay" for="__search"></label>
<div class="md-search__inner" role="search">
<form class="md-search__form" name="search">
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
<label class="md-search__icon md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
</label>
<nav class="md-search__options" aria-label="Search">
<button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
</button>
</nav>
</form>
<div class="md-search__output">
<div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
<div class="md-search-result" data-md-component="search-result">
<div class="md-search-result__meta">
Initializing search
</div>
<ol class="md-search-result__list" role="presentation"></ol>
</div>
</div>
</div>
</div>
</div>
<div class="md-header__source">
<a href="https://github.com/aliasrobotics/cai" title="Go to repository" class="md-source" data-md-component="source">
<div class="md-source__icon md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><!--! Font Awesome Free 6.7.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6m-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3m44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9M244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8M97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1m-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7m32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1m-11.4-14.7c-1.6 1-1.6 3.6 0 5.9s4.3 3.3 5.6 2.3c1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2"/></svg>
</div>
<div class="md-source__repository">
aliasrobotics/cai
</div>
</a>
</div>
</nav>
</header>
<div class="md-container" data-md-component="container">
<main class="md-main" data-md-component="main">
<div class="md-main__inner md-grid">
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
<label class="md-nav__title" for="__drawer">
<a href="../.." title="CAI" class="md-nav__button md-logo" aria-label="CAI" data-md-component="logo">
<img src="../../assets/imago.png" alt="logo">
</a>
CAI
</label>
<div class="md-nav__source">
<a href="https://github.com/aliasrobotics/cai" title="Go to repository" class="md-source" data-md-component="source">
<div class="md-source__icon md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><!--! Font Awesome Free 6.7.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6m-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3m44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9M244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8M97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1m-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7m32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1m-11.4-14.7c-1.6 1-1.6 3.6 0 5.9s4.3 3.3 5.6 2.3c1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2"/></svg>
</div>
<div class="md-source__repository">
aliasrobotics/cai
</div>
</a>
</div>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_1" >
<label class="md-nav__link" for="__nav_1" id="__nav_1_label" tabindex="">
<span class="md-ellipsis">
Getting Started
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_1_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_1">
<span class="md-nav__icon md-icon"></span>
Getting Started
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../.." class="md-nav__link">
<span class="md-ellipsis">
Welcome
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../cai_installation/" class="md-nav__link">
<span class="md-ellipsis">
Installation
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../cai_quickstart/" class="md-nav__link">
<span class="md-ellipsis">
Quickstart
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../cai_list_of_models/" class="md-nav__link">
<span class="md-ellipsis">
Available Models
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_1_5" >
<label class="md-nav__link" for="__nav_1_5" id="__nav_1_5_label" tabindex="0">
<span class="md-ellipsis">
Model Providers
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_1_5_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_1_5">
<span class="md-nav__icon md-icon"></span>
Model Providers
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../providers/openrouter.md" class="md-nav__link">
<span class="md-ellipsis">
OpenRouter
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../providers/ollama/" class="md-nav__link">
<span class="md-ellipsis">
Ollama
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../providers/azure.md" class="md-nav__link">
<span class="md-ellipsis">
Azure OpenAI
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../../cai_pro/" class="md-nav__link">
<span class="md-ellipsis">
🚀 CAI PRO
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_3" >
<label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="">
<span class="md-ellipsis">
Core Concepts
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_3">
<span class="md-nav__icon md-icon"></span>
Core Concepts
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../cai_architecture/" class="md-nav__link">
<span class="md-ellipsis">
Architecture
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../agents/" class="md-nav__link">
<span class="md-ellipsis">
Agents
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../tools/" class="md-nav__link">
<span class="md-ellipsis">
Tools
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../handoffs/" class="md-nav__link">
<span class="md-ellipsis">
Handoffs
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../multi_agent/" class="md-nav__link">
<span class="md-ellipsis">
Multi-Agent Systems
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4" checked>
<label class="md-nav__link" for="__nav_4" id="__nav_4_label" tabindex="">
<span class="md-ellipsis">
Benchmarking
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="true">
<label class="md-nav__title" for="__nav_4">
<span class="md-nav__icon md-icon"></span>
Benchmarking
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../overview/" class="md-nav__link">
<span class="md-ellipsis">
Overview
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../running_benchmarks/" class="md-nav__link">
<span class="md-ellipsis">
Running Benchmarks
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../attack_defense/" class="md-nav__link">
<span class="md-ellipsis">
Attack & Defense CTFs
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--active">
<input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
<label class="md-nav__link md-nav__link--active" for="__toc">
<span class="md-ellipsis">
Jeopardy CTFs
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<a href="./" class="md-nav__link md-nav__link--active">
<span class="md-ellipsis">
Jeopardy CTFs
</span>
</a>
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#available-benchmarks" class="md-nav__link">
<span class="md-ellipsis">
📊 Available Benchmarks
</span>
</a>
<nav class="md-nav" aria-label="📊 Available Benchmarks">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#base-benchmark" class="md-nav__link">
<span class="md-ellipsis">
Base Benchmark
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#cybench-framework" class="md-nav__link">
<span class="md-ellipsis">
Cybench Framework
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#rctf2-robotics-ctf" class="md-nav__link">
<span class="md-ellipsis">
RCTF2 - Robotics CTF
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#challenge-categories" class="md-nav__link">
<span class="md-ellipsis">
🎯 Challenge Categories
</span>
</a>
<nav class="md-nav" aria-label="🎯 Challenge Categories">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#web-exploitation" class="md-nav__link">
<span class="md-ellipsis">
Web Exploitation
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#binary-exploitation-pwn" class="md-nav__link">
<span class="md-ellipsis">
Binary Exploitation (Pwn)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#cryptography" class="md-nav__link">
<span class="md-ellipsis">
Cryptography
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#reverse-engineering" class="md-nav__link">
<span class="md-ellipsis">
Reverse Engineering
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#forensics" class="md-nav__link">
<span class="md-ellipsis">
Forensics
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#miscellaneous" class="md-nav__link">
<span class="md-ellipsis">
Miscellaneous
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#alias1-performance" class="md-nav__link">
<span class="md-ellipsis">
🏆 alias1 Performance
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#running-jeopardy-ctf-benchmarks" class="md-nav__link">
<span class="md-ellipsis">
🚀 Running Jeopardy CTF Benchmarks
</span>
</a>
<nav class="md-nav" aria-label="🚀 Running Jeopardy CTF Benchmarks">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#for-cai-pro-subscribers" class="md-nav__link">
<span class="md-ellipsis">
For CAI PRO Subscribers
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#benchmark-configuration" class="md-nav__link">
<span class="md-ellipsis">
📊 Benchmark Configuration
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#why-jeopardy-ctfs-matter" class="md-nav__link">
<span class="md-ellipsis">
🎓 Why Jeopardy CTFs Matter
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#research-papers" class="md-nav__link">
<span class="md-ellipsis">
📚 Research Papers
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#related-benchmarks" class="md-nav__link">
<span class="md-ellipsis">
🔗 Related Benchmarks
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../cyber_ranges/" class="md-nav__link">
<span class="md-ellipsis">
Cyber Ranges
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../knowledge_benchmarks/" class="md-nav__link">
<span class="md-ellipsis">
Knowledge Benchmarks
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../privacy_benchmarks/" class="md-nav__link">
<span class="md-ellipsis">
Privacy Benchmarks
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_5" >
<label class="md-nav__link" for="__nav_5" id="__nav_5_label" tabindex="">
<span class="md-ellipsis">
Guides
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_5_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_5">
<span class="md-nav__icon md-icon"></span>
Guides
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../running_agents/" class="md-nav__link">
<span class="md-ellipsis">
Running Agents
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../continue_mode/" class="md-nav__link">
<span class="md-ellipsis">
Continue Mode
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../results/" class="md-nav__link">
<span class="md-ellipsis">
Working with Results
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../streaming/" class="md-nav__link">
<span class="md-ellipsis">
Streaming
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../tracing/" class="md-nav__link">
<span class="md-ellipsis">
Tracing & Debugging
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../context/" class="md-nav__link">
<span class="md-ellipsis">
Context Management
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../guardrails/" class="md-nav__link">
<span class="md-ellipsis">
Guardrails & Security
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../environment_variables/" class="md-nav__link">
<span class="md-ellipsis">
Environment Variables
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../cai/getting-started/packet_capture_wsl/" class="md-nav__link">
<span class="md-ellipsis">
Packet Capture on WSL2
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_6" >
<label class="md-nav__link" for="__nav_6" id="__nav_6_label" tabindex="">
<span class="md-ellipsis">
Troubleshooting
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_6_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_6">
<span class="md-nav__icon md-icon"></span>
Troubleshooting
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../cai/troubleshooting/operator_feedback_reproduction/" class="md-nav__link">
<span class="md-ellipsis">
Operator Feedback Reproduction
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../cai/troubleshooting/platform_limitations/" class="md-nav__link">
<span class="md-ellipsis">
Platform Limitations
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_7" >
<label class="md-nav__link" for="__nav_7" id="__nav_7_label" tabindex="">
<span class="md-ellipsis">
Case Studies
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_7_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_7">
<span class="md-nav__icon md-icon"></span>
Case Studies
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../cai/case-studies/operator-artifact-evidence/" class="md-nav__link">
<span class="md-ellipsis">
Operator Artifact Evidence
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_8" >
<label class="md-nav__link" for="__nav_8" id="__nav_8_label" tabindex="">
<span class="md-ellipsis">
Mobile UI (iOS)
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_8">
<span class="md-nav__icon md-icon"></span>
Mobile UI (iOS)
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../mui/mui_index/" class="md-nav__link">
<span class="md-ellipsis">
Overview
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../mui/getting_started/" class="md-nav__link">
<span class="md-ellipsis">
Getting Started
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../mui/user_interface/" class="md-nav__link">
<span class="md-ellipsis">
User Interface
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../mui/gestures_shortcuts/" class="md-nav__link">
<span class="md-ellipsis">
Gestures & Shortcuts
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../mui/chat_features/" class="md-nav__link">
<span class="md-ellipsis">
Chat Features
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_9" >
<label class="md-nav__link" for="__nav_9" id="__nav_9_label" tabindex="">
<span class="md-ellipsis">
Terminal UI (TUI) - Deprecated
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_9">
<span class="md-nav__icon md-icon"></span>
Terminal UI (TUI) - Deprecated
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../tui/tui_index/" class="md-nav__link">
<span class="md-ellipsis">
Overview
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../tui/getting_started/" class="md-nav__link">
<span class="md-ellipsis">
Getting Started
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../tui/user_interface/" class="md-nav__link">
<span class="md-ellipsis">
User Interface
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../tui/terminals_management/" class="md-nav__link">
<span class="md-ellipsis">
Terminals Management
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../tui/teams_and_parallel_execution/" class="md-nav__link">
<span class="md-ellipsis">
Teams & Parallel Execution
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../tui/sidebar_features/" class="md-nav__link">
<span class="md-ellipsis">
Sidebar Features
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../tui/keyboard_shortcuts/" class="md-nav__link">
<span class="md-ellipsis">
Keyboard Shortcuts
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../tui/commands_reference/" class="md-nav__link">
<span class="md-ellipsis">
Commands Reference
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../tui/advanced_features/" class="md-nav__link">
<span class="md-ellipsis">
Advanced Features
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../tui/troubleshooting/" class="md-nav__link">
<span class="md-ellipsis">
Troubleshooting
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_10" >
<label class="md-nav__link" for="__nav_10" id="__nav_10_label" tabindex="">
<span class="md-ellipsis">
API Reference
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_10">
<span class="md-nav__icon md-icon"></span>
API Reference
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_10_1" >
<label class="md-nav__link" for="__nav_10_1" id="__nav_10_1_label" tabindex="0">
<span class="md-ellipsis">
Agents
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_1_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_10_1">
<span class="md-nav__icon md-icon"></span>
Agents
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../ref/agent/" class="md-nav__link">
<span class="md-ellipsis">
Agent
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../ref/run/" class="md-nav__link">
<span class="md-ellipsis">
Runner
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../ref/tool/" class="md-nav__link">
<span class="md-ellipsis">
Tool
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../ref/result/" class="md-nav__link">
<span class="md-ellipsis">
Result
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../ref/stream_events/" class="md-nav__link">
<span class="md-ellipsis">
Stream Events
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../ref/handoffs/" class="md-nav__link">
<span class="md-ellipsis">
Handoffs
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../ref/lifecycle/" class="md-nav__link">
<span class="md-ellipsis">
Lifecycle
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../ref/items/" class="md-nav__link">
<span class="md-ellipsis">
Items
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../ref/run_context/" class="md-nav__link">
<span class="md-ellipsis">
Run Context
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../ref/usage/" class="md-nav__link">
<span class="md-ellipsis">
Usage
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../ref/exceptions/" class="md-nav__link">
<span class="md-ellipsis">
Exceptions
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../ref/guardrail/" class="md-nav__link">
<span class="md-ellipsis">
Guardrail
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../ref/model_settings/" class="md-nav__link">
<span class="md-ellipsis">
Model Settings
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../ref/agent_output/" class="md-nav__link">
<span class="md-ellipsis">
Agent Output
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../ref/function_schema/" class="md-nav__link">
<span class="md-ellipsis">
Function Schema
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_10_2" >
<label class="md-nav__link" for="__nav_10_2" id="__nav_10_2_label" tabindex="0">
<span class="md-ellipsis">
Models
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_2_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_10_2">
<span class="md-nav__icon md-icon"></span>
Models
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../ref/models/interface/" class="md-nav__link">
<span class="md-ellipsis">
Interface
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../ref/models/openai_chatcompletions/" class="md-nav__link">
<span class="md-ellipsis">
OpenAI Chat Completions
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../ref/models/openai_responses/" class="md-nav__link">
<span class="md-ellipsis">
OpenAI Responses
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_10_3" >
<label class="md-nav__link" for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
<span class="md-ellipsis">
Extensions
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_10_3">
<span class="md-nav__icon md-icon"></span>
Extensions
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../ref/extensions/handoff_filters/" class="md-nav__link">
<span class="md-ellipsis">
Handoff Filters
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../ref/extensions/handoff_prompt/" class="md-nav__link">
<span class="md-ellipsis">
Handoff Prompt
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_11" >
<label class="md-nav__link" for="__nav_11" id="__nav_11_label" tabindex="">
<span class="md-ellipsis">
Advanced
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_11">
<span class="md-nav__icon md-icon"></span>
Advanced
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../cai_development/" class="md-nav__link">
<span class="md-ellipsis">
Development
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_12" >
<label class="md-nav__link" for="__nav_12" id="__nav_12_label" tabindex="">
<span class="md-ellipsis">
Research
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_12_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_12">
<span class="md-nav__icon md-icon"></span>
Research
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../research/" class="md-nav__link">
<span class="md-ellipsis">
Overview
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_13" >
<label class="md-nav__link" for="__nav_13" id="__nav_13_label" tabindex="">
<span class="md-ellipsis">
Resources
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_13_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_13">
<span class="md-nav__icon md-icon"></span>
Resources
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../cai_faq/" class="md-nav__link">
<span class="md-ellipsis">
FAQ
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../cai_find_us/" class="md-nav__link">
<span class="md-ellipsis">
Find Us
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../cai_citation_and_acknowledgments/" class="md-nav__link">
<span class="md-ellipsis">
Citation & Acknowledgments
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#available-benchmarks" class="md-nav__link">
<span class="md-ellipsis">
📊 Available Benchmarks
</span>
</a>
<nav class="md-nav" aria-label="📊 Available Benchmarks">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#base-benchmark" class="md-nav__link">
<span class="md-ellipsis">
Base Benchmark
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#cybench-framework" class="md-nav__link">
<span class="md-ellipsis">
Cybench Framework
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#rctf2-robotics-ctf" class="md-nav__link">
<span class="md-ellipsis">
RCTF2 - Robotics CTF
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#challenge-categories" class="md-nav__link">
<span class="md-ellipsis">
🎯 Challenge Categories
</span>
</a>
<nav class="md-nav" aria-label="🎯 Challenge Categories">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#web-exploitation" class="md-nav__link">
<span class="md-ellipsis">
Web Exploitation
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#binary-exploitation-pwn" class="md-nav__link">
<span class="md-ellipsis">
Binary Exploitation (Pwn)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#cryptography" class="md-nav__link">
<span class="md-ellipsis">
Cryptography
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#reverse-engineering" class="md-nav__link">
<span class="md-ellipsis">
Reverse Engineering
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#forensics" class="md-nav__link">
<span class="md-ellipsis">
Forensics
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#miscellaneous" class="md-nav__link">
<span class="md-ellipsis">
Miscellaneous
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#alias1-performance" class="md-nav__link">
<span class="md-ellipsis">
🏆 alias1 Performance
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#running-jeopardy-ctf-benchmarks" class="md-nav__link">
<span class="md-ellipsis">
🚀 Running Jeopardy CTF Benchmarks
</span>
</a>
<nav class="md-nav" aria-label="🚀 Running Jeopardy CTF Benchmarks">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#for-cai-pro-subscribers" class="md-nav__link">
<span class="md-ellipsis">
For CAI PRO Subscribers
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#benchmark-configuration" class="md-nav__link">
<span class="md-ellipsis">
📊 Benchmark Configuration
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#why-jeopardy-ctfs-matter" class="md-nav__link">
<span class="md-ellipsis">
🎓 Why Jeopardy CTFs Matter
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#research-papers" class="md-nav__link">
<span class="md-ellipsis">
📚 Research Papers
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#related-benchmarks" class="md-nav__link">
<span class="md-ellipsis">
🔗 Related Benchmarks
</span>
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-content" data-md-component="content">
<article class="md-content__inner md-typeset">
<h1 id="jeopardy-style-ctf-benchmarks">Jeopardy-style CTF Benchmarks</h1>
<p>Jeopardy-style Capture The Flag (CTF) challenges evaluate AI agents on independent security tasks across multiple domains: cryptography, web exploitation, binary reversing, forensics, and pwn.</p>
<hr />
<h2 id="available-benchmarks">📊 Available Benchmarks</h2>
<h3 id="base-benchmark">Base Benchmark</h3>
<p><strong>21 curated CTF challenges</strong> measuring initial penetration testing capabilities.</p>
<ul>
<li><strong>Difficulty</strong>: 🚩 Very Easy to 🚩🚩🚩 Medium</li>
<li><strong>Categories</strong>: Reversing, Miscellaneous, Pwn, Web, Crypto, Forensics</li>
<li><strong>Status</strong>: ⚠️ <em>Saturated</em> - Frontier cybersecurity models (like alias1) conquer most challenges</li>
</ul>
<table>
<tr>
<th style="text-align:center;"><b>Model Performance in Jeopardy CTFs Base Benchmark</b></th>
</tr>
<tr>
<td align="center"><img src="/assets/images/base_1col.png" alt="Base Benchmark Results" /></td>
</tr>
</table>
<div class="admonition success">
<p class="admonition-title">alias1 Performance</p>
<p><code>alias1</code> achieves <strong>near-perfect scores</strong> on the Base benchmark, demonstrating mastery of fundamental cybersecurity concepts and techniques.</p>
</div>
<h3 id="cybench-framework">Cybench Framework</h3>
<p><strong>35 CTF challenges</strong> from the comprehensive Cybench evaluation framework.</p>
<ul>
<li><strong>Difficulty</strong>: 🚩 Very Easy to 🚩🚩🚩🚩🚩 Very Hard</li>
<li><strong>Source</strong>: <a href="https://github.com/andyzorigin/cybench">Cybench Framework for Evaluating Cybersecurity Capabilities and Risk</a></li>
<li><strong>Coverage</strong>: Broad spectrum of real-world security scenarios</li>
<li><strong>Note</strong>: 35 of 40 Cybench scenarios (reduced for infrastructure/reproducibility)</li>
</ul>
<h3 id="rctf2-robotics-ctf">RCTF2 - Robotics CTF</h3>
<p><strong>27 robotics-focused challenges</strong> for attacking and defending robots and robotic frameworks.</p>
<ul>
<li><strong>Difficulty</strong>: 🚩 Very Easy to 🚩🚩🚩🚩🚩 Very Hard</li>
<li><strong>Systems Covered</strong>: ROS, ROS 2, manipulators, AGVs, AMRs, collaborative robots, legged robots, humanoids</li>
<li><strong>Unique Focus</strong>: Only benchmark evaluating AI capabilities against robotic systems</li>
</ul>
<hr />
<h2 id="challenge-categories">🎯 Challenge Categories</h2>
<h3 id="web-exploitation">Web Exploitation</h3>
<p>Vulnerabilities in web applications and services:
- SQL Injection
- Cross-Site Scripting (XSS)
- Server-Side Template Injection (SSTI)
- Authentication bypasses
- API vulnerabilities</p>
<h3 id="binary-exploitation-pwn">Binary Exploitation (Pwn)</h3>
<p>Memory corruption and exploitation:
- Buffer overflows
- Format string vulnerabilities
- Return-oriented programming (ROP)
- Heap exploitation
- Use-after-free</p>
<h3 id="cryptography">Cryptography</h3>
<p>Breaking or exploiting cryptographic implementations:
- Weak encryption algorithms
- Poor key management
- Custom cryptography flaws
- Hash collisions
- Padding oracle attacks</p>
<h3 id="reverse-engineering">Reverse Engineering</h3>
<p>Analyzing and understanding compiled binaries:
- Assembly code analysis
- Decompilation and deobfuscation
- Anti-debugging techniques
- Packed/encrypted binaries
- Firmware analysis</p>
<h3 id="forensics">Forensics</h3>
<p>Investigating and extracting information from data:
- File carving
- Steganography
- Memory forensics
- Network traffic analysis
- Log analysis</p>
<h3 id="miscellaneous">Miscellaneous</h3>
<p>Challenges that don't fit standard categories:
- OSINT (Open Source Intelligence)
- Scripting and automation
- Logic puzzles
- Unconventional attack vectors</p>
<hr />
<h2 id="alias1-performance">🏆 alias1 Performance</h2>
<div class="admonition success">
<p class="admonition-title">Superior Jeopardy CTF Performance</p>
<p><strong><code>alias1</code> consistently outperforms all other AI models</strong> in Jeopardy-style CTF benchmarks:</p>
<ul>
<li>🥇 <strong>Highest solve rate</strong> across all difficulty levels</li>
<li>🥇 <strong>Fastest time to solve</strong> for timed challenges</li>
<li>🥇 <strong>Best multi-category performance</strong> - Excels in web, pwn, crypto, forensics, and reversing</li>
<li>🥇 <strong>Zero refusals</strong> - Unrestricted responses for all CTF challenges</li>
</ul>
<p><strong>General-purpose models</strong> (GPT-4o, Claude 3.5) show:
- ❌ High refusal rates on pwn/exploitation challenges
- ❌ Inconsistent performance across categories
- ❌ Limited success on medium+ difficulty challenges</p>
</div>
<p><strong><a href="../../cai_pro/">Get alias1 with CAI PRO →</a></strong></p>
<hr />
<h2 id="running-jeopardy-ctf-benchmarks">🚀 Running Jeopardy CTF Benchmarks</h2>
<div class="admonition warning">
<p class="admonition-title">CAI PRO Exclusive</p>
<p>Jeopardy-style CTF benchmarks are available exclusively with <strong><a href="../../cai_pro/">CAI PRO</a></strong> subscriptions.</p>
<p>General users can access:
- <a href="../knowledge_benchmarks/">Knowledge benchmarks</a>
- <a href="../privacy_benchmarks/">Privacy benchmarks</a></p>
</div>
<h3 id="for-cai-pro-subscribers">For CAI PRO Subscribers</h3>
<p>Docker-based CTF environments can be launched individually or in batches:</p>
<div class="language-bash highlight"><pre><span></span><code><span id="__span-0-1"><span class="c1"># Run single CTF challenge</span>
</span><span id="__span-0-2">docker<span class="w"> </span>run<span class="w"> </span>-it<span class="w"> </span>cai-ctf/base:challenge-01
</span><span id="__span-0-3">
</span><span id="__span-0-4"><span class="c1"># Run full Base benchmark suite</span>
</span><span id="__span-0-5">python<span class="w"> </span>benchmarks/eval_ctf.py<span class="w"> </span>--benchmark<span class="w"> </span>base<span class="w"> </span>--model<span class="w"> </span>alias1
</span><span id="__span-0-6">
</span><span id="__span-0-7"><span class="c1"># Run Cybench evaluation</span>
</span><span id="__span-0-8">python<span class="w"> </span>benchmarks/eval_ctf.py<span class="w"> </span>--benchmark<span class="w"> </span>cybench<span class="w"> </span>--model<span class="w"> </span>alias1
</span><span id="__span-0-9">
</span><span id="__span-0-10"><span class="c1"># Run RCTF2 robotics challenges</span>
</span><span id="__span-0-11">python<span class="w"> </span>benchmarks/eval_ctf.py<span class="w"> </span>--benchmark<span class="w"> </span>rctf2<span class="w"> </span>--model<span class="w"> </span>alias1
</span></code></pre></div>
<p>Contact research@aliasrobotics.com for detailed setup instructions.</p>
<hr />
<h2 id="benchmark-configuration">📊 Benchmark Configuration</h2>
<p>CTF configurations are defined in <a href="https://github.com/aliasrobotics/cai/blob/main/src/cai/caibench/ctf-jsons/ctf_configs.jsonl"><code>ctf_configs.jsonl</code></a>:</p>
<div class="language-json highlight"><pre><span></span><code><span id="__span-1-1"><span class="p">{</span>
</span><span id="__span-1-2"><span class="w"> </span><span class="nt">&quot;name&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;example-ctf&quot;</span><span class="p">,</span>
</span><span id="__span-1-3"><span class="w"> </span><span class="nt">&quot;category&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;web&quot;</span><span class="p">,</span>
</span><span id="__span-1-4"><span class="w"> </span><span class="nt">&quot;difficulty&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;medium&quot;</span><span class="p">,</span>
</span><span id="__span-1-5"><span class="w"> </span><span class="nt">&quot;points&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">100</span><span class="p">,</span>
</span><span id="__span-1-6"><span class="w"> </span><span class="nt">&quot;flag_format&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;CTF{...}&quot;</span><span class="p">,</span>
</span><span id="__span-1-7"><span class="w"> </span><span class="nt">&quot;docker_image&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;cai-ctf/web-01:latest&quot;</span><span class="p">,</span>
</span><span id="__span-1-8"><span class="w"> </span><span class="nt">&quot;timeout&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">3600</span>
</span><span id="__span-1-9"><span class="p">}</span>
</span></code></pre></div>
<hr />
<h2 id="why-jeopardy-ctfs-matter">🎓 Why Jeopardy CTFs Matter</h2>
<p>Jeopardy-style CTFs are essential for evaluating cybersecurity AI because:</p>
<ol>
<li><strong>Diverse Skillset</strong> - Tests wide range of security knowledge and techniques</li>
<li><strong>Independent Challenges</strong> - Isolates specific capabilities without dependencies</li>
<li><strong>Scalable Difficulty</strong> - From beginner to elite-level challenges</li>
<li><strong>Real-world Relevance</strong> - Based on actual vulnerabilities and attack patterns</li>
<li><strong>Objective Measurement</strong> - Clear success criteria (flag captured or not)</li>
</ol>
<p>Unlike traditional benchmarks that test general knowledge, CTFs require <strong>active exploitation and problem-solving</strong> - skills critical for real-world penetration testing.</p>
<hr />
<h2 id="research-papers">📚 Research Papers</h2>
<ul>
<li>
<p>📊 <a href="https://arxiv.org/pdf/2510.24317"><strong>CAIBench: Cybersecurity AI Benchmark</strong></a> (2025)
Meta-benchmark framework including Jeopardy CTF evaluation methodology.</p>
</li>
<li>
<p>🚀 <a href="https://arxiv.org/pdf/2504.06017"><strong>Cybersecurity AI (CAI) Framework</strong></a> (2025)
Core framework demonstrating 3,600× performance improvement using CTF scenarios.</p>
</li>
</ul>
<p><strong><a href="https://aliasrobotics.com/research-security.php#papers">View all research →</a></strong></p>
<hr />
<h2 id="related-benchmarks">🔗 Related Benchmarks</h2>
<ul>
<li><strong><a href="../attack_defense/">Attack &amp; Defense CTFs</a></strong> - Real-time competitive environments</li>
<li><strong><a href="../cyber_ranges/">Cyber Ranges</a></strong> - Complex multi-system scenarios</li>
<li><strong><a href="../running_benchmarks/">Running Benchmarks</a></strong> - Setup and usage guide</li>
</ul>
<hr />
<p>🚀 <strong><a href="../../cai_pro/">Upgrade to CAI PRO for access to Jeopardy CTF benchmarks →</a></strong></p>
</article>
</div>
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
</div>
</main>
<footer class="md-footer">
<div class="md-footer-meta md-typeset">
<div class="md-footer-meta__inner md-grid">
<div class="md-copyright">
</div>
</div>
</div>
</footer>
</div>
<div class="md-dialog" data-md-component="dialog">
<div class="md-dialog__inner md-typeset"></div>
</div>
<script id="__config" type="application/json">{"base": "../..", "features": ["content.code.copy", "content.code.select", "navigation.path", "navigation.sections", "navigation.expand", "content.code.annotate"], "search": "../../assets/javascripts/workers/search.f8cc74c7.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
<script src="../../assets/javascripts/bundle.c8b220af.min.js"></script>
</body>
</html>
Reference in New Issue View Git Blame Copy Permalink