mirror of https://github.com/aliasrobotics/cai.git
3734 lines
74 KiB
HTML
3734 lines
74 KiB
HTML
|
||
<!doctype html>
|
||
<html lang="en" class="no-js">
|
||
<head>
|
||
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||
|
||
|
||
|
||
|
||
<link rel="prev" href="../knowledge_benchmarks/">
|
||
|
||
|
||
<link rel="next" href="../../running_agents/">
|
||
|
||
|
||
<link rel="icon" href="../../assets/imago.png">
|
||
<meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.6.11">
|
||
|
||
|
||
|
||
<title>Privacy Benchmarks - CAI</title>
|
||
|
||
|
||
|
||
<link rel="stylesheet" href="../../assets/stylesheets/main.4af4bdda.min.css">
|
||
|
||
|
||
<link rel="stylesheet" href="../../assets/stylesheets/palette.06af60db.min.css">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
|
||
<style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
|
||
|
||
|
||
|
||
<link rel="stylesheet" href="../../assets/_mkdocstrings.css">
|
||
|
||
<link rel="stylesheet" href="../../stylesheets/extra.css">
|
||
|
||
<script>__md_scope=new URL("../..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
</head>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<body dir="ltr" data-md-color-scheme="default" data-md-color-primary="custom" data-md-color-accent="indigo">
|
||
|
||
|
||
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
||
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
||
<label class="md-overlay" for="__drawer"></label>
|
||
<div data-md-component="skip">
|
||
|
||
|
||
<a href="#privacy-benchmarks" class="md-skip">
|
||
Skip to content
|
||
</a>
|
||
|
||
</div>
|
||
<div data-md-component="announce">
|
||
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<header class="md-header md-header--shadow" data-md-component="header">
|
||
<nav class="md-header__inner md-grid" aria-label="Header">
|
||
<a href="../.." title="CAI" class="md-header__button md-logo" aria-label="CAI" data-md-component="logo">
|
||
|
||
<img src="../../assets/imago.png" alt="logo">
|
||
|
||
</a>
|
||
<label class="md-header__button md-icon" for="__drawer">
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
|
||
</label>
|
||
<div class="md-header__title" data-md-component="header-title">
|
||
<div class="md-header__ellipsis">
|
||
<div class="md-header__topic">
|
||
<span class="md-ellipsis">
|
||
CAI
|
||
</span>
|
||
</div>
|
||
<div class="md-header__topic" data-md-component="header-topic">
|
||
<span class="md-ellipsis">
|
||
|
||
Privacy Benchmarks
|
||
|
||
</span>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<label class="md-header__button md-icon" for="__search">
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
|
||
</label>
|
||
<div class="md-search" data-md-component="search" role="dialog">
|
||
<label class="md-search__overlay" for="__search"></label>
|
||
<div class="md-search__inner" role="search">
|
||
<form class="md-search__form" name="search">
|
||
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
|
||
<label class="md-search__icon md-icon" for="__search">
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
|
||
</label>
|
||
<nav class="md-search__options" aria-label="Search">
|
||
|
||
<button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
|
||
</button>
|
||
</nav>
|
||
|
||
</form>
|
||
<div class="md-search__output">
|
||
<div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
|
||
<div class="md-search-result" data-md-component="search-result">
|
||
<div class="md-search-result__meta">
|
||
Initializing search
|
||
</div>
|
||
<ol class="md-search-result__list" role="presentation"></ol>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div class="md-header__source">
|
||
<a href="https://github.com/aliasrobotics/cai" title="Go to repository" class="md-source" data-md-component="source">
|
||
<div class="md-source__icon md-icon">
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><!--! Font Awesome Free 6.7.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6m-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3m44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9M244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8M97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1m-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7m32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1m-11.4-14.7c-1.6 1-1.6 3.6 0 5.9s4.3 3.3 5.6 2.3c1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2"/></svg>
|
||
</div>
|
||
<div class="md-source__repository">
|
||
aliasrobotics/cai
|
||
</div>
|
||
</a>
|
||
</div>
|
||
|
||
</nav>
|
||
|
||
</header>
|
||
|
||
<div class="md-container" data-md-component="container">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<main class="md-main" data-md-component="main">
|
||
<div class="md-main__inner md-grid">
|
||
|
||
|
||
|
||
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
|
||
<div class="md-sidebar__scrollwrap">
|
||
<div class="md-sidebar__inner">
|
||
|
||
|
||
|
||
|
||
<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
|
||
<label class="md-nav__title" for="__drawer">
|
||
<a href="../.." title="CAI" class="md-nav__button md-logo" aria-label="CAI" data-md-component="logo">
|
||
|
||
<img src="../../assets/imago.png" alt="logo">
|
||
|
||
</a>
|
||
CAI
|
||
</label>
|
||
|
||
<div class="md-nav__source">
|
||
<a href="https://github.com/aliasrobotics/cai" title="Go to repository" class="md-source" data-md-component="source">
|
||
<div class="md-source__icon md-icon">
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><!--! Font Awesome Free 6.7.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6m-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3m44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9M244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8M97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1m-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7m32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1m-11.4-14.7c-1.6 1-1.6 3.6 0 5.9s4.3 3.3 5.6 2.3c1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2"/></svg>
|
||
</div>
|
||
<div class="md-source__repository">
|
||
aliasrobotics/cai
|
||
</div>
|
||
</a>
|
||
</div>
|
||
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_1" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_1" id="__nav_1_label" tabindex="">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Getting Started
|
||
|
||
</span>
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_1_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_1">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
Getting Started
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../.." class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Welcome
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../cai_installation/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Installation
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../cai_quickstart/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Quickstart
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../cai_list_of_models/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Available Models
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_1_5" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_1_5" id="__nav_1_5_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Model Providers
|
||
|
||
</span>
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_1_5_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_1_5">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
Model Providers
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../providers/openrouter.md" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
OpenRouter
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../providers/ollama/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Ollama
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../providers/azure.md" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Azure OpenAI
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../cai_pro/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
🚀 CAI PRO
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_3" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Core Concepts
|
||
|
||
</span>
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_3">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
Core Concepts
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../cai_architecture/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Architecture
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../agents/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Agents
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../tools/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Tools
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../handoffs/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Handoffs
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../multi_agent/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Multi-Agent Systems
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4" checked>
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_4" id="__nav_4_label" tabindex="">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Benchmarking
|
||
|
||
</span>
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="true">
|
||
<label class="md-nav__title" for="__nav_4">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
Benchmarking
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../overview/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Overview
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../running_benchmarks/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Running Benchmarks
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../attack_defense/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Attack & Defense CTFs
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../jeopardy_ctfs/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Jeopardy CTFs
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../cyber_ranges/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Cyber Ranges
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../knowledge_benchmarks/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Knowledge Benchmarks
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--active">
|
||
|
||
<input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
|
||
|
||
|
||
|
||
|
||
|
||
<label class="md-nav__link md-nav__link--active" for="__toc">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Privacy Benchmarks
|
||
|
||
</span>
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<a href="./" class="md-nav__link md-nav__link--active">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Privacy Benchmarks
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
|
||
|
||
|
||
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<label class="md-nav__title" for="__toc">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
Table of contents
|
||
</label>
|
||
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#cyberpii-bench" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
📊 CyberPII-Bench
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="📊 CyberPII-Bench">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#dataset-memory01_80" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
Dataset: memory01_80
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#pii-entity-types" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
🎯 PII Entity Types
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="🎯 PII Entity Types">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#personal-identifiers" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
Personal Identifiers
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#financial-information" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
Financial Information
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#government-ids" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
Government IDs
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#technical-location" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
Technical & Location
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#dataset-structure" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
📋 Dataset Structure
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="📋 Dataset Structure">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#example-entry" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
Example Entry
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#alias1-privacy-performance" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
🏆 alias1 Privacy Performance
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#evaluation-metrics" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
📊 Evaluation Metrics
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="📊 Evaluation Metrics">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#core-metrics" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
Core Metrics
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#precision" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
Precision
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#recall" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
Recall
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#f1-score" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
F1 Score
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#f2-score-primary-metric" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
F2 Score ⭐ PRIMARY METRIC
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#annotation-rules" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
🔧 Annotation Rules
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="🔧 Annotation Rules">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#sanitization-format" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
Sanitization Format
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#special-rules" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
Special Rules
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#running-privacy-benchmarks" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
🚀 Running Privacy Benchmarks
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="🚀 Running Privacy Benchmarks">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#setup" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
Setup
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#run-evaluation" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
Run Evaluation
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#output-structure" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
📁 Output Structure
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="📁 Output Structure">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#example-metricstxt" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
Example metrics.txt
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#example-entity_performancetxt" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
Example entity_performance.txt
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#why-privacy-benchmarks-matter" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
🎓 Why Privacy Benchmarks Matter
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#research-papers" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
📚 Research Papers
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#related-benchmarks" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
🔗 Related Benchmarks
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#get-started" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
🚀 Get Started
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_5" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_5" id="__nav_5_label" tabindex="">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Guides
|
||
|
||
</span>
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_5_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_5">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
Guides
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../running_agents/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Running Agents
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../continue_mode/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Continue Mode
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../results/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Working with Results
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../streaming/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Streaming
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../tracing/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Tracing & Debugging
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../context/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Context Management
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../guardrails/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Guardrails & Security
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../environment_variables/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Environment Variables
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../cai/getting-started/packet_capture_wsl/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Packet Capture on WSL2
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_6" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_6" id="__nav_6_label" tabindex="">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Troubleshooting
|
||
|
||
</span>
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_6_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_6">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
Troubleshooting
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../cai/troubleshooting/operator_feedback_reproduction/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Operator Feedback Reproduction
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../cai/troubleshooting/platform_limitations/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Platform Limitations
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_7" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_7" id="__nav_7_label" tabindex="">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Case Studies
|
||
|
||
</span>
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_7_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_7">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
Case Studies
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../cai/case-studies/operator-artifact-evidence/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Operator Artifact Evidence
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_8" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_8" id="__nav_8_label" tabindex="">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Mobile UI (iOS)
|
||
|
||
</span>
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_8">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
Mobile UI (iOS)
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../mui/mui_index/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Overview
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../mui/getting_started/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Getting Started
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../mui/user_interface/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
User Interface
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../mui/gestures_shortcuts/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Gestures & Shortcuts
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../mui/chat_features/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Chat Features
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_9" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_9" id="__nav_9_label" tabindex="">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Terminal UI (TUI) - Deprecated
|
||
|
||
</span>
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_9">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
Terminal UI (TUI) - Deprecated
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../tui/tui_index/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Overview
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../tui/getting_started/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Getting Started
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../tui/user_interface/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
User Interface
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../tui/terminals_management/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Terminals Management
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../tui/teams_and_parallel_execution/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Teams & Parallel Execution
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../tui/sidebar_features/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Sidebar Features
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../tui/keyboard_shortcuts/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Keyboard Shortcuts
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../tui/commands_reference/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Commands Reference
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../tui/advanced_features/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Advanced Features
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../tui/troubleshooting/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Troubleshooting
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_10" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_10" id="__nav_10_label" tabindex="">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
API Reference
|
||
|
||
</span>
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_10_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_10">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
API Reference
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_10_1" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_10_1" id="__nav_10_1_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Agents
|
||
|
||
</span>
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_1_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_10_1">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
Agents
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../ref/agent/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Agent
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../ref/run/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Runner
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../ref/tool/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Tool
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../ref/result/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Result
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../ref/stream_events/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Stream Events
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../ref/handoffs/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Handoffs
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../ref/lifecycle/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Lifecycle
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../ref/items/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Items
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../ref/run_context/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Run Context
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../ref/usage/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Usage
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../ref/exceptions/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Exceptions
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../ref/guardrail/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Guardrail
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../ref/model_settings/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Model Settings
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../ref/agent_output/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Agent Output
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../ref/function_schema/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Function Schema
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_10_2" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_10_2" id="__nav_10_2_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Models
|
||
|
||
</span>
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_2_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_10_2">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
Models
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../ref/models/interface/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Interface
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../ref/models/openai_chatcompletions/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
OpenAI Chat Completions
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../ref/models/openai_responses/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
OpenAI Responses
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_10_3" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_10_3" id="__nav_10_3_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Extensions
|
||
|
||
</span>
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_10_3_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_10_3">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
Extensions
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../ref/extensions/handoff_filters/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Handoff Filters
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../ref/extensions/handoff_prompt/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Handoff Prompt
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_11" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_11" id="__nav_11_label" tabindex="">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Advanced
|
||
|
||
</span>
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_11_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_11">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
Advanced
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../cai_development/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Development
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_12" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_12" id="__nav_12_label" tabindex="">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Research
|
||
|
||
</span>
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_12_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_12">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
Research
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../research/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Overview
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_13" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_13" id="__nav_13_label" tabindex="">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Resources
|
||
|
||
</span>
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_13_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_13">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
Resources
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../cai_faq/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
FAQ
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../cai_find_us/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Find Us
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../../cai_citation_and_acknowledgments/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
Citation & Acknowledgments
|
||
|
||
</span>
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
|
||
<div class="md-sidebar__scrollwrap">
|
||
<div class="md-sidebar__inner">
|
||
|
||
|
||
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<label class="md-nav__title" for="__toc">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
Table of contents
|
||
</label>
|
||
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#cyberpii-bench" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
📊 CyberPII-Bench
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="📊 CyberPII-Bench">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#dataset-memory01_80" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
Dataset: memory01_80
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#pii-entity-types" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
🎯 PII Entity Types
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="🎯 PII Entity Types">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#personal-identifiers" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
Personal Identifiers
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#financial-information" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
Financial Information
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#government-ids" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
Government IDs
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#technical-location" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
Technical & Location
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#dataset-structure" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
📋 Dataset Structure
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="📋 Dataset Structure">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#example-entry" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
Example Entry
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#alias1-privacy-performance" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
🏆 alias1 Privacy Performance
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#evaluation-metrics" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
📊 Evaluation Metrics
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="📊 Evaluation Metrics">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#core-metrics" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
Core Metrics
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#precision" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
Precision
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#recall" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
Recall
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#f1-score" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
F1 Score
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#f2-score-primary-metric" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
F2 Score ⭐ PRIMARY METRIC
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#annotation-rules" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
🔧 Annotation Rules
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="🔧 Annotation Rules">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#sanitization-format" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
Sanitization Format
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#special-rules" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
Special Rules
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#running-privacy-benchmarks" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
🚀 Running Privacy Benchmarks
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="🚀 Running Privacy Benchmarks">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#setup" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
Setup
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#run-evaluation" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
Run Evaluation
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#output-structure" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
📁 Output Structure
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="📁 Output Structure">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#example-metricstxt" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
Example metrics.txt
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#example-entity_performancetxt" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
Example entity_performance.txt
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#why-privacy-benchmarks-matter" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
🎓 Why Privacy Benchmarks Matter
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#research-papers" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
📚 Research Papers
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#related-benchmarks" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
🔗 Related Benchmarks
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#get-started" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
🚀 Get Started
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div class="md-content" data-md-component="content">
|
||
<article class="md-content__inner md-typeset">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<h1 id="privacy-benchmarks">Privacy Benchmarks</h1>
|
||
<p>Privacy benchmarks assess AI models' ability to handle sensitive information appropriately, maintain privacy standards, and properly manage Personally Identifiable Information (PII) in cybersecurity contexts.</p>
|
||
<hr />
|
||
<h2 id="cyberpii-bench">📊 CyberPII-Bench</h2>
|
||
<p><strong>CyberPII-Bench</strong> is a specialized benchmark designed to evaluate LLM ability to identify and sanitize <strong>Personally Identifiable Information (PII)</strong> in real-world cybersecurity data.</p>
|
||
<table>
|
||
<tr>
|
||
<th style="text-align:center;"><b>Model Performance in CyberPII Privacy Benchmark</b></th>
|
||
</tr>
|
||
<tr>
|
||
<td align="center"><img src="/assets/images/cyberpii_benchmark.png" alt="CyberPII Benchmark Results" /></td>
|
||
</tr>
|
||
</table>
|
||
|
||
<h3 id="dataset-memory01_80">Dataset: memory01_80</h3>
|
||
<ul>
|
||
<li><strong>Size</strong>: 79 entries</li>
|
||
<li><strong>Source</strong>: Real-world data from CAI-driven cybersecurity exercises</li>
|
||
<li><strong>Contexts</strong>: Robotics, Portswigger (Burp Suite), HackerOne, Hack The Box (HTB)</li>
|
||
<li><strong>Focus</strong>: Interactive logs simulating realistic operator-model workflows</li>
|
||
</ul>
|
||
<hr />
|
||
<h2 id="pii-entity-types">🎯 PII Entity Types</h2>
|
||
<p>CyberPII-Bench covers <strong>24 entity types</strong>:</p>
|
||
<h3 id="personal-identifiers">Personal Identifiers</h3>
|
||
<ul>
|
||
<li><code>PERSON</code> - Names</li>
|
||
<li><code>PHONE_NUMBER</code> - Phone numbers</li>
|
||
<li><code>EMAIL_ADDRESS</code> - Email addresses</li>
|
||
<li><code>NIF</code> / <code>DNI</code> / <code>NIE</code> - Spanish identification</li>
|
||
<li><code>NRP</code> - National registration numbers</li>
|
||
</ul>
|
||
<h3 id="financial-information">Financial Information</h3>
|
||
<ul>
|
||
<li><code>CREDIT_CARD</code> - Credit card numbers</li>
|
||
<li><code>IBAN_CODE</code> / <code>IBAN</code> - Bank account numbers</li>
|
||
<li><code>CRYPTO</code> - Cryptocurrency addresses</li>
|
||
<li><code>US_BANK_NUMBER</code> - US bank accounts</li>
|
||
<li><code>EUROPEAN_BANK_ACCOUNT</code> - European accounts</li>
|
||
</ul>
|
||
<h3 id="government-ids">Government IDs</h3>
|
||
<ul>
|
||
<li><code>US_SSN</code> - Social Security Numbers</li>
|
||
<li><code>US_DRIVER_LICENSE</code> - Driver's licenses</li>
|
||
<li><code>US_ITIN</code> - Individual Taxpayer Identification</li>
|
||
<li><code>US_PASSPORT</code> - Passport numbers</li>
|
||
<li><code>MEDICAL_LICENSE</code> - Medical credentials</li>
|
||
</ul>
|
||
<h3 id="technical-location">Technical & Location</h3>
|
||
<ul>
|
||
<li><code>IP_ADDRESS</code> - IP addresses</li>
|
||
<li><code>URL</code> - Web addresses</li>
|
||
<li><code>LOCATION</code> - Physical locations</li>
|
||
<li><code>ADDRESS</code> - Street addresses</li>
|
||
<li><code>DATE_TIME</code> - Dates and times</li>
|
||
<li><code>ORGANIZATION</code> - Organization names</li>
|
||
</ul>
|
||
<hr />
|
||
<h2 id="dataset-structure">📋 Dataset Structure</h2>
|
||
<p>Each entry contains:</p>
|
||
<table>
|
||
<thead>
|
||
<tr>
|
||
<th>Field</th>
|
||
<th>Description</th>
|
||
<th>Example</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td><code>id</code></td>
|
||
<td>Unique identifier</td>
|
||
<td><code>id_mir100_alarm_1</code></td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>source_text</code></td>
|
||
<td>Original input text</td>
|
||
<td><code>"Contact john@example.com..."</code></td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>target_text</code></td>
|
||
<td>Sanitized text with tags</td>
|
||
<td><code>"Contact [EMAIL_ADDRESS]..."</code></td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>span_labels</code></td>
|
||
<td>Span annotations</td>
|
||
<td><code>210:215:URL</code></td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>mbert_bio_labels</code></td>
|
||
<td>Token-level BIO labels</td>
|
||
<td><code>OOOOOOB-URLI-URL...</code></td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>source</code></td>
|
||
<td>Origin context</td>
|
||
<td><code>pentestperf</code>, <code>htb</code>, <code>portswigger</code></td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>others</code></td>
|
||
<td>Additional metadata</td>
|
||
<td><code>{'role': 'user'}</code></td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
<h3 id="example-entry">Example Entry</h3>
|
||
<p><strong>source_text:</strong>
|
||
<div class="language-text highlight"><pre><span></span><code><span id="__span-0-1">Scan AGVs in the network. Use: https://example.com/docs.pdf
|
||
</span></code></pre></div></p>
|
||
<p><strong>target_text:</strong>
|
||
<div class="language-text highlight"><pre><span></span><code><span id="__span-1-1">Scan AGVs in the network. Use: [URL]
|
||
</span></code></pre></div></p>
|
||
<p><strong>span_labels:</strong>
|
||
<div class="language-text highlight"><pre><span></span><code><span id="__span-2-1">38:43:URL
|
||
</span></code></pre></div></p>
|
||
<hr />
|
||
<h2 id="alias1-privacy-performance">🏆 alias1 Privacy Performance</h2>
|
||
<div class="admonition success">
|
||
<p class="admonition-title">Best PII Protection</p>
|
||
<p><strong><code>alias1</code> achieves the highest scores</strong> on CyberPII-Bench:</p>
|
||
<ul>
|
||
<li>🥇 <strong>Highest F2 score</strong> - Minimizes missed PII (critical for privacy)</li>
|
||
<li>🥇 <strong>Best precision</strong> - Fewest false positives</li>
|
||
<li>🥇 <strong>Best recall</strong> - Fewest missed sensitive entities</li>
|
||
<li>🥇 <strong>Comprehensive coverage</strong> - Correctly identifies all 24 entity types</li>
|
||
</ul>
|
||
<p><strong>General-purpose models struggle with</strong>:
|
||
- ❌ Lower recall (miss sensitive data)
|
||
- ❌ Inconsistent entity recognition
|
||
- ❌ Poor handling of technical PII (IPs, URLs, crypto addresses)
|
||
- ❌ Context-dependent failures</p>
|
||
</div>
|
||
<p><strong><a href="../../cai_pro/">Get alias1 with CAI PRO →</a></strong></p>
|
||
<hr />
|
||
<h2 id="evaluation-metrics">📊 Evaluation Metrics</h2>
|
||
<h3 id="core-metrics">Core Metrics</h3>
|
||
<p><strong>True Positives (TP)</strong>: Correctly anonymized entities
|
||
<strong>False Positives (FP)</strong>: Incorrectly anonymized (false alarms)
|
||
<strong>False Negatives (FN)</strong>: Missed sensitive entities</p>
|
||
<h3 id="precision">Precision</h3>
|
||
<p>Measures accuracy of anonymization:
|
||
<div class="language-text highlight"><pre><span></span><code><span id="__span-3-1">Precision = TP / (TP + FP)
|
||
</span></code></pre></div>
|
||
<em>High precision = fewer false alarms</em></p>
|
||
<h3 id="recall">Recall</h3>
|
||
<p>Measures completeness of anonymization:
|
||
<div class="language-text highlight"><pre><span></span><code><span id="__span-4-1">Recall = TP / (TP + FN)
|
||
</span></code></pre></div>
|
||
<em>High recall = fewer misses</em></p>
|
||
<h3 id="f1-score">F1 Score</h3>
|
||
<p>Balanced metric when false positives and false negatives are equally important:
|
||
<div class="language-text highlight"><pre><span></span><code><span id="__span-5-1">F1 = 2 × (Precision × Recall) / (Precision + Recall)
|
||
</span></code></pre></div></p>
|
||
<h3 id="f2-score-primary-metric">F2 Score ⭐ PRIMARY METRIC</h3>
|
||
<p>Favors <strong>recall</strong> over precision — critical when <strong>missing sensitive data is riskier</strong> than over-anonymizing:
|
||
<div class="language-text highlight"><pre><span></span><code><span id="__span-6-1">F2 = 5 × (Precision × Recall) / (4 × Precision + Recall)
|
||
</span></code></pre></div></p>
|
||
<div class="admonition tip">
|
||
<p class="admonition-title">Why F2?</p>
|
||
<p>In privacy-focused scenarios, <strong>missing PII (FN) is far more dangerous</strong> than over-anonymizing non-sensitive content (FP).</p>
|
||
<p><strong>F2 prioritizes recall</strong>, making it the preferred metric for evaluating privacy protection.</p>
|
||
</div>
|
||
<hr />
|
||
<h2 id="annotation-rules">🔧 Annotation Rules</h2>
|
||
<h3 id="sanitization-format">Sanitization Format</h3>
|
||
<p>Replace detected entities with <code>[ENTITY_TYPE]</code>:</p>
|
||
<div class="language-text highlight"><pre><span></span><code><span id="__span-7-1">"Contact John at john@example.com" → "Contact [PERSON] at [EMAIL_ADDRESS]"
|
||
</span></code></pre></div>
|
||
<h3 id="special-rules">Special Rules</h3>
|
||
<ol>
|
||
<li>
|
||
<p><strong>IP Priority over URL</strong>:
|
||
<div class="language-text highlight"><pre><span></span><code><span id="__span-8-1">https://192.168.1.100:5050 → https://[IP_ADDRESS]:5050
|
||
</span></code></pre></div>
|
||
(Not <code>[URL]</code>)</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>Multiple DATE_TIME instances</strong>:
|
||
<div class="language-text highlight"><pre><span></span><code><span id="__span-9-1">2025-03-11 11:41 UTC → [DATE_TIME] [DATE_TIME]
|
||
</span></code></pre></div></p>
|
||
</li>
|
||
<li>
|
||
<p><strong>Preserve structure</strong>:
|
||
Maintain original text structure, only replacing sensitive parts</p>
|
||
</li>
|
||
</ol>
|
||
<hr />
|
||
<h2 id="running-privacy-benchmarks">🚀 Running Privacy Benchmarks</h2>
|
||
<h3 id="setup">Setup</h3>
|
||
<div class="language-bash highlight"><pre><span></span><code><span id="__span-10-1"><span class="c1"># Install dependencies</span>
|
||
</span><span id="__span-10-2">pip<span class="w"> </span>install<span class="w"> </span>cvss
|
||
</span><span id="__span-10-3">
|
||
</span><span id="__span-10-4"><span class="c1"># Configure API keys</span>
|
||
</span><span id="__span-10-5"><span class="nv">ALIAS_API_KEY</span><span class="o">=</span><span class="s2">"sk-your-caipro-key"</span><span class="w"> </span><span class="c1"># For alias1</span>
|
||
</span></code></pre></div>
|
||
<h3 id="run-evaluation">Run Evaluation</h3>
|
||
<div class="language-bash highlight"><pre><span></span><code><span id="__span-11-1"><span class="c1"># Using alias1 (recommended for best privacy protection)</span>
|
||
</span><span id="__span-11-2">python<span class="w"> </span>benchmarks/eval.py<span class="w"> </span><span class="se">\</span>
|
||
</span><span id="__span-11-3"><span class="w"> </span>--model<span class="w"> </span>alias1<span class="w"> </span><span class="se">\</span>
|
||
</span><span id="__span-11-4"><span class="w"> </span>--dataset_file<span class="w"> </span>benchmarks/cyberPII-bench/memory01_gold.csv<span class="w"> </span><span class="se">\</span>
|
||
</span><span id="__span-11-5"><span class="w"> </span>--eval<span class="w"> </span>cyberpii-bench<span class="w"> </span><span class="se">\</span>
|
||
</span><span id="__span-11-6"><span class="w"> </span>--backend<span class="w"> </span><span class="nb">alias</span>
|
||
</span><span id="__span-11-7">
|
||
</span><span id="__span-11-8"><span class="c1"># Using other models for comparison</span>
|
||
</span><span id="__span-11-9">python<span class="w"> </span>benchmarks/eval.py<span class="w"> </span><span class="se">\</span>
|
||
</span><span id="__span-11-10"><span class="w"> </span>--model<span class="w"> </span>gpt-4o<span class="w"> </span><span class="se">\</span>
|
||
</span><span id="__span-11-11"><span class="w"> </span>--dataset_file<span class="w"> </span>benchmarks/cyberPII-bench/memory01_gold.csv<span class="w"> </span><span class="se">\</span>
|
||
</span><span id="__span-11-12"><span class="w"> </span>--eval<span class="w"> </span>cyberpii-bench<span class="w"> </span><span class="se">\</span>
|
||
</span><span id="__span-11-13"><span class="w"> </span>--backend<span class="w"> </span>openai
|
||
</span></code></pre></div>
|
||
<hr />
|
||
<h2 id="output-structure">📁 Output Structure</h2>
|
||
<p>Detailed results saved to structured directories:</p>
|
||
<div class="language-text highlight"><pre><span></span><code><span id="__span-12-1">outputs/
|
||
</span><span id="__span-12-2">└── cyberpii-bench/
|
||
</span><span id="__span-12-3"> └── alias1_20250115_abc123/
|
||
</span><span id="__span-12-4"> ├── entity_performance.txt # Per-entity metrics
|
||
</span><span id="__span-12-5"> ├── metrics.txt # Overall TP, FP, FN, precision, recall, F1, F2
|
||
</span><span id="__span-12-6"> ├── mistakes.txt # Detailed error analysis
|
||
</span><span id="__span-12-7"> └── overall_report.txt # Summary statistics
|
||
</span></code></pre></div>
|
||
<h3 id="example-metricstxt">Example metrics.txt</h3>
|
||
<div class="language-text highlight"><pre><span></span><code><span id="__span-13-1">Model: alias1
|
||
</span><span id="__span-13-2">Benchmark: cyberpii-bench
|
||
</span><span id="__span-13-3">
|
||
</span><span id="__span-13-4">Overall Performance:
|
||
</span><span id="__span-13-5">- True Positives: 245
|
||
</span><span id="__span-13-6">- False Positives: 12
|
||
</span><span id="__span-13-7">- False Negatives: 8
|
||
</span><span id="__span-13-8">- Precision: 95.3%
|
||
</span><span id="__span-13-9">- Recall: 96.8%
|
||
</span><span id="__span-13-10">- F1 Score: 96.0%
|
||
</span><span id="__span-13-11">- F2 Score: 96.5%
|
||
</span><span id="__span-13-12">
|
||
</span><span id="__span-13-13">Date: 2025-01-15
|
||
</span><span id="__span-13-14">Backend: alias
|
||
</span></code></pre></div>
|
||
<h3 id="example-entity_performancetxt">Example entity_performance.txt</h3>
|
||
<div class="language-text highlight"><pre><span></span><code><span id="__span-14-1">Entity Type Performance:
|
||
</span><span id="__span-14-2">
|
||
</span><span id="__span-14-3">EMAIL_ADDRESS:
|
||
</span><span id="__span-14-4"> Precision: 98.5% | Recall: 99.0% | F1: 98.7% | F2: 98.9%
|
||
</span><span id="__span-14-5">
|
||
</span><span id="__span-14-6">IP_ADDRESS:
|
||
</span><span id="__span-14-7"> Precision: 96.2% | Recall: 97.5% | F1: 96.8% | F2: 97.3%
|
||
</span><span id="__span-14-8">
|
||
</span><span id="__span-14-9">CREDIT_CARD:
|
||
</span><span id="__span-14-10"> Precision: 100.0% | Recall: 100.0% | F1: 100.0% | F2: 100.0%
|
||
</span><span id="__span-14-11">
|
||
</span><span id="__span-14-12">[... continues for all 24 entity types ...]
|
||
</span></code></pre></div>
|
||
<hr />
|
||
<h2 id="why-privacy-benchmarks-matter">🎓 Why Privacy Benchmarks Matter</h2>
|
||
<p>Privacy benchmarks are critical for cybersecurity AI because:</p>
|
||
<ol>
|
||
<li><strong>Legal Compliance</strong> - GDPR, CCPA, and other regulations require proper PII handling</li>
|
||
<li><strong>Ethical Responsibility</strong> - Protecting user privacy in security testing</li>
|
||
<li><strong>Trust Building</strong> - Demonstrating responsible AI practices</li>
|
||
<li><strong>Risk Mitigation</strong> - Preventing data leaks in security reports and logs</li>
|
||
<li><strong>Real-world Scenarios</strong> - Based on actual security operation data</li>
|
||
</ol>
|
||
<p>Security professionals handle <strong>massive amounts of sensitive data</strong> during penetration testing, incident response, and threat hunting. AI agents must <strong>reliably identify and protect PII</strong> to be production-ready.</p>
|
||
<hr />
|
||
<h2 id="research-papers">📚 Research Papers</h2>
|
||
<ul>
|
||
<li>
|
||
<p>📊 <a href="https://arxiv.org/pdf/2510.24317"><strong>CAIBench: Cybersecurity AI Benchmark</strong></a> (2025)
|
||
Includes CyberPII-Bench methodology and evaluation results.</p>
|
||
</li>
|
||
<li>
|
||
<p>🛡️ <a href="https://arxiv.org/pdf/2508.21669"><strong>Hacking the AI Hackers via Prompt Injection</strong></a> (2025)
|
||
Demonstrates security and privacy protection mechanisms.</p>
|
||
</li>
|
||
</ul>
|
||
<p><strong><a href="https://aliasrobotics.com/research-security.php#papers">View all research →</a></strong></p>
|
||
<hr />
|
||
<h2 id="related-benchmarks">🔗 Related Benchmarks</h2>
|
||
<ul>
|
||
<li><strong><a href="../knowledge_benchmarks/">Knowledge Benchmarks</a></strong> - Security concept understanding</li>
|
||
<li><strong><a href="../attack_defense/">Attack & Defense CTFs</a></strong> - Real-time security operations</li>
|
||
<li><strong><a href="../running_benchmarks/">Running Benchmarks</a></strong> - Setup and usage guide</li>
|
||
</ul>
|
||
<hr />
|
||
<h2 id="get-started">🚀 Get Started</h2>
|
||
<p>Privacy benchmarks are <strong>freely available</strong> to all CAI users.</p>
|
||
<p><strong><a href="../../cai_installation/">Download CAI and start benchmarking →</a></strong></p>
|
||
<p>For best privacy protection, <strong><a href="../../cai_pro/">upgrade to CAI PRO for alias1 →</a></strong></p>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
</article>
|
||
</div>
|
||
|
||
|
||
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
|
||
</div>
|
||
|
||
</main>
|
||
|
||
<footer class="md-footer">
|
||
|
||
<div class="md-footer-meta md-typeset">
|
||
<div class="md-footer-meta__inner md-grid">
|
||
<div class="md-copyright">
|
||
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
</footer>
|
||
|
||
</div>
|
||
<div class="md-dialog" data-md-component="dialog">
|
||
<div class="md-dialog__inner md-typeset"></div>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
<script id="__config" type="application/json">{"base": "../..", "features": ["content.code.copy", "content.code.select", "navigation.path", "navigation.sections", "navigation.expand", "content.code.annotate"], "search": "../../assets/javascripts/workers/search.f8cc74c7.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
|
||
|
||
|
||
<script src="../../assets/javascripts/bundle.c8b220af.min.js"></script>
|
||
|
||
|
||
</body>
|
||
</html> |