diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/Cargo.lock b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/Cargo.lock index 0bc75fc4..2ca28d4d 100644 --- a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/Cargo.lock +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/Cargo.lock @@ -79,6 +79,12 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "1.0.0" @@ -286,6 +292,12 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.2.63" @@ -302,6 +314,33 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "cipher" version = "0.4.4" @@ -400,6 +439,46 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools", + "num-traits", + "once_cell", + "oorandom", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.1.7" @@ -452,6 +531,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + [[package]] name = "equivalent" version = "1.0.2" @@ -662,6 +747,17 @@ dependencies = [ "polyval", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -701,6 +797,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + [[package]] name = "hex" version = "0.4.3" @@ -838,12 +940,32 @@ dependencies = [ "generic-array", ] +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -1044,6 +1166,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "opaque-debug" version = "0.3.1" @@ -1316,6 +1444,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -1538,6 +1675,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tlsfp" version = "0.1.0" @@ -1571,6 +1718,7 @@ version = "0.1.0" dependencies = [ "aes", "aes-gcm", + "criterion", "etherparse", "hex", "hkdf", @@ -1839,6 +1987,16 @@ dependencies = [ "libc", ] +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -1958,6 +2116,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/Cargo.toml b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/Cargo.toml index febc79e0..6e79636d 100644 --- a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/Cargo.toml +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/Cargo.toml @@ -31,3 +31,8 @@ aes-gcm.workspace = true hex.workspace = true proptest = "1" serde_json.workspace = true +criterion = { version = "0.5", default-features = false, features = ["cargo_bench_support"] } + +[[bench]] +name = "fingerprint" +harness = false diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/benches/fingerprint.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/benches/fingerprint.rs new file mode 100644 index 00000000..e3cdca35 --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/benches/fingerprint.rs @@ -0,0 +1,215 @@ +// ©AngelaMos | 2026 +// fingerprint.rs + +//! Throughput benchmarks for the fingerprinting hot path. +//! +//! Two questions matter for a passive sensor: how fast it parses one handshake, +//! and how fast it carries a whole capture through the pipeline. The first +//! benchmark isolates parsing and hashing a single ClientHello; the second +//! replays a vendored capture frame by frame, reporting fingerprints per second +//! against the project target of ten thousand. The capture is read into memory +//! once at setup so the file system never enters the measured loop. + +use std::hint::black_box; +use std::path::Path; + +use criterion::{Criterion, Throughput, criterion_group, criterion_main}; + +use tlsfp_core::ja3::ja3; +use tlsfp_core::ja4::{Transport, ja4}; +use tlsfp_core::parse::parse_client_hello; +use tlsfp_core::pipeline::source::{PacketSource, PcapFileSource, RawFrame}; +use tlsfp_core::pipeline::{Pipeline, PipelineConfig}; + +/// Captures replayed by the pipeline benchmark, named for the report. +const PCAPS: [(&str, &str); 2] = [ + ( + "tls-handshake", + concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../testdata/pcap/tls-handshake.pcapng" + ), + ), + ( + "browsers-x509", + concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../testdata/pcap/browsers-x509.pcapng" + ), + ), +]; + +/// One captured frame copied into an owned buffer for replay. +struct OwnedFrame { + ts_nanos: u64, + link_type: i32, + data: Vec, +} + +/// Reads every frame of a capture into memory so the benchmark loop measures +/// the pipeline rather than the file system. +fn load_frames(path: &str) -> Vec { + let mut source = PcapFileSource::open(Path::new(path)).expect("open capture"); + let mut frames = Vec::new(); + while let Some(frame) = source.next_frame().expect("read frame") { + frames.push(OwnedFrame { + ts_nanos: frame.ts_nanos, + link_type: frame.link_type, + data: frame.data.to_vec(), + }); + } + frames +} + +/// Replays preloaded frames through a fresh pipeline, returning the event count. +fn replay(frames: &[OwnedFrame]) -> u64 { + let mut pipeline = Pipeline::new(PipelineConfig::default()); + for frame in frames { + let raw = RawFrame { + ts_nanos: frame.ts_nanos, + link_type: frame.link_type, + data: &frame.data, + }; + pipeline.feed(&raw, &mut |event| { + black_box(&event); + }); + } + pipeline.finish(); + pipeline.counters().events +} + +fn bench_pipeline(c: &mut Criterion) { + let mut group = c.benchmark_group("pipeline"); + for (name, path) in PCAPS { + let frames = load_frames(path); + let events = replay(&frames).max(1); + group.throughput(Throughput::Elements(events)); + group.bench_function(name, |b| { + b.iter(|| black_box(replay(black_box(&frames)))); + }); + } + group.finish(); +} + +fn bench_fingerprint(c: &mut Criterion) { + let body = client_hello_body(); + let hello = parse_client_hello(&body).expect("the benchmark hello parses"); + + let mut group = c.benchmark_group("fingerprint"); + group.bench_function("parse_client_hello", |b| { + b.iter(|| black_box(parse_client_hello(black_box(&body)).expect("parse"))); + }); + group.bench_function("ja3", |b| b.iter(|| black_box(ja3(black_box(&hello))))); + group.bench_function("ja4", |b| { + b.iter(|| black_box(ja4(black_box(&hello), Transport::Tcp))); + }); + group.finish(); +} + +/// Assembles a representative TLS 1.3 ClientHello body, the input the parser and +/// the hash functions take. The exact bytes are not pinned to any published +/// vector here: this is a throughput fixture, and the conformance vectors live +/// in the integration tests. +fn client_hello_body() -> Vec { + const CIPHERS: [u16; 15] = [ + 0x1301, 0x1302, 0x1303, 0xc02b, 0xc02f, 0xc02c, 0xc030, 0xcca9, 0xcca8, 0xc013, 0xc014, + 0x009c, 0x009d, 0x002f, 0x0035, + ]; + const GROUPS: [u16; 4] = [0x001d, 0x0017, 0x0018, 0x0019]; + const SIG_ALGS: [u16; 8] = [ + 0x0403, 0x0804, 0x0401, 0x0503, 0x0805, 0x0501, 0x0806, 0x0601, + ]; + const VERSIONS: [u16; 2] = [0x0304, 0x0303]; + const OPAQUE: [u16; 4] = [0x0005, 0x0017, 0x0023, 0xff01]; + + let mut body = Vec::new(); + body.extend_from_slice(&0x0303u16.to_be_bytes()); + body.extend_from_slice(&[0u8; 32]); + body.push(0); + + let mut ciphers = Vec::new(); + for suite in CIPHERS { + ciphers.extend_from_slice(&suite.to_be_bytes()); + } + push_u16(&mut body, &ciphers); + push_u8(&mut body, &[0]); + + let mut exts = Vec::new(); + add_ext(&mut exts, 0x0000, &sni("example.com")); + add_ext(&mut exts, 0x000a, &u16_list_u16(&GROUPS)); + add_ext(&mut exts, 0x000b, &u8_list(&[0])); + add_ext(&mut exts, 0x000d, &u16_list_u16(&SIG_ALGS)); + add_ext(&mut exts, 0x0010, &alpn(&[b"h2", b"http/1.1"])); + add_ext(&mut exts, 0x002b, &u8_list_u16(&VERSIONS)); + for ext in OPAQUE { + add_ext(&mut exts, ext, &[]); + } + push_u16(&mut body, &exts); + body +} + +fn push_u8(out: &mut Vec, data: &[u8]) { + out.push(u8::try_from(data.len()).expect("length fits a byte")); + out.extend_from_slice(data); +} + +fn push_u16(out: &mut Vec, data: &[u8]) { + out.extend_from_slice( + &u16::try_from(data.len()) + .expect("length fits two bytes") + .to_be_bytes(), + ); + out.extend_from_slice(data); +} + +fn add_ext(out: &mut Vec, ext_type: u16, data: &[u8]) { + out.extend_from_slice(&ext_type.to_be_bytes()); + push_u16(out, data); +} + +fn sni(host: &str) -> Vec { + let mut entry = vec![0u8]; + push_u16(&mut entry, host.as_bytes()); + let mut data = Vec::new(); + push_u16(&mut data, &entry); + data +} + +fn alpn(protocols: &[&[u8]]) -> Vec { + let mut list = Vec::new(); + for protocol in protocols { + push_u8(&mut list, protocol); + } + let mut data = Vec::new(); + push_u16(&mut data, &list); + data +} + +fn u16_list_u16(values: &[u16]) -> Vec { + let mut list = Vec::new(); + for value in values { + list.extend_from_slice(&value.to_be_bytes()); + } + let mut data = Vec::new(); + push_u16(&mut data, &list); + data +} + +fn u8_list_u16(values: &[u16]) -> Vec { + let mut list = Vec::new(); + for value in values { + list.extend_from_slice(&value.to_be_bytes()); + } + let mut data = Vec::new(); + push_u8(&mut data, &list); + data +} + +fn u8_list(values: &[u8]) -> Vec { + let mut data = Vec::new(); + push_u8(&mut data, values); + data +} + +criterion_group!(benches, bench_pipeline, bench_fingerprint); +criterion_main!(benches); diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/pipeline/mod.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/pipeline/mod.rs index 83e951a2..989b8cd6 100644 --- a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/pipeline/mod.rs +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/pipeline/mod.rs @@ -102,6 +102,10 @@ pub struct Counters { pub segments_dropped: u64, pub events: u64, pub streams_capped: u64, + /// TCP streams recognized as TLS that yielded a complete ClientHello or + /// ServerHello. The denominator, with `unfinished_tls_streams`, of the miss + /// rate: how many handshakes the tool actually read. + pub tls_handshakes_fingerprinted: u64, pub unfinished_tls_streams: u64, /// QUIC long header Initial packets observed, both directions. pub quic_initials: u64, @@ -113,6 +117,28 @@ pub struct Counters { pub quic_version_unsupported: u64, } +impl Counters { + /// The share of recognized TLS streams the capture clipped before a + /// complete handshake message could be read. + /// + /// A fingerprinting tool that cannot say what it failed to read is one + /// whose silence gets mistaken for absence. A truncated or multi segment + /// ClientHello whose later segments never arrived counts as a miss here, so + /// an operator can tell a clean link from a clipped capture before trusting + /// the count of fingerprints. Zero recognized streams reports a zero rate + /// rather than dividing by nothing. + #[must_use] + #[allow(clippy::cast_precision_loss)] + pub fn tls_miss_rate(&self) -> f64 { + let recognized = self.tls_handshakes_fingerprinted + self.unfinished_tls_streams; + if recognized == 0 { + 0.0 + } else { + self.unfinished_tls_streams as f64 / recognized as f64 + } + } +} + /// One direction of one tracked flow. struct StreamHalf { reassembler: StreamReassembler, @@ -300,8 +326,15 @@ impl Pipeline { if outcome == PushOutcome::Grew { let mut emitted = 0u64; + let mut tls_fingerprinted = 0u64; tls::advance(&mut half.protocol, half.reassembler.data(), &mut |event| { emitted += 1; + if matches!( + event, + StreamEvent::ClientHello { .. } | StreamEvent::ServerHello { .. } + ) { + tls_fingerprinted += 1; + } sink(FingerprintEvent { ts_nanos: frame.ts_nanos, src, @@ -310,6 +343,7 @@ impl Pipeline { }); }); self.counters.events += emitted; + self.counters.tls_handshakes_fingerprinted += tls_fingerprinted; if half.protocol.finished() && !half.reassembler.released() { half.reassembler.release(); } @@ -595,6 +629,47 @@ mod tests { assert!(events[1].to_string().contains("http_request")); } + #[test] + fn miss_rate_arithmetic_is_misses_over_recognized() { + use super::Counters; + let none = Counters::default(); + assert!(none.tls_miss_rate().abs() < 1e-9); + + let clean = Counters { + tls_handshakes_fingerprinted: 4, + unfinished_tls_streams: 0, + ..Counters::default() + }; + assert!(clean.tls_miss_rate().abs() < 1e-9); + + let clipped = Counters { + tls_handshakes_fingerprinted: 3, + unfinished_tls_streams: 1, + ..Counters::default() + }; + assert!((clipped.tls_miss_rate() - 0.25).abs() < 1e-9); + } + + #[test] + fn clipped_tls_handshake_counts_as_a_miss() { + let client = ([10, 0, 0, 1], 40002); + let server = ([10, 0, 0, 2], 443); + let clipped_client_hello = [ + 0x16, 0x03, 0x01, 0x01, 0x00, 0x01, 0x00, 0x00, 0xfa, 0x03, 0x03, + ]; + + let frames = vec![tcp_frame(client, server, 1000, &clipped_client_hello)]; + let mut pipeline = Pipeline::new(PipelineConfig::default()); + let events = feed_all(&mut pipeline, &frames); + pipeline.finish(); + + assert!(events.is_empty()); + let counters = pipeline.counters(); + assert_eq!(counters.tls_handshakes_fingerprinted, 0); + assert_eq!(counters.unfinished_tls_streams, 1); + assert!((counters.tls_miss_rate() - 1.0).abs() < 1e-9); + } + #[test] fn pressure_eviction_keeps_the_table_bounded() { let config = PipelineConfig { diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp/src/cli.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp/src/cli.rs index 2f0993fe..00733510 100644 --- a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp/src/cli.rs +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp/src/cli.rs @@ -3,7 +3,7 @@ use std::io::Write as _; use std::path::{Path, PathBuf}; -use std::time::Duration; +use std::time::{Duration, Instant}; use anyhow::{Context, Result}; use clap::{Parser, Subcommand}; @@ -12,6 +12,7 @@ use tracing_subscriber::EnvFilter; use tlsfp_core::{FingerprintEvent, PcapFileSource, Pipeline, PipelineConfig, SourceError}; use crate::live::{DEFAULT_BPF_FILTER, LiveConfig, LiveSource}; +use crate::report::ReportBuilder; use crate::serve::{self, ServeConfig, Source}; use tlsfp_intel::{Alert, FpKind, IntelStore, MatchReport, MatchStrength, default_db_path}; @@ -54,6 +55,16 @@ pub enum Command { #[arg(long)] detect: bool, + /// Print a single forensic summary of the whole capture instead of one + /// line per handshake. When an intelligence database is present it also + /// folds in match verdicts and detection alerts. + #[arg(long)] + report: bool, + + /// How many rows each ranked section of the report shows. + #[arg(long, default_value_t = crate::report::DEFAULT_TOP)] + top: usize, + /// Path to the intelligence database, defaulting to the data directory. #[arg(long)] db: Option, @@ -251,8 +262,10 @@ impl Cli { json, intel, detect, + report, + top, db, - } => run_pcap(&path, json, intel, detect, db.as_deref()), + } => run_pcap(&path, json, intel, detect, report, top, db.as_deref()), Command::Live { interface, json, @@ -318,7 +331,19 @@ impl IntelCommand { /// The summary goes to the log rather than stdout so that piping the output /// into a tool sees only events, while a human still learns how much of the /// capture was readable and whether the file was cut short mid packet. -fn run_pcap(path: &Path, json: bool, intel: bool, detect: bool, db: Option<&Path>) -> Result<()> { +#[allow(clippy::fn_params_excessive_bools)] +fn run_pcap( + path: &Path, + json: bool, + intel: bool, + detect: bool, + report: bool, + top: usize, + db: Option<&Path>, +) -> Result<()> { + if report { + return run_pcap_report(path, json, top, db); + } let mut source = PcapFileSource::open(path) .with_context(|| format!("cannot open capture {}", path.display()))?; let mut store = open_for_run(intel, detect, db)?; @@ -356,7 +381,9 @@ fn run_pcap(path: &Path, json: bool, intel: bool, detect: bool, db: Option<&Path quic_initials = counters.quic_initials, quic_decrypted = counters.quic_decrypted, quic_version_unsupported = counters.quic_version_unsupported, + tls_handshakes = counters.tls_handshakes_fingerprinted, unfinished_tls_streams = counters.unfinished_tls_streams, + tls_miss_rate = counters.tls_miss_rate(), segments_dropped = counters.segments_dropped, "capture processed" ); @@ -366,6 +393,70 @@ fn run_pcap(path: &Path, json: bool, intel: bool, detect: bool, db: Option<&Path Ok(()) } +/// Reads a whole capture and prints one forensic summary instead of a stream +/// of events. +/// +/// Unlike the streaming path, the report builds an in process picture of every +/// endpoint, fingerprint, and miss before printing, so it folds intelligence +/// and detection in automatically whenever a database is present rather than +/// asking for the flags. With no database it falls back to a pure fingerprint +/// inventory. The intel lookups and detection writes are the same calls the +/// streaming path makes, so the report can never disagree with a live sensor +/// pointed at the same store. +fn run_pcap_report(path: &Path, json: bool, top: usize, db: Option<&Path>) -> Result<()> { + let mut source = PcapFileSource::open(path) + .with_context(|| format!("cannot open capture {}", path.display()))?; + let mut store = open_for_report(db)?; + let enabled = store.is_some(); + let mut pipeline = Pipeline::new(PipelineConfig::default()); + let mut builder = ReportBuilder::new(&path.display().to_string()); + + let started = Instant::now(); + pipeline.run(&mut source, |event| { + let reports = if enabled { + enrich(store.as_ref(), &event) + } else { + Vec::new() + }; + let alerts = detect_event(store.as_mut(), enabled, &event); + builder.observe(&event, &reports, &alerts); + })?; + let elapsed = started.elapsed(); + + let report = builder.finish(*pipeline.counters(), source.truncated(), elapsed, top); + let stdout = std::io::stdout().lock(); + let mut out = std::io::BufWriter::new(stdout); + if json { + serde_json::to_writer_pretty(&mut out, &report).context("writing report as JSON")?; + writeln!(out).context("writing report as JSON")?; + } else { + write!(out, "{}", report.render_text()).context("writing report")?; + } + out.flush().context("flushing the report to stdout")?; + Ok(()) +} + +/// Opens the store for a report run. A report folds in intelligence and +/// detection whenever a database is there to answer, so this opens an existing +/// database but never creates one: a forensic read of a capture should not +/// quietly leave a new database behind, and with none present the report is +/// still a complete fingerprint inventory. +fn open_for_report(db: Option<&Path>) -> Result> { + let path = db.map_or_else(default_db_path, Path::to_path_buf); + if path.exists() { + tracing::info!( + path = %path.display(), + "matching and recording detections against the intelligence database" + ); + return Ok(Some(open_or_create(&path)?)); + } + tracing::info!( + path = %path.display(), + "no intelligence database found; reporting fingerprints only, run 'tlsfp intel seed' to add verdicts" + ); + Ok(None) +} + /// Builds the dashboard configuration from the command line and starts the /// server. The live feed is a replayed capture, a live interface, or, by /// default, the alerts an external sensor tails into the same database. @@ -507,7 +598,9 @@ async fn drive_live( quic_initials = counters.quic_initials, quic_decrypted = counters.quic_decrypted, quic_version_unsupported = counters.quic_version_unsupported, + tls_handshakes = counters.tls_handshakes_fingerprinted, unfinished_tls_streams = counters.unfinished_tls_streams, + tls_miss_rate = counters.tls_miss_rate(), segments_dropped = counters.segments_dropped, "live capture stopped" ); diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp/src/main.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp/src/main.rs index 2c216670..d16dddbe 100644 --- a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp/src/main.rs +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp/src/main.rs @@ -3,6 +3,7 @@ mod cli; mod live; +mod report; mod serve; use anyhow::Result; diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp/src/report.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp/src/report.rs new file mode 100644 index 00000000..5ddac4e0 --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp/src/report.rs @@ -0,0 +1,727 @@ +// ©AngelaMos | 2026 +// report.rs + +//! The forensic batch report for a capture file. +//! +//! Where the streaming `pcap` output prints one line per handshake as it is +//! read, the report holds the whole capture in mind and answers the questions +//! an analyst asks after the fact: who spoke, what they presented, which +//! fingerprints the intelligence database recognised, what the detection rules +//! flagged, and, just as important, how much of the capture the tool could not +//! read. The last part is the honesty section: a miss rate and a throughput +//! that let a reader tell a clean capture from a clipped one before trusting an +//! absence of fingerprints. +//! +//! The aggregation is fed one event at a time so a multi gigabyte capture never +//! has to be held in memory at once; only the distinct fingerprints, endpoints, +//! and names survive between events. The [`Report`] it produces serialises +//! straight to JSON and renders to aligned text, and the builder is unit tested +//! on synthetic events with no capture file in the picture. + +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::fmt::Write as _; +use std::net::IpAddr; +use std::time::Duration; + +use serde::Serialize; + +use tlsfp_core::{Counters, FingerprintEvent, StreamEvent}; +use tlsfp_intel::{Alert, MatchReport, Verdict}; + +/// How many rows each ranked section shows by default. +pub const DEFAULT_TOP: usize = 15; + +/// One endpoint's running inventory while the capture is being read. +#[derive(Default)] +struct EndpointAgg { + events: u64, + ja4: BTreeSet, + ja4s: BTreeSet, + ja4t: BTreeSet, + ja4h: BTreeSet, + ja4x: BTreeSet, + sni: BTreeSet, + user_agents: BTreeSet, + worst_verdict: Option, + alerts: u64, +} + +/// One intelligence finding's running aggregate, keyed by fingerprint. +struct IntelAgg { + verdict: Verdict, + threat_score: f64, + label: String, + source: String, + endpoints: BTreeSet, +} + +/// Accumulates a capture into a [`Report`], one event at a time. +pub struct ReportBuilder { + source: String, + first_ts: Option, + last_ts: Option, + endpoints: HashMap, + by_kind: BTreeMap<&'static str, u64>, + ja4_counts: HashMap, + ja4_endpoints: HashMap>, + sni_counts: HashMap, + intel: HashMap<(String, String), IntelAgg>, + alerts: Vec, +} + +impl ReportBuilder { + #[must_use] + pub fn new(source: &str) -> Self { + Self { + source: source.to_owned(), + first_ts: None, + last_ts: None, + endpoints: HashMap::new(), + by_kind: BTreeMap::new(), + ja4_counts: HashMap::new(), + ja4_endpoints: HashMap::new(), + sni_counts: HashMap::new(), + intel: HashMap::new(), + alerts: Vec::new(), + } + } + + /// Folds one event, with any intelligence and alerts it produced, into the + /// running aggregates. + pub fn observe(&mut self, event: &FingerprintEvent, reports: &[MatchReport], alerts: &[Alert]) { + self.first_ts = Some( + self.first_ts + .map_or(event.ts_nanos, |t| t.min(event.ts_nanos)), + ); + self.last_ts = Some( + self.last_ts + .map_or(event.ts_nanos, |t| t.max(event.ts_nanos)), + ); + + let ip = event.src.ip(); + *self.by_kind.entry(kind_label(&event.event)).or_insert(0) += 1; + let endpoint = self.endpoints.entry(ip).or_default(); + endpoint.events += 1; + + match &event.event { + StreamEvent::ClientHello { ja4, sni, .. } => { + endpoint.ja4.insert(ja4.hash.clone()); + *self.ja4_counts.entry(ja4.hash.clone()).or_insert(0) += 1; + self.ja4_endpoints + .entry(ja4.hash.clone()) + .or_default() + .insert(ip); + if let Some(name) = sni { + endpoint.sni.insert(name.clone()); + *self.sni_counts.entry(name.clone()).or_insert(0) += 1; + } + } + StreamEvent::ServerHello { ja4s, .. } => { + endpoint.ja4s.insert(ja4s.hash.clone()); + } + StreamEvent::Certificate { ja4x } => { + endpoint.ja4x.insert(ja4x.clone()); + } + StreamEvent::HttpRequest { + ja4h, user_agent, .. + } => { + endpoint.ja4h.insert(ja4h.hash.clone()); + if let Some(agent) = user_agent { + endpoint.user_agents.insert(agent.clone()); + } + } + StreamEvent::TcpSyn { ja4t } => { + endpoint.ja4t.insert(ja4t.clone()); + } + StreamEvent::TcpSynAck { .. } => {} + } + + for report in reports { + endpoint.worst_verdict = Some(worse(endpoint.worst_verdict, report.verdict)); + let key = (report.kind.as_str().to_owned(), report.observed.clone()); + let best = report + .hits + .iter() + .max_by(|a, b| a.strength.weight().total_cmp(&b.strength.weight())); + let agg = self.intel.entry(key).or_insert_with(|| IntelAgg { + verdict: report.verdict, + threat_score: report.threat_score, + label: best.map_or_else(|| "-".to_owned(), |hit| hit.label.clone()), + source: best.map_or_else(|| "-".to_owned(), |hit| hit.source.clone()), + endpoints: BTreeSet::new(), + }); + agg.threat_score = agg.threat_score.max(report.threat_score); + agg.endpoints.insert(ip.to_string()); + } + + endpoint.alerts += u64::try_from(alerts.len()).unwrap_or(u64::MAX); + self.alerts.extend_from_slice(alerts); + } + + /// Closes the books and produces the report from the running aggregates, + /// the pipeline's final counters, and the wall clock the run took. + #[must_use] + pub fn finish( + self, + counters: Counters, + truncated: bool, + elapsed: Duration, + top: usize, + ) -> Report { + let duration_secs = match (self.first_ts, self.last_ts) { + (Some(first), Some(last)) => nanos_to_secs(last.saturating_sub(first)), + _ => 0.0, + }; + + let mut endpoints: Vec = self + .endpoints + .into_iter() + .map(|(ip, agg)| EndpointSummary { + ip: ip.to_string(), + events: agg.events, + ja4: agg.ja4.into_iter().collect(), + ja4s: agg.ja4s.into_iter().collect(), + ja4t: agg.ja4t.into_iter().collect(), + ja4h: agg.ja4h.into_iter().collect(), + ja4x: agg.ja4x.into_iter().collect(), + sni: agg.sni.into_iter().collect(), + user_agents: agg.user_agents.into_iter().collect(), + verdict: agg.worst_verdict.map(|v| v.as_str().to_owned()), + alerts: agg.alerts, + }) + .collect(); + endpoints.sort_by(|a, b| b.events.cmp(&a.events).then_with(|| a.ip.cmp(&b.ip))); + + let top_ja4 = rank(&self.ja4_counts, top, |value| FpCount { + value: value.clone(), + count: self.ja4_counts[value], + endpoints: self.ja4_endpoints.get(value).map_or(0, HashSet::len), + }); + let top_sni = rank(&self.sni_counts, top, |value| NameCount { + name: value.clone(), + count: self.sni_counts[value], + }); + let by_kind = self + .by_kind + .iter() + .map(|(kind, count)| KindCount { + kind: (*kind).to_owned(), + count: *count, + }) + .collect(); + + let mut intel: Vec = self + .intel + .into_iter() + .map(|((kind, value), agg)| IntelFinding { + kind, + value, + verdict: agg.verdict.as_str().to_owned(), + threat_score: agg.threat_score, + label: agg.label, + source: agg.source, + endpoints: agg.endpoints.into_iter().collect(), + }) + .collect(); + intel.sort_by(|a, b| { + b.threat_score + .total_cmp(&a.threat_score) + .then_with(|| a.value.cmp(&b.value)) + }); + intel.truncate(top); + + let alerts = AlertSummary::from_alerts(&self.alerts, top); + + Report { + capture: CaptureSummary { + source: self.source, + frames: counters.frames, + bytes: counters.bytes, + tcp_segments: counters.tcp_segments, + udp_datagrams: counters.udp_datagrams, + events: counters.events, + flows: counters.flows_created, + duration_secs, + truncated, + }, + distribution: Distribution { + by_kind, + top_ja4, + top_sni, + }, + endpoints, + intel, + alerts, + coverage: Coverage::derive(&counters, elapsed), + } + } +} + +/// The full forensic report, ready to serialise or render. +#[derive(Debug, Serialize)] +pub struct Report { + pub capture: CaptureSummary, + pub distribution: Distribution, + pub endpoints: Vec, + pub intel: Vec, + pub alerts: AlertSummary, + pub coverage: Coverage, +} + +#[derive(Debug, Serialize)] +pub struct CaptureSummary { + pub source: String, + pub frames: u64, + pub bytes: u64, + pub tcp_segments: u64, + pub udp_datagrams: u64, + pub events: u64, + pub flows: u64, + pub duration_secs: f64, + pub truncated: bool, +} + +#[derive(Debug, Serialize)] +pub struct Distribution { + pub by_kind: Vec, + pub top_ja4: Vec, + pub top_sni: Vec, +} + +#[derive(Debug, Serialize)] +pub struct KindCount { + pub kind: String, + pub count: u64, +} + +#[derive(Debug, Serialize)] +pub struct FpCount { + pub value: String, + pub count: u64, + pub endpoints: usize, +} + +#[derive(Debug, Serialize)] +pub struct NameCount { + pub name: String, + pub count: u64, +} + +#[derive(Debug, Serialize)] +pub struct EndpointSummary { + pub ip: String, + pub events: u64, + pub ja4: Vec, + pub ja4s: Vec, + pub ja4t: Vec, + pub ja4h: Vec, + pub ja4x: Vec, + pub sni: Vec, + pub user_agents: Vec, + pub verdict: Option, + pub alerts: u64, +} + +#[derive(Debug, Serialize)] +pub struct IntelFinding { + pub kind: String, + pub value: String, + pub verdict: String, + pub threat_score: f64, + pub label: String, + pub source: String, + pub endpoints: Vec, +} + +#[derive(Debug, Serialize)] +pub struct AlertSummary { + pub total: u64, + pub by_rule: Vec, + pub recent: Vec, +} + +impl AlertSummary { + fn from_alerts(alerts: &[Alert], top: usize) -> Self { + let mut by_rule: BTreeMap<&'static str, u64> = BTreeMap::new(); + for alert in alerts { + *by_rule.entry(alert.rule.as_str()).or_insert(0) += 1; + } + let mut counts: Vec = by_rule + .into_iter() + .map(|(rule, count)| RuleCount { + rule: rule.to_owned(), + count, + }) + .collect(); + counts.sort_by(|a, b| b.count.cmp(&a.count).then_with(|| a.rule.cmp(&b.rule))); + + let recent = alerts.iter().rev().take(top).cloned().collect(); + Self { + total: u64::try_from(alerts.len()).unwrap_or(u64::MAX), + by_rule: counts, + recent, + } + } +} + +#[derive(Debug, Serialize)] +pub struct RuleCount { + pub rule: String, + pub count: u64, +} + +#[derive(Debug, Serialize)] +pub struct Coverage { + pub counters: Counters, + pub tls_miss_rate: f64, + pub quic_decrypt_rate: f64, + pub events_per_sec: f64, + pub frames_per_sec: f64, + pub megabits_per_sec: f64, +} + +impl Coverage { + #[allow(clippy::cast_precision_loss)] + fn derive(counters: &Counters, elapsed: Duration) -> Self { + let secs = elapsed.as_secs_f64(); + let per_sec = |n: u64| if secs > 0.0 { n as f64 / secs } else { 0.0 }; + let quic_decrypt_rate = if counters.quic_initials == 0 { + 0.0 + } else { + counters.quic_decrypted as f64 / counters.quic_initials as f64 + }; + Self { + counters: *counters, + tls_miss_rate: counters.tls_miss_rate(), + quic_decrypt_rate, + events_per_sec: per_sec(counters.events), + frames_per_sec: per_sec(counters.frames), + megabits_per_sec: if secs > 0.0 { + (counters.bytes as f64 * 8.0) / 1_000_000.0 / secs + } else { + 0.0 + }, + } + } +} + +impl Report { + /// Renders the report as aligned, sectioned text for a terminal reader. + #[must_use] + pub fn render_text(&self) -> String { + let mut out = String::new(); + self.render_capture(&mut out); + self.render_distribution(&mut out); + self.render_endpoints(&mut out); + self.render_intel(&mut out); + self.render_alerts(&mut out); + self.render_coverage(&mut out); + out + } + + fn render_capture(&self, out: &mut String) { + let c = &self.capture; + let _ = writeln!(out, "== capture =="); + let _ = writeln!(out, " source {}", c.source); + let _ = writeln!(out, " frames {}", c.frames); + let _ = writeln!(out, " bytes {}", c.bytes); + let _ = writeln!(out, " tcp segments {}", c.tcp_segments); + let _ = writeln!(out, " udp datagrams {}", c.udp_datagrams); + let _ = writeln!(out, " flows {}", c.flows); + let _ = writeln!(out, " fingerprints {}", c.events); + let _ = writeln!(out, " capture span {:.3}s", c.duration_secs); + if c.truncated { + let _ = writeln!(out, " truncated yes (file ended mid packet)"); + } + } + + fn render_distribution(&self, out: &mut String) { + let d = &self.distribution; + let _ = writeln!(out, "\n== fingerprints by kind =="); + if d.by_kind.is_empty() { + let _ = writeln!(out, " none"); + } + for row in &d.by_kind { + let _ = writeln!(out, " {:<16} {}", row.kind, row.count); + } + + let _ = writeln!(out, "\n== top client ja4 =="); + if d.top_ja4.is_empty() { + let _ = writeln!(out, " none"); + } + for row in &d.top_ja4 { + let _ = writeln!( + out, + " {:<40} {:>5} across {} endpoint(s)", + row.value, row.count, row.endpoints + ); + } + + if !d.top_sni.is_empty() { + let _ = writeln!(out, "\n== top server names =="); + for row in &d.top_sni { + let _ = writeln!(out, " {:<40} {:>5}", row.name, row.count); + } + } + } + + fn render_endpoints(&self, out: &mut String) { + let _ = writeln!(out, "\n== endpoints =="); + if self.endpoints.is_empty() { + let _ = writeln!(out, " none"); + } + for endpoint in &self.endpoints { + let verdict = endpoint.verdict.as_deref().unwrap_or("-"); + let _ = writeln!( + out, + " {} ({} event(s), verdict {}, {} alert(s))", + endpoint.ip, endpoint.events, verdict, endpoint.alerts + ); + write_list(out, "ja4", &endpoint.ja4); + write_list(out, "ja4s", &endpoint.ja4s); + write_list(out, "ja4t", &endpoint.ja4t); + write_list(out, "ja4h", &endpoint.ja4h); + write_list(out, "ja4x", &endpoint.ja4x); + write_list(out, "sni", &endpoint.sni); + write_list(out, "ua", &endpoint.user_agents); + } + } + + fn render_intel(&self, out: &mut String) { + let _ = writeln!(out, "\n== intelligence =="); + if self.intel.is_empty() { + let _ = writeln!(out, " no fingerprints matched the database"); + return; + } + for finding in &self.intel { + let _ = writeln!( + out, + " [{}] {} {} (threat {:.2})", + finding.verdict, finding.kind, finding.value, finding.threat_score + ); + let _ = writeln!( + out, + " {} ({}), {} endpoint(s)", + finding.label, + finding.source, + finding.endpoints.len() + ); + } + } + + fn render_alerts(&self, out: &mut String) { + let _ = writeln!(out, "\n== alerts =="); + if self.alerts.total == 0 { + let _ = writeln!(out, " none raised"); + return; + } + let _ = writeln!(out, " {} total", self.alerts.total); + for row in &self.alerts.by_rule { + let _ = writeln!(out, " {:<14} {}", row.rule, row.count); + } + let _ = writeln!(out, " most recent:"); + for alert in &self.alerts.recent { + let target = alert.ip.as_deref().unwrap_or("-"); + let _ = writeln!( + out, + " [{}] {} {} {}", + alert.severity.as_str(), + alert.rule.as_str(), + target, + alert.title + ); + } + } + + fn render_coverage(&self, out: &mut String) { + let c = &self.coverage; + let _ = writeln!(out, "\n== coverage =="); + let _ = writeln!( + out, + " tls miss rate {:.1}% ({} read, {} clipped)", + c.tls_miss_rate * 100.0, + c.counters.tls_handshakes_fingerprinted, + c.counters.unfinished_tls_streams + ); + let _ = writeln!(out, " streams capped {}", c.counters.streams_capped); + let _ = writeln!(out, " segments dropped {}", c.counters.segments_dropped); + let _ = writeln!( + out, + " quic initials {} ({} decrypted, {:.0}%, {} unsupported version)", + c.counters.quic_initials, + c.counters.quic_decrypted, + c.quic_decrypt_rate * 100.0, + c.counters.quic_version_unsupported + ); + let _ = writeln!( + out, + " throughput {:.0} fp/s, {:.0} frames/s, {:.1} Mb/s", + c.events_per_sec, c.frames_per_sec, c.megabits_per_sec + ); + } +} + +/// Writes one labelled line listing a set's values, skipping an empty set. +fn write_list(out: &mut String, label: &str, values: &[String]) { + if values.is_empty() { + return; + } + let _ = writeln!(out, " {:<5} {}", label, values.join(", ")); +} + +/// The snake case name a stream event carries, matching its serialised tag. +fn kind_label(event: &StreamEvent) -> &'static str { + match event { + StreamEvent::ClientHello { .. } => "client_hello", + StreamEvent::ServerHello { .. } => "server_hello", + StreamEvent::Certificate { .. } => "certificate", + StreamEvent::HttpRequest { .. } => "http_request", + StreamEvent::TcpSyn { .. } => "tcp_syn", + StreamEvent::TcpSynAck { .. } => "tcp_syn_ack", + } +} + +/// Ranks the keys of a count map by count descending then key ascending, +/// keeping the top `n` and mapping each through `build`. +fn rank(counts: &HashMap, n: usize, build: impl Fn(&String) -> T) -> Vec { + let mut keys: Vec<&String> = counts.keys().collect(); + keys.sort_by(|a, b| counts[*b].cmp(&counts[*a]).then_with(|| a.cmp(b))); + keys.into_iter().take(n).map(build).collect() +} + +/// The more alarming of two verdicts, used to fold an endpoint's worst case. +fn worse(current: Option, next: Verdict) -> Verdict { + let rank = |v: Verdict| match v { + Verdict::Malicious => 3, + Verdict::Suspicious => 2, + Verdict::Unknown => 1, + Verdict::Benign => 0, + }; + match current { + Some(existing) if rank(existing) >= rank(next) => existing, + _ => next, + } +} + +/// Converts whole nanoseconds to fractional seconds without the precision loss +/// lint, accepting that a capture span beyond `u32::MAX` seconds is unreachable. +#[allow(clippy::cast_precision_loss)] +fn nanos_to_secs(nanos: u64) -> f64 { + nanos as f64 / 1_000_000_000.0 +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use tlsfp_core::{Counters, FingerprintEvent, Ja3, Ja4Family, StreamEvent}; + use tlsfp_intel::{Alert, AlertSeverity, FpKind, IntelHit, MatchReport, MatchStrength, Rule}; + + use super::ReportBuilder; + + fn client_hello(ip: &str, ja4: &str, sni: &str) -> FingerprintEvent { + FingerprintEvent { + ts_nanos: 1_000_000_000, + src: format!("{ip}:40000").parse().unwrap(), + dst: "10.0.0.2:443".parse().unwrap(), + event: StreamEvent::ClientHello { + ja3: Ja3::from_digest([0u8; 16]), + ja3_raw: String::new(), + ja4: Ja4Family::new(ja4.to_owned(), String::new()), + sni: Some(sni.to_owned()), + alpn: None, + }, + } + } + + #[test] + fn aggregates_endpoints_and_distribution() { + let mut builder = ReportBuilder::new("test.pcap"); + builder.observe( + &client_hello("10.0.0.1", "t13d1516h2_aaaa_bbbb", "a.example"), + &[], + &[], + ); + builder.observe( + &client_hello("10.0.0.1", "t13d1516h2_aaaa_bbbb", "b.example"), + &[], + &[], + ); + builder.observe( + &client_hello("10.0.0.9", "t13d1516h2_aaaa_bbbb", "a.example"), + &[], + &[], + ); + + let report = builder.finish(Counters::default(), false, Duration::from_millis(10), 15); + + assert_eq!(report.endpoints.len(), 2); + assert_eq!(report.endpoints[0].ip, "10.0.0.1"); + assert_eq!(report.endpoints[0].events, 2); + assert_eq!(report.distribution.top_ja4.len(), 1); + assert_eq!(report.distribution.top_ja4[0].count, 3); + assert_eq!(report.distribution.top_ja4[0].endpoints, 2); + assert_eq!(report.distribution.top_sni[0].count, 2); + } + + #[test] + fn folds_intel_and_alerts_into_worst_verdict() { + let mut builder = ReportBuilder::new("test.pcap"); + let event = client_hello("10.0.0.1", "t13d1516h2_aaaa_bbbb", "evil.example"); + let report = MatchReport::from_hits( + FpKind::Ja4, + "t13d1516h2_aaaa_bbbb".to_owned(), + vec![IntelHit { + kind: FpKind::Ja4, + value: "t13d1516h2_aaaa_bbbb".to_owned(), + label: "Cobalt Strike".to_owned(), + category: tlsfp_intel::Category::Malware, + source: "test-feed".to_owned(), + reference: None, + strength: MatchStrength::Exact, + }], + ); + let alert = Alert { + ts_nanos: 1_000_000_000, + rule: Rule::KnownBad, + severity: AlertSeverity::Critical, + ip: Some("10.0.0.1".to_owned()), + fp_kind: Some(FpKind::Ja4), + fp_value: Some("t13d1516h2_aaaa_bbbb".to_owned()), + title: "known bad fingerprint".to_owned(), + detail: "matched test-feed".to_owned(), + score: Some(1.0), + }; + builder.observe(&event, &[report], &[alert]); + + let built = builder.finish(Counters::default(), false, Duration::from_millis(5), 15); + assert_eq!(built.endpoints[0].verdict.as_deref(), Some("malicious")); + assert_eq!(built.endpoints[0].alerts, 1); + assert_eq!(built.intel.len(), 1); + assert_eq!(built.intel[0].label, "Cobalt Strike"); + assert_eq!(built.alerts.total, 1); + assert_eq!(built.alerts.by_rule[0].rule, "known_bad"); + + let text = built.render_text(); + assert!(text.contains("== coverage ==")); + assert!(text.contains("Cobalt Strike")); + } + + #[test] + fn miss_rate_and_throughput_reach_the_report() { + let counters = Counters { + frames: 1000, + bytes: 1_000_000, + events: 200, + tls_handshakes_fingerprinted: 3, + unfinished_tls_streams: 1, + ..Counters::default() + }; + let builder = ReportBuilder::new("test.pcap"); + let report = builder.finish(counters, true, Duration::from_secs(1), 15); + assert!((report.coverage.tls_miss_rate - 0.25).abs() < 1e-9); + assert!((report.coverage.events_per_sec - 200.0).abs() < 1e-9); + assert!(report.capture.truncated); + assert!(report.render_text().contains("truncated")); + } +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp/tests/report.rs b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp/tests/report.rs new file mode 100644 index 00000000..54c33e67 --- /dev/null +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp/tests/report.rs @@ -0,0 +1,118 @@ +// ©AngelaMos | 2026 +// report.rs + +//! End to end coverage of the forensic report mode. +//! +//! These drive the real binary against a vendored capture the way an analyst +//! would, then assert on what it prints. The capture is read with a database +//! path that does not exist, so the report runs as a pure fingerprint inventory +//! with no intelligence side effects and no dependence on a seeded store, which +//! keeps the test hermetic. + +use std::path::PathBuf; +use std::process::Command; + +/// The vendored capture used throughout: a browser session with TLS, QUIC, and +/// certificate handshakes, enough to exercise every section of the report. +fn capture() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../testdata/pcap/tls-handshake.pcapng") +} + +/// A database path under the temp directory that is guaranteed not to exist, so +/// the report stays a pure inventory and never writes a store behind the test. +fn absent_db() -> PathBuf { + std::env::temp_dir().join(format!( + "tlsfp-report-test-{}-absent.db", + std::process::id() + )) +} + +fn run(args: &[&str]) -> std::process::Output { + Command::new(env!("CARGO_BIN_EXE_tlsfp")) + .args(args) + .output() + .expect("running the tlsfp binary") +} + +#[test] +fn text_report_has_every_section() { + let db = absent_db(); + let output = run(&[ + "pcap", + capture().to_str().unwrap(), + "--report", + "--db", + db.to_str().unwrap(), + ]); + assert!(output.status.success(), "report run failed"); + let text = String::from_utf8(output.stdout).expect("utf8 report"); + + for section in [ + "== capture ==", + "== fingerprints by kind ==", + "== top client ja4 ==", + "== endpoints ==", + "== intelligence ==", + "== alerts ==", + "== coverage ==", + ] { + assert!(text.contains(section), "missing section {section}"); + } + assert!(text.contains("t13d1516h2_8daaf6152771_e5627efa2ab1")); + assert!(text.contains("q13d0310h3_55b375c5d22e_cd85d2d88918")); + assert!(text.contains("tls miss rate")); + assert!(text.contains("fp/s")); + assert!(!db.exists(), "report must not create a database"); +} + +#[test] +fn json_report_is_well_formed() { + let db = absent_db(); + let output = run(&[ + "pcap", + capture().to_str().unwrap(), + "--report", + "--json", + "--db", + db.to_str().unwrap(), + ]); + assert!(output.status.success(), "json report run failed"); + let value: serde_json::Value = + serde_json::from_slice(&output.stdout).expect("report is valid JSON"); + + assert!(value["capture"]["frames"].as_u64().unwrap() > 0); + assert!( + value["coverage"]["counters"]["tls_handshakes_fingerprinted"] + .as_u64() + .unwrap() + > 0 + ); + assert!(value["coverage"]["events_per_sec"].as_f64().unwrap() > 0.0); + assert!( + !value["distribution"]["top_ja4"] + .as_array() + .unwrap() + .is_empty() + ); + let endpoints = value["endpoints"].as_array().unwrap(); + assert!(endpoints.iter().any(|e| e["ip"] == "192.168.1.168")); +} + +#[test] +fn top_flag_caps_ranked_rows() { + let db = absent_db(); + let output = run(&[ + "pcap", + capture().to_str().unwrap(), + "--report", + "--json", + "--top", + "2", + "--db", + db.to_str().unwrap(), + ]); + assert!(output.status.success(), "capped report run failed"); + let value: serde_json::Value = + serde_json::from_slice(&output.stdout).expect("report is valid JSON"); + assert!(value["distribution"]["top_sni"].as_array().unwrap().len() <= 2); +} diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/frontend/index.html b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/frontend/index.html index 6f8ff938..24f7bb19 100644 --- a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/frontend/index.html +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/frontend/index.html @@ -28,7 +28,7 @@ http-equiv="Content-Security-Policy" content="default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval'; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob:; font-src 'self'; connect-src 'self'; form-action 'self'; worker-src 'self' blob:;" /> - JA3/JA4 TLS Fingerprinting + MKUtra Valedictorian tls dossier / passive - specimen 001 / handshake + specimen 006 / handshake
- surveillance aesthetics + sleep is counter surveillance

TLSFP

the fingerprint of every client that speaks tls diff --git a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/justfile b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/justfile index 53103bce..89540373 100644 --- a/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/justfile +++ b/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/justfile @@ -59,6 +59,10 @@ run-backend *ARGS: test-backend: cargo test --workspace +[group('backend')] +bench *ARGS: + cargo bench --bench fingerprint {{ARGS}} + [group('backend')] clippy: cargo clippy --workspace --all-targets -- -D warnings