feat(ja3-ja4-tls-fingerprinting): JA3/JA4 TLS fingerprinting tool in Rust (M0-M5)
Passive TLS client and server fingerprinting from packet captures. Two-crate workspace: tlsfp-core (parsing and fingerprint logic, no I/O, unsafe forbidden) and tlsfp (clap CLI). Edition 2024, MSRV 1.85. Core (M0-M4): a hand-rolled bounds-checked TLS parser for ClientHello, ServerHello, and the TLS 1.2 Certificate chain, plus the full fingerprint family computed byte-exact against published vectors: JA3/JA3S (MD5), JA4/JA4_r, JA4S, JA4H, JA4X, and JA4T. GREASE is stripped, wire order and unknown cipher and extension values are preserved. Pipeline (M5): a pcap and pcapng file reader behind a PacketSource trait, etherparse L2-L4 decode (Ethernet, raw IP, BSD loopback, Linux SLL and SLL2, VLAN and QinQ), and a hand-rolled per-flow TCP reassembler with wrapping serial arithmetic, out-of-order parking, first-write-wins overlap resolution, and bounded memory. The tlsfp pcap subcommand fingerprints every handshake in a capture, as readable lines or NDJSON. Verified three ways: 94 tests including unit KATs, nine real-pcap full-pipeline KATs against the FoxIO sample corpus, and property tests that reassemble arbitrarily segmented and reordered streams. Builds clean on stable and MSRV 1.85; clippy -D warnings and rustfmt clean. Test captures under testdata/pcap are vendored unmodified from the FoxIO ja4 repository with per-file SHA-256 provenance.
This commit is contained in:
CarterPerez-dev
parent
d03b9a37ad
commit
e4207b343a
|
|
@ -0,0 +1,46 @@
|
|||
# ©AngelaMos | 2026
|
||||
# ci.yml
|
||||
|
||||
name: ci
|
||||
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- "PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/**"
|
||||
pull_request:
|
||||
paths:
|
||||
- "PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/**"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: PROJECTS/intermediate/ja3-ja4-tls-fingerprinting
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
rust: [stable, "1.85"]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Install libpcap
|
||||
run: sudo apt-get update && sudo apt-get install -y libpcap-dev
|
||||
- name: Install Rust ${{ matrix.rust }}
|
||||
run: rustup toolchain install ${{ matrix.rust }} --component clippy rustfmt --profile minimal
|
||||
- name: Build
|
||||
run: cargo +${{ matrix.rust }} build --workspace --all-targets
|
||||
- name: Test
|
||||
run: cargo +${{ matrix.rust }} test --workspace
|
||||
|
||||
lint:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Install libpcap
|
||||
run: sudo apt-get update && sudo apt-get install -y libpcap-dev
|
||||
- name: Install Rust
|
||||
run: rustup toolchain install stable --component clippy rustfmt --profile minimal
|
||||
- name: Format
|
||||
run: cargo fmt --all --check
|
||||
- name: Clippy
|
||||
run: cargo clippy --workspace --all-targets -- -D warnings
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
# ©AngelaMos | 2026
|
||||
# .gitignore
|
||||
|
||||
docs/
|
||||
target/
|
||||
*.db
|
||||
*.sqlite
|
||||
fuzz/corpus/
|
||||
fuzz/artifacts/
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,66 @@
|
|||
# ©AngelaMos | 2026
|
||||
# Cargo.toml
|
||||
|
||||
[workspace]
|
||||
resolver = "3"
|
||||
members = ["crates/tlsfp-core", "crates/tlsfp"]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
rust-version = "1.85"
|
||||
license = "MIT"
|
||||
authors = ["Carter Perez"]
|
||||
repository = "https://github.com/CarterPerez-dev/Cybersecurity-Projects"
|
||||
|
||||
[workspace.dependencies]
|
||||
tlsfp-core = { path = "crates/tlsfp-core" }
|
||||
|
||||
winnow = "0.7"
|
||||
md-5 = "0.10"
|
||||
sha2 = "0.10"
|
||||
hex = "0.4"
|
||||
smallvec = "1"
|
||||
thiserror = "2"
|
||||
anyhow = "1"
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
|
||||
etherparse = "0.18"
|
||||
pcap-parser = "0.16"
|
||||
pcap = "2"
|
||||
|
||||
tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "signal"] }
|
||||
flume = "0.11"
|
||||
|
||||
rusqlite = { version = "0.32", features = ["bundled"] }
|
||||
|
||||
axum = "0.8"
|
||||
tower-http = { version = "0.6", features = ["fs", "trace", "cors"] }
|
||||
tower = "0.5"
|
||||
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
|
||||
hkdf = "0.12"
|
||||
aes = "0.8"
|
||||
aes-gcm = "0.10"
|
||||
ctr = "0.9"
|
||||
|
||||
[workspace.lints.rust]
|
||||
unsafe_code = "forbid"
|
||||
|
||||
[workspace.lints.clippy]
|
||||
pedantic = { level = "warn", priority = -1 }
|
||||
module_name_repetitions = "allow"
|
||||
missing_errors_doc = "allow"
|
||||
missing_panics_doc = "allow"
|
||||
must_use_candidate = "allow"
|
||||
doc_markdown = "allow"
|
||||
|
||||
[profile.release]
|
||||
opt-level = 3
|
||||
lto = "thin"
|
||||
codegen-units = 1
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
MIT License
|
||||
|
||||
Copyright (c) 2026 Carter Perez
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
# Third-party fingerprint algorithm licensing
|
||||
|
||||
This project implements several published TLS fingerprinting algorithms. They do
|
||||
not all carry the same license, and the split matters. This notice records what
|
||||
applies to what.
|
||||
|
||||
## Our own code
|
||||
|
||||
Everything under `crates/` is original work licensed under the MIT License (see
|
||||
`LICENSE`). No source code from any reference implementation was copied. The
|
||||
algorithms were implemented from their published specifications.
|
||||
|
||||
## JA3 and JA3S
|
||||
|
||||
JA3 and JA3S were created by John Althouse, Jeff Atkinson, and Josh Atkins at
|
||||
Salesforce and released under the BSD 3-Clause license at
|
||||
`https://github.com/salesforce/ja3`. The algorithm is free to implement and use.
|
||||
That repository was archived in May 2025 and is no longer maintained.
|
||||
|
||||
## JA4 (TLS client fingerprint)
|
||||
|
||||
JA4, the TLS client fingerprint, is licensed by FoxIO under the BSD 3-Clause
|
||||
license, separately from the rest of the JA4+ suite. FoxIO has stated it holds
|
||||
no patents on JA4 TLS client fingerprinting. The canonical license text is at
|
||||
`https://github.com/FoxIO-LLC/ja4/blob/main/LICENSE-JA4`. JA4 may be implemented
|
||||
and used without restriction.
|
||||
|
||||
## JA4S, JA4H, JA4X, JA4T (the rest of the JA4+ suite)
|
||||
|
||||
The remaining JA4+ fingerprints implemented here are licensed under the FoxIO
|
||||
License 1.1, and the methods are patent pending. The canonical license text is
|
||||
at `https://github.com/FoxIO-LLC/ja4/blob/main/LICENSE`.
|
||||
|
||||
The FoxIO License 1.1 grants the right to use and modify these methods for non
|
||||
commercial purposes, which it defines to include personal use, academic research
|
||||
and development, and internal evaluation. It excludes any use for which a fee is
|
||||
charged and excludes providing the methods as a hosted or managed service to
|
||||
others.
|
||||
|
||||
This project is a free, open source, educational tool. It is not sold, not
|
||||
monetized, and not offered as a service. It therefore falls within the non
|
||||
commercial grant of the FoxIO License 1.1. Anyone who forks this project and
|
||||
intends to monetize it, or to offer it as a hosted service, must obtain an OEM
|
||||
license from FoxIO for the JA4+ methods first.
|
||||
|
||||
This is the same boundary that led the Suricata project to ship only JA4 and not
|
||||
the rest of the JA4+ suite: their GPL licensing is incompatible with the FoxIO
|
||||
License 1.1. This project is MIT licensed and non commercial, so both the GPL
|
||||
incompatibility and the monetization restriction are avoided.
|
||||
|
||||
## Seeded fingerprint data
|
||||
|
||||
The intelligence database is seeded from public sources with their own terms:
|
||||
|
||||
- abuse.ch SSLBL JA3 feed is released under CC0 and a snapshot may be
|
||||
redistributed. Its known limitation, that the fingerprints have not been
|
||||
tested against benign traffic and may produce false positives, is surfaced in
|
||||
the tool output.
|
||||
- The salesforce/ja3 application lists are MIT licensed.
|
||||
- The ja4db.com database has no stated redistribution license, so it is fetched
|
||||
at install time rather than bundled, and entries are validated on import.
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
# ©AngelaMos | 2026
|
||||
# Cargo.toml
|
||||
|
||||
[package]
|
||||
name = "tlsfp-core"
|
||||
description = "TLS handshake parsing and JA3/JA4 fingerprint computation"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
authors.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
md-5.workspace = true
|
||||
sha2.workspace = true
|
||||
hex.workspace = true
|
||||
smallvec.workspace = true
|
||||
thiserror.workspace = true
|
||||
serde.workspace = true
|
||||
etherparse.workspace = true
|
||||
pcap-parser.workspace = true
|
||||
hkdf.workspace = true
|
||||
aes.workspace = true
|
||||
aes-gcm.workspace = true
|
||||
ctr.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
hex.workspace = true
|
||||
proptest = "1"
|
||||
serde_json.workspace = true
|
||||
|
|
@ -0,0 +1,110 @@
|
|||
// ©AngelaMos | 2026
|
||||
// der.rs
|
||||
|
||||
use crate::error::{ParseError, Result};
|
||||
|
||||
/// DER tag bytes that the certificate walk needs to recognize.
|
||||
pub mod tag {
|
||||
pub const OBJECT_IDENTIFIER: u8 = 0x06;
|
||||
pub const SEQUENCE: u8 = 0x30;
|
||||
pub const SET: u8 = 0x31;
|
||||
pub const CONTEXT_3: u8 = 0xa3;
|
||||
}
|
||||
|
||||
/// A minimal reader for the subset of DER that X.509 certificates use.
|
||||
///
|
||||
/// It decodes one tag length value triple at a time and hands back the content
|
||||
/// as a borrowed slice. It deliberately does not understand the meaning of any
|
||||
/// structure; the JA4X walk drives it, deciding which fields to descend into and
|
||||
/// which to skip. Keeping the reader this small keeps it auditable, which
|
||||
/// matters because certificate parsers are a classic source of memory safety
|
||||
/// bugs and this one has no `unsafe` and cannot read out of bounds.
|
||||
pub struct Der<'a> {
|
||||
buf: &'a [u8],
|
||||
pos: usize,
|
||||
}
|
||||
|
||||
impl<'a> Der<'a> {
|
||||
#[must_use]
|
||||
pub const fn new(buf: &'a [u8]) -> Self {
|
||||
Self { buf, pos: 0 }
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn is_empty(&self) -> bool {
|
||||
self.pos >= self.buf.len()
|
||||
}
|
||||
|
||||
fn byte(&mut self) -> Result<u8> {
|
||||
if self.pos >= self.buf.len() {
|
||||
return Err(ParseError::Truncated { needed: 1, have: 0 });
|
||||
}
|
||||
let b = self.buf[self.pos];
|
||||
self.pos += 1;
|
||||
Ok(b)
|
||||
}
|
||||
|
||||
/// Reads one tag length value triple and returns the tag and the content
|
||||
/// slice, advancing past the whole triple.
|
||||
pub fn read_tlv(&mut self) -> Result<(u8, &'a [u8])> {
|
||||
let tag = self.byte()?;
|
||||
let len = self.read_length()?;
|
||||
if self.buf.len() - self.pos < len {
|
||||
return Err(ParseError::LengthOverrun {
|
||||
field: "der",
|
||||
declared: len,
|
||||
available: self.buf.len() - self.pos,
|
||||
});
|
||||
}
|
||||
let content = &self.buf[self.pos..self.pos + len];
|
||||
self.pos += len;
|
||||
Ok((tag, content))
|
||||
}
|
||||
|
||||
fn read_length(&mut self) -> Result<usize> {
|
||||
let first = self.byte()?;
|
||||
if first & 0x80 == 0 {
|
||||
return Ok(first as usize);
|
||||
}
|
||||
let count = (first & 0x7f) as usize;
|
||||
if count == 0 || count > core::mem::size_of::<usize>() {
|
||||
return Err(ParseError::Misaligned(count));
|
||||
}
|
||||
let mut len = 0usize;
|
||||
for _ in 0..count {
|
||||
len = (len << 8) | self.byte()? as usize;
|
||||
}
|
||||
Ok(len)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{Der, tag};
|
||||
|
||||
#[test]
|
||||
fn reads_short_form_sequence() {
|
||||
let data = [0x30, 0x03, 0x06, 0x01, 0x55];
|
||||
let mut der = Der::new(&data);
|
||||
let (t, content) = der.read_tlv().unwrap();
|
||||
assert_eq!(t, tag::SEQUENCE);
|
||||
assert_eq!(content, &[0x06, 0x01, 0x55]);
|
||||
assert!(der.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_long_form_length() {
|
||||
let mut data = vec![0x04, 0x82, 0x01, 0x00];
|
||||
data.extend(std::iter::repeat_n(0xaa, 256));
|
||||
let mut der = Der::new(&data);
|
||||
let (_t, content) = der.read_tlv().unwrap();
|
||||
assert_eq!(content.len(), 256);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_length_overrun() {
|
||||
let data = [0x06, 0x05, 0x55, 0x04];
|
||||
let mut der = Der::new(&data);
|
||||
assert!(der.read_tlv().is_err());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
// ©AngelaMos | 2026
|
||||
// error.rs
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
/// Errors produced while parsing TLS records, handshake messages, or QUIC
|
||||
/// initial packets.
|
||||
///
|
||||
/// Every variant is stack only. No variant carries a heap allocation, so the
|
||||
/// malformed packet path never touches the allocator. This matters because a
|
||||
/// fingerprinting engine spends most of its time rejecting traffic that is not
|
||||
/// a clean handshake, and allocating on each rejection adds jitter at line rate.
|
||||
#[derive(Debug, Error, Clone, PartialEq, Eq)]
|
||||
#[non_exhaustive]
|
||||
pub enum ParseError {
|
||||
#[error("buffer too short: needed {needed} bytes, had {have}")]
|
||||
Truncated { needed: usize, have: usize },
|
||||
|
||||
#[error("not a TLS handshake record (content type {0:#04x})")]
|
||||
NotHandshake(u8),
|
||||
|
||||
#[error("unexpected handshake message type {0:#04x}")]
|
||||
UnexpectedHandshake(u8),
|
||||
|
||||
#[error("length field {field} declares {declared} bytes but {available} remain")]
|
||||
LengthOverrun {
|
||||
field: &'static str,
|
||||
declared: usize,
|
||||
available: usize,
|
||||
},
|
||||
|
||||
#[error("vector length {0} is not a whole number of elements")]
|
||||
Misaligned(usize),
|
||||
|
||||
#[error("trailing {0} bytes after a complete message")]
|
||||
Trailing(usize),
|
||||
|
||||
#[error("handshake message spans more bytes than the reassembly cap allows")]
|
||||
OversizedHandshake,
|
||||
|
||||
#[error("malformed extension {ext_type:#06x}")]
|
||||
BadExtension { ext_type: u16 },
|
||||
|
||||
#[error("not a QUIC long header initial packet")]
|
||||
NotQuicInitial,
|
||||
|
||||
#[error("unsupported QUIC version {0:#010x}")]
|
||||
UnsupportedQuicVersion(u32),
|
||||
|
||||
#[error("QUIC header protection or AEAD removal failed")]
|
||||
QuicCryptoFailure,
|
||||
|
||||
#[error("no CRYPTO frame carrying a ClientHello was present")]
|
||||
NoCryptoClientHello,
|
||||
}
|
||||
|
||||
pub type Result<T> = core::result::Result<T, ParseError>;
|
||||
|
|
@ -0,0 +1,100 @@
|
|||
// ©AngelaMos | 2026
|
||||
// fingerprint.rs
|
||||
|
||||
use core::fmt;
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
/// A JA3 or JA3S fingerprint: the MD5 digest of the pre hash string.
|
||||
///
|
||||
/// JA3 is carried as the raw sixteen byte digest rather than a hex string so
|
||||
/// that equality, hashing, and database keys operate on the compact binary form
|
||||
/// and the hex rendering happens only at display boundaries.
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize)]
|
||||
#[serde(into = "String")]
|
||||
pub struct Ja3([u8; 16]);
|
||||
|
||||
impl Ja3 {
|
||||
#[must_use]
|
||||
pub const fn from_digest(digest: [u8; 16]) -> Self {
|
||||
Self(digest)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn bytes(&self) -> &[u8; 16] {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for Ja3 {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
for byte in self.0 {
|
||||
write!(f, "{byte:02x}")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for Ja3 {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "Ja3({self})")
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Ja3> for String {
|
||||
fn from(value: Ja3) -> Self {
|
||||
value.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
/// A fingerprint from the JA4 family, carried in both its hashed canonical form
|
||||
/// and its raw pre hash form.
|
||||
///
|
||||
/// The raw form is the unhashed list of cipher and extension values. It is kept
|
||||
/// alongside the hash because it is the form an analyst reads when explaining
|
||||
/// why two clients differ, and because clustering on the raw lists is more
|
||||
/// informative than clustering on opaque digests.
|
||||
#[derive(Clone, PartialEq, Eq, Hash, Serialize)]
|
||||
pub struct Ja4Family {
|
||||
pub hash: String,
|
||||
pub raw: String,
|
||||
}
|
||||
|
||||
impl Ja4Family {
|
||||
#[must_use]
|
||||
pub fn new(hash: String, raw: String) -> Self {
|
||||
Self { hash, raw }
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for Ja4Family {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(&self.hash)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for Ja4Family {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "Ja4Family(hash={}, raw={})", self.hash, self.raw)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{Ja3, Ja4Family};
|
||||
|
||||
#[test]
|
||||
fn ja3_display_is_lowercase_hex() {
|
||||
let fp = Ja3::from_digest([
|
||||
0xad, 0xa7, 0x02, 0x06, 0xe4, 0x06, 0x42, 0xa3, 0xe4, 0x46, 0x1f, 0x35, 0x50, 0x32,
|
||||
0x41, 0xd5,
|
||||
]);
|
||||
assert_eq!(fp.to_string(), "ada70206e40642a3e4461f35503241d5");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ja4_family_displays_hash() {
|
||||
let fp = Ja4Family::new("t13d1516h2_8daaf6152771_e5627efa2ab1".into(), "raw".into());
|
||||
assert_eq!(fp.to_string(), "t13d1516h2_8daaf6152771_e5627efa2ab1");
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
// ©AngelaMos | 2026
|
||||
// grease.rs
|
||||
|
||||
/// The sixteen GREASE values reserved by RFC 8701.
|
||||
///
|
||||
/// GREASE (Generate Random Extensions And Sustain Extensibility) values are
|
||||
/// inserted by clients into cipher lists, extension lists, supported groups,
|
||||
/// supported versions, and signature algorithms. They are deliberate noise
|
||||
/// whose only purpose is to keep middleboxes tolerant of unknown values. Both
|
||||
/// JA3 and JA4 strip them before hashing so that the same client produces a
|
||||
/// stable fingerprint regardless of which GREASE values it happened to pick.
|
||||
///
|
||||
/// Every value has the form `0xZaZa` where the two bytes are equal and the low
|
||||
/// nibble of each is `a`.
|
||||
pub const GREASE_VALUES: [u16; 16] = [
|
||||
0x0a0a, 0x1a1a, 0x2a2a, 0x3a3a, 0x4a4a, 0x5a5a, 0x6a6a, 0x7a7a, 0x8a8a, 0x9a9a, 0xaaaa, 0xbaba,
|
||||
0xcaca, 0xdada, 0xeaea, 0xfafa,
|
||||
];
|
||||
|
||||
/// Returns true when `value` is one of the sixteen RFC 8701 GREASE values.
|
||||
///
|
||||
/// The check exploits the structural regularity of the GREASE set rather than
|
||||
/// scanning the table: both bytes must be equal and the low nibble of each must
|
||||
/// be `0xa`. This is a single pair of comparisons rather than a sixteen way
|
||||
/// branch, which keeps it cheap on the per packet path.
|
||||
#[inline]
|
||||
#[must_use]
|
||||
pub const fn is_grease(value: u16) -> bool {
|
||||
let high = (value >> 8) as u8;
|
||||
let low = (value & 0x00ff) as u8;
|
||||
high == low && (low & 0x0f) == 0x0a
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{GREASE_VALUES, is_grease};
|
||||
|
||||
#[test]
|
||||
fn table_matches_structural_check() {
|
||||
for v in 0..=u16::MAX {
|
||||
let in_table = GREASE_VALUES.contains(&v);
|
||||
assert_eq!(in_table, is_grease(v), "mismatch for {v:#06x}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn known_non_grease_values() {
|
||||
for v in [0x0000_u16, 0x0010, 0x1301, 0x00ff, 0x5600, 0xc02f] {
|
||||
assert!(!is_grease(v), "{v:#06x} should not be grease");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
// ©AngelaMos | 2026
|
||||
// hash.rs
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
/// Returns the first twelve hex characters of the SHA256 digest of `bytes`.
|
||||
///
|
||||
/// The whole JA4 family truncates SHA256 to twelve hex characters, which is six
|
||||
/// bytes of digest. Twelve characters is enough to make accidental collisions
|
||||
/// vanishingly unlikely across the fingerprint space while keeping the
|
||||
/// fingerprint short enough to read and to paste into a search box.
|
||||
#[must_use]
|
||||
pub fn sha256_hex12(bytes: &[u8]) -> String {
|
||||
let digest = Sha256::digest(bytes);
|
||||
hex::encode(&digest[..6])
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::sha256_hex12;
|
||||
|
||||
#[test]
|
||||
fn known_digest_prefix() {
|
||||
assert_eq!(sha256_hex12(b""), "e3b0c44298fc");
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,125 @@
|
|||
// ©AngelaMos | 2026
|
||||
// ja3.rs
|
||||
|
||||
use std::fmt::Write as _;
|
||||
|
||||
use md5::{Digest, Md5};
|
||||
use smallvec::SmallVec;
|
||||
|
||||
use crate::fingerprint::Ja3;
|
||||
use crate::grease::is_grease;
|
||||
use crate::parse::{ClientHello, ServerHello};
|
||||
|
||||
/// Computes the JA3 string and digest for a ClientHello.
|
||||
///
|
||||
/// JA3 concatenates five fields, in order: the legacy version, the cipher
|
||||
/// suites, the extension types, the supported groups, and the elliptic curve
|
||||
/// point formats. Within a field the values are decimal and joined with
|
||||
/// hyphens; the fields themselves are joined with commas. GREASE values are
|
||||
/// removed from every list field so that the deliberate noise a client inserts
|
||||
/// does not change its fingerprint. The MD5 of that string is the JA3 hash.
|
||||
///
|
||||
/// JA3 is kept here despite being effectively dead for modern browser traffic,
|
||||
/// because malware fingerprints in public feeds are still expressed as JA3 and
|
||||
/// because watching JA3 collapse next to JA4 is the clearest way to understand
|
||||
/// why JA4 exists.
|
||||
#[must_use]
|
||||
pub fn ja3_string(ch: &ClientHello) -> String {
|
||||
let ext_types: SmallVec<[u16; 16]> = ch.extensions.iter().map(|e| e.ext_type).collect();
|
||||
let groups = ch.supported_groups();
|
||||
let formats = ch.ec_point_formats();
|
||||
|
||||
let mut s = String::new();
|
||||
let _ = write!(s, "{}", ch.legacy_version);
|
||||
s.push(',');
|
||||
append_u16_hyphenated(&mut s, &ch.cipher_suites);
|
||||
s.push(',');
|
||||
append_u16_hyphenated(&mut s, &ext_types);
|
||||
s.push(',');
|
||||
append_u16_hyphenated(&mut s, &groups);
|
||||
s.push(',');
|
||||
append_u8_hyphenated(&mut s, &formats);
|
||||
s
|
||||
}
|
||||
|
||||
/// Computes the JA3 digest for a ClientHello.
|
||||
#[must_use]
|
||||
pub fn ja3(ch: &ClientHello) -> Ja3 {
|
||||
digest(&ja3_string(ch))
|
||||
}
|
||||
|
||||
/// Computes the JA3S string for a ServerHello.
|
||||
///
|
||||
/// JA3S mirrors JA3 on the server side with three fields: the version, the
|
||||
/// single chosen cipher suite, and the extension types. A server and the exact
|
||||
/// ClientHello it answered together identify a deployment more tightly than
|
||||
/// either side alone.
|
||||
#[must_use]
|
||||
pub fn ja3s_string(sh: &ServerHello) -> String {
|
||||
let ext_types: SmallVec<[u16; 16]> = sh.extensions.iter().map(|e| e.ext_type).collect();
|
||||
let mut s = String::new();
|
||||
let _ = write!(s, "{},{},", sh.legacy_version, sh.cipher_suite);
|
||||
append_u16_hyphenated(&mut s, &ext_types);
|
||||
s
|
||||
}
|
||||
|
||||
/// Computes the JA3S digest for a ServerHello.
|
||||
#[must_use]
|
||||
pub fn ja3s(sh: &ServerHello) -> Ja3 {
|
||||
digest(&ja3s_string(sh))
|
||||
}
|
||||
|
||||
fn digest(pre_hash: &str) -> Ja3 {
|
||||
let out = Md5::digest(pre_hash.as_bytes());
|
||||
let mut bytes = [0u8; 16];
|
||||
bytes.copy_from_slice(&out);
|
||||
Ja3::from_digest(bytes)
|
||||
}
|
||||
|
||||
fn append_u16_hyphenated(out: &mut String, values: &[u16]) {
|
||||
let mut first = true;
|
||||
for &v in values {
|
||||
if is_grease(v) {
|
||||
continue;
|
||||
}
|
||||
if !first {
|
||||
out.push('-');
|
||||
}
|
||||
first = false;
|
||||
let _ = write!(out, "{v}");
|
||||
}
|
||||
}
|
||||
|
||||
fn append_u8_hyphenated(out: &mut String, values: &[u8]) {
|
||||
let mut first = true;
|
||||
for &v in values {
|
||||
if !first {
|
||||
out.push('-');
|
||||
}
|
||||
first = false;
|
||||
let _ = write!(out, "{v}");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::digest;
|
||||
|
||||
#[test]
|
||||
fn salesforce_client_vector_one() {
|
||||
let pre = "769,47-53-5-10-49161-49162-49171-49172-50-56-19-4,0-10-11,23-24-25,0";
|
||||
assert_eq!(digest(pre).to_string(), "ada70206e40642a3e4461f35503241d5");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn salesforce_client_vector_two_empty_fields() {
|
||||
let pre = "769,4-5-10-9-100-98-3-6-19-18-99,,,";
|
||||
assert_eq!(digest(pre).to_string(), "de350869b8c85de67a350c8d186f11e6");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn server_vector_round_trips_through_md5() {
|
||||
let pre = "769,47,65281-0-11-35-5-16";
|
||||
assert_eq!(digest(pre).to_string(), "836ce314215654b5b1f85f97c73e506f");
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,263 @@
|
|||
// ©AngelaMos | 2026
|
||||
// ja4.rs
|
||||
|
||||
use std::fmt::Write as _;
|
||||
|
||||
use smallvec::SmallVec;
|
||||
|
||||
use crate::fingerprint::Ja4Family;
|
||||
use crate::grease::is_grease;
|
||||
use crate::hash::sha256_hex12;
|
||||
use crate::parse::{ClientHello, ServerHello};
|
||||
use crate::registry::{extension, ja4_version_code};
|
||||
|
||||
/// The transport that carried the handshake.
|
||||
///
|
||||
/// JA4 records the transport in its first character because the same TLS stack
|
||||
/// produces a recognizably different ClientHello over QUIC than over TCP, and an
|
||||
/// analyst wants to see that difference at a glance rather than infer it.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Transport {
|
||||
Tcp,
|
||||
Quic,
|
||||
Dtls,
|
||||
}
|
||||
|
||||
impl Transport {
|
||||
const fn marker(self) -> char {
|
||||
match self {
|
||||
Transport::Tcp => 't',
|
||||
Transport::Quic => 'q',
|
||||
Transport::Dtls => 'd',
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const EMPTY_HASH: &str = "000000000000";
|
||||
|
||||
/// Computes the JA4 fingerprint of a ClientHello in both hashed and raw forms.
|
||||
///
|
||||
/// The fingerprint has three underscore separated sections. The first is ten
|
||||
/// human readable characters: transport, version, whether SNI was present,
|
||||
/// cipher count, extension count, and two characters derived from the first
|
||||
/// ALPN value. The second is a truncated SHA256 of the cipher list sorted by
|
||||
/// value. The third is a truncated SHA256 of the extension list sorted by value,
|
||||
/// with SNI and ALPN removed and the signature algorithms appended in their
|
||||
/// original order.
|
||||
///
|
||||
/// Sorting the cipher and extension lists before hashing is the whole reason
|
||||
/// JA4 survived what killed JA3. When Chrome began shuffling its extension order
|
||||
/// on every connection, JA3, which hashes extensions in wire order, produced a
|
||||
/// fresh hash for every Chrome request. JA4 sorts first, so order shuffling
|
||||
/// leaves the fingerprint unchanged.
|
||||
#[must_use]
|
||||
pub fn ja4(ch: &ClientHello, transport: Transport) -> Ja4Family {
|
||||
let prefix = ja4_prefix(ch, transport);
|
||||
|
||||
let (cipher_csv, _) = sorted_hex_csv(&ch.cipher_suites, &[]);
|
||||
let cipher_hash = truncated_sha256(&cipher_csv);
|
||||
|
||||
let ext_raw = ja4_extension_raw(ch);
|
||||
let ext_hash = truncated_sha256(&ext_raw);
|
||||
|
||||
let hash = format!("{prefix}_{cipher_hash}_{ext_hash}");
|
||||
let raw = format!("{prefix}_{cipher_csv}_{ext_raw}");
|
||||
Ja4Family::new(hash, raw)
|
||||
}
|
||||
|
||||
/// Computes the JA4S fingerprint of a ServerHello.
|
||||
///
|
||||
/// JA4S mirrors JA4 on the server side, with three differences that follow from
|
||||
/// what a server controls. The server picks exactly one cipher suite, so the
|
||||
/// cipher section is that single value in hex rather than a hash of a list. The
|
||||
/// extensions are hashed in the order the server sent them, not sorted, because
|
||||
/// a server does not shuffle its own extensions. And there is no SNI field,
|
||||
/// because the server is not the party naming a host.
|
||||
#[must_use]
|
||||
pub fn ja4s(sh: &ServerHello, transport: Transport) -> Ja4Family {
|
||||
let version = ja4_version_code(sh.selected_version());
|
||||
let alpn = alpn_chars(sh.alpn_protocol());
|
||||
let ext_count = sh.extensions.len().min(99);
|
||||
|
||||
let mut prefix = String::with_capacity(7);
|
||||
prefix.push(transport.marker());
|
||||
prefix.push_str(version);
|
||||
let _ = write!(prefix, "{ext_count:02}");
|
||||
prefix.push_str(&alpn);
|
||||
|
||||
let cipher_hex = format!("{:04x}", sh.cipher_suite);
|
||||
|
||||
let ext_csv = wire_order_hex_csv(
|
||||
&sh.extensions
|
||||
.iter()
|
||||
.map(|e| e.ext_type)
|
||||
.collect::<SmallVec<[u16; 16]>>(),
|
||||
);
|
||||
let ext_hash = truncated_sha256(&ext_csv);
|
||||
|
||||
let hash = format!("{prefix}_{cipher_hex}_{ext_hash}");
|
||||
let raw = format!("{prefix}_{cipher_hex}_{ext_csv}");
|
||||
Ja4Family::new(hash, raw)
|
||||
}
|
||||
|
||||
fn wire_order_hex_csv(values: &[u16]) -> String {
|
||||
let hexed: SmallVec<[String; 16]> = values.iter().map(|v| format!("{v:04x}")).collect();
|
||||
hexed.join(",")
|
||||
}
|
||||
|
||||
fn ja4_prefix(ch: &ClientHello, transport: Transport) -> String {
|
||||
let version = select_version(ch);
|
||||
let sni = if ch.has_extension(extension::SERVER_NAME) {
|
||||
'd'
|
||||
} else {
|
||||
'i'
|
||||
};
|
||||
let cipher_count = capped_count(ch.cipher_suites.iter().copied());
|
||||
let ext_count = capped_count(ch.extensions.iter().map(|e| e.ext_type));
|
||||
let alpn = ja4_alpn(ch);
|
||||
|
||||
let mut prefix = String::with_capacity(10);
|
||||
prefix.push(transport.marker());
|
||||
prefix.push_str(ja4_version_code(version));
|
||||
prefix.push(sni);
|
||||
let _ = write!(prefix, "{cipher_count:02}{ext_count:02}");
|
||||
prefix.push_str(&alpn);
|
||||
prefix
|
||||
}
|
||||
|
||||
/// Selects the JA4 version word: the highest non GREASE value from the supported
|
||||
/// versions extension when present, otherwise the legacy record version.
|
||||
fn select_version(ch: &ClientHello) -> u16 {
|
||||
ch.supported_versions()
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|v| !is_grease(*v))
|
||||
.max()
|
||||
.unwrap_or(ch.legacy_version)
|
||||
}
|
||||
|
||||
fn capped_count(values: impl Iterator<Item = u16>) -> usize {
|
||||
values.filter(|v| !is_grease(*v)).count().min(99)
|
||||
}
|
||||
|
||||
/// Derives the two ALPN characters.
|
||||
///
|
||||
/// The implementation follows the published JA4 specification rather than the
|
||||
/// FoxIO Python reference. The two diverge for ALPN values whose first or last
|
||||
/// byte is not an ASCII alphanumeric: the specification says to print the first
|
||||
/// and last characters of the hex encoding of the value, while the Python
|
||||
/// reference emits a fixed fallback. The specification is the more informative
|
||||
/// and more portable choice, so it is what this code does.
|
||||
fn ja4_alpn(ch: &ClientHello) -> String {
|
||||
alpn_chars(ch.alpn_protocols().first().copied())
|
||||
}
|
||||
|
||||
fn alpn_chars(first: Option<&[u8]>) -> String {
|
||||
let Some(first) = first else {
|
||||
return "00".to_string();
|
||||
};
|
||||
if first.is_empty() {
|
||||
return "00".to_string();
|
||||
}
|
||||
|
||||
let first_byte = first[0];
|
||||
let last_byte = first[first.len() - 1];
|
||||
|
||||
if is_ascii_alphanumeric(first_byte) && is_ascii_alphanumeric(last_byte) {
|
||||
let mut out = String::with_capacity(2);
|
||||
out.push(first_byte as char);
|
||||
out.push(last_byte as char);
|
||||
out
|
||||
} else {
|
||||
let encoded = hex::encode(first);
|
||||
let mut chars = encoded.chars();
|
||||
let first_char = chars.next().unwrap_or('0');
|
||||
let last_char = encoded.chars().last().unwrap_or('0');
|
||||
let mut out = String::with_capacity(2);
|
||||
out.push(first_char);
|
||||
out.push(last_char);
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
const fn is_ascii_alphanumeric(byte: u8) -> bool {
|
||||
byte.is_ascii_digit() || byte.is_ascii_uppercase() || byte.is_ascii_lowercase()
|
||||
}
|
||||
|
||||
/// Builds the raw, pre hash extension string for section three.
|
||||
///
|
||||
/// The extension types are sorted by hex value after removing GREASE, SNI, and
|
||||
/// ALPN. If a signature algorithms extension is present, its values are appended
|
||||
/// after an underscore, in their original order. If it is absent, the string
|
||||
/// ends without a trailing underscore, which is the behavior the specification
|
||||
/// requires and which changes the resulting hash.
|
||||
fn ja4_extension_raw(ch: &ClientHello) -> String {
|
||||
let excluded = [extension::SERVER_NAME, extension::ALPN];
|
||||
let ext_types: SmallVec<[u16; 16]> = ch.extensions.iter().map(|e| e.ext_type).collect();
|
||||
let (ext_csv, _) = sorted_hex_csv(&ext_types, &excluded);
|
||||
|
||||
let sig_algs = ch.signature_algorithms();
|
||||
if ch.has_extension(extension::SIGNATURE_ALGORITHMS) {
|
||||
let sig_csv = unsorted_hex_csv(&sig_algs);
|
||||
format!("{ext_csv}_{sig_csv}")
|
||||
} else {
|
||||
ext_csv
|
||||
}
|
||||
}
|
||||
|
||||
fn sorted_hex_csv(values: &[u16], excluded: &[u16]) -> (String, usize) {
|
||||
let mut hexed: SmallVec<[String; 32]> = values
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|v| !is_grease(*v) && !excluded.contains(v))
|
||||
.map(|v| format!("{v:04x}"))
|
||||
.collect();
|
||||
hexed.sort_unstable();
|
||||
(hexed.join(","), hexed.len())
|
||||
}
|
||||
|
||||
fn unsorted_hex_csv(values: &[u16]) -> String {
|
||||
let hexed: SmallVec<[String; 16]> = values
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|v| !is_grease(*v))
|
||||
.map(|v| format!("{v:04x}"))
|
||||
.collect();
|
||||
hexed.join(",")
|
||||
}
|
||||
|
||||
fn truncated_sha256(input: &str) -> String {
|
||||
if input.is_empty() {
|
||||
return EMPTY_HASH.to_string();
|
||||
}
|
||||
sha256_hex12(input.as_bytes())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{EMPTY_HASH, truncated_sha256};
|
||||
|
||||
#[test]
|
||||
fn empty_input_is_the_zero_hash() {
|
||||
assert_eq!(truncated_sha256(""), EMPTY_HASH);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncation_is_twelve_hex_chars() {
|
||||
let h = truncated_sha256("1301,1302,1303");
|
||||
assert_eq!(h.len(), 12);
|
||||
assert!(h.chars().all(|c| c.is_ascii_hexdigit()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn foxio_cipher_section_vector() {
|
||||
let ciphers = "002f,0035,009c,009d,1301,1302,1303,c013,c014,c02b,c02c,c02f,c030,cca8,cca9";
|
||||
assert_eq!(truncated_sha256(ciphers), "8daaf6152771");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn foxio_extension_section_vector() {
|
||||
let exts = "0005,000a,000b,000d,0012,0015,0017,001b,0023,002b,002d,0033,4469,ff01_0403,0804,0401,0503,0805,0501,0806,0601";
|
||||
assert_eq!(truncated_sha256(exts), "e5627efa2ab1");
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,195 @@
|
|||
// ©AngelaMos | 2026
|
||||
// ja4h.rs
|
||||
|
||||
use crate::fingerprint::Ja4Family;
|
||||
use crate::hash::sha256_hex12;
|
||||
|
||||
/// A parsed HTTP request, holding what JA4H reads.
|
||||
///
|
||||
/// JA4H fingerprints an HTTP client from one request: its method, version,
|
||||
/// whether it carries cookies and a referer, the names of its other headers in
|
||||
/// the order they were sent, its accept language, and its cookie names and
|
||||
/// values. A request that omits an accept language and sends no cookies is far
|
||||
/// more likely to be a script than a person, and JA4H makes that visible in the
|
||||
/// first few characters.
|
||||
///
|
||||
/// This applies to cleartext HTTP only. Over HTTPS the request is encrypted and
|
||||
/// invisible to a passive observer, and HTTP/2 carries its headers in HPACK,
|
||||
/// which a passive tool cannot decode without following the whole connection.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HttpRequest {
|
||||
pub method: String,
|
||||
pub version: String,
|
||||
pub headers: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
/// Parses an HTTP/1.x request from the start of a reassembled byte stream.
|
||||
///
|
||||
/// Returns `None` when the bytes are not a well formed request line followed by
|
||||
/// headers. Header values keep their original bytes; header names keep their
|
||||
/// original case because JA4H hashes them as sent.
|
||||
#[must_use]
|
||||
pub fn parse_http_request(bytes: &[u8]) -> Option<HttpRequest> {
|
||||
let text = std::str::from_utf8(bytes).ok()?;
|
||||
let mut lines = text.split("\r\n");
|
||||
|
||||
let request_line = lines.next()?;
|
||||
let mut parts = request_line.split(' ');
|
||||
let method = parts.next()?.to_string();
|
||||
let _target = parts.next()?;
|
||||
let http_token = parts.next()?;
|
||||
if !http_token.starts_with("HTTP/") {
|
||||
return None;
|
||||
}
|
||||
let version = http_token.trim_start_matches("HTTP/").replace('.', "");
|
||||
|
||||
let mut headers = Vec::new();
|
||||
for line in lines {
|
||||
if line.is_empty() {
|
||||
break;
|
||||
}
|
||||
let (name, value) = line.split_once(':')?;
|
||||
headers.push((name.to_string(), value.trim_start().to_string()));
|
||||
}
|
||||
|
||||
Some(HttpRequest {
|
||||
method,
|
||||
version,
|
||||
headers,
|
||||
})
|
||||
}
|
||||
|
||||
/// Computes the JA4H fingerprint for a parsed HTTP request.
|
||||
#[must_use]
|
||||
pub fn ja4h(req: &HttpRequest) -> Ja4Family {
|
||||
let method = method_code(&req.method);
|
||||
let version = version_code(&req.version);
|
||||
let cookie_flag = if has_header(req, "cookie") { 'c' } else { 'n' };
|
||||
let referer_flag = if has_named_header(req, "referer") {
|
||||
'r'
|
||||
} else {
|
||||
'n'
|
||||
};
|
||||
|
||||
let counted: Vec<&str> = req
|
||||
.headers
|
||||
.iter()
|
||||
.map(|(name, _)| name.as_str())
|
||||
.filter(|name| is_counted_header(name))
|
||||
.collect();
|
||||
let header_len = counted.len().min(99);
|
||||
let lang = accept_language(req);
|
||||
|
||||
let prefix = format!("{method}{version}{cookie_flag}{referer_flag}{header_len:02}{lang}");
|
||||
let header_hash = sha12(&counted.join(","));
|
||||
|
||||
let cookies = cookie_pairs(req);
|
||||
let (cookie_hash, value_hash, raw_cookie_tail) = if let Some(mut pairs) = cookies {
|
||||
pairs.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
let names: Vec<&str> = pairs.iter().map(|p| p.0.as_str()).collect();
|
||||
let entries: Vec<&str> = pairs.iter().map(|p| p.1.as_str()).collect();
|
||||
let tail = format!("{}_{}", names.join(","), entries.join(","));
|
||||
(sha12(&names.join(",")), sha12(&entries.join(",")), tail)
|
||||
} else {
|
||||
(ZERO_HASH.to_string(), ZERO_HASH.to_string(), String::new())
|
||||
};
|
||||
|
||||
let hash = format!("{prefix}_{header_hash}_{cookie_hash}_{value_hash}");
|
||||
let raw = format!("{prefix}_{}_{raw_cookie_tail}", counted.join(","));
|
||||
Ja4Family::new(hash, raw)
|
||||
}
|
||||
|
||||
const ZERO_HASH: &str = "000000000000";
|
||||
|
||||
fn method_code(method: &str) -> String {
|
||||
method.to_lowercase().chars().take(2).collect()
|
||||
}
|
||||
|
||||
fn version_code(version: &str) -> String {
|
||||
match version {
|
||||
"2" | "20" => "20".to_string(),
|
||||
"3" | "30" => "30".to_string(),
|
||||
"10" => "10".to_string(),
|
||||
_ => "11".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn has_header(req: &HttpRequest, prefix_lower: &str) -> bool {
|
||||
req.headers
|
||||
.iter()
|
||||
.any(|(name, _)| name.to_lowercase().starts_with(prefix_lower))
|
||||
}
|
||||
|
||||
fn has_named_header(req: &HttpRequest, name_lower: &str) -> bool {
|
||||
req.headers
|
||||
.iter()
|
||||
.any(|(name, _)| name.to_lowercase() == name_lower)
|
||||
}
|
||||
|
||||
fn is_counted_header(name: &str) -> bool {
|
||||
let lower = name.to_lowercase();
|
||||
!name.starts_with(':') && !lower.starts_with("cookie") && lower != "referer" && !name.is_empty()
|
||||
}
|
||||
|
||||
fn accept_language(req: &HttpRequest) -> String {
|
||||
let Some((_, value)) = req
|
||||
.headers
|
||||
.iter()
|
||||
.find(|(name, _)| name.to_lowercase() == "accept-language")
|
||||
else {
|
||||
return "0000".to_string();
|
||||
};
|
||||
let normalized = value.replace('-', "").replace(';', ",").to_lowercase();
|
||||
let first = normalized.split(',').next().unwrap_or("");
|
||||
let mut code: String = first.chars().take(4).collect();
|
||||
while code.len() < 4 {
|
||||
code.push('0');
|
||||
}
|
||||
code
|
||||
}
|
||||
|
||||
fn cookie_pairs(req: &HttpRequest) -> Option<Vec<(String, String)>> {
|
||||
let (_, value) = req
|
||||
.headers
|
||||
.iter()
|
||||
.find(|(name, _)| name.to_lowercase() == "cookie")?;
|
||||
let pairs = value
|
||||
.split(';')
|
||||
.map(str::trim)
|
||||
.filter(|p| !p.is_empty())
|
||||
.map(|p| {
|
||||
let name = p.split('=').next().unwrap_or(p).trim().to_string();
|
||||
(name, p.to_string())
|
||||
})
|
||||
.collect();
|
||||
Some(pairs)
|
||||
}
|
||||
|
||||
fn sha12(joined: &str) -> String {
|
||||
sha256_hex12(joined.as_bytes())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{parse_http_request, version_code};
|
||||
|
||||
#[test]
|
||||
fn parses_request_line_and_headers() {
|
||||
let raw = b"GET /index.html HTTP/1.1\r\nHost: example.com\r\nAccept: */*\r\n\r\n";
|
||||
let req = parse_http_request(raw).unwrap();
|
||||
assert_eq!(req.method, "GET");
|
||||
assert_eq!(req.version, "11");
|
||||
assert_eq!(req.headers.len(), 2);
|
||||
assert_eq!(
|
||||
req.headers[0],
|
||||
("Host".to_string(), "example.com".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_codes() {
|
||||
assert_eq!(version_code("11"), "11");
|
||||
assert_eq!(version_code("10"), "10");
|
||||
assert_eq!(version_code("2"), "20");
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
// ©AngelaMos | 2026
|
||||
// ja4t.rs
|
||||
|
||||
use std::fmt::Write as _;
|
||||
|
||||
use smallvec::SmallVec;
|
||||
|
||||
/// The TCP layer inputs to a JA4T fingerprint, read from a SYN or SYN ACK.
|
||||
///
|
||||
/// JA4T fingerprints the TCP stack rather than the TLS stack. It is computed
|
||||
/// from fields that an operating system sets the same way on every connection
|
||||
/// but that differ between operating systems: the advertised window size, the
|
||||
/// kinds and order of TCP options, the maximum segment size, and the window
|
||||
/// scale factor. Pairing it with JA4 exposes a class of evasion that TLS only
|
||||
/// fingerprinting misses, where a tool wears a browser's TLS clothing while its
|
||||
/// host operating system speaks with a different TCP accent.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct TcpFingerprintInput {
|
||||
pub window_size: u16,
|
||||
pub option_kinds: SmallVec<[u8; 8]>,
|
||||
pub mss: u16,
|
||||
pub window_scale: u8,
|
||||
}
|
||||
|
||||
/// Computes the JA4T or JA4TS string.
|
||||
///
|
||||
/// The format is the window size, then the option kinds joined with hyphens,
|
||||
/// then the maximum segment size, then the window scale, with the four parts
|
||||
/// separated by underscores. A missing MSS or window scale option is reported as
|
||||
/// zero. The same function serves both the client SYN, which yields JA4T, and
|
||||
/// the server SYN ACK, which yields JA4TS.
|
||||
#[must_use]
|
||||
pub fn ja4t(input: &TcpFingerprintInput) -> String {
|
||||
let mut options = String::new();
|
||||
let mut first = true;
|
||||
for kind in &input.option_kinds {
|
||||
if !first {
|
||||
options.push('-');
|
||||
}
|
||||
first = false;
|
||||
let _ = write!(options, "{kind}");
|
||||
}
|
||||
|
||||
format!(
|
||||
"{}_{}_{}_{}",
|
||||
input.window_size, options, input.mss, input.window_scale
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{TcpFingerprintInput, ja4t};
|
||||
use smallvec::SmallVec;
|
||||
|
||||
fn input(window: u16, kinds: &[u8], mss: u16, scale: u8) -> TcpFingerprintInput {
|
||||
TcpFingerprintInput {
|
||||
window_size: window,
|
||||
option_kinds: SmallVec::from_slice(kinds),
|
||||
mss,
|
||||
window_scale: scale,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn foxio_linux_default_vector() {
|
||||
let i = input(64240, &[2, 1, 3, 1, 1, 4], 1460, 8);
|
||||
assert_eq!(ja4t(&i), "64240_2-1-3-1-1-4_1460_8");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn foxio_windowed_vector() {
|
||||
let i = input(65535, &[2, 1, 3, 1, 1, 8, 4, 0, 0], 1346, 6);
|
||||
assert_eq!(ja4t(&i), "65535_2-1-3-1-1-8-4-0-0_1346_6");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_mss_and_scale_report_zero() {
|
||||
let i = input(8192, &[2, 4], 0, 0);
|
||||
assert_eq!(ja4t(&i), "8192_2-4_0_0");
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,175 @@
|
|||
// ©AngelaMos | 2026
|
||||
// ja4x.rs
|
||||
|
||||
use smallvec::SmallVec;
|
||||
|
||||
use crate::der::{Der, tag};
|
||||
use crate::error::{ParseError, Result};
|
||||
use crate::hash::sha256_hex12;
|
||||
|
||||
/// Computes the JA4X fingerprint of one DER encoded X.509 certificate.
|
||||
///
|
||||
/// JA4X does not fingerprint the contents of a certificate. It fingerprints how
|
||||
/// the certificate was built: which object identifiers appear in the issuer
|
||||
/// name, which appear in the subject name, and which appear among the
|
||||
/// extensions, each in the order they were written. Two certificates minted by
|
||||
/// the same software with the same template share a JA4X even when every name
|
||||
/// and serial differs, which is what makes it useful for clustering certificates
|
||||
/// from one toolchain or one malware family.
|
||||
///
|
||||
/// Passively this only works on TLS 1.2 and earlier, where the certificate
|
||||
/// travels in the clear. TLS 1.3 encrypts the certificate message, so a passive
|
||||
/// observer never sees it.
|
||||
pub fn ja4x(cert_der: &[u8]) -> Result<String> {
|
||||
let (issuer_oids, subject_oids, ext_oids) = certificate_oids(cert_der)?;
|
||||
|
||||
let issuer_hash = sha256_hex12(issuer_oids.join(",").as_bytes());
|
||||
let subject_hash = sha256_hex12(subject_oids.join(",").as_bytes());
|
||||
let ext_hash = sha256_hex12(ext_oids.join(",").as_bytes());
|
||||
|
||||
Ok(format!("{issuer_hash}_{subject_hash}_{ext_hash}"))
|
||||
}
|
||||
|
||||
type OidList = SmallVec<[String; 8]>;
|
||||
|
||||
fn certificate_oids(cert_der: &[u8]) -> Result<(OidList, OidList, OidList)> {
|
||||
let mut outer = Der::new(cert_der);
|
||||
let (_, certificate) = outer.read_tlv()?;
|
||||
let mut cert = Der::new(certificate);
|
||||
let (_, tbs) = cert.read_tlv()?;
|
||||
|
||||
let mut fields: SmallVec<[(u8, &[u8]); 10]> = SmallVec::new();
|
||||
let mut walker = Der::new(tbs);
|
||||
while !walker.is_empty() {
|
||||
fields.push(walker.read_tlv()?);
|
||||
}
|
||||
|
||||
let mut idx = 0;
|
||||
if fields.first().is_some_and(|(t, _)| *t == 0xa0) {
|
||||
idx += 1;
|
||||
}
|
||||
idx += 2;
|
||||
let issuer = field_content(&fields, idx)?;
|
||||
idx += 1;
|
||||
idx += 1;
|
||||
let subject = field_content(&fields, idx)?;
|
||||
idx += 1;
|
||||
idx += 1;
|
||||
|
||||
let extensions = fields[idx..]
|
||||
.iter()
|
||||
.find(|(t, _)| *t == tag::CONTEXT_3)
|
||||
.map(|(_, c)| *c);
|
||||
|
||||
let issuer_oids = name_oids(issuer)?;
|
||||
let subject_oids = name_oids(subject)?;
|
||||
let ext_oids = match extensions {
|
||||
Some(content) => extension_oids(content)?,
|
||||
None => OidList::new(),
|
||||
};
|
||||
|
||||
Ok((issuer_oids, subject_oids, ext_oids))
|
||||
}
|
||||
|
||||
fn field_content<'a>(fields: &[(u8, &'a [u8])], idx: usize) -> Result<&'a [u8]> {
|
||||
fields
|
||||
.get(idx)
|
||||
.map(|(_, c)| *c)
|
||||
.ok_or(ParseError::Truncated { needed: 1, have: 0 })
|
||||
}
|
||||
|
||||
/// Collects the attribute type object identifiers from a Name, in order.
|
||||
///
|
||||
/// A Name is a sequence of relative distinguished names, each a set of attribute
|
||||
/// type and value pairs. The fingerprint reads only the attribute type, the
|
||||
/// leading object identifier of each pair, and renders it as the hex of its DER
|
||||
/// content bytes, which is exactly the form JA4X hashes.
|
||||
fn name_oids(name: &[u8]) -> Result<OidList> {
|
||||
let mut oids = OidList::new();
|
||||
let mut rdns = Der::new(name);
|
||||
while !rdns.is_empty() {
|
||||
let (_, rdn) = rdns.read_tlv()?;
|
||||
let mut set = Der::new(rdn);
|
||||
while !set.is_empty() {
|
||||
let (_, atv) = set.read_tlv()?;
|
||||
let mut pair = Der::new(atv);
|
||||
let (oid_tag, oid) = pair.read_tlv()?;
|
||||
if oid_tag == tag::OBJECT_IDENTIFIER {
|
||||
oids.push(hex::encode(oid));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(oids)
|
||||
}
|
||||
|
||||
/// Collects the extension object identifiers, in order.
|
||||
fn extension_oids(context: &[u8]) -> Result<OidList> {
|
||||
let mut oids = OidList::new();
|
||||
let mut wrapper = Der::new(context);
|
||||
let (_, sequence) = wrapper.read_tlv()?;
|
||||
let mut exts = Der::new(sequence);
|
||||
while !exts.is_empty() {
|
||||
let (_, ext) = exts.read_tlv()?;
|
||||
let mut fields = Der::new(ext);
|
||||
let (oid_tag, oid) = fields.read_tlv()?;
|
||||
if oid_tag == tag::OBJECT_IDENTIFIER {
|
||||
oids.push(hex::encode(oid));
|
||||
}
|
||||
}
|
||||
Ok(oids)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{extension_oids, name_oids};
|
||||
|
||||
fn oid(content: &[u8]) -> Vec<u8> {
|
||||
let mut v = vec![0x06, u8::try_from(content.len()).unwrap()];
|
||||
v.extend_from_slice(content);
|
||||
v
|
||||
}
|
||||
|
||||
fn tlv(tag: u8, content: &[u8]) -> Vec<u8> {
|
||||
let mut v = vec![tag, u8::try_from(content.len()).unwrap()];
|
||||
v.extend_from_slice(content);
|
||||
v
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_issuer_oids_in_order() {
|
||||
let mut name = Vec::new();
|
||||
for content in [
|
||||
&[0x55u8, 0x04, 0x06][..],
|
||||
&[0x55, 0x04, 0x0a],
|
||||
&[0x55, 0x04, 0x0b],
|
||||
&[0x55, 0x04, 0x03],
|
||||
] {
|
||||
let atv = tlv(0x30, &{
|
||||
let mut a = oid(content);
|
||||
a.extend(tlv(0x13, b"x"));
|
||||
a
|
||||
});
|
||||
name.extend(tlv(0x31, &atv));
|
||||
}
|
||||
|
||||
let oids = name_oids(&name).unwrap();
|
||||
assert_eq!(oids.as_slice(), &["550406", "55040a", "55040b", "550403"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_extension_oids() {
|
||||
let mut sequence = Vec::new();
|
||||
for content in [&[0x55u8, 0x1d, 0x23][..], &[0x55, 0x1d, 0x0e]] {
|
||||
let ext = tlv(0x30, &{
|
||||
let mut e = oid(content);
|
||||
e.extend(tlv(0x04, b"v"));
|
||||
e
|
||||
});
|
||||
sequence.extend(ext);
|
||||
}
|
||||
let context = tlv(0x30, &sequence);
|
||||
|
||||
let oids = extension_oids(&context).unwrap();
|
||||
assert_eq!(oids.as_slice(), &["551d23", "551d0e"]);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
// ©AngelaMos | 2026
|
||||
// lib.rs
|
||||
|
||||
//! TLS handshake parsing and JA3/JA4 family fingerprint computation.
|
||||
//!
|
||||
//! This crate is the engine. It parses TLS records and handshake messages, the
|
||||
//! TLS carried inside QUIC initial packets, and computes the JA3, JA3S, JA4,
|
||||
//! JA4S, JA4H, JA4X, and JA4T fingerprints. It depends on nothing that touches
|
||||
//! a network interface, a database, or an async runtime, so it can be embedded,
|
||||
//! fuzzed, and unit tested in isolation.
|
||||
|
||||
pub mod der;
|
||||
pub mod error;
|
||||
pub mod fingerprint;
|
||||
pub mod grease;
|
||||
pub mod hash;
|
||||
pub mod ja3;
|
||||
pub mod ja4;
|
||||
pub mod ja4h;
|
||||
pub mod ja4t;
|
||||
pub mod ja4x;
|
||||
pub mod parse;
|
||||
pub mod pipeline;
|
||||
pub mod registry;
|
||||
|
||||
pub use error::{ParseError, Result};
|
||||
pub use fingerprint::{Ja3, Ja4Family};
|
||||
pub use grease::{GREASE_VALUES, is_grease};
|
||||
pub use ja3::{ja3, ja3_string, ja3s, ja3s_string};
|
||||
pub use ja4::{Transport, ja4, ja4s};
|
||||
pub use ja4h::{HttpRequest, ja4h, parse_http_request};
|
||||
pub use ja4t::{TcpFingerprintInput, ja4t};
|
||||
pub use ja4x::ja4x;
|
||||
pub use parse::{ClientHello, Extension, ServerHello};
|
||||
pub use pipeline::event::{FingerprintEvent, StreamEvent};
|
||||
pub use pipeline::source::{PacketSource, PcapFileSource, RawFrame, SourceError};
|
||||
pub use pipeline::{Counters, Pipeline, PipelineConfig};
|
||||
|
|
@ -0,0 +1,70 @@
|
|||
// ©AngelaMos | 2026
|
||||
// cert.rs
|
||||
|
||||
use smallvec::SmallVec;
|
||||
|
||||
use crate::error::Result;
|
||||
use crate::parse::reader::Reader;
|
||||
|
||||
/// The DER certificates carried by one TLS Certificate handshake message.
|
||||
pub type CertificateList<'pkt> = SmallVec<[&'pkt [u8]; 4]>;
|
||||
|
||||
/// Extracts the DER encoded certificates from a Certificate message body.
|
||||
///
|
||||
/// The body is a three byte total length followed by entries that are each a
|
||||
/// three byte length and the raw DER bytes. This is the TLS 1.2 framing; it is
|
||||
/// the only framing a passive observer ever parses, because TLS 1.3 moved the
|
||||
/// Certificate message inside the encrypted part of the handshake. The
|
||||
/// certificates come back in wire order, leaf first, which is the order JA4X
|
||||
/// reports them in.
|
||||
pub fn certificate_der_list(body: &[u8]) -> Result<CertificateList<'_>> {
|
||||
let mut r = Reader::new(body);
|
||||
let mut list = r.sub_u24_vec()?;
|
||||
let mut certs = CertificateList::new();
|
||||
while !list.is_empty() {
|
||||
certs.push(list.take_u24_vec()?);
|
||||
}
|
||||
Ok(certs)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::certificate_der_list;
|
||||
use crate::error::ParseError;
|
||||
|
||||
fn message(certs: &[&[u8]]) -> Vec<u8> {
|
||||
let total: usize = certs.iter().map(|c| c.len() + 3).sum();
|
||||
let mut v = Vec::new();
|
||||
v.extend_from_slice(&u32::try_from(total).unwrap().to_be_bytes()[1..]);
|
||||
for cert in certs {
|
||||
v.extend_from_slice(&u32::try_from(cert.len()).unwrap().to_be_bytes()[1..]);
|
||||
v.extend_from_slice(cert);
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn splits_a_two_certificate_chain() {
|
||||
let body = message(&[&[0x30, 0x01, 0xaa], &[0x30, 0x02, 0xbb, 0xcc]]);
|
||||
let certs = certificate_der_list(&body).unwrap();
|
||||
assert_eq!(certs.len(), 2);
|
||||
assert_eq!(certs[0], &[0x30, 0x01, 0xaa]);
|
||||
assert_eq!(certs[1], &[0x30, 0x02, 0xbb, 0xcc]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncated_entry_is_an_error_not_a_panic() {
|
||||
let mut body = message(&[&[0x30, 0x01, 0xaa]]);
|
||||
body.truncate(body.len() - 1);
|
||||
assert!(matches!(
|
||||
certificate_der_list(&body),
|
||||
Err(ParseError::Truncated { .. })
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_chain_is_empty() {
|
||||
let certs = certificate_der_list(&[0, 0, 0]).unwrap();
|
||||
assert!(certs.is_empty());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,246 @@
|
|||
// ©AngelaMos | 2026
|
||||
// hello.rs
|
||||
|
||||
use smallvec::SmallVec;
|
||||
|
||||
use crate::error::Result;
|
||||
use crate::parse::reader::Reader;
|
||||
use crate::registry::extension;
|
||||
|
||||
/// A single TLS extension as it appeared on the wire.
|
||||
///
|
||||
/// The extension body is borrowed from the packet, not copied. Specific
|
||||
/// extensions are decoded on demand through the accessor methods on
|
||||
/// [`ClientHello`] so that the common parse path does no work for extensions a
|
||||
/// given fingerprint does not read.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct Extension<'pkt> {
|
||||
pub ext_type: u16,
|
||||
pub data: &'pkt [u8],
|
||||
}
|
||||
|
||||
/// A parsed ClientHello, holding exactly what the fingerprint algorithms read.
|
||||
///
|
||||
/// Cipher suites and extensions are stored in wire order. Order is preserved
|
||||
/// because JA3 hashes extensions in their original order, and because the
|
||||
/// difference between wire order and sorted order is itself a signal: a client
|
||||
/// that permutes its extension order on every connection is doing something a
|
||||
/// fixed order client is not.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ClientHello<'pkt> {
|
||||
pub legacy_version: u16,
|
||||
pub cipher_suites: SmallVec<[u16; 32]>,
|
||||
pub extensions: SmallVec<[Extension<'pkt>; 16]>,
|
||||
pub is_sslv2: bool,
|
||||
}
|
||||
|
||||
/// A parsed ServerHello.
|
||||
///
|
||||
/// The server selects exactly one cipher suite, so `cipher_suite` is a single
|
||||
/// value rather than a list. Extensions are again kept in wire order.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ServerHello<'pkt> {
|
||||
pub legacy_version: u16,
|
||||
pub cipher_suite: u16,
|
||||
pub extensions: SmallVec<[Extension<'pkt>; 16]>,
|
||||
}
|
||||
|
||||
fn parse_extensions<'pkt>(r: &mut Reader<'pkt>) -> Result<SmallVec<[Extension<'pkt>; 16]>> {
|
||||
let mut exts = SmallVec::new();
|
||||
if r.is_empty() {
|
||||
return Ok(exts);
|
||||
}
|
||||
let mut block = r.sub_u16_vec()?;
|
||||
while !block.is_empty() {
|
||||
let ext_type = block.u16()?;
|
||||
let data = block.take_u16_vec()?;
|
||||
exts.push(Extension { ext_type, data });
|
||||
}
|
||||
Ok(exts)
|
||||
}
|
||||
|
||||
/// Parses the body of a ClientHello handshake message.
|
||||
///
|
||||
/// The input is the message body, meaning the four byte handshake header has
|
||||
/// already been stripped. The session id, which fingerprinting ignores, is
|
||||
/// skipped, as are the compression methods; getting those two skips right is a
|
||||
/// classic source of bugs because a parser that forgets them reads the wrong
|
||||
/// bytes as cipher suites.
|
||||
pub fn parse_client_hello(body: &[u8]) -> Result<ClientHello<'_>> {
|
||||
let mut r = Reader::new(body);
|
||||
let legacy_version = r.u16()?;
|
||||
let _random = r.take(32)?;
|
||||
let _session_id = r.take_u8_vec()?;
|
||||
|
||||
let mut cipher_reader = r.sub_u16_vec()?;
|
||||
let mut cipher_suites = SmallVec::new();
|
||||
while !cipher_reader.is_empty() {
|
||||
cipher_suites.push(cipher_reader.u16()?);
|
||||
}
|
||||
|
||||
let _compression = r.take_u8_vec()?;
|
||||
let extensions = parse_extensions(&mut r)?;
|
||||
|
||||
Ok(ClientHello {
|
||||
legacy_version,
|
||||
cipher_suites,
|
||||
extensions,
|
||||
is_sslv2: false,
|
||||
})
|
||||
}
|
||||
|
||||
/// Parses the body of a ServerHello handshake message.
|
||||
pub fn parse_server_hello(body: &[u8]) -> Result<ServerHello<'_>> {
|
||||
let mut r = Reader::new(body);
|
||||
let legacy_version = r.u16()?;
|
||||
let _random = r.take(32)?;
|
||||
let _session_id = r.take_u8_vec()?;
|
||||
let cipher_suite = r.u16()?;
|
||||
let _compression = r.u8()?;
|
||||
let extensions = parse_extensions(&mut r)?;
|
||||
|
||||
Ok(ServerHello {
|
||||
legacy_version,
|
||||
cipher_suite,
|
||||
extensions,
|
||||
})
|
||||
}
|
||||
|
||||
impl<'pkt> ServerHello<'pkt> {
|
||||
fn extension(&self, ext_type: u16) -> Option<&Extension<'pkt>> {
|
||||
self.extensions.iter().find(|e| e.ext_type == ext_type)
|
||||
}
|
||||
|
||||
/// Returns the version the server selected.
|
||||
///
|
||||
/// In TLS 1.3 the negotiated version lives in the supported versions
|
||||
/// extension as a single value rather than in the legacy version word, which
|
||||
/// the server pins to TLS 1.2 for compatibility. JA4S reads the real version
|
||||
/// from the extension when it is present.
|
||||
#[must_use]
|
||||
pub fn selected_version(&self) -> u16 {
|
||||
self.extension(extension::SUPPORTED_VERSIONS)
|
||||
.and_then(|ext| {
|
||||
let mut r = Reader::new(ext.data);
|
||||
r.u16().ok()
|
||||
})
|
||||
.unwrap_or(self.legacy_version)
|
||||
}
|
||||
|
||||
/// Returns the ALPN protocol the server chose, if any.
|
||||
#[must_use]
|
||||
pub fn alpn_protocol(&self) -> Option<&'pkt [u8]> {
|
||||
let ext = self.extension(extension::ALPN)?;
|
||||
let mut r = Reader::new(ext.data);
|
||||
let mut list = r.sub_u16_vec().ok()?;
|
||||
list.take_u8_vec().ok().filter(|p| !p.is_empty())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'pkt> ClientHello<'pkt> {
|
||||
fn extension(&self, ext_type: u16) -> Option<&Extension<'pkt>> {
|
||||
self.extensions.iter().find(|e| e.ext_type == ext_type)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn has_extension(&self, ext_type: u16) -> bool {
|
||||
self.extension(ext_type).is_some()
|
||||
}
|
||||
|
||||
/// Returns the server name from the SNI extension, if a host name entry is
|
||||
/// present.
|
||||
#[must_use]
|
||||
pub fn server_name(&self) -> Option<&'pkt str> {
|
||||
let ext = self.extension(extension::SERVER_NAME)?;
|
||||
let mut list = Reader::new(ext.data).sub_u16_vec().ok()?;
|
||||
while !list.is_empty() {
|
||||
let name_type = list.u8().ok()?;
|
||||
let name = list.take_u16_vec().ok()?;
|
||||
if name_type == 0 {
|
||||
return core::str::from_utf8(name).ok();
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Returns the supported groups, the field JA3 calls elliptic curves.
|
||||
#[must_use]
|
||||
pub fn supported_groups(&self) -> SmallVec<[u16; 16]> {
|
||||
self.u16_list(extension::SUPPORTED_GROUPS)
|
||||
}
|
||||
|
||||
/// Returns the elliptic curve point formats.
|
||||
#[must_use]
|
||||
pub fn ec_point_formats(&self) -> SmallVec<[u8; 4]> {
|
||||
let mut out = SmallVec::new();
|
||||
let Some(ext) = self.extension(extension::EC_POINT_FORMATS) else {
|
||||
return out;
|
||||
};
|
||||
let mut r = Reader::new(ext.data);
|
||||
let Ok(list) = r.take_u8_vec() else {
|
||||
return out;
|
||||
};
|
||||
out.extend_from_slice(list);
|
||||
out
|
||||
}
|
||||
|
||||
/// Returns the protocol versions the client offers in the supported versions
|
||||
/// extension. JA4 selects its version field from the highest non GREASE
|
||||
/// value here when the extension is present.
|
||||
#[must_use]
|
||||
pub fn supported_versions(&self) -> SmallVec<[u16; 8]> {
|
||||
let mut out = SmallVec::new();
|
||||
let Some(ext) = self.extension(extension::SUPPORTED_VERSIONS) else {
|
||||
return out;
|
||||
};
|
||||
let mut r = Reader::new(ext.data);
|
||||
let Ok(list) = r.take_u8_vec() else {
|
||||
return out;
|
||||
};
|
||||
let mut inner = Reader::new(list);
|
||||
while let Ok(v) = inner.u16() {
|
||||
out.push(v);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Returns the signature algorithms in their original order. JA4 appends
|
||||
/// these, unsorted, to the extension hash input.
|
||||
#[must_use]
|
||||
pub fn signature_algorithms(&self) -> SmallVec<[u16; 16]> {
|
||||
self.u16_list(extension::SIGNATURE_ALGORITHMS)
|
||||
}
|
||||
|
||||
/// Returns the ALPN protocol identifiers in order. JA4 uses the first one.
|
||||
#[must_use]
|
||||
pub fn alpn_protocols(&self) -> SmallVec<[&'pkt [u8]; 4]> {
|
||||
let mut out = SmallVec::new();
|
||||
let Some(ext) = self.extension(extension::ALPN) else {
|
||||
return out;
|
||||
};
|
||||
let mut r = Reader::new(ext.data);
|
||||
let Ok(mut list) = r.sub_u16_vec() else {
|
||||
return out;
|
||||
};
|
||||
while let Ok(proto) = list.take_u8_vec() {
|
||||
out.push(proto);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn u16_list(&self, ext_type: u16) -> SmallVec<[u16; 16]> {
|
||||
let mut out = SmallVec::new();
|
||||
let Some(ext) = self.extension(ext_type) else {
|
||||
return out;
|
||||
};
|
||||
let mut r = Reader::new(ext.data);
|
||||
let Ok(list) = r.take_u16_vec() else {
|
||||
return out;
|
||||
};
|
||||
let mut inner = Reader::new(list);
|
||||
while let Ok(v) = inner.u16() {
|
||||
out.push(v);
|
||||
}
|
||||
out
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
// ©AngelaMos | 2026
|
||||
// mod.rs
|
||||
|
||||
//! Hand written, bounds checked TLS parsing.
|
||||
//!
|
||||
//! The parser is deliberately not built on a parser combinator framework. It is
|
||||
//! the security critical core of the tool, and the byte by byte reader here maps
|
||||
//! directly onto the wire format described in the TLS RFCs, which makes it easy
|
||||
//! to audit against the specification. Reassembly of fragmented handshakes is
|
||||
//! handled before parsing, so every parse function sees a complete message and
|
||||
//! never has to model partial input.
|
||||
|
||||
pub mod cert;
|
||||
pub mod hello;
|
||||
pub mod reader;
|
||||
pub mod record;
|
||||
|
||||
pub use cert::{CertificateList, certificate_der_list};
|
||||
pub use hello::{ClientHello, Extension, ServerHello, parse_client_hello, parse_server_hello};
|
||||
pub use reader::Reader;
|
||||
pub use record::{
|
||||
first_handshake_message, handshake_bytes, is_sslv2_client_hello, parse_sslv2_client_hello,
|
||||
};
|
||||
|
|
@ -0,0 +1,173 @@
|
|||
// ©AngelaMos | 2026
|
||||
// reader.rs
|
||||
|
||||
use crate::error::{ParseError, Result};
|
||||
|
||||
/// A forward only cursor over a borrowed byte slice with bounds checked reads.
|
||||
///
|
||||
/// Every read advances the cursor and returns an error rather than panicking
|
||||
/// when the buffer is too short. This is the foundation the whole parser stands
|
||||
/// on: because the cursor can never read past the end of the slice, the parser
|
||||
/// has no `unsafe`, cannot index out of bounds, and treats a truncated or
|
||||
/// hostile packet as an ordinary error instead of a crash. The slices it hands
|
||||
/// back borrow from the original packet buffer, so parsing copies nothing.
|
||||
pub struct Reader<'pkt> {
|
||||
buf: &'pkt [u8],
|
||||
pos: usize,
|
||||
}
|
||||
|
||||
impl<'pkt> Reader<'pkt> {
|
||||
#[must_use]
|
||||
pub const fn new(buf: &'pkt [u8]) -> Self {
|
||||
Self { buf, pos: 0 }
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn remaining(&self) -> usize {
|
||||
self.buf.len() - self.pos
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn is_empty(&self) -> bool {
|
||||
self.remaining() == 0
|
||||
}
|
||||
|
||||
fn need(&self, n: usize) -> Result<()> {
|
||||
if self.remaining() < n {
|
||||
return Err(ParseError::Truncated {
|
||||
needed: n,
|
||||
have: self.remaining(),
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn u8(&mut self) -> Result<u8> {
|
||||
self.need(1)?;
|
||||
let v = self.buf[self.pos];
|
||||
self.pos += 1;
|
||||
Ok(v)
|
||||
}
|
||||
|
||||
pub fn u16(&mut self) -> Result<u16> {
|
||||
self.need(2)?;
|
||||
let v = u16::from_be_bytes([self.buf[self.pos], self.buf[self.pos + 1]]);
|
||||
self.pos += 2;
|
||||
Ok(v)
|
||||
}
|
||||
|
||||
/// Reads a 24 bit big endian length, the width TLS uses for handshake
|
||||
/// message bodies and certificate entries.
|
||||
pub fn u24(&mut self) -> Result<u32> {
|
||||
self.need(3)?;
|
||||
let v = u32::from_be_bytes([
|
||||
0,
|
||||
self.buf[self.pos],
|
||||
self.buf[self.pos + 1],
|
||||
self.buf[self.pos + 2],
|
||||
]);
|
||||
self.pos += 3;
|
||||
Ok(v)
|
||||
}
|
||||
|
||||
pub fn u32(&mut self) -> Result<u32> {
|
||||
self.need(4)?;
|
||||
let v = u32::from_be_bytes([
|
||||
self.buf[self.pos],
|
||||
self.buf[self.pos + 1],
|
||||
self.buf[self.pos + 2],
|
||||
self.buf[self.pos + 3],
|
||||
]);
|
||||
self.pos += 4;
|
||||
Ok(v)
|
||||
}
|
||||
|
||||
/// Borrows the next `n` bytes and advances past them.
|
||||
pub fn take(&mut self, n: usize) -> Result<&'pkt [u8]> {
|
||||
self.need(n)?;
|
||||
let slice = &self.buf[self.pos..self.pos + n];
|
||||
self.pos += n;
|
||||
Ok(slice)
|
||||
}
|
||||
|
||||
/// Reads a one byte length prefix, then borrows that many bytes.
|
||||
pub fn take_u8_vec(&mut self) -> Result<&'pkt [u8]> {
|
||||
let len = self.u8()? as usize;
|
||||
self.take(len)
|
||||
}
|
||||
|
||||
/// Reads a two byte length prefix, then borrows that many bytes.
|
||||
pub fn take_u16_vec(&mut self) -> Result<&'pkt [u8]> {
|
||||
let len = self.u16()? as usize;
|
||||
self.take(len)
|
||||
}
|
||||
|
||||
/// Reads a three byte length prefix, then borrows that many bytes.
|
||||
pub fn take_u24_vec(&mut self) -> Result<&'pkt [u8]> {
|
||||
let len = self.u24()? as usize;
|
||||
self.take(len)
|
||||
}
|
||||
|
||||
/// Returns a sub reader over a two byte length prefixed region.
|
||||
///
|
||||
/// This is the workhorse for nested vectors such as the extensions block,
|
||||
/// where an outer length governs a region that itself contains a sequence of
|
||||
/// length prefixed elements.
|
||||
pub fn sub_u16_vec(&mut self) -> Result<Reader<'pkt>> {
|
||||
let len = self.u16()? as usize;
|
||||
let slice = self.take(len)?;
|
||||
Ok(Reader::new(slice))
|
||||
}
|
||||
|
||||
/// Returns a sub reader over a three byte length prefixed region, the width
|
||||
/// the Certificate message uses for its certificate list.
|
||||
pub fn sub_u24_vec(&mut self) -> Result<Reader<'pkt>> {
|
||||
let len = self.u24()? as usize;
|
||||
let slice = self.take(len)?;
|
||||
Ok(Reader::new(slice))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::Reader;
|
||||
use crate::error::ParseError;
|
||||
|
||||
#[test]
|
||||
fn reads_widths_in_order() {
|
||||
let data = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06];
|
||||
let mut r = Reader::new(&data);
|
||||
assert_eq!(r.u8().unwrap(), 0x01);
|
||||
assert_eq!(r.u16().unwrap(), 0x0203);
|
||||
assert_eq!(r.u24().unwrap(), 0x0004_0506);
|
||||
assert!(r.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn short_read_is_an_error_not_a_panic() {
|
||||
let data = [0x01];
|
||||
let mut r = Reader::new(&data);
|
||||
assert_eq!(
|
||||
r.u16().unwrap_err(),
|
||||
ParseError::Truncated { needed: 2, have: 1 }
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn length_prefixed_take_respects_bounds() {
|
||||
let data = [0x03, 0xaa, 0xbb, 0xcc, 0xff];
|
||||
let mut r = Reader::new(&data);
|
||||
assert_eq!(r.take_u8_vec().unwrap(), &[0xaa, 0xbb, 0xcc]);
|
||||
assert_eq!(r.u8().unwrap(), 0xff);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sub_vector_isolates_a_region() {
|
||||
let data = [0x00, 0x02, 0x11, 0x22, 0x33];
|
||||
let mut r = Reader::new(&data);
|
||||
let mut sub = r.sub_u16_vec().unwrap();
|
||||
assert_eq!(sub.remaining(), 2);
|
||||
assert_eq!(sub.u16().unwrap(), 0x1122);
|
||||
assert_eq!(r.u8().unwrap(), 0x33);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,179 @@
|
|||
// ©AngelaMos | 2026
|
||||
// record.rs
|
||||
|
||||
use std::borrow::Cow;
|
||||
|
||||
use smallvec::SmallVec;
|
||||
|
||||
use crate::error::{ParseError, Result};
|
||||
use crate::parse::hello::ClientHello;
|
||||
use crate::parse::reader::Reader;
|
||||
use crate::registry::content_type;
|
||||
|
||||
/// Reassembles the cleartext handshake flight from a TLS record stream.
|
||||
///
|
||||
/// A handshake message can be split across several TLS records, and several
|
||||
/// short messages can share one record. This walks the record framing and
|
||||
/// concatenates the payloads of the handshake records so the caller sees one
|
||||
/// contiguous handshake byte stream. The common case is a single record holding
|
||||
/// a single ClientHello, and that case borrows the original bytes with no copy.
|
||||
/// Only genuinely fragmented flights allocate.
|
||||
///
|
||||
/// Records carrying anything other than handshake data are ignored. In TLS 1.3
|
||||
/// the later handshake messages travel inside records typed as application data
|
||||
/// and are encrypted, so they never reach this function, which is correct: the
|
||||
/// only handshake bytes we can read in the clear are the first flight.
|
||||
pub fn handshake_bytes(stream: &[u8]) -> Result<Cow<'_, [u8]>> {
|
||||
let mut segments: SmallVec<[&[u8]; 4]> = SmallVec::new();
|
||||
let mut r = Reader::new(stream);
|
||||
|
||||
while r.remaining() >= 5 {
|
||||
let ctype = r.u8()?;
|
||||
let _version = r.u16()?;
|
||||
let payload = r.take_u16_vec()?;
|
||||
if ctype == content_type::HANDSHAKE {
|
||||
segments.push(payload);
|
||||
} else if !segments.is_empty() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
match segments.as_slice() {
|
||||
[] => Err(ParseError::Truncated {
|
||||
needed: 5,
|
||||
have: stream.len(),
|
||||
}),
|
||||
[only] => Ok(Cow::Borrowed(*only)),
|
||||
many => {
|
||||
let mut joined = Vec::with_capacity(many.iter().map(|s| s.len()).sum());
|
||||
for seg in many {
|
||||
joined.extend_from_slice(seg);
|
||||
}
|
||||
Ok(Cow::Owned(joined))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the body of the first handshake message of the requested type.
|
||||
///
|
||||
/// The handshake header is a one byte type and a three byte length. This walks
|
||||
/// the messages in the reassembled flight and returns the body slice of the
|
||||
/// first one whose type matches, so the caller never has to reason about the
|
||||
/// header widths.
|
||||
pub fn first_handshake_message(handshake: &[u8], want_type: u8) -> Result<&[u8]> {
|
||||
let mut r = Reader::new(handshake);
|
||||
while r.remaining() >= 4 {
|
||||
let msg_type = r.u8()?;
|
||||
let len = r.u24()? as usize;
|
||||
let body = r.take(len)?;
|
||||
if msg_type == want_type {
|
||||
return Ok(body);
|
||||
}
|
||||
}
|
||||
Err(ParseError::UnexpectedHandshake(want_type))
|
||||
}
|
||||
|
||||
/// Returns true when the stream begins with an SSLv2 style ClientHello.
|
||||
///
|
||||
/// SSLv2 framing sets the high bit of the first length byte and places the
|
||||
/// message type in the first body byte. Type 1 is CLIENT-HELLO. Some old
|
||||
/// malware opens with this backward compatible hello even when it intends to
|
||||
/// negotiate TLS, so detecting it keeps the TLS parser from misreading the
|
||||
/// SSLv2 header as a TLS record.
|
||||
#[must_use]
|
||||
pub fn is_sslv2_client_hello(stream: &[u8]) -> bool {
|
||||
stream.len() >= 3 && (stream[0] & 0x80) != 0 && stream[2] == 1
|
||||
}
|
||||
|
||||
/// Parses an SSLv2 style ClientHello into the common ClientHello shape.
|
||||
///
|
||||
/// SSLv2 carries no extensions, supported groups, or point formats, so those
|
||||
/// stay empty, which matches the community consensus for fingerprinting an
|
||||
/// SSLv2 hello. Cipher specs are three bytes each. Specs that begin with a zero
|
||||
/// byte are SSLv3 and TLS cipher suites carried in the backward compatible
|
||||
/// hello, and those are the values a fingerprint cares about, so they are
|
||||
/// extracted as their two byte suite numbers. True SSLv2 only specs are
|
||||
/// counted but cannot be expressed as two byte suites and are skipped.
|
||||
pub fn parse_sslv2_client_hello(stream: &[u8]) -> Result<ClientHello<'static>> {
|
||||
let mut r = Reader::new(stream);
|
||||
let len_hi = r.u8()? & 0x7f;
|
||||
let len_lo = r.u8()?;
|
||||
let _record_len = (u16::from(len_hi) << 8) | u16::from(len_lo);
|
||||
|
||||
let msg_type = r.u8()?;
|
||||
if msg_type != 1 {
|
||||
return Err(ParseError::UnexpectedHandshake(msg_type));
|
||||
}
|
||||
|
||||
let legacy_version = r.u16()?;
|
||||
let cipher_spec_len = r.u16()? as usize;
|
||||
let session_id_len = r.u16()? as usize;
|
||||
let challenge_len = r.u16()? as usize;
|
||||
|
||||
let cipher_specs = r.take(cipher_spec_len)?;
|
||||
let _session_id = r.take(session_id_len)?;
|
||||
let _challenge = r.take(challenge_len)?;
|
||||
|
||||
let mut cipher_suites = SmallVec::new();
|
||||
let mut specs = Reader::new(cipher_specs);
|
||||
while specs.remaining() >= 3 {
|
||||
let kind = specs.u8()?;
|
||||
let suite = specs.u16()?;
|
||||
if kind == 0 {
|
||||
cipher_suites.push(suite);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(ClientHello {
|
||||
legacy_version,
|
||||
cipher_suites,
|
||||
extensions: SmallVec::new(),
|
||||
is_sslv2: true,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{first_handshake_message, handshake_bytes, is_sslv2_client_hello};
|
||||
use crate::registry::handshake_type;
|
||||
|
||||
fn record(ctype: u8, payload: &[u8]) -> Vec<u8> {
|
||||
let mut v = vec![ctype, 0x03, 0x03];
|
||||
let len = u16::try_from(payload.len()).unwrap();
|
||||
v.extend_from_slice(&len.to_be_bytes());
|
||||
v.extend_from_slice(payload);
|
||||
v
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_record_borrows() {
|
||||
let stream = record(22, &[0x01, 0x00, 0x00, 0x00]);
|
||||
let hs = handshake_bytes(&stream).unwrap();
|
||||
assert!(matches!(hs, std::borrow::Cow::Borrowed(_)));
|
||||
assert_eq!(hs.as_ref(), &[0x01, 0x00, 0x00, 0x00]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fragmented_records_join() {
|
||||
let mut stream = record(22, &[0x01, 0x00, 0x00, 0x06, 0xaa, 0xbb]);
|
||||
stream.extend(record(22, &[0xcc, 0xdd, 0xee, 0xff]));
|
||||
let hs = handshake_bytes(&stream).unwrap();
|
||||
assert!(matches!(hs, std::borrow::Cow::Owned(_)));
|
||||
assert_eq!(hs.as_ref().len(), 10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn finds_the_requested_message() {
|
||||
let hs = [
|
||||
0x02, 0x00, 0x00, 0x01, 0x99, 0x01, 0x00, 0x00, 0x02, 0xaa, 0xbb,
|
||||
];
|
||||
let body = first_handshake_message(&hs, handshake_type::CLIENT_HELLO).unwrap();
|
||||
assert_eq!(body, &[0xaa, 0xbb]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sslv2_detection() {
|
||||
assert!(is_sslv2_client_hello(&[0x80, 0x2e, 0x01, 0x00, 0x02]));
|
||||
assert!(!is_sslv2_client_hello(&[0x16, 0x03, 0x01, 0x00]));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,347 @@
|
|||
// ©AngelaMos | 2026
|
||||
// decode.rs
|
||||
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
|
||||
use etherparse::{EtherType, NetSlice, SlicedPacket, TransportSlice};
|
||||
use smallvec::SmallVec;
|
||||
|
||||
use crate::ja4t::TcpFingerprintInput;
|
||||
|
||||
/// LINKTYPE registry numbers this decoder understands.
|
||||
///
|
||||
/// The values come from the tcpdump link layer header type registry. They are
|
||||
/// redeclared here as plain constants because pcap file readers and live
|
||||
/// captures both report them as bare integers, and the decoder is the single
|
||||
/// place that interprets them.
|
||||
pub mod link_type {
|
||||
pub const NULL: i32 = 0;
|
||||
pub const ETHERNET: i32 = 1;
|
||||
pub const RAW: i32 = 101;
|
||||
pub const LOOP: i32 = 108;
|
||||
pub const LINUX_SLL: i32 = 113;
|
||||
pub const IPV4: i32 = 228;
|
||||
pub const IPV6: i32 = 229;
|
||||
pub const LINUX_SLL2: i32 = 276;
|
||||
}
|
||||
|
||||
/// The BSD null and loopback link headers are four bytes of address family.
|
||||
const NULL_HEADER_LEN: usize = 4;
|
||||
|
||||
/// The Linux cooked capture v2 header is twenty bytes with the protocol in the
|
||||
/// first two.
|
||||
const SLL2_HEADER_LEN: usize = 20;
|
||||
|
||||
/// The TCP kind numbers the JA4T walk extracts values from.
|
||||
const TCP_OPT_END: u8 = 0;
|
||||
const TCP_OPT_NOP: u8 = 1;
|
||||
const TCP_OPT_MSS: u8 = 2;
|
||||
const TCP_OPT_WSCALE: u8 = 3;
|
||||
const TCP_OPT_MSS_LEN: u8 = 4;
|
||||
const TCP_OPT_WSCALE_LEN: u8 = 3;
|
||||
|
||||
/// The TCP flag bits, exactly as byte thirteen of the header carries them.
|
||||
///
|
||||
/// Keeping the flags as the wire bitfield instead of a fistful of bools means
|
||||
/// the struct mirrors the protocol and reads the byte the packet already
|
||||
/// holds, rather than rebuilding it from a handful of accessor calls. Adding
|
||||
/// a flag later is then a constant, not a field.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct TcpFlags(u8);
|
||||
|
||||
impl TcpFlags {
|
||||
pub const FIN: u8 = 0x01;
|
||||
pub const SYN: u8 = 0x02;
|
||||
pub const RST: u8 = 0x04;
|
||||
pub const ACK: u8 = 0x10;
|
||||
|
||||
/// The offset of the flags byte within a TCP header.
|
||||
const FLAGS_BYTE: usize = 13;
|
||||
|
||||
#[must_use]
|
||||
pub const fn new(bits: u8) -> Self {
|
||||
Self(bits)
|
||||
}
|
||||
|
||||
/// Reads the flags byte from a TCP header slice.
|
||||
///
|
||||
/// A missing byte cannot happen for a slice the decoder hands in, since
|
||||
/// the transport layer is only present when a full header parsed, but the
|
||||
/// bounds checked read keeps this honest under direct unit testing.
|
||||
#[must_use]
|
||||
fn from_header(header: &[u8]) -> Self {
|
||||
Self(header.get(Self::FLAGS_BYTE).copied().unwrap_or(0))
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn syn(self) -> bool {
|
||||
self.0 & Self::SYN != 0
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn ack(self) -> bool {
|
||||
self.0 & Self::ACK != 0
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn fin(self) -> bool {
|
||||
self.0 & Self::FIN != 0
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn rst(self) -> bool {
|
||||
self.0 & Self::RST != 0
|
||||
}
|
||||
}
|
||||
|
||||
/// The TCP level facts about one decoded segment.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct TcpMeta {
|
||||
pub seq: u32,
|
||||
pub flags: TcpFlags,
|
||||
pub window_size: u16,
|
||||
}
|
||||
|
||||
/// One TCP segment decoded out of a captured frame.
|
||||
///
|
||||
/// Addresses are directional: `src` sent this segment to `dst`. The JA4T
|
||||
/// input is walked eagerly, but only for SYN packets, because those are the
|
||||
/// only packets whose options JA4T reads and the walk needs the option bytes
|
||||
/// that do not outlive the decode.
|
||||
#[derive(Debug)]
|
||||
pub struct DecodedSegment<'pkt> {
|
||||
pub src: SocketAddr,
|
||||
pub dst: SocketAddr,
|
||||
pub tcp: TcpMeta,
|
||||
pub syn_fingerprint: Option<TcpFingerprintInput>,
|
||||
pub payload: &'pkt [u8],
|
||||
}
|
||||
|
||||
/// Why a frame produced no segment. The distinction only feeds counters, but
|
||||
/// the counters are how an operator learns what a capture was made of.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Skip {
|
||||
UnsupportedLinkType,
|
||||
NotIp,
|
||||
NotTcp,
|
||||
Malformed,
|
||||
}
|
||||
|
||||
/// Decodes a captured frame down to its TCP segment, if it has one.
|
||||
///
|
||||
/// VLAN tags, including stacked QinQ, are stepped over by etherparse. Frames
|
||||
/// the decoder does not understand are skipped with a reason rather than
|
||||
/// failing the capture: a fingerprinting pipeline must shrug off the GRE
|
||||
/// tunnel, the ARP chatter, and the malformed frame that share every real
|
||||
/// network with the TLS it cares about.
|
||||
pub fn decode_frame(link: i32, data: &[u8]) -> Result<DecodedSegment<'_>, Skip> {
|
||||
let sliced = match link {
|
||||
link_type::ETHERNET => SlicedPacket::from_ethernet(data),
|
||||
link_type::LINUX_SLL => SlicedPacket::from_linux_sll(data),
|
||||
link_type::RAW | link_type::IPV4 | link_type::IPV6 => SlicedPacket::from_ip(data),
|
||||
link_type::NULL | link_type::LOOP => {
|
||||
let Some(inner) = data.get(NULL_HEADER_LEN..) else {
|
||||
return Err(Skip::Malformed);
|
||||
};
|
||||
SlicedPacket::from_ip(inner)
|
||||
}
|
||||
link_type::LINUX_SLL2 => {
|
||||
let Some(proto) = data.first_chunk::<2>() else {
|
||||
return Err(Skip::Malformed);
|
||||
};
|
||||
let Some(inner) = data.get(SLL2_HEADER_LEN..) else {
|
||||
return Err(Skip::Malformed);
|
||||
};
|
||||
SlicedPacket::from_ether_type(EtherType(u16::from_be_bytes(*proto)), inner)
|
||||
}
|
||||
_ => return Err(Skip::UnsupportedLinkType),
|
||||
};
|
||||
let sliced = sliced.map_err(|_| Skip::Malformed)?;
|
||||
|
||||
let (src_ip, dst_ip): (IpAddr, IpAddr) = match &sliced.net {
|
||||
Some(NetSlice::Ipv4(v4)) => (
|
||||
IpAddr::V4(v4.header().source_addr()),
|
||||
IpAddr::V4(v4.header().destination_addr()),
|
||||
),
|
||||
Some(NetSlice::Ipv6(v6)) => (
|
||||
IpAddr::V6(v6.header().source_addr()),
|
||||
IpAddr::V6(v6.header().destination_addr()),
|
||||
),
|
||||
Some(NetSlice::Arp(_)) | None => return Err(Skip::NotIp),
|
||||
};
|
||||
|
||||
let Some(TransportSlice::Tcp(tcp)) = &sliced.transport else {
|
||||
return Err(Skip::NotTcp);
|
||||
};
|
||||
|
||||
let flags = TcpFlags::from_header(tcp.slice());
|
||||
let syn_fingerprint = flags
|
||||
.syn()
|
||||
.then(|| tcp_fingerprint_input(tcp.window_size(), tcp.options()));
|
||||
|
||||
Ok(DecodedSegment {
|
||||
src: SocketAddr::new(src_ip, tcp.source_port()),
|
||||
dst: SocketAddr::new(dst_ip, tcp.destination_port()),
|
||||
tcp: TcpMeta {
|
||||
seq: tcp.sequence_number(),
|
||||
flags,
|
||||
window_size: tcp.window_size(),
|
||||
},
|
||||
syn_fingerprint,
|
||||
payload: tcp.payload(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Walks raw TCP options into the JA4T input.
|
||||
///
|
||||
/// JA4T records every option kind in order, including each NOP and each
|
||||
/// trailing end of list byte, because the padding pattern is part of how an
|
||||
/// operating system's stack writes a SYN. The walk is deliberately tolerant:
|
||||
/// a truncated or nonsense length byte ends the walk after recording the kind
|
||||
/// it was found on, so a hostile SYN cannot push the parser out of bounds.
|
||||
pub fn tcp_fingerprint_input(window_size: u16, options: &[u8]) -> TcpFingerprintInput {
|
||||
let mut kinds: SmallVec<[u8; 8]> = SmallVec::new();
|
||||
let mut mss = 0u16;
|
||||
let mut window_scale = 0u8;
|
||||
|
||||
let mut i = 0;
|
||||
while i < options.len() {
|
||||
let kind = options[i];
|
||||
kinds.push(kind);
|
||||
if kind == TCP_OPT_END || kind == TCP_OPT_NOP {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
let Some(&len) = options.get(i + 1) else {
|
||||
break;
|
||||
};
|
||||
if len < 2 {
|
||||
break;
|
||||
}
|
||||
let Some(body) = options.get(i + 2..i + usize::from(len)) else {
|
||||
break;
|
||||
};
|
||||
if kind == TCP_OPT_MSS && len == TCP_OPT_MSS_LEN {
|
||||
if let Some(value) = body.first_chunk::<2>() {
|
||||
mss = u16::from_be_bytes(*value);
|
||||
}
|
||||
}
|
||||
if kind == TCP_OPT_WSCALE && len == TCP_OPT_WSCALE_LEN {
|
||||
if let Some(&value) = body.first() {
|
||||
window_scale = value;
|
||||
}
|
||||
}
|
||||
i += usize::from(len);
|
||||
}
|
||||
|
||||
TcpFingerprintInput {
|
||||
window_size,
|
||||
option_kinds: kinds,
|
||||
mss,
|
||||
window_scale,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{Skip, TcpFlags, decode_frame, link_type, tcp_fingerprint_input};
|
||||
use crate::ja4t::ja4t;
|
||||
use etherparse::PacketBuilder;
|
||||
|
||||
fn tcp_frame(payload: &[u8]) -> Vec<u8> {
|
||||
let builder = PacketBuilder::ethernet2([1; 6], [2; 6])
|
||||
.ipv4([10, 0, 0, 1], [10, 0, 0, 2], 64)
|
||||
.tcp(40000, 443, 1000, 64240);
|
||||
let mut out = Vec::with_capacity(builder.size(payload.len()));
|
||||
builder.write(&mut out, payload).unwrap();
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flags_byte_decodes_to_the_right_bits() {
|
||||
let syn = TcpFlags::from_header(&[0u8; 14]);
|
||||
assert!(!syn.syn());
|
||||
|
||||
let mut header = [0u8; 20];
|
||||
header[13] = TcpFlags::SYN | TcpFlags::ACK;
|
||||
let flags = TcpFlags::from_header(&header);
|
||||
assert!(flags.syn() && flags.ack());
|
||||
assert!(!flags.fin() && !flags.rst());
|
||||
|
||||
assert!(!TcpFlags::from_header(&[]).syn());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decodes_an_ethernet_tcp_frame() {
|
||||
let frame = tcp_frame(b"hello");
|
||||
let seg = decode_frame(link_type::ETHERNET, &frame).unwrap();
|
||||
assert_eq!(seg.src.to_string(), "10.0.0.1:40000");
|
||||
assert_eq!(seg.dst.to_string(), "10.0.0.2:443");
|
||||
assert_eq!(seg.tcp.seq, 1000);
|
||||
assert_eq!(seg.payload, b"hello");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decodes_a_vlan_tagged_frame() {
|
||||
let builder = PacketBuilder::ethernet2([1; 6], [2; 6])
|
||||
.single_vlan(etherparse::VlanId::try_new(7).unwrap())
|
||||
.ipv4([10, 0, 0, 1], [10, 0, 0, 2], 64)
|
||||
.tcp(40000, 443, 1, 64240);
|
||||
let mut frame = Vec::with_capacity(builder.size(0));
|
||||
builder.write(&mut frame, &[]).unwrap();
|
||||
|
||||
let seg = decode_frame(link_type::ETHERNET, &frame).unwrap();
|
||||
assert_eq!(seg.dst.port(), 443);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_tcp_and_garbage_are_skips_not_panics() {
|
||||
let builder = PacketBuilder::ethernet2([1; 6], [2; 6])
|
||||
.ipv4([10, 0, 0, 1], [10, 0, 0, 2], 64)
|
||||
.udp(5000, 53);
|
||||
let mut udp = Vec::with_capacity(builder.size(4));
|
||||
builder.write(&mut udp, &[0xde, 0xad, 0xbe, 0xef]).unwrap();
|
||||
|
||||
assert!(matches!(
|
||||
decode_frame(link_type::ETHERNET, &udp),
|
||||
Err(Skip::NotTcp)
|
||||
));
|
||||
assert!(matches!(
|
||||
decode_frame(link_type::ETHERNET, &[0x01, 0x02]),
|
||||
Err(Skip::Malformed)
|
||||
));
|
||||
assert!(matches!(
|
||||
decode_frame(147, &udp),
|
||||
Err(Skip::UnsupportedLinkType)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ja4t_walk_reproduces_the_linux_default_vector() {
|
||||
let options = [
|
||||
0x02, 0x04, 0x05, 0xb4, 0x01, 0x03, 0x03, 0x08, 0x01, 0x01, 0x04, 0x02,
|
||||
];
|
||||
let input = tcp_fingerprint_input(64240, &options);
|
||||
assert_eq!(ja4t(&input), "64240_2-1-3-1-1-4_1460_8");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ja4t_walk_counts_trailing_end_of_list_padding() {
|
||||
let options = [
|
||||
0x02, 0x04, 0x05, 0x42, 0x01, 0x03, 0x03, 0x06, 0x01, 0x01, 0x08, 0x0a, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x02, 0x00, 0x00,
|
||||
];
|
||||
let input = tcp_fingerprint_input(65535, &options);
|
||||
assert_eq!(ja4t(&input), "65535_2-1-3-1-1-8-4-0-0_1346_6");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ja4t_walk_survives_truncated_options() {
|
||||
let input = tcp_fingerprint_input(1024, &[0x02, 0x04, 0x05]);
|
||||
assert_eq!(input.option_kinds.as_slice(), &[0x02]);
|
||||
assert_eq!(input.mss, 0);
|
||||
|
||||
let zero_len = tcp_fingerprint_input(1024, &[0x05, 0x00, 0x02]);
|
||||
assert_eq!(zero_len.option_kinds.as_slice(), &[0x05]);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,135 @@
|
|||
// ©AngelaMos | 2026
|
||||
// event.rs
|
||||
|
||||
use std::fmt;
|
||||
use std::net::SocketAddr;
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::fingerprint::{Ja3, Ja4Family};
|
||||
|
||||
/// A fingerprint produced by one direction of one flow, without addressing.
|
||||
///
|
||||
/// The protocol layer emits these; the pipeline wraps them with the flow's
|
||||
/// addresses and timestamp to make a [`FingerprintEvent`]. Keeping the two
|
||||
/// layers apart means the protocol extractor can be tested with bare byte
|
||||
/// streams, no packets required.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
pub enum StreamEvent {
|
||||
ClientHello {
|
||||
ja3: Ja3,
|
||||
ja3_raw: String,
|
||||
ja4: Ja4Family,
|
||||
sni: Option<String>,
|
||||
alpn: Option<String>,
|
||||
},
|
||||
ServerHello {
|
||||
ja3s: Ja3,
|
||||
ja3s_raw: String,
|
||||
ja4s: Ja4Family,
|
||||
},
|
||||
Certificate {
|
||||
ja4x: String,
|
||||
},
|
||||
HttpRequest {
|
||||
ja4h: Ja4Family,
|
||||
method: String,
|
||||
host: Option<String>,
|
||||
},
|
||||
TcpSyn {
|
||||
ja4t: String,
|
||||
},
|
||||
TcpSynAck {
|
||||
ja4ts: String,
|
||||
},
|
||||
}
|
||||
|
||||
/// One fingerprint observation, addressed and timestamped.
|
||||
///
|
||||
/// `src` is always the party that sent the fingerprinted bytes: the client
|
||||
/// for a ClientHello or SYN, the server for a ServerHello or certificate.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct FingerprintEvent {
|
||||
pub ts_nanos: u64,
|
||||
pub src: SocketAddr,
|
||||
pub dst: SocketAddr,
|
||||
#[serde(flatten)]
|
||||
pub event: StreamEvent,
|
||||
}
|
||||
|
||||
impl fmt::Display for FingerprintEvent {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
let secs = self.ts_nanos / 1_000_000_000;
|
||||
let millis = self.ts_nanos % 1_000_000_000 / 1_000_000;
|
||||
write!(f, "{secs}.{millis:03} {} -> {} ", self.src, self.dst)?;
|
||||
match &self.event {
|
||||
StreamEvent::ClientHello {
|
||||
ja3,
|
||||
ja4,
|
||||
sni,
|
||||
alpn,
|
||||
..
|
||||
} => {
|
||||
write!(f, "client_hello ja4={} ja3={ja3}", ja4.hash)?;
|
||||
if let Some(sni) = sni {
|
||||
write!(f, " sni={sni}")?;
|
||||
}
|
||||
if let Some(alpn) = alpn {
|
||||
write!(f, " alpn={alpn}")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
StreamEvent::ServerHello { ja3s, ja4s, .. } => {
|
||||
write!(f, "server_hello ja4s={} ja3s={ja3s}", ja4s.hash)
|
||||
}
|
||||
StreamEvent::Certificate { ja4x } => write!(f, "certificate ja4x={ja4x}"),
|
||||
StreamEvent::HttpRequest { ja4h, method, host } => {
|
||||
write!(f, "http_request ja4h={} method={method}", ja4h.hash)?;
|
||||
if let Some(host) = host {
|
||||
write!(f, " host={host}")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
StreamEvent::TcpSyn { ja4t } => write!(f, "tcp_syn ja4t={ja4t}"),
|
||||
StreamEvent::TcpSynAck { ja4ts } => write!(f, "tcp_syn_ack ja4ts={ja4ts}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{FingerprintEvent, StreamEvent};
|
||||
|
||||
#[test]
|
||||
fn json_shape_is_tagged_and_flat() {
|
||||
let event = FingerprintEvent {
|
||||
ts_nanos: 1_500_000_000,
|
||||
src: "10.0.0.1:40000".parse().unwrap(),
|
||||
dst: "10.0.0.2:443".parse().unwrap(),
|
||||
event: StreamEvent::Certificate {
|
||||
ja4x: "7d5dbb3783b4_ba7ce0880c07_7bf9a7bf7029".into(),
|
||||
},
|
||||
};
|
||||
let json = serde_json::to_value(&event).unwrap();
|
||||
assert_eq!(json["kind"], "certificate");
|
||||
assert_eq!(json["ja4x"], "7d5dbb3783b4_ba7ce0880c07_7bf9a7bf7029");
|
||||
assert_eq!(json["src"], "10.0.0.1:40000");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn display_is_one_greppable_line() {
|
||||
let event = FingerprintEvent {
|
||||
ts_nanos: 1_234_000_000,
|
||||
src: "10.0.0.1:40000".parse().unwrap(),
|
||||
dst: "10.0.0.2:443".parse().unwrap(),
|
||||
event: StreamEvent::TcpSyn {
|
||||
ja4t: "64240_2-1-3-1-1-4_1460_8".into(),
|
||||
},
|
||||
};
|
||||
assert_eq!(
|
||||
event.to_string(),
|
||||
"1.234 10.0.0.1:40000 -> 10.0.0.2:443 tcp_syn ja4t=64240_2-1-3-1-1-4_1460_8"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,430 @@
|
|||
// ©AngelaMos | 2026
|
||||
// flow.rs
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::net::SocketAddr;
|
||||
|
||||
/// A bidirectional flow identity.
|
||||
///
|
||||
/// The two endpoints are stored in sorted order so that a packet and its reply
|
||||
/// hash to the same key. Which endpoint is the client is a separate question,
|
||||
/// answered by who sent the SYN or, failing that, who spoke a ClientHello, and
|
||||
/// it is deliberately not baked into the key: captures routinely start in the
|
||||
/// middle of connections, and a key that guessed wrong would split one
|
||||
/// conversation into two.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub struct FlowKey {
|
||||
pub lo: SocketAddr,
|
||||
pub hi: SocketAddr,
|
||||
}
|
||||
|
||||
/// Which endpoint of a [`FlowKey`] sent a given segment.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Direction {
|
||||
FromLo,
|
||||
FromHi,
|
||||
}
|
||||
|
||||
impl Direction {
|
||||
#[must_use]
|
||||
pub const fn index(self) -> usize {
|
||||
match self {
|
||||
Direction::FromLo => 0,
|
||||
Direction::FromHi => 1,
|
||||
}
|
||||
}
|
||||
|
||||
/// The source and destination addresses of traffic flowing this way.
|
||||
#[must_use]
|
||||
pub const fn addresses(self, key: &FlowKey) -> (SocketAddr, SocketAddr) {
|
||||
match self {
|
||||
Direction::FromLo => (key.lo, key.hi),
|
||||
Direction::FromHi => (key.hi, key.lo),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FlowKey {
|
||||
/// Normalizes a directional (source, destination) pair into a key plus the
|
||||
/// direction the packet travelled.
|
||||
#[must_use]
|
||||
pub fn from_pair(src: SocketAddr, dst: SocketAddr) -> (Self, Direction) {
|
||||
if src <= dst {
|
||||
(Self { lo: src, hi: dst }, Direction::FromLo)
|
||||
} else {
|
||||
(Self { lo: dst, hi: src }, Direction::FromHi)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The midpoint of the sequence space. In TCP serial arithmetic an offset at
|
||||
/// or beyond this point is read as the segment sitting behind the anchor, not
|
||||
/// absurdly far ahead of it.
|
||||
const HALF_SERIAL_SPACE: u32 = 0x8000_0000;
|
||||
|
||||
/// Resource limits for one reassembled direction of one flow.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct ReassemblyLimits {
|
||||
/// Most contiguous bytes kept. Everything a passive fingerprinter reads
|
||||
/// sits in the first kilobytes of a stream, so this is a cap on patience,
|
||||
/// not on correctness.
|
||||
pub max_assembled_bytes: usize,
|
||||
/// Most bytes parked in the out of order buffer.
|
||||
pub max_pending_bytes: usize,
|
||||
/// Most segments parked in the out of order buffer.
|
||||
pub max_pending_segments: usize,
|
||||
}
|
||||
|
||||
/// What [`StreamReassembler::push`] did with a segment.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum PushOutcome {
|
||||
/// The contiguous stream grew; the protocol layer should look again.
|
||||
Grew,
|
||||
/// Nothing new: a duplicate, pure overlap, or empty segment.
|
||||
Unchanged,
|
||||
/// The segment was parked out of order for later.
|
||||
Parked,
|
||||
/// The segment fell outside the window this reassembler is willing to
|
||||
/// track, or a buffer limit was hit, and it was dropped.
|
||||
Dropped,
|
||||
}
|
||||
|
||||
/// Reassembles one direction of a TCP stream into contiguous bytes.
|
||||
///
|
||||
/// This is the piece most toy fingerprinting tools skip, and skipping it is
|
||||
/// why they miss handshakes: a ClientHello, and even more so a certificate
|
||||
/// chain, regularly spans several segments, and those segments arrive
|
||||
/// reordered on any path with packet loss. The reassembler anchors at the
|
||||
/// sequence number the SYN names or, on a flow whose start the capture
|
||||
/// missed, at the first segment it sees. Everything else is a relative
|
||||
/// offset from that anchor in wrapping serial arithmetic: in order segments
|
||||
/// append to one contiguous buffer, out of order segments park in a map
|
||||
/// keyed by offset until the gap before them fills.
|
||||
///
|
||||
/// Data from before the anchor on a SYN-less flow is gone; a streaming
|
||||
/// engine cannot retroactively prepend, and accepting that loss explicitly
|
||||
/// is what Suricata does for midstream pickup too. A segment that straddles
|
||||
/// the anchor is trimmed to its useful part rather than discarded.
|
||||
///
|
||||
/// Overlaps resolve first write wins: bytes already accepted are never
|
||||
/// rewritten by a later segment. A passive observer cannot know which copy
|
||||
/// the receiver kept, and the CVE-2018-6794 capture in the test corpus exists
|
||||
/// precisely because inconsistent overlap handling let attackers show an IDS
|
||||
/// a different stream than the one the victim read. First write wins is one
|
||||
/// deterministic, documented answer.
|
||||
#[derive(Debug)]
|
||||
pub struct StreamReassembler {
|
||||
limits: ReassemblyLimits,
|
||||
anchor: Option<u32>,
|
||||
assembled: Vec<u8>,
|
||||
pending: BTreeMap<u32, Vec<u8>>,
|
||||
pending_bytes: usize,
|
||||
released: bool,
|
||||
capped: bool,
|
||||
}
|
||||
|
||||
impl StreamReassembler {
|
||||
#[must_use]
|
||||
pub fn new(limits: ReassemblyLimits) -> Self {
|
||||
Self {
|
||||
limits,
|
||||
anchor: None,
|
||||
assembled: Vec::new(),
|
||||
pending: BTreeMap::new(),
|
||||
pending_bytes: 0,
|
||||
released: false,
|
||||
capped: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// The contiguous bytes assembled so far, from the anchor onward.
|
||||
#[must_use]
|
||||
pub fn data(&self) -> &[u8] {
|
||||
&self.assembled
|
||||
}
|
||||
|
||||
/// Pins the stream start, used when a SYN reveals the true initial
|
||||
/// sequence number before any data arrives. Later anchors are ignored.
|
||||
pub fn anchor(&mut self, seq: u32) {
|
||||
if self.anchor.is_none() {
|
||||
self.anchor = Some(seq);
|
||||
}
|
||||
}
|
||||
|
||||
/// Drops every buffer and refuses all future data.
|
||||
///
|
||||
/// Called once the protocol layer has what it needs, or knows it never
|
||||
/// will. This is what keeps memory flat when a capture contains long
|
||||
/// lived flows: the flow entry stays, the payload buffers do not.
|
||||
pub fn release(&mut self) {
|
||||
self.assembled = Vec::new();
|
||||
self.pending = BTreeMap::new();
|
||||
self.pending_bytes = 0;
|
||||
self.released = true;
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn released(&self) -> bool {
|
||||
self.released
|
||||
}
|
||||
|
||||
/// True when the assembled cap was hit and the tail of the stream is gone.
|
||||
#[must_use]
|
||||
pub fn capped(&self) -> bool {
|
||||
self.capped
|
||||
}
|
||||
|
||||
/// Offers one segment to the stream.
|
||||
pub fn push(&mut self, seq: u32, payload: &[u8]) -> PushOutcome {
|
||||
if self.released || payload.is_empty() {
|
||||
return PushOutcome::Unchanged;
|
||||
}
|
||||
if self.capped {
|
||||
return PushOutcome::Dropped;
|
||||
}
|
||||
let anchor = *self.anchor.get_or_insert(seq);
|
||||
|
||||
let offset = seq.wrapping_sub(anchor);
|
||||
if offset >= HALF_SERIAL_SPACE {
|
||||
let stale = offset.wrapping_neg() as usize;
|
||||
if stale >= payload.len() {
|
||||
return PushOutcome::Unchanged;
|
||||
}
|
||||
return self.push(anchor, &payload[stale..]);
|
||||
}
|
||||
let window_end = self
|
||||
.limits
|
||||
.max_assembled_bytes
|
||||
.saturating_add(self.limits.max_pending_bytes);
|
||||
if offset as usize > window_end {
|
||||
return PushOutcome::Dropped;
|
||||
}
|
||||
|
||||
let assembled_len = self.assembled.len();
|
||||
if (offset as usize) < assembled_len {
|
||||
let overlap = assembled_len - offset as usize;
|
||||
if overlap >= payload.len() {
|
||||
return PushOutcome::Unchanged;
|
||||
}
|
||||
return self.append_in_order(&payload[overlap..]);
|
||||
}
|
||||
if offset as usize == assembled_len {
|
||||
return self.append_in_order(payload);
|
||||
}
|
||||
|
||||
if self.pending.len() >= self.limits.max_pending_segments
|
||||
|| self.pending_bytes.saturating_add(payload.len()) > self.limits.max_pending_bytes
|
||||
{
|
||||
return PushOutcome::Dropped;
|
||||
}
|
||||
match self.pending.entry(offset) {
|
||||
std::collections::btree_map::Entry::Occupied(existing) => {
|
||||
if existing.get().len() >= payload.len() {
|
||||
return PushOutcome::Unchanged;
|
||||
}
|
||||
self.pending_bytes += payload.len() - existing.get().len();
|
||||
*existing.into_mut() = payload.to_vec();
|
||||
}
|
||||
std::collections::btree_map::Entry::Vacant(slot) => {
|
||||
self.pending_bytes += payload.len();
|
||||
slot.insert(payload.to_vec());
|
||||
}
|
||||
}
|
||||
PushOutcome::Parked
|
||||
}
|
||||
|
||||
fn append_in_order(&mut self, payload: &[u8]) -> PushOutcome {
|
||||
let room = self
|
||||
.limits
|
||||
.max_assembled_bytes
|
||||
.saturating_sub(self.assembled.len());
|
||||
if room == 0 {
|
||||
self.mark_capped();
|
||||
return PushOutcome::Dropped;
|
||||
}
|
||||
let take = payload.len().min(room);
|
||||
self.assembled.extend_from_slice(&payload[..take]);
|
||||
if take < payload.len() {
|
||||
self.mark_capped();
|
||||
} else {
|
||||
self.drain_pending();
|
||||
}
|
||||
PushOutcome::Grew
|
||||
}
|
||||
|
||||
/// Splices parked segments onto the contiguous buffer while they touch it.
|
||||
fn drain_pending(&mut self) {
|
||||
while let Some(entry) = self.pending.first_entry() {
|
||||
let offset = *entry.key() as usize;
|
||||
if offset > self.assembled.len() {
|
||||
break;
|
||||
}
|
||||
let segment = entry.remove();
|
||||
self.pending_bytes -= segment.len();
|
||||
let overlap = self.assembled.len() - offset;
|
||||
if overlap >= segment.len() {
|
||||
continue;
|
||||
}
|
||||
let room = self
|
||||
.limits
|
||||
.max_assembled_bytes
|
||||
.saturating_sub(self.assembled.len());
|
||||
let take = (segment.len() - overlap).min(room);
|
||||
self.assembled
|
||||
.extend_from_slice(&segment[overlap..overlap + take]);
|
||||
if take < segment.len() - overlap {
|
||||
self.mark_capped();
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Once the assembled cap is hit nothing later can ever become contiguous,
|
||||
/// so the parked segments are garbage. Drop them and refuse new data, but
|
||||
/// keep the assembled prefix: it is still a valid stream head and whatever
|
||||
/// the protocol layer already read from it stands.
|
||||
fn mark_capped(&mut self) {
|
||||
self.capped = true;
|
||||
self.pending = BTreeMap::new();
|
||||
self.pending_bytes = 0;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{FlowKey, PushOutcome, ReassemblyLimits, StreamReassembler};
|
||||
|
||||
fn limits() -> ReassemblyLimits {
|
||||
ReassemblyLimits {
|
||||
max_assembled_bytes: 64,
|
||||
max_pending_bytes: 64,
|
||||
max_pending_segments: 4,
|
||||
}
|
||||
}
|
||||
|
||||
fn reasm() -> StreamReassembler {
|
||||
StreamReassembler::new(limits())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn both_directions_share_one_key() {
|
||||
let a: std::net::SocketAddr = "10.0.0.1:40000".parse().unwrap();
|
||||
let b: std::net::SocketAddr = "10.0.0.2:443".parse().unwrap();
|
||||
let (forward, fwd_dir) = FlowKey::from_pair(a, b);
|
||||
let (reverse, rev_dir) = FlowKey::from_pair(b, a);
|
||||
assert_eq!(forward, reverse);
|
||||
assert_ne!(fwd_dir, rev_dir);
|
||||
assert_eq!(fwd_dir.addresses(&forward), (a, b));
|
||||
assert_eq!(rev_dir.addresses(&reverse), (b, a));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn in_order_segments_concatenate() {
|
||||
let mut r = reasm();
|
||||
assert_eq!(r.push(100, b"hell"), PushOutcome::Grew);
|
||||
assert_eq!(r.push(104, b"o"), PushOutcome::Grew);
|
||||
assert_eq!(r.data(), b"hello");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn out_of_order_segments_wait_for_the_gap() {
|
||||
let mut r = reasm();
|
||||
assert_eq!(r.push(100, b"hell"), PushOutcome::Grew);
|
||||
assert_eq!(r.push(107, b"orld"), PushOutcome::Parked);
|
||||
assert_eq!(r.data(), b"hell");
|
||||
assert_eq!(r.push(104, b"o w"), PushOutcome::Grew);
|
||||
assert_eq!(r.data(), b"hello world");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn anchor_from_syn_orders_data_arriving_backwards() {
|
||||
let mut r = reasm();
|
||||
r.anchor(1000);
|
||||
assert_eq!(r.push(1004, b"data"), PushOutcome::Parked);
|
||||
assert_eq!(r.push(1000, b"more"), PushOutcome::Grew);
|
||||
assert_eq!(r.data(), b"moredata");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retransmissions_change_nothing() {
|
||||
let mut r = reasm();
|
||||
r.push(100, b"abcdef");
|
||||
assert_eq!(r.push(100, b"abcdef"), PushOutcome::Unchanged);
|
||||
assert_eq!(r.push(102, b"cd"), PushOutcome::Unchanged);
|
||||
assert_eq!(r.data(), b"abcdef");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn overlapping_segment_keeps_the_first_write() {
|
||||
let mut r = reasm();
|
||||
r.push(100, b"abcdef");
|
||||
assert_eq!(r.push(103, b"XXXghi"), PushOutcome::Grew);
|
||||
assert_eq!(r.data(), b"abcdefghi");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parked_overlap_keeps_the_first_write_too() {
|
||||
let mut r = reasm();
|
||||
r.anchor(100);
|
||||
assert_eq!(r.push(104, b"efgh"), PushOutcome::Parked);
|
||||
assert_eq!(r.push(100, b"abcdEFG"), PushOutcome::Grew);
|
||||
assert_eq!(r.data(), b"abcdEFGh");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sequence_numbers_wrap_around_zero() {
|
||||
let mut r = reasm();
|
||||
let anchor = u32::MAX - 1;
|
||||
assert_eq!(r.push(anchor, b"ab"), PushOutcome::Grew);
|
||||
assert_eq!(r.push(0, b"cd"), PushOutcome::Grew);
|
||||
assert_eq!(r.push(2, b"ef"), PushOutcome::Grew);
|
||||
assert_eq!(r.data(), b"abcdef");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_pre_anchor_data_is_ignored_and_far_future_dropped() {
|
||||
let mut r = reasm();
|
||||
r.push(1000, b"ab");
|
||||
assert_eq!(r.push(990, b"old"), PushOutcome::Unchanged);
|
||||
assert_eq!(r.push(100_000, b"far"), PushOutcome::Dropped);
|
||||
assert_eq!(r.data(), b"ab");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn segment_straddling_the_anchor_is_trimmed_not_lost() {
|
||||
let mut r = reasm();
|
||||
r.anchor(1000);
|
||||
assert_eq!(r.push(996, b"oldNEW"), PushOutcome::Grew);
|
||||
assert_eq!(r.data(), b"EW");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assembled_cap_truncates_but_keeps_the_prefix() {
|
||||
let mut r = reasm();
|
||||
let big = vec![0x41u8; 100];
|
||||
assert_eq!(r.push(0, &big), PushOutcome::Grew);
|
||||
assert_eq!(r.data().len(), limits().max_assembled_bytes);
|
||||
assert_eq!(r.push(100, b"more"), PushOutcome::Dropped);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pending_limits_drop_excess_segments() {
|
||||
let mut r = reasm();
|
||||
r.anchor(0);
|
||||
assert_eq!(r.push(10, b"a"), PushOutcome::Parked);
|
||||
assert_eq!(r.push(20, b"b"), PushOutcome::Parked);
|
||||
assert_eq!(r.push(30, b"c"), PushOutcome::Parked);
|
||||
assert_eq!(r.push(40, b"d"), PushOutcome::Parked);
|
||||
assert_eq!(r.push(50, b"e"), PushOutcome::Dropped);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn release_drops_buffers_and_refuses_data() {
|
||||
let mut r = reasm();
|
||||
r.push(0, b"abc");
|
||||
r.release();
|
||||
assert!(r.released());
|
||||
assert_eq!(r.data(), b"");
|
||||
assert_eq!(r.push(3, b"def"), PushOutcome::Unchanged);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,433 @@
|
|||
// ©AngelaMos | 2026
|
||||
// mod.rs
|
||||
|
||||
//! The passive fingerprinting pipeline: frames in, fingerprint events out.
|
||||
//!
|
||||
//! The stages are deliberately separable. A [`PacketSource`] yields raw link
|
||||
//! layer frames from a capture file or, later, a live interface. The decoder
|
||||
//! strips the frame down to a TCP segment. The flow table reassembles each
|
||||
//! direction of each conversation into a contiguous byte stream, surviving
|
||||
//! reordering, retransmission, and overlap. The protocol layer watches each
|
||||
//! stream until it recognizes a TLS flight or an HTTP request head and emits
|
||||
//! fingerprints. Nothing in here touches a network interface, so the whole
|
||||
//! pipeline runs byte exact in tests against vendored captures.
|
||||
|
||||
pub mod decode;
|
||||
pub mod event;
|
||||
pub mod flow;
|
||||
pub mod source;
|
||||
pub mod tls;
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::ja4t::ja4t;
|
||||
use crate::pipeline::decode::{Skip, decode_frame};
|
||||
use crate::pipeline::event::{FingerprintEvent, StreamEvent};
|
||||
use crate::pipeline::flow::{FlowKey, PushOutcome, ReassemblyLimits, StreamReassembler};
|
||||
use crate::pipeline::source::{PacketSource, RawFrame, SourceError};
|
||||
use crate::pipeline::tls::StreamProtocol;
|
||||
|
||||
/// Tuning knobs for the pipeline.
|
||||
///
|
||||
/// The defaults are sized for handshake harvesting: generous enough that a
|
||||
/// fat certificate chain or a multi segment ClientHello always fits, small
|
||||
/// enough that an adversarial capture cannot turn the flow table into a
|
||||
/// memory bomb.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct PipelineConfig {
|
||||
/// Flows tracked at once before the table sheds its oldest entries.
|
||||
pub max_flows: usize,
|
||||
/// A flow untouched for this long is eligible for eviction.
|
||||
pub idle_timeout_nanos: u64,
|
||||
/// Contiguous bytes kept per direction.
|
||||
pub max_assembled_bytes: usize,
|
||||
/// Out of order bytes parked per direction.
|
||||
pub max_pending_bytes: usize,
|
||||
/// Out of order segments parked per direction.
|
||||
pub max_pending_segments: usize,
|
||||
}
|
||||
|
||||
impl PipelineConfig {
|
||||
pub const DEFAULT_MAX_FLOWS: usize = 65_536;
|
||||
pub const DEFAULT_IDLE_TIMEOUT_NANOS: u64 = 60 * 1_000_000_000;
|
||||
pub const DEFAULT_MAX_ASSEMBLED_BYTES: usize = 256 * 1024;
|
||||
pub const DEFAULT_MAX_PENDING_BYTES: usize = 256 * 1024;
|
||||
pub const DEFAULT_MAX_PENDING_SEGMENTS: usize = 128;
|
||||
|
||||
fn limits(&self) -> ReassemblyLimits {
|
||||
ReassemblyLimits {
|
||||
max_assembled_bytes: self.max_assembled_bytes,
|
||||
max_pending_bytes: self.max_pending_bytes,
|
||||
max_pending_segments: self.max_pending_segments,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for PipelineConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_flows: Self::DEFAULT_MAX_FLOWS,
|
||||
idle_timeout_nanos: Self::DEFAULT_IDLE_TIMEOUT_NANOS,
|
||||
max_assembled_bytes: Self::DEFAULT_MAX_ASSEMBLED_BYTES,
|
||||
max_pending_bytes: Self::DEFAULT_MAX_PENDING_BYTES,
|
||||
max_pending_segments: Self::DEFAULT_MAX_PENDING_SEGMENTS,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// What the pipeline saw, for the operator and for the miss rate honesty
|
||||
/// check: a fingerprinting tool that cannot say what it failed to read is a
|
||||
/// tool whose silence gets mistaken for absence.
|
||||
#[derive(Debug, Default, Clone, Copy, Serialize)]
|
||||
pub struct Counters {
|
||||
pub frames: u64,
|
||||
pub bytes: u64,
|
||||
pub tcp_segments: u64,
|
||||
pub skipped_unsupported_link_type: u64,
|
||||
pub skipped_not_ip: u64,
|
||||
pub skipped_not_tcp: u64,
|
||||
pub skipped_malformed: u64,
|
||||
pub flows_created: u64,
|
||||
pub flows_evicted_idle: u64,
|
||||
pub flows_evicted_pressure: u64,
|
||||
pub segments_dropped: u64,
|
||||
pub events: u64,
|
||||
pub streams_capped: u64,
|
||||
pub unfinished_tls_streams: u64,
|
||||
}
|
||||
|
||||
/// One direction of one tracked flow.
|
||||
struct StreamHalf {
|
||||
reassembler: StreamReassembler,
|
||||
protocol: StreamProtocol,
|
||||
syn_fingerprint_emitted: bool,
|
||||
}
|
||||
|
||||
impl StreamHalf {
|
||||
fn new(limits: ReassemblyLimits) -> Self {
|
||||
Self {
|
||||
reassembler: StreamReassembler::new(limits),
|
||||
protocol: StreamProtocol::Undecided,
|
||||
syn_fingerprint_emitted: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct FlowState {
|
||||
halves: [StreamHalf; 2],
|
||||
last_seen_nanos: u64,
|
||||
}
|
||||
|
||||
impl FlowState {
|
||||
fn new(limits: ReassemblyLimits) -> Self {
|
||||
Self {
|
||||
halves: [StreamHalf::new(limits), StreamHalf::new(limits)],
|
||||
last_seen_nanos: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn finished(&self) -> bool {
|
||||
self.halves.iter().all(|h| h.protocol.finished())
|
||||
}
|
||||
}
|
||||
|
||||
/// The passive fingerprinting engine.
|
||||
///
|
||||
/// Feed it frames, take events out through the sink closure. The pipeline is
|
||||
/// synchronous and single threaded by design: one pipeline owns its flow
|
||||
/// table outright, and running one per worker beats sharing a locked table
|
||||
/// between workers.
|
||||
pub struct Pipeline {
|
||||
config: PipelineConfig,
|
||||
flows: HashMap<FlowKey, FlowState>,
|
||||
counters: Counters,
|
||||
}
|
||||
|
||||
impl Pipeline {
|
||||
#[must_use]
|
||||
pub fn new(config: PipelineConfig) -> Self {
|
||||
Self {
|
||||
config,
|
||||
flows: HashMap::new(),
|
||||
counters: Counters::default(),
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn counters(&self) -> &Counters {
|
||||
&self.counters
|
||||
}
|
||||
|
||||
/// Drains a source through the pipeline, sending every event to `sink`.
|
||||
pub fn run<S: PacketSource>(
|
||||
&mut self,
|
||||
source: &mut S,
|
||||
mut sink: impl FnMut(FingerprintEvent),
|
||||
) -> Result<(), SourceError> {
|
||||
while let Some(frame) = source.next_frame()? {
|
||||
self.feed(&frame, &mut sink);
|
||||
}
|
||||
self.finish();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Processes one captured frame.
|
||||
pub fn feed(&mut self, frame: &RawFrame<'_>, sink: &mut impl FnMut(FingerprintEvent)) {
|
||||
self.counters.frames += 1;
|
||||
self.counters.bytes += frame.data.len() as u64;
|
||||
|
||||
let segment = match decode_frame(frame.link_type, frame.data) {
|
||||
Ok(segment) => segment,
|
||||
Err(skip) => {
|
||||
match skip {
|
||||
Skip::UnsupportedLinkType => {
|
||||
self.counters.skipped_unsupported_link_type += 1;
|
||||
}
|
||||
Skip::NotIp => self.counters.skipped_not_ip += 1,
|
||||
Skip::NotTcp => self.counters.skipped_not_tcp += 1,
|
||||
Skip::Malformed => self.counters.skipped_malformed += 1,
|
||||
}
|
||||
return;
|
||||
}
|
||||
};
|
||||
self.counters.tcp_segments += 1;
|
||||
|
||||
let (key, direction) = FlowKey::from_pair(segment.src, segment.dst);
|
||||
if !self.flows.contains_key(&key) {
|
||||
if self.flows.len() >= self.config.max_flows {
|
||||
self.evict(frame.ts_nanos);
|
||||
}
|
||||
self.flows.insert(key, FlowState::new(self.config.limits()));
|
||||
self.counters.flows_created += 1;
|
||||
}
|
||||
let Some(flow) = self.flows.get_mut(&key) else {
|
||||
return;
|
||||
};
|
||||
flow.last_seen_nanos = flow.last_seen_nanos.max(frame.ts_nanos);
|
||||
|
||||
let (src, dst) = direction.addresses(&key);
|
||||
let half = &mut flow.halves[direction.index()];
|
||||
|
||||
if !half.syn_fingerprint_emitted {
|
||||
if let Some(input) = &segment.syn_fingerprint {
|
||||
half.syn_fingerprint_emitted = true;
|
||||
let fingerprint = ja4t(input);
|
||||
let event = if segment.tcp.flags.ack() {
|
||||
StreamEvent::TcpSynAck { ja4ts: fingerprint }
|
||||
} else {
|
||||
StreamEvent::TcpSyn { ja4t: fingerprint }
|
||||
};
|
||||
self.counters.events += 1;
|
||||
sink(FingerprintEvent {
|
||||
ts_nanos: frame.ts_nanos,
|
||||
src,
|
||||
dst,
|
||||
event,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if segment.tcp.flags.syn() {
|
||||
half.reassembler.anchor(segment.tcp.seq.wrapping_add(1));
|
||||
}
|
||||
|
||||
let payload_seq = if segment.tcp.flags.syn() {
|
||||
segment.tcp.seq.wrapping_add(1)
|
||||
} else {
|
||||
segment.tcp.seq
|
||||
};
|
||||
let outcome = half.reassembler.push(payload_seq, segment.payload);
|
||||
if outcome == PushOutcome::Dropped {
|
||||
self.counters.segments_dropped += 1;
|
||||
}
|
||||
|
||||
if outcome == PushOutcome::Grew {
|
||||
let mut emitted = 0u64;
|
||||
tls::advance(&mut half.protocol, half.reassembler.data(), &mut |event| {
|
||||
emitted += 1;
|
||||
sink(FingerprintEvent {
|
||||
ts_nanos: frame.ts_nanos,
|
||||
src,
|
||||
dst,
|
||||
event,
|
||||
});
|
||||
});
|
||||
self.counters.events += emitted;
|
||||
if half.protocol.finished() && !half.reassembler.released() {
|
||||
half.reassembler.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Settles the books at end of capture.
|
||||
///
|
||||
/// Streams that were recognized as TLS but never produced a complete
|
||||
/// handshake message are counted: each one is a handshake the capture
|
||||
/// clipped, which is exactly the number an operator needs before trusting
|
||||
/// an absence of fingerprints.
|
||||
pub fn finish(&mut self) {
|
||||
for flow in self.flows.values() {
|
||||
for half in &flow.halves {
|
||||
if half.protocol.unfinished_tls() {
|
||||
self.counters.unfinished_tls_streams += 1;
|
||||
}
|
||||
if half.reassembler.capped() {
|
||||
self.counters.streams_capped += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
self.flows.clear();
|
||||
}
|
||||
|
||||
/// Sheds flows when the table is full: everything idle past the timeout
|
||||
/// or fully harvested goes, and if nothing qualifies, the single stalest
|
||||
/// flow goes, so the table never refuses a brand new conversation in
|
||||
/// favor of a dead one.
|
||||
fn evict(&mut self, now_nanos: u64) {
|
||||
let timeout = self.config.idle_timeout_nanos;
|
||||
let idle: Vec<FlowKey> = self
|
||||
.flows
|
||||
.iter()
|
||||
.filter(|(_, flow)| {
|
||||
now_nanos.saturating_sub(flow.last_seen_nanos) > timeout || flow.finished()
|
||||
})
|
||||
.map(|(key, _)| *key)
|
||||
.collect();
|
||||
|
||||
if idle.is_empty() {
|
||||
let stalest = self
|
||||
.flows
|
||||
.iter()
|
||||
.min_by_key(|(_, flow)| flow.last_seen_nanos)
|
||||
.map(|(key, _)| *key);
|
||||
if let Some(key) = stalest {
|
||||
self.drop_flow(key);
|
||||
self.counters.flows_evicted_pressure += 1;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
for key in idle {
|
||||
self.drop_flow(key);
|
||||
self.counters.flows_evicted_idle += 1;
|
||||
}
|
||||
}
|
||||
|
||||
fn drop_flow(&mut self, key: FlowKey) {
|
||||
if let Some(flow) = self.flows.remove(&key) {
|
||||
for half in &flow.halves {
|
||||
if half.protocol.unfinished_tls() {
|
||||
self.counters.unfinished_tls_streams += 1;
|
||||
}
|
||||
if half.reassembler.capped() {
|
||||
self.counters.streams_capped += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{Pipeline, PipelineConfig};
|
||||
use crate::pipeline::event::FingerprintEvent;
|
||||
use crate::pipeline::source::RawFrame;
|
||||
use etherparse::PacketBuilder;
|
||||
|
||||
fn tcp_frame(src: ([u8; 4], u16), dst: ([u8; 4], u16), seq: u32, payload: &[u8]) -> Vec<u8> {
|
||||
let builder = PacketBuilder::ethernet2([1; 6], [2; 6])
|
||||
.ipv4(src.0, dst.0, 64)
|
||||
.tcp(src.1, dst.1, seq, 64240);
|
||||
let mut out = Vec::with_capacity(builder.size(payload.len()));
|
||||
builder.write(&mut out, payload).unwrap();
|
||||
out
|
||||
}
|
||||
|
||||
fn feed_all(pipeline: &mut Pipeline, frames: &[Vec<u8>]) -> Vec<FingerprintEvent> {
|
||||
let mut events = Vec::new();
|
||||
for (i, data) in frames.iter().enumerate() {
|
||||
let frame = RawFrame {
|
||||
ts_nanos: u64::try_from(i).unwrap() * 1_000_000,
|
||||
link_type: 1,
|
||||
data,
|
||||
};
|
||||
pipeline.feed(&frame, &mut |e| events.push(e));
|
||||
}
|
||||
events
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn http_request_split_across_segments_fingerprints_once() {
|
||||
let client = ([10, 0, 0, 1], 40000);
|
||||
let server = ([10, 0, 0, 2], 80);
|
||||
let request = b"GET / HTTP/1.1\r\nHost: example.com\r\nAccept: */*\r\n\r\n";
|
||||
let (a, b) = request.split_at(20);
|
||||
|
||||
let frames = vec![
|
||||
tcp_frame(client, server, 1000, a),
|
||||
tcp_frame(client, server, 1000 + u32::try_from(a.len()).unwrap(), b),
|
||||
];
|
||||
|
||||
let mut pipeline = Pipeline::new(PipelineConfig::default());
|
||||
let events = feed_all(&mut pipeline, &frames);
|
||||
assert_eq!(events.len(), 1);
|
||||
assert_eq!(events[0].src.to_string(), "10.0.0.1:40000");
|
||||
assert_eq!(pipeline.counters().tcp_segments, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn out_of_order_delivery_after_a_syn_still_fingerprints() {
|
||||
let client = ([10, 0, 0, 1], 40001);
|
||||
let server = ([10, 0, 0, 2], 80);
|
||||
let request = b"GET / HTTP/1.1\r\nHost: example.com\r\nAccept: */*\r\n\r\n";
|
||||
let (a, b) = request.split_at(20);
|
||||
|
||||
let syn = {
|
||||
let builder = PacketBuilder::ethernet2([1; 6], [2; 6])
|
||||
.ipv4(client.0, server.0, 64)
|
||||
.tcp(client.1, server.1, 999, 64240)
|
||||
.syn();
|
||||
let mut out = Vec::with_capacity(builder.size(0));
|
||||
builder.write(&mut out, &[]).unwrap();
|
||||
out
|
||||
};
|
||||
|
||||
let frames = vec![
|
||||
syn,
|
||||
tcp_frame(client, server, 1000 + u32::try_from(a.len()).unwrap(), b),
|
||||
tcp_frame(client, server, 1000, a),
|
||||
];
|
||||
|
||||
let mut pipeline = Pipeline::new(PipelineConfig::default());
|
||||
let events = feed_all(&mut pipeline, &frames);
|
||||
assert_eq!(events.len(), 2);
|
||||
assert!(events[0].to_string().contains("tcp_syn ja4t="));
|
||||
assert!(events[1].to_string().contains("http_request"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pressure_eviction_keeps_the_table_bounded() {
|
||||
let config = PipelineConfig {
|
||||
max_flows: 4,
|
||||
..PipelineConfig::default()
|
||||
};
|
||||
let mut pipeline = Pipeline::new(config);
|
||||
|
||||
let mut frames = Vec::new();
|
||||
for i in 0..8u16 {
|
||||
let port = 40000 + i;
|
||||
frames.push(tcp_frame(
|
||||
([10, 0, 0, 1], port),
|
||||
([10, 0, 0, 2], 80),
|
||||
1,
|
||||
b"x",
|
||||
));
|
||||
}
|
||||
feed_all(&mut pipeline, &frames);
|
||||
|
||||
assert_eq!(pipeline.counters().flows_created, 8);
|
||||
assert!(pipeline.counters().flows_evicted_pressure >= 4);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,362 @@
|
|||
// ©AngelaMos | 2026
|
||||
// source.rs
|
||||
|
||||
use std::fs::File;
|
||||
use std::io::Read;
|
||||
use std::path::Path;
|
||||
|
||||
use pcap_parser::traits::PcapReaderIterator;
|
||||
use pcap_parser::{Block, PcapBlockOwned, PcapError, PcapHeader, create_reader};
|
||||
use thiserror::Error;
|
||||
|
||||
/// How many bytes of buffer the file reader starts with.
|
||||
///
|
||||
/// A single capture block must fit in the buffer. Offload features such as TSO
|
||||
/// can put frames far larger than an MTU into a capture, so the buffer starts
|
||||
/// generous and can still grow up to [`MAX_BUFFER_CAPACITY`] if a bigger block
|
||||
/// appears.
|
||||
const INITIAL_BUFFER_CAPACITY: usize = 1024 * 1024;
|
||||
|
||||
/// The ceiling for buffer growth. A block larger than this is treated as a
|
||||
/// malformed capture rather than a reason to exhaust memory.
|
||||
const MAX_BUFFER_CAPACITY: usize = 64 * 1024 * 1024;
|
||||
|
||||
/// Timestamp units per second when a capture does not say otherwise.
|
||||
///
|
||||
/// Both the legacy pcap format and the pcapng default are microsecond
|
||||
/// resolution.
|
||||
const DEFAULT_UNITS_PER_SECOND: u64 = 1_000_000;
|
||||
|
||||
const NANOS_PER_SECOND: u64 = 1_000_000_000;
|
||||
const NANOS_PER_MICRO: u64 = 1_000;
|
||||
|
||||
/// Errors produced while reading frames from a capture source.
|
||||
#[derive(Debug, Error)]
|
||||
#[non_exhaustive]
|
||||
pub enum SourceError {
|
||||
#[error("failed to read capture: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
|
||||
#[error("not a pcap or pcapng capture")]
|
||||
NotACapture,
|
||||
|
||||
#[error("capture block exceeds the {MAX_BUFFER_CAPACITY} byte buffer ceiling")]
|
||||
BlockTooLarge,
|
||||
|
||||
#[error("malformed capture: {0}")]
|
||||
Malformed(String),
|
||||
}
|
||||
|
||||
/// One link layer frame as captured, with the metadata needed to decode it.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct RawFrame<'src> {
|
||||
/// Capture timestamp in nanoseconds since the epoch. Zero when the capture
|
||||
/// format carries no timestamp for this frame.
|
||||
pub ts_nanos: u64,
|
||||
/// The link layer type, using the tcpdump LINKTYPE registry numbers.
|
||||
pub link_type: i32,
|
||||
pub data: &'src [u8],
|
||||
}
|
||||
|
||||
/// A source of captured frames.
|
||||
///
|
||||
/// The trait is a lending iterator: each frame borrows from the source and is
|
||||
/// only valid until the next call. That shape fits both file readers, which
|
||||
/// hand out windows into an internal buffer, and live captures, which hand out
|
||||
/// the kernel's buffer. A consumer that needs to keep a frame longer copies
|
||||
/// it, and that decision stays visible at the call site.
|
||||
pub trait PacketSource {
|
||||
/// Returns the next frame, or `None` when the source is exhausted.
|
||||
fn next_frame(&mut self) -> Result<Option<RawFrame<'_>>, SourceError>;
|
||||
}
|
||||
|
||||
/// Per interface metadata from a pcapng interface description block.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct InterfaceInfo {
|
||||
link_type: i32,
|
||||
units_per_second: u64,
|
||||
ts_offset_seconds: i64,
|
||||
}
|
||||
|
||||
/// Everything in the source except the parser, split out so the borrow of the
|
||||
/// parser's buffer held by a block and the mutable borrow needed to stage a
|
||||
/// frame land on different fields.
|
||||
#[derive(Default)]
|
||||
struct SourceState {
|
||||
interfaces: Vec<InterfaceInfo>,
|
||||
legacy: Option<InterfaceInfo>,
|
||||
legacy_nanos: bool,
|
||||
frame: Vec<u8>,
|
||||
frame_ts_nanos: u64,
|
||||
frame_link_type: i32,
|
||||
}
|
||||
|
||||
impl SourceState {
|
||||
/// Copies a frame out of the parser's buffer so the borrow on the parser
|
||||
/// can end before the block is consumed.
|
||||
fn stage(&mut self, ts_nanos: u64, link_type: i32, data: &[u8]) {
|
||||
self.frame.clear();
|
||||
self.frame.extend_from_slice(data);
|
||||
self.frame_ts_nanos = ts_nanos;
|
||||
self.frame_link_type = link_type;
|
||||
}
|
||||
|
||||
fn handle_legacy_header(&mut self, header: &PcapHeader) {
|
||||
self.legacy = Some(InterfaceInfo {
|
||||
link_type: header.network.0,
|
||||
units_per_second: DEFAULT_UNITS_PER_SECOND,
|
||||
ts_offset_seconds: 0,
|
||||
});
|
||||
self.legacy_nanos = header.is_nanosecond_precision();
|
||||
}
|
||||
|
||||
/// Stages a packet block. Returns false for metadata blocks.
|
||||
fn handle_block(&mut self, block: &PcapBlockOwned<'_>) -> bool {
|
||||
match block {
|
||||
PcapBlockOwned::LegacyHeader(header) => {
|
||||
self.handle_legacy_header(header);
|
||||
false
|
||||
}
|
||||
PcapBlockOwned::Legacy(frame) => {
|
||||
let Some(meta) = self.legacy else {
|
||||
return false;
|
||||
};
|
||||
let fraction = if self.legacy_nanos {
|
||||
u64::from(frame.ts_usec)
|
||||
} else {
|
||||
u64::from(frame.ts_usec) * NANOS_PER_MICRO
|
||||
};
|
||||
let ts = u64::from(frame.ts_sec)
|
||||
.saturating_mul(NANOS_PER_SECOND)
|
||||
.saturating_add(fraction);
|
||||
let len = frame.data.len().min(frame.caplen as usize);
|
||||
self.stage(ts, meta.link_type, &frame.data[..len]);
|
||||
true
|
||||
}
|
||||
PcapBlockOwned::NG(Block::SectionHeader(_)) => {
|
||||
self.interfaces.clear();
|
||||
false
|
||||
}
|
||||
PcapBlockOwned::NG(Block::InterfaceDescription(idb)) => {
|
||||
self.interfaces.push(InterfaceInfo {
|
||||
link_type: idb.linktype.0,
|
||||
units_per_second: idb.ts_resolution().unwrap_or(DEFAULT_UNITS_PER_SECOND),
|
||||
ts_offset_seconds: idb.ts_offset(),
|
||||
});
|
||||
false
|
||||
}
|
||||
PcapBlockOwned::NG(Block::EnhancedPacket(epb)) => {
|
||||
let Some(meta) = self.interfaces.get(epb.if_id as usize).copied() else {
|
||||
return false;
|
||||
};
|
||||
let units = (u64::from(epb.ts_high) << 32) | u64::from(epb.ts_low);
|
||||
let ts = scale_to_nanos(units, meta.units_per_second, meta.ts_offset_seconds);
|
||||
let len = epb.data.len().min(epb.caplen as usize);
|
||||
self.stage(ts, meta.link_type, &epb.data[..len]);
|
||||
true
|
||||
}
|
||||
PcapBlockOwned::NG(Block::SimplePacket(spb)) => {
|
||||
let Some(meta) = self.interfaces.first().copied() else {
|
||||
return false;
|
||||
};
|
||||
let len = spb.data.len().min(spb.origlen as usize);
|
||||
self.stage(0, meta.link_type, &spb.data[..len]);
|
||||
true
|
||||
}
|
||||
PcapBlockOwned::NG(_) => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads frames from a pcap or pcapng file.
|
||||
///
|
||||
/// The two formats are probed automatically. pcapng is handled with its full
|
||||
/// generality: every interface carries its own link type and timestamp
|
||||
/// resolution, multiple sections reset the interface list, and metadata blocks
|
||||
/// such as name resolution and decryption secrets are skipped rather than
|
||||
/// treated as packets. A truncated final packet, the signature of a capture
|
||||
/// that was stopped rather than closed, ends iteration cleanly and is reported
|
||||
/// through [`PcapFileSource::truncated`] instead of failing the whole file.
|
||||
pub struct PcapFileSource {
|
||||
reader: Box<dyn PcapReaderIterator>,
|
||||
state: SourceState,
|
||||
buffer_capacity: usize,
|
||||
truncated: bool,
|
||||
finished: bool,
|
||||
}
|
||||
|
||||
impl PcapFileSource {
|
||||
/// Opens a capture file from a path.
|
||||
pub fn open(path: impl AsRef<Path>) -> Result<Self, SourceError> {
|
||||
Self::from_reader(File::open(path)?)
|
||||
}
|
||||
|
||||
/// Builds a source from any byte reader holding pcap or pcapng data.
|
||||
pub fn from_reader(reader: impl Read + 'static) -> Result<Self, SourceError> {
|
||||
let reader = create_reader(INITIAL_BUFFER_CAPACITY, reader).map_err(|e| match e {
|
||||
PcapError::HeaderNotRecognized | PcapError::Eof => SourceError::NotACapture,
|
||||
PcapError::ReadError => SourceError::Io(std::io::Error::other("read failed")),
|
||||
other => SourceError::Malformed(other.to_string()),
|
||||
})?;
|
||||
Ok(Self {
|
||||
reader,
|
||||
state: SourceState::default(),
|
||||
buffer_capacity: INITIAL_BUFFER_CAPACITY,
|
||||
truncated: false,
|
||||
finished: false,
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns true when the file ended in the middle of a block.
|
||||
pub fn truncated(&self) -> bool {
|
||||
self.truncated
|
||||
}
|
||||
}
|
||||
|
||||
impl PacketSource for PcapFileSource {
|
||||
fn next_frame(&mut self) -> Result<Option<RawFrame<'_>>, SourceError> {
|
||||
if self.finished {
|
||||
return Ok(None);
|
||||
}
|
||||
loop {
|
||||
let staged = match self.reader.next() {
|
||||
Ok((offset, block)) => {
|
||||
let staged = self.state.handle_block(&block);
|
||||
self.reader.consume(offset);
|
||||
staged
|
||||
}
|
||||
Err(PcapError::Eof) => {
|
||||
self.finished = true;
|
||||
return Ok(None);
|
||||
}
|
||||
Err(PcapError::UnexpectedEof) => {
|
||||
self.finished = true;
|
||||
self.truncated = true;
|
||||
return Ok(None);
|
||||
}
|
||||
Err(PcapError::Incomplete(_)) => {
|
||||
self.reader
|
||||
.refill()
|
||||
.map_err(|e| SourceError::Malformed(e.to_string()))?;
|
||||
continue;
|
||||
}
|
||||
Err(PcapError::BufferTooSmall) => {
|
||||
let grown = self.buffer_capacity.saturating_mul(2);
|
||||
if grown > MAX_BUFFER_CAPACITY || !self.reader.grow(grown) {
|
||||
self.finished = true;
|
||||
return Err(SourceError::BlockTooLarge);
|
||||
}
|
||||
self.buffer_capacity = grown;
|
||||
continue;
|
||||
}
|
||||
Err(PcapError::ReadError) => {
|
||||
self.finished = true;
|
||||
return Err(SourceError::Io(std::io::Error::other("read failed")));
|
||||
}
|
||||
Err(
|
||||
e @ (PcapError::HeaderNotRecognized
|
||||
| PcapError::NomError(..)
|
||||
| PcapError::OwnedNomError(..)),
|
||||
) => {
|
||||
self.finished = true;
|
||||
return Err(SourceError::Malformed(e.to_string()));
|
||||
}
|
||||
};
|
||||
if staged {
|
||||
return Ok(Some(RawFrame {
|
||||
ts_nanos: self.state.frame_ts_nanos,
|
||||
link_type: self.state.frame_link_type,
|
||||
data: &self.state.frame,
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts a timestamp in interface units to nanoseconds since the epoch.
|
||||
///
|
||||
/// The arithmetic runs in 128 bits so the conversion stays exact for every
|
||||
/// resolution pcapng can express, including nanosecond counts that already
|
||||
/// fill most of a u64.
|
||||
fn scale_to_nanos(units: u64, units_per_second: u64, offset_seconds: i64) -> u64 {
|
||||
if units_per_second == 0 {
|
||||
return 0;
|
||||
}
|
||||
let nanos = u128::from(units) * u128::from(NANOS_PER_SECOND) / u128::from(units_per_second);
|
||||
let offset = i128::from(offset_seconds) * i128::from(NANOS_PER_SECOND);
|
||||
u64::try_from(
|
||||
i128::try_from(nanos)
|
||||
.unwrap_or(i128::MAX)
|
||||
.saturating_add(offset),
|
||||
)
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{PacketSource, PcapFileSource, SourceError, scale_to_nanos};
|
||||
|
||||
fn legacy_pcap(frames: &[&[u8]]) -> Vec<u8> {
|
||||
let mut v = Vec::new();
|
||||
v.extend_from_slice(&0xa1b2_c3d4_u32.to_le_bytes());
|
||||
v.extend_from_slice(&2u16.to_le_bytes());
|
||||
v.extend_from_slice(&4u16.to_le_bytes());
|
||||
v.extend_from_slice(&0i32.to_le_bytes());
|
||||
v.extend_from_slice(&0u32.to_le_bytes());
|
||||
v.extend_from_slice(&65535u32.to_le_bytes());
|
||||
v.extend_from_slice(&1u32.to_le_bytes());
|
||||
for (i, frame) in frames.iter().enumerate() {
|
||||
let len = u32::try_from(frame.len()).unwrap();
|
||||
v.extend_from_slice(&u32::try_from(i + 1).unwrap().to_le_bytes());
|
||||
v.extend_from_slice(&500_000u32.to_le_bytes());
|
||||
v.extend_from_slice(&len.to_le_bytes());
|
||||
v.extend_from_slice(&len.to_le_bytes());
|
||||
v.extend_from_slice(frame);
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_legacy_frames_with_timestamps() {
|
||||
let data = legacy_pcap(&[&[0xaa; 14], &[0xbb; 20]]);
|
||||
let mut source = PcapFileSource::from_reader(std::io::Cursor::new(data)).unwrap();
|
||||
|
||||
let one = source.next_frame().unwrap().unwrap();
|
||||
assert_eq!(one.link_type, 1);
|
||||
assert_eq!(one.ts_nanos, 1_500_000_000);
|
||||
assert_eq!(one.data.len(), 14);
|
||||
|
||||
let two = source.next_frame().unwrap().unwrap();
|
||||
assert_eq!(two.data, &[0xbb; 20]);
|
||||
|
||||
assert!(source.next_frame().unwrap().is_none());
|
||||
assert!(!source.truncated());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncated_final_frame_ends_cleanly() {
|
||||
let mut data = legacy_pcap(&[&[0xaa; 14], &[0xbb; 20]]);
|
||||
data.truncate(data.len() - 5);
|
||||
let mut source = PcapFileSource::from_reader(std::io::Cursor::new(data)).unwrap();
|
||||
|
||||
assert!(source.next_frame().unwrap().is_some());
|
||||
assert!(source.next_frame().unwrap().is_none());
|
||||
assert!(source.truncated());
|
||||
assert!(source.next_frame().unwrap().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn garbage_is_not_a_capture() {
|
||||
let err = PcapFileSource::from_reader(std::io::Cursor::new(vec![0x55; 64]))
|
||||
.err()
|
||||
.unwrap();
|
||||
assert!(matches!(err, SourceError::NotACapture));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn timestamp_scaling_is_exact_for_common_resolutions() {
|
||||
assert_eq!(scale_to_nanos(1_500_000, 1_000_000, 0), 1_500_000_000);
|
||||
assert_eq!(scale_to_nanos(7, 1_000_000_000, 0), 7);
|
||||
assert_eq!(scale_to_nanos(1, 1, 1), 2_000_000_000);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,401 @@
|
|||
// ©AngelaMos | 2026
|
||||
// tls.rs
|
||||
|
||||
use std::borrow::Cow;
|
||||
|
||||
use crate::ja3::{ja3, ja3_string, ja3s, ja3s_string};
|
||||
use crate::ja4::{Transport, ja4, ja4s};
|
||||
use crate::ja4h::{ja4h, parse_http_request};
|
||||
use crate::ja4x::ja4x;
|
||||
use crate::parse::reader::Reader;
|
||||
use crate::parse::{
|
||||
certificate_der_list, is_sslv2_client_hello, parse_client_hello, parse_server_hello,
|
||||
parse_sslv2_client_hello,
|
||||
};
|
||||
use crate::pipeline::event::StreamEvent;
|
||||
use crate::registry::{content_type, handshake_type, version};
|
||||
|
||||
/// How many bytes the sniffer needs before it gives up on classifying a
|
||||
/// stream. Every protocol this pipeline recognizes shows its hand within the
|
||||
/// first few bytes; eight covers the longest HTTP method prefix.
|
||||
const SNIFF_DECISION_LEN: usize = 8;
|
||||
|
||||
/// The HTTP methods the sniffer accepts as the start of a cleartext request.
|
||||
const HTTP_METHOD_PREFIXES: [&[u8]; 9] = [
|
||||
b"GET ",
|
||||
b"POST ",
|
||||
b"PUT ",
|
||||
b"HEAD ",
|
||||
b"DELETE ",
|
||||
b"OPTIONS ",
|
||||
b"PATCH ",
|
||||
b"TRACE ",
|
||||
b"CONNECT ",
|
||||
];
|
||||
|
||||
/// A stream whose HTTP request head has not finished inside this many bytes
|
||||
/// is not worth waiting on.
|
||||
const HTTP_HEAD_CAP: usize = 8 * 1024;
|
||||
|
||||
const HTTP_HEAD_TERMINATOR: &[u8] = b"\r\n\r\n";
|
||||
|
||||
/// A TLS record payload cannot exceed 2^14 plus expansion; RFC 8446 allows
|
||||
/// 255 bytes of expansion on top of the 16384 byte plaintext limit. A length
|
||||
/// beyond that means the stream is not actually TLS record framing.
|
||||
const MAX_TLS_RECORD_LEN: usize = 16384 + 255;
|
||||
|
||||
/// What one direction of a flow is, as far as the protocol layer can tell.
|
||||
#[derive(Debug)]
|
||||
pub enum StreamProtocol {
|
||||
/// Not enough bytes yet to say.
|
||||
Undecided,
|
||||
/// TLS record framing; the cleartext first flight is being extracted.
|
||||
Tls(TlsFlight),
|
||||
/// A cleartext HTTP/1.x request head is being accumulated.
|
||||
Http,
|
||||
/// Recognized and fully harvested; the stream needs no more buffering.
|
||||
Done,
|
||||
/// Unrecognized or unparseable; the stream is ignored.
|
||||
Ignored,
|
||||
}
|
||||
|
||||
impl StreamProtocol {
|
||||
/// True when this direction will never produce another event, which is
|
||||
/// the signal to drop its reassembly buffers.
|
||||
#[must_use]
|
||||
pub fn finished(&self) -> bool {
|
||||
matches!(self, StreamProtocol::Done | StreamProtocol::Ignored)
|
||||
}
|
||||
|
||||
/// True when the stream was recognized as TLS but the capture ended
|
||||
/// before a complete hello could be read. Feeds the miss rate counter
|
||||
/// that tells an operator their capture is clipping handshakes.
|
||||
#[must_use]
|
||||
pub fn unfinished_tls(&self) -> bool {
|
||||
match self {
|
||||
StreamProtocol::Tls(flight) => !flight.saw_any_message,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Incremental extraction state for one direction's cleartext TLS flight.
|
||||
///
|
||||
/// Only what must survive between walks lives here. A ClientHello ends its
|
||||
/// direction immediately, so it needs no flag; a ServerHello does not, since
|
||||
/// a TLS 1.2 Certificate may still be in flight behind it, so the emission
|
||||
/// guard for it persists.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct TlsFlight {
|
||||
emitted_server_hello: bool,
|
||||
saw_any_message: bool,
|
||||
}
|
||||
|
||||
/// Drives protocol detection and extraction over one direction of a stream.
|
||||
///
|
||||
/// `stream` is always the full contiguous bytes from the start of the
|
||||
/// direction; the extractor re-walks them on each call. That sounds wasteful
|
||||
/// and is not: the walk is linear over at most the reassembly cap, the
|
||||
/// interesting messages sit in the first packets, and re-walking from the
|
||||
/// start is what makes a message that arrives split across three segments
|
||||
/// parse correctly with no incremental parser state to get wrong.
|
||||
///
|
||||
/// Returns events through `sink` and updates `proto` in place.
|
||||
pub fn advance(proto: &mut StreamProtocol, stream: &[u8], sink: &mut impl FnMut(StreamEvent)) {
|
||||
if matches!(proto, StreamProtocol::Undecided) {
|
||||
*proto = sniff(stream);
|
||||
}
|
||||
match proto {
|
||||
StreamProtocol::Undecided | StreamProtocol::Done | StreamProtocol::Ignored => {}
|
||||
StreamProtocol::Tls(_) => advance_tls(proto, stream, sink),
|
||||
StreamProtocol::Http => advance_http(proto, stream, sink),
|
||||
}
|
||||
}
|
||||
|
||||
/// Classifies the first bytes of a stream.
|
||||
fn sniff(stream: &[u8]) -> StreamProtocol {
|
||||
if stream.len() >= 3 {
|
||||
if stream[0] == content_type::HANDSHAKE && stream[1] == 0x03 && stream[2] <= 0x04 {
|
||||
return StreamProtocol::Tls(TlsFlight::default());
|
||||
}
|
||||
if is_sslv2_client_hello(stream) {
|
||||
return StreamProtocol::Tls(TlsFlight::default());
|
||||
}
|
||||
}
|
||||
if stream.len() >= SNIFF_DECISION_LEN {
|
||||
if HTTP_METHOD_PREFIXES.iter().any(|m| stream.starts_with(m)) {
|
||||
return StreamProtocol::Http;
|
||||
}
|
||||
return StreamProtocol::Ignored;
|
||||
}
|
||||
StreamProtocol::Undecided
|
||||
}
|
||||
|
||||
fn advance_http(proto: &mut StreamProtocol, stream: &[u8], sink: &mut impl FnMut(StreamEvent)) {
|
||||
let head_end = stream
|
||||
.windows(HTTP_HEAD_TERMINATOR.len())
|
||||
.position(|w| w == HTTP_HEAD_TERMINATOR);
|
||||
let Some(head_end) = head_end else {
|
||||
if stream.len() > HTTP_HEAD_CAP {
|
||||
*proto = StreamProtocol::Ignored;
|
||||
}
|
||||
return;
|
||||
};
|
||||
|
||||
let head = &stream[..head_end + HTTP_HEAD_TERMINATOR.len()];
|
||||
if let Some(request) = parse_http_request(head) {
|
||||
let host = request
|
||||
.headers
|
||||
.iter()
|
||||
.find(|(name, _)| name.eq_ignore_ascii_case("host"))
|
||||
.map(|(_, value)| value.clone());
|
||||
sink(StreamEvent::HttpRequest {
|
||||
ja4h: ja4h(&request),
|
||||
method: request.method.clone(),
|
||||
host,
|
||||
});
|
||||
*proto = StreamProtocol::Done;
|
||||
} else {
|
||||
*proto = StreamProtocol::Ignored;
|
||||
}
|
||||
}
|
||||
|
||||
fn advance_tls(proto: &mut StreamProtocol, stream: &[u8], sink: &mut impl FnMut(StreamEvent)) {
|
||||
if is_sslv2_client_hello(stream) {
|
||||
advance_sslv2(proto, stream, sink);
|
||||
return;
|
||||
}
|
||||
|
||||
let Some(flight_bytes) = collect_flight(stream) else {
|
||||
*proto = StreamProtocol::Ignored;
|
||||
return;
|
||||
};
|
||||
|
||||
let StreamProtocol::Tls(flight) = proto else {
|
||||
return;
|
||||
};
|
||||
let mut done = walk_messages(flight, flight_bytes.handshake.as_ref(), sink);
|
||||
if flight_bytes.flight_closed && !done {
|
||||
done = true;
|
||||
}
|
||||
if done {
|
||||
*proto = StreamProtocol::Done;
|
||||
}
|
||||
}
|
||||
|
||||
fn advance_sslv2(proto: &mut StreamProtocol, stream: &[u8], sink: &mut impl FnMut(StreamEvent)) {
|
||||
match parse_sslv2_client_hello(stream) {
|
||||
Ok(hello) => {
|
||||
sink(StreamEvent::ClientHello {
|
||||
ja3: ja3(&hello),
|
||||
ja3_raw: ja3_string(&hello),
|
||||
ja4: ja4(&hello, Transport::Tcp),
|
||||
sni: None,
|
||||
alpn: None,
|
||||
});
|
||||
*proto = StreamProtocol::Done;
|
||||
}
|
||||
Err(crate::error::ParseError::Truncated { .. }) => {}
|
||||
Err(_) => *proto = StreamProtocol::Ignored,
|
||||
}
|
||||
}
|
||||
|
||||
struct FlightBytes<'stream> {
|
||||
handshake: Cow<'stream, [u8]>,
|
||||
/// True when a non handshake record followed the handshake records, which
|
||||
/// in cleartext TLS means the readable part of the flight is over.
|
||||
flight_closed: bool,
|
||||
}
|
||||
|
||||
/// Collects the payloads of the leading complete handshake records.
|
||||
///
|
||||
/// Unlike the strict reassembled flight walker in the parse module, this
|
||||
/// tolerates a trailing partial record, because the stream is still growing.
|
||||
/// Returns `None` when the bytes stop looking like TLS record framing at all.
|
||||
fn collect_flight(stream: &[u8]) -> Option<FlightBytes<'_>> {
|
||||
let mut segments: Vec<&[u8]> = Vec::new();
|
||||
let mut r = Reader::new(stream);
|
||||
let mut flight_closed = false;
|
||||
|
||||
while r.remaining() >= 5 {
|
||||
let Ok(ctype) = r.u8() else { break };
|
||||
let Ok(record_version) = r.u16() else { break };
|
||||
if !matches!(
|
||||
ctype,
|
||||
content_type::CHANGE_CIPHER_SPEC
|
||||
| content_type::ALERT
|
||||
| content_type::HANDSHAKE
|
||||
| content_type::APPLICATION_DATA
|
||||
) {
|
||||
return None;
|
||||
}
|
||||
if (record_version & 0xff00) != 0x0300 || (record_version & 0x00ff) > 0x04 {
|
||||
return None;
|
||||
}
|
||||
let Ok(declared) = r.u16() else { break };
|
||||
if declared as usize > MAX_TLS_RECORD_LEN {
|
||||
return None;
|
||||
}
|
||||
let Ok(payload) = r.take(declared as usize) else {
|
||||
break;
|
||||
};
|
||||
if ctype == content_type::HANDSHAKE {
|
||||
segments.push(payload);
|
||||
} else if !segments.is_empty() {
|
||||
flight_closed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let handshake = match segments.as_slice() {
|
||||
[] => Cow::Borrowed(&[][..]),
|
||||
[only] => Cow::Borrowed(*only),
|
||||
many => {
|
||||
let mut joined = Vec::with_capacity(many.iter().map(|s| s.len()).sum());
|
||||
for seg in many {
|
||||
joined.extend_from_slice(seg);
|
||||
}
|
||||
Cow::Owned(joined)
|
||||
}
|
||||
};
|
||||
Some(FlightBytes {
|
||||
handshake,
|
||||
flight_closed,
|
||||
})
|
||||
}
|
||||
|
||||
/// Walks the complete handshake messages in the flight, emitting fingerprints
|
||||
/// for the ones that carry them. Returns true when this direction has yielded
|
||||
/// everything it ever will.
|
||||
fn walk_messages(
|
||||
flight: &mut TlsFlight,
|
||||
handshake: &[u8],
|
||||
sink: &mut impl FnMut(StreamEvent),
|
||||
) -> bool {
|
||||
let mut r = Reader::new(handshake);
|
||||
while r.remaining() >= 4 {
|
||||
let Ok(msg_type) = r.u8() else { break };
|
||||
let Ok(len) = r.u24() else { break };
|
||||
let Ok(body) = r.take(len as usize) else {
|
||||
break;
|
||||
};
|
||||
flight.saw_any_message = true;
|
||||
|
||||
match msg_type {
|
||||
handshake_type::CLIENT_HELLO => {
|
||||
if let Ok(hello) = parse_client_hello(body) {
|
||||
sink(StreamEvent::ClientHello {
|
||||
ja3: ja3(&hello),
|
||||
ja3_raw: ja3_string(&hello),
|
||||
ja4: ja4(&hello, Transport::Tcp),
|
||||
sni: hello.server_name().map(str::to_owned),
|
||||
alpn: hello
|
||||
.alpn_protocols()
|
||||
.first()
|
||||
.map(|p| String::from_utf8_lossy(p).into_owned()),
|
||||
});
|
||||
}
|
||||
return true;
|
||||
}
|
||||
handshake_type::SERVER_HELLO => {
|
||||
if !flight.emitted_server_hello {
|
||||
if let Ok(hello) = parse_server_hello(body) {
|
||||
flight.emitted_server_hello = true;
|
||||
let negotiated_tls13 = hello.selected_version() == version::TLS_1_3;
|
||||
sink(StreamEvent::ServerHello {
|
||||
ja3s: ja3s(&hello),
|
||||
ja3s_raw: ja3s_string(&hello),
|
||||
ja4s: ja4s(&hello, Transport::Tcp),
|
||||
});
|
||||
if negotiated_tls13 {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
handshake_type::CERTIFICATE => {
|
||||
if let Ok(certs) = certificate_der_list(body) {
|
||||
for cert in certs {
|
||||
if let Ok(fingerprint) = ja4x(cert) {
|
||||
sink(StreamEvent::Certificate { ja4x: fingerprint });
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{StreamProtocol, advance, sniff};
|
||||
use crate::pipeline::event::StreamEvent;
|
||||
|
||||
fn record(payload: &[u8]) -> Vec<u8> {
|
||||
let mut v = vec![0x16, 0x03, 0x01];
|
||||
v.extend_from_slice(&u16::try_from(payload.len()).unwrap().to_be_bytes());
|
||||
v.extend_from_slice(payload);
|
||||
v
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sniffs_tls_http_and_garbage() {
|
||||
assert!(matches!(
|
||||
sniff(&[0x16, 0x03, 0x01, 0x00, 0x05]),
|
||||
StreamProtocol::Tls(_)
|
||||
));
|
||||
assert!(matches!(sniff(b"GET / HTTP/1.1\r\n"), StreamProtocol::Http));
|
||||
assert!(matches!(
|
||||
sniff(b"SSH-2.0-OpenSSH_9.7"),
|
||||
StreamProtocol::Ignored
|
||||
));
|
||||
assert!(matches!(sniff(b"GE"), StreamProtocol::Undecided));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn http_request_yields_ja4h_once_head_completes() {
|
||||
let mut proto = StreamProtocol::Undecided;
|
||||
let mut events = Vec::new();
|
||||
|
||||
let partial = b"GET / HTTP/1.1\r\nHost: example.com\r\n";
|
||||
advance(&mut proto, partial, &mut |e| events.push(e));
|
||||
assert!(events.is_empty());
|
||||
assert!(matches!(proto, StreamProtocol::Http));
|
||||
|
||||
let full = b"GET / HTTP/1.1\r\nHost: example.com\r\nAccept: */*\r\n\r\n";
|
||||
advance(&mut proto, full, &mut |e| events.push(e));
|
||||
assert_eq!(events.len(), 1);
|
||||
assert!(matches!(
|
||||
&events[0],
|
||||
StreamEvent::HttpRequest { method, host: Some(h), .. }
|
||||
if method == "GET" && h == "example.com"
|
||||
));
|
||||
assert!(proto.finished());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_tls_record_waits_for_more_bytes() {
|
||||
let mut proto = StreamProtocol::Undecided;
|
||||
let mut events = Vec::new();
|
||||
|
||||
let full = record(&[0x01, 0x00, 0x00, 0x02, 0xaa, 0xbb]);
|
||||
advance(&mut proto, &full[..7], &mut |e| events.push(e));
|
||||
assert!(events.is_empty());
|
||||
assert!(matches!(proto, StreamProtocol::Tls(_)));
|
||||
assert!(!proto.finished());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nonsense_record_length_poisons_the_stream() {
|
||||
let mut proto = StreamProtocol::Undecided;
|
||||
let mut events = Vec::new();
|
||||
|
||||
let stream = [0x16, 0x03, 0x01, 0xff, 0xff, 0x00, 0x00, 0x00];
|
||||
advance(&mut proto, &stream, &mut |e| events.push(e));
|
||||
assert!(matches!(proto, StreamProtocol::Ignored));
|
||||
assert!(events.is_empty());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,91 @@
|
|||
// ©AngelaMos | 2026
|
||||
// registry.rs
|
||||
|
||||
//! Named constants for the slices of the TLS and DTLS registries that the
|
||||
//! fingerprint algorithms reference by number.
|
||||
//!
|
||||
//! Wire values are kept as raw `u16` everywhere in this crate rather than being
|
||||
//! decoded into a closed enum. A fingerprinting engine must preserve cipher and
|
||||
//! extension values it has never seen, because a future RFC value or a vendor
|
||||
//! specific extension is exactly the kind of detail that makes one client
|
||||
//! distinguishable from another. Decoding into a closed enum would collapse all
|
||||
//! unknown values into a single bucket and corrupt the fingerprint.
|
||||
|
||||
/// TLS and DTLS record content types (the first byte of a record).
|
||||
pub mod content_type {
|
||||
pub const CHANGE_CIPHER_SPEC: u8 = 20;
|
||||
pub const ALERT: u8 = 21;
|
||||
pub const HANDSHAKE: u8 = 22;
|
||||
pub const APPLICATION_DATA: u8 = 23;
|
||||
}
|
||||
|
||||
/// Handshake message types (the first byte of a handshake message body).
|
||||
pub mod handshake_type {
|
||||
pub const CLIENT_HELLO: u8 = 1;
|
||||
pub const SERVER_HELLO: u8 = 2;
|
||||
pub const CERTIFICATE: u8 = 11;
|
||||
}
|
||||
|
||||
/// Extension type numbers that the fingerprint algorithms treat specially.
|
||||
pub mod extension {
|
||||
pub const SERVER_NAME: u16 = 0x0000;
|
||||
pub const SUPPORTED_GROUPS: u16 = 0x000a;
|
||||
pub const EC_POINT_FORMATS: u16 = 0x000b;
|
||||
pub const SIGNATURE_ALGORITHMS: u16 = 0x000d;
|
||||
pub const ALPN: u16 = 0x0010;
|
||||
pub const SUPPORTED_VERSIONS: u16 = 0x002b;
|
||||
}
|
||||
|
||||
/// Legacy and negotiated protocol version words.
|
||||
pub mod version {
|
||||
pub const SSL_2_0: u16 = 0x0002;
|
||||
pub const SSL_3_0: u16 = 0x0300;
|
||||
pub const TLS_1_0: u16 = 0x0301;
|
||||
pub const TLS_1_1: u16 = 0x0302;
|
||||
pub const TLS_1_2: u16 = 0x0303;
|
||||
pub const TLS_1_3: u16 = 0x0304;
|
||||
pub const DTLS_1_0: u16 = 0xfeff;
|
||||
pub const DTLS_1_2: u16 = 0xfefd;
|
||||
pub const DTLS_1_3: u16 = 0xfefc;
|
||||
}
|
||||
|
||||
/// Maps a protocol version word to the two character JA4 version code.
|
||||
///
|
||||
/// The mapping is taken verbatim from the FoxIO JA4 specification. Unknown
|
||||
/// words collapse to `00`, which is the specified fallback. DTLS words are
|
||||
/// included even though the Python reference omits them, because the published
|
||||
/// specification lists them and the Wireshark and Rust references honor them.
|
||||
#[must_use]
|
||||
pub fn ja4_version_code(word: u16) -> &'static str {
|
||||
match word {
|
||||
version::TLS_1_3 => "13",
|
||||
version::TLS_1_2 => "12",
|
||||
version::TLS_1_1 => "11",
|
||||
version::TLS_1_0 => "10",
|
||||
version::SSL_3_0 => "s3",
|
||||
version::SSL_2_0 => "s2",
|
||||
version::DTLS_1_0 => "d1",
|
||||
version::DTLS_1_2 => "d2",
|
||||
version::DTLS_1_3 => "d3",
|
||||
_ => "00",
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::ja4_version_code;
|
||||
use super::version;
|
||||
|
||||
#[test]
|
||||
fn known_versions_map() {
|
||||
assert_eq!(ja4_version_code(version::TLS_1_3), "13");
|
||||
assert_eq!(ja4_version_code(version::TLS_1_2), "12");
|
||||
assert_eq!(ja4_version_code(version::SSL_3_0), "s3");
|
||||
assert_eq!(ja4_version_code(version::DTLS_1_2), "d2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_version_falls_back() {
|
||||
assert_eq!(ja4_version_code(0x7f1d), "00");
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,161 @@
|
|||
// ©AngelaMos | 2026
|
||||
// mod.rs
|
||||
|
||||
//! Builders that assemble TLS handshake bytes for tests.
|
||||
//!
|
||||
//! Hand assembling wire bytes keeps the fingerprint tests honest. Rather than
|
||||
//! trusting a higher level library to produce a ClientHello, the tests state the
|
||||
//! exact cipher list, extension list, and extension bodies, then assert that the
|
||||
//! parser and the fingerprint algorithms read them back the way the
|
||||
//! specifications require.
|
||||
//!
|
||||
//! Each integration test binary links this module independently, so a builder
|
||||
//! method that one binary does not call reads as dead code there even though
|
||||
//! another binary uses it. The allow keeps that shared infrastructure honest
|
||||
//! without scattering per method annotations.
|
||||
#![allow(dead_code)]
|
||||
|
||||
/// Assembles a ClientHello handshake message body.
|
||||
#[derive(Default)]
|
||||
pub struct ClientHelloBuilder {
|
||||
legacy_version: u16,
|
||||
cipher_suites: Vec<u16>,
|
||||
extensions: Vec<(u16, Vec<u8>)>,
|
||||
}
|
||||
|
||||
impl ClientHelloBuilder {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
legacy_version: 0x0303,
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn legacy_version(mut self, v: u16) -> Self {
|
||||
self.legacy_version = v;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn ciphers(mut self, suites: &[u16]) -> Self {
|
||||
self.cipher_suites = suites.to_vec();
|
||||
self
|
||||
}
|
||||
|
||||
pub fn extension(mut self, ext_type: u16, data: Vec<u8>) -> Self {
|
||||
self.extensions.push((ext_type, data));
|
||||
self
|
||||
}
|
||||
|
||||
/// Adds a server name indication extension for the given host.
|
||||
pub fn sni(self, host: &str) -> Self {
|
||||
let host = host.as_bytes();
|
||||
let mut entry = vec![0u8];
|
||||
push_u16_vec(&mut entry, host);
|
||||
let mut data = Vec::new();
|
||||
push_u16_vec(&mut data, &entry);
|
||||
self.extension(0x0000, data)
|
||||
}
|
||||
|
||||
/// Adds a supported groups extension.
|
||||
pub fn supported_groups(self, groups: &[u16]) -> Self {
|
||||
let mut list = Vec::new();
|
||||
for g in groups {
|
||||
list.extend_from_slice(&g.to_be_bytes());
|
||||
}
|
||||
let mut data = Vec::new();
|
||||
push_u16_vec(&mut data, &list);
|
||||
self.extension(0x000a, data)
|
||||
}
|
||||
|
||||
/// Adds an elliptic curve point formats extension.
|
||||
pub fn ec_point_formats(self, formats: &[u8]) -> Self {
|
||||
let mut data = Vec::new();
|
||||
push_u8_vec(&mut data, formats);
|
||||
self.extension(0x000b, data)
|
||||
}
|
||||
|
||||
/// Adds a signature algorithms extension.
|
||||
pub fn signature_algorithms(self, algs: &[u16]) -> Self {
|
||||
let mut list = Vec::new();
|
||||
for a in algs {
|
||||
list.extend_from_slice(&a.to_be_bytes());
|
||||
}
|
||||
let mut data = Vec::new();
|
||||
push_u16_vec(&mut data, &list);
|
||||
self.extension(0x000d, data)
|
||||
}
|
||||
|
||||
/// Adds an ALPN extension advertising the given protocols.
|
||||
pub fn alpn(self, protos: &[&[u8]]) -> Self {
|
||||
let mut list = Vec::new();
|
||||
for p in protos {
|
||||
push_u8_vec(&mut list, p);
|
||||
}
|
||||
let mut data = Vec::new();
|
||||
push_u16_vec(&mut data, &list);
|
||||
self.extension(0x0010, data)
|
||||
}
|
||||
|
||||
/// Adds a supported versions extension.
|
||||
pub fn supported_versions(self, versions: &[u16]) -> Self {
|
||||
let mut list = Vec::new();
|
||||
for v in versions {
|
||||
list.extend_from_slice(&v.to_be_bytes());
|
||||
}
|
||||
let mut data = Vec::new();
|
||||
push_u8_vec(&mut data, &list);
|
||||
self.extension(0x002b, data)
|
||||
}
|
||||
|
||||
/// Returns the handshake message body, the input to `parse_client_hello`.
|
||||
pub fn build_body(&self) -> Vec<u8> {
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&self.legacy_version.to_be_bytes());
|
||||
body.extend_from_slice(&[0u8; 32]);
|
||||
body.push(0);
|
||||
|
||||
let mut ciphers = Vec::new();
|
||||
for c in &self.cipher_suites {
|
||||
ciphers.extend_from_slice(&c.to_be_bytes());
|
||||
}
|
||||
push_u16_vec(&mut body, &ciphers);
|
||||
|
||||
push_u8_vec(&mut body, &[0]);
|
||||
|
||||
let mut exts = Vec::new();
|
||||
for (ext_type, data) in &self.extensions {
|
||||
exts.extend_from_slice(&ext_type.to_be_bytes());
|
||||
push_u16_vec(&mut exts, data);
|
||||
}
|
||||
push_u16_vec(&mut body, &exts);
|
||||
body
|
||||
}
|
||||
|
||||
/// Wraps the handshake body in a handshake header and a TLS record so the
|
||||
/// result is a complete record stream.
|
||||
pub fn build_record(&self) -> Vec<u8> {
|
||||
let body = self.build_body();
|
||||
let mut msg = vec![1u8];
|
||||
push_u24_vec(&mut msg, &body);
|
||||
|
||||
let mut record = vec![22u8, 0x03, 0x01];
|
||||
push_u16_vec(&mut record, &msg);
|
||||
record
|
||||
}
|
||||
}
|
||||
|
||||
fn push_u8_vec(out: &mut Vec<u8>, data: &[u8]) {
|
||||
out.push(u8::try_from(data.len()).unwrap());
|
||||
out.extend_from_slice(data);
|
||||
}
|
||||
|
||||
fn push_u16_vec(out: &mut Vec<u8>, data: &[u8]) {
|
||||
out.extend_from_slice(&u16::try_from(data.len()).unwrap().to_be_bytes());
|
||||
out.extend_from_slice(data);
|
||||
}
|
||||
|
||||
fn push_u24_vec(out: &mut Vec<u8>, data: &[u8]) {
|
||||
let len = u32::try_from(data.len()).unwrap();
|
||||
out.extend_from_slice(&len.to_be_bytes()[1..]);
|
||||
out.extend_from_slice(data);
|
||||
}
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
// ©AngelaMos | 2026
|
||||
// ja3.rs
|
||||
|
||||
mod common;
|
||||
|
||||
use common::ClientHelloBuilder;
|
||||
use tlsfp_core::ja3::{ja3, ja3_string};
|
||||
use tlsfp_core::parse::parse_client_hello;
|
||||
|
||||
/// Rebuilds the exact ClientHello that produces the first published Salesforce
|
||||
/// JA3 vector, then checks both the pre hash string and the digest end to end.
|
||||
/// The SNI body content is arbitrary because JA3 records only that extension
|
||||
/// type zero was present, not its value.
|
||||
#[test]
|
||||
fn end_to_end_matches_salesforce_vector_one() {
|
||||
let body = ClientHelloBuilder::new()
|
||||
.legacy_version(0x0301)
|
||||
.ciphers(&[47, 53, 5, 10, 49161, 49162, 49171, 49172, 50, 56, 19, 4])
|
||||
.extension(0x0000, vec![0x00, 0x00])
|
||||
.supported_groups(&[23, 24, 25])
|
||||
.ec_point_formats(&[0])
|
||||
.build_body();
|
||||
|
||||
let ch = parse_client_hello(&body).unwrap();
|
||||
assert_eq!(
|
||||
ja3_string(&ch),
|
||||
"769,47-53-5-10-49161-49162-49171-49172-50-56-19-4,0-10-11,23-24-25,0"
|
||||
);
|
||||
assert_eq!(ja3(&ch).to_string(), "ada70206e40642a3e4461f35503241d5");
|
||||
}
|
||||
|
||||
/// A ClientHello with GREASE in ciphers, extensions, and supported groups must
|
||||
/// produce the same JA3 as the same hello without GREASE, because GREASE is
|
||||
/// stripped from every list field before hashing.
|
||||
#[test]
|
||||
fn grease_does_not_change_the_fingerprint() {
|
||||
let clean = ClientHelloBuilder::new()
|
||||
.legacy_version(0x0303)
|
||||
.ciphers(&[0x1301, 0x1302])
|
||||
.supported_groups(&[0x001d, 0x0017])
|
||||
.ec_point_formats(&[0])
|
||||
.build_body();
|
||||
|
||||
let greasy = ClientHelloBuilder::new()
|
||||
.legacy_version(0x0303)
|
||||
.ciphers(&[0x0a0a, 0x1301, 0x1302])
|
||||
.extension(0x1a1a, vec![])
|
||||
.supported_groups(&[0x2a2a, 0x001d, 0x0017])
|
||||
.ec_point_formats(&[0])
|
||||
.build_body();
|
||||
|
||||
let clean = parse_client_hello(&clean).unwrap();
|
||||
let greasy = parse_client_hello(&greasy).unwrap();
|
||||
assert_eq!(ja3(&clean), ja3(&greasy));
|
||||
}
|
||||
|
|
@ -0,0 +1,88 @@
|
|||
// ©AngelaMos | 2026
|
||||
// ja4.rs
|
||||
|
||||
mod common;
|
||||
|
||||
use common::ClientHelloBuilder;
|
||||
use tlsfp_core::ja4::{Transport, ja4};
|
||||
use tlsfp_core::parse::parse_client_hello;
|
||||
|
||||
const FOXIO_HASH: &str = "t13d1516h2_8daaf6152771_e5627efa2ab1";
|
||||
const FOXIO_RAW: &str = "t13d1516h2_002f,0035,009c,009d,1301,1302,1303,c013,c014,c02b,c02c,c02f,c030,cca8,cca9_0005,000a,000b,000d,0012,0015,0017,001b,0023,002b,002d,0033,4469,ff01_0403,0804,0401,0503,0805,0501,0806,0601";
|
||||
|
||||
const CIPHERS_ORIGINAL_ORDER: [u16; 15] = [
|
||||
0x1301, 0x1302, 0x1303, 0xc02b, 0xc02f, 0xc02c, 0xc030, 0xcca9, 0xcca8, 0xc013, 0xc014, 0x009c,
|
||||
0x009d, 0x002f, 0x0035,
|
||||
];
|
||||
|
||||
const OPAQUE_EXTENSIONS: [u16; 10] = [
|
||||
0x0005, 0x0012, 0x0015, 0x0017, 0x001b, 0x0023, 0x002d, 0x0033, 0x4469, 0xff01,
|
||||
];
|
||||
|
||||
const SIG_ALGS: [u16; 8] = [
|
||||
0x0403, 0x0804, 0x0401, 0x0503, 0x0805, 0x0501, 0x0806, 0x0601,
|
||||
];
|
||||
|
||||
/// Assembles the handshake body for the canonical FoxIO example.
|
||||
///
|
||||
/// The cipher suites are supplied out of order with an injected GREASE value, to
|
||||
/// prove that sorting and GREASE removal happen before hashing. When `reversed`
|
||||
/// is set the opaque extensions are added in reverse, which must not change the
|
||||
/// fingerprint because JA4 sorts the extension list.
|
||||
fn foxio_body(reversed: bool) -> Vec<u8> {
|
||||
let mut ciphers = vec![0x0a0a];
|
||||
ciphers.extend_from_slice(&CIPHERS_ORIGINAL_ORDER);
|
||||
|
||||
let mut builder = ClientHelloBuilder::new()
|
||||
.ciphers(&ciphers)
|
||||
.sni("example.com")
|
||||
.supported_groups(&[0x001d, 0x0017])
|
||||
.ec_point_formats(&[0x00])
|
||||
.signature_algorithms(&SIG_ALGS)
|
||||
.alpn(&[b"h2"])
|
||||
.supported_versions(&[0x2a2a, 0x0304, 0x0303]);
|
||||
|
||||
if reversed {
|
||||
for ext in OPAQUE_EXTENSIONS.iter().rev() {
|
||||
builder = builder.extension(*ext, vec![]);
|
||||
}
|
||||
} else {
|
||||
for ext in OPAQUE_EXTENSIONS {
|
||||
builder = builder.extension(ext, vec![]);
|
||||
}
|
||||
}
|
||||
|
||||
builder.build_body()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reproduces_foxio_canonical_example() {
|
||||
let body = foxio_body(false);
|
||||
let ch = parse_client_hello(&body).unwrap();
|
||||
let fp = ja4(&ch, Transport::Tcp);
|
||||
assert_eq!(fp.raw, FOXIO_RAW);
|
||||
assert_eq!(fp.hash, FOXIO_HASH);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extension_order_does_not_change_the_hash() {
|
||||
let normal = foxio_body(false);
|
||||
let reversed = foxio_body(true);
|
||||
let normal = parse_client_hello(&normal).unwrap();
|
||||
let reversed = parse_client_hello(&reversed).unwrap();
|
||||
assert_eq!(
|
||||
ja4(&normal, Transport::Tcp).hash,
|
||||
ja4(&reversed, Transport::Tcp).hash
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transport_marker_switches_first_character() {
|
||||
let body = foxio_body(false);
|
||||
let ch = parse_client_hello(&body).unwrap();
|
||||
let tcp = ja4(&ch, Transport::Tcp);
|
||||
let quic = ja4(&ch, Transport::Quic);
|
||||
assert!(tcp.hash.starts_with('t'));
|
||||
assert!(quic.hash.starts_with('q'));
|
||||
assert_eq!(&tcp.hash[1..], &quic.hash[1..]);
|
||||
}
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
// ©AngelaMos | 2026
|
||||
// ja4_suite.rs
|
||||
|
||||
use tlsfp_core::ja4::{Transport, ja4s};
|
||||
use tlsfp_core::ja4h::{ja4h, parse_http_request};
|
||||
use tlsfp_core::parse::parse_server_hello;
|
||||
|
||||
/// Builds the ServerHello behind the FoxIO JA4S example: TLS 1.3 selected via the
|
||||
/// supported versions extension, the cipher 0x1301, and two extensions in the
|
||||
/// order key share then supported versions.
|
||||
fn foxio_server_hello() -> Vec<u8> {
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&0x0303u16.to_be_bytes());
|
||||
body.extend_from_slice(&[0u8; 32]);
|
||||
body.push(0);
|
||||
body.extend_from_slice(&0x1301u16.to_be_bytes());
|
||||
body.push(0);
|
||||
|
||||
let mut exts = Vec::new();
|
||||
exts.extend_from_slice(&0x0033u16.to_be_bytes());
|
||||
exts.extend_from_slice(&0u16.to_be_bytes());
|
||||
exts.extend_from_slice(&0x002bu16.to_be_bytes());
|
||||
exts.extend_from_slice(&2u16.to_be_bytes());
|
||||
exts.extend_from_slice(&0x0304u16.to_be_bytes());
|
||||
|
||||
body.extend_from_slice(&u16::try_from(exts.len()).unwrap().to_be_bytes());
|
||||
body.extend_from_slice(&exts);
|
||||
body
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ja4s_reproduces_foxio_example() {
|
||||
let body = foxio_server_hello();
|
||||
let sh = parse_server_hello(&body).unwrap();
|
||||
let fp = ja4s(&sh, Transport::Tcp);
|
||||
assert_eq!(fp.hash, "t130200_1301_234ea6891581");
|
||||
assert_eq!(fp.raw, "t130200_1301_0033,002b");
|
||||
}
|
||||
|
||||
/// The published JA4H example for a request with four uppercase headers, no
|
||||
/// cookies, no referer, and no accept language. The header hash is the SHA256 of
|
||||
/// the comma joined header names in wire order.
|
||||
#[test]
|
||||
fn ja4h_reproduces_published_example() {
|
||||
let raw = b"GET * HTTP/1.1\r\nHOST: a\r\nMAN: b\r\nMX: c\r\nST: d\r\n\r\n";
|
||||
let req = parse_http_request(raw).unwrap();
|
||||
let fp = ja4h(&req);
|
||||
assert_eq!(
|
||||
fp.hash,
|
||||
"ge11nn040000_a3c882e23515_000000000000_000000000000"
|
||||
);
|
||||
}
|
||||
|
||||
/// Cookies and a referer must flip their flags and stop being counted as plain
|
||||
/// headers, while the accept language must populate the language characters.
|
||||
#[test]
|
||||
fn ja4h_flags_cookies_referer_and_language() {
|
||||
let raw = b"GET / HTTP/1.1\r\nHost: x\r\nAccept-Language: en-US,en;q=0.9\r\nReferer: http://x\r\nCookie: a=1; b=2\r\n\r\n";
|
||||
let req = parse_http_request(raw).unwrap();
|
||||
let fp = ja4h(&req);
|
||||
let prefix = fp.hash.split('_').next().unwrap();
|
||||
assert_eq!(prefix, "ge11cr02enus");
|
||||
assert!(!fp.hash.contains("_000000000000_000000000000"));
|
||||
}
|
||||
|
|
@ -0,0 +1,105 @@
|
|||
// ©AngelaMos | 2026
|
||||
// parse.rs
|
||||
|
||||
mod common;
|
||||
|
||||
use common::ClientHelloBuilder;
|
||||
use proptest::prelude::*;
|
||||
use tlsfp_core::parse::{
|
||||
first_handshake_message, handshake_bytes, parse_client_hello, parse_server_hello,
|
||||
};
|
||||
use tlsfp_core::registry::handshake_type;
|
||||
|
||||
fn sample() -> ClientHelloBuilder {
|
||||
ClientHelloBuilder::new()
|
||||
.ciphers(&[0x0a0a, 0x1301, 0x1302, 0x1303, 0xc02b])
|
||||
.sni("example.com")
|
||||
.supported_groups(&[0x1a1a, 0x001d, 0x0017])
|
||||
.ec_point_formats(&[0x00])
|
||||
.signature_algorithms(&[0x0403, 0x0804, 0x0401])
|
||||
.alpn(&[b"h2", b"http/1.1"])
|
||||
.supported_versions(&[0x2a2a, 0x0304, 0x0303])
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_every_field_a_fingerprint_reads() {
|
||||
let body = sample().build_body();
|
||||
let ch = parse_client_hello(&body).unwrap();
|
||||
|
||||
assert_eq!(ch.legacy_version, 0x0303);
|
||||
assert_eq!(
|
||||
ch.cipher_suites.as_slice(),
|
||||
&[0x0a0a, 0x1301, 0x1302, 0x1303, 0xc02b]
|
||||
);
|
||||
assert_eq!(ch.extensions.len(), 6);
|
||||
assert_eq!(ch.server_name(), Some("example.com"));
|
||||
assert_eq!(ch.supported_groups().as_slice(), &[0x1a1a, 0x001d, 0x0017]);
|
||||
assert_eq!(ch.ec_point_formats().as_slice(), &[0x00]);
|
||||
assert_eq!(
|
||||
ch.signature_algorithms().as_slice(),
|
||||
&[0x0403, 0x0804, 0x0401]
|
||||
);
|
||||
assert_eq!(ch.alpn_protocols().first().copied(), Some(b"h2".as_slice()));
|
||||
assert_eq!(
|
||||
ch.supported_versions().as_slice(),
|
||||
&[0x2a2a, 0x0304, 0x0303]
|
||||
);
|
||||
assert!(!ch.is_sslv2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_client_hello_through_the_record_layer() {
|
||||
let stream = sample().build_record();
|
||||
let hs = handshake_bytes(&stream).unwrap();
|
||||
let body = first_handshake_message(&hs, handshake_type::CLIENT_HELLO).unwrap();
|
||||
let ch = parse_client_hello(body).unwrap();
|
||||
assert_eq!(ch.cipher_suites.len(), 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn server_hello_parses_single_cipher() {
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&0x0303u16.to_be_bytes());
|
||||
body.extend_from_slice(&[0u8; 32]);
|
||||
body.push(0);
|
||||
body.extend_from_slice(&0x1301u16.to_be_bytes());
|
||||
body.push(0);
|
||||
body.extend_from_slice(&0u16.to_be_bytes());
|
||||
|
||||
let sh = parse_server_hello(&body).unwrap();
|
||||
assert_eq!(sh.cipher_suite, 0x1301);
|
||||
assert_eq!(sh.legacy_version, 0x0303);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn client_hello_without_extensions_parses() {
|
||||
let body = ClientHelloBuilder::new()
|
||||
.ciphers(&[0x002f, 0x0035])
|
||||
.build_body();
|
||||
let body = &body[..body.len() - 2];
|
||||
let ch = parse_client_hello(body).unwrap();
|
||||
assert_eq!(ch.cipher_suites.as_slice(), &[0x002f, 0x0035]);
|
||||
assert!(ch.extensions.is_empty());
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#[test]
|
||||
fn parser_never_panics_on_arbitrary_bytes(bytes in proptest::collection::vec(any::<u8>(), 0..2048)) {
|
||||
let _ = parse_client_hello(&bytes);
|
||||
let _ = parse_server_hello(&bytes);
|
||||
if let Ok(hs) = handshake_bytes(&bytes) {
|
||||
let _ = first_handshake_message(&hs, handshake_type::CLIENT_HELLO);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn well_formed_prefix_with_random_extension_tail_is_stable(
|
||||
tail in proptest::collection::vec(any::<u8>(), 0..256)
|
||||
) {
|
||||
let mut ch = ClientHelloBuilder::new().ciphers(&[0x1301, 0x1302]);
|
||||
ch = ch.extension(0xabcd, tail);
|
||||
let body = ch.build_body();
|
||||
let parsed = parse_client_hello(&body).unwrap();
|
||||
prop_assert_eq!(parsed.cipher_suites.as_slice(), &[0x1301, 0x1302]);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,196 @@
|
|||
// ©AngelaMos | 2026
|
||||
// pipeline.rs
|
||||
|
||||
//! Full pipeline known answer tests over the vendored FoxIO captures.
|
||||
//!
|
||||
//! These read a real capture file off disk, run it through the entire stack
|
||||
//! the binary uses, and assert the fingerprints against the values FoxIO
|
||||
//! published for the same files. A unit test proves an algorithm in isolation;
|
||||
//! these prove that the file reader, the link layer decoder, the TCP
|
||||
//! reassembler, the protocol sniffer, and the algorithms together still land
|
||||
//! on the published answer. Every hash asserted here was independently
|
||||
//! recomputed from its raw pre hash string with a shell tool before being
|
||||
//! pinned, so a regression in any stage shows up as a mismatch rather than a
|
||||
//! silently wrong but self consistent number.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use tlsfp_core::pipeline::event::StreamEvent;
|
||||
use tlsfp_core::{FingerprintEvent, PcapFileSource, Pipeline, PipelineConfig};
|
||||
|
||||
fn pcap_path(name: &str) -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("../../testdata/pcap")
|
||||
.join(name)
|
||||
}
|
||||
|
||||
/// Runs one capture all the way through the pipeline and returns its events.
|
||||
fn fingerprint(name: &str) -> Vec<FingerprintEvent> {
|
||||
let mut source =
|
||||
PcapFileSource::open(pcap_path(name)).unwrap_or_else(|e| panic!("opening {name}: {e}"));
|
||||
let mut pipeline = Pipeline::new(PipelineConfig::default());
|
||||
let mut events = Vec::new();
|
||||
pipeline
|
||||
.run(&mut source, |event| events.push(event))
|
||||
.unwrap_or_else(|e| panic!("running {name}: {e}"));
|
||||
events
|
||||
}
|
||||
|
||||
fn client_hellos(events: &[FingerprintEvent]) -> Vec<(&str, &str)> {
|
||||
events
|
||||
.iter()
|
||||
.filter_map(|e| match &e.event {
|
||||
StreamEvent::ClientHello { ja4, .. } => Some((ja4.hash.as_str(), ja4.raw.as_str())),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn ja4s_hashes(events: &[FingerprintEvent]) -> Vec<(&str, &str)> {
|
||||
events
|
||||
.iter()
|
||||
.filter_map(|e| match &e.event {
|
||||
StreamEvent::ServerHello { ja4s, .. } => Some((ja4s.hash.as_str(), ja4s.raw.as_str())),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn ja4x_hashes(events: &[FingerprintEvent]) -> Vec<&str> {
|
||||
events
|
||||
.iter()
|
||||
.filter_map(|e| match &e.event {
|
||||
StreamEvent::Certificate { ja4x } => Some(ja4x.as_str()),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn ja4h_hashes(events: &[FingerprintEvent]) -> Vec<&str> {
|
||||
events
|
||||
.iter()
|
||||
.filter_map(|e| match &e.event {
|
||||
StreamEvent::HttpRequest { ja4h, .. } => Some(ja4h.hash.as_str()),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tls_alpn_h2_reproduces_published_ja4_and_ja4x() {
|
||||
let events = fingerprint("tls-alpn-h2.pcap");
|
||||
|
||||
let hellos = client_hellos(&events);
|
||||
assert_eq!(hellos.len(), 1);
|
||||
assert_eq!(hellos[0].0, "t12d4605h2_85626a9a5f7f_aaf95bb78ec9");
|
||||
|
||||
let certs = ja4x_hashes(&events);
|
||||
assert_eq!(
|
||||
certs.first().copied(),
|
||||
Some("7d5dbb3783b4_ba7ce0880c07_7bf9a7bf7029")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chrome_cloudflare_tcp_handshake_reproduces_published_ja4() {
|
||||
let events = fingerprint("chrome-cloudflare-quic-with-secrets.pcapng");
|
||||
|
||||
let hellos = client_hellos(&events);
|
||||
assert!(
|
||||
hellos
|
||||
.iter()
|
||||
.any(|(hash, _)| *hash == "t13d1516h2_8daaf6152771_e5627efa2ab1"),
|
||||
"expected the published chrome JA4, saw {hellos:?}"
|
||||
);
|
||||
|
||||
let servers = ja4s_hashes(&events);
|
||||
assert!(
|
||||
servers
|
||||
.iter()
|
||||
.any(|(hash, _)| *hash == "t130200_1301_234ea6891581"),
|
||||
"expected the published cloudflare JA4S, saw {servers:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tls_handshake_reproduces_published_ja4s_raw_and_hash() {
|
||||
let events = fingerprint("tls-handshake.pcapng");
|
||||
|
||||
// The capture holds many flows to many Google and Cloudflare endpoints, so
|
||||
// several distinct JA4S values appear. The published vector is the one for
|
||||
// the first flow, and it must carry both its published hash and raw form.
|
||||
let servers = ja4s_hashes(&events);
|
||||
assert!(
|
||||
servers
|
||||
.iter()
|
||||
.any(|(hash, raw)| *hash == "t130200_1301_234ea6891581"
|
||||
&& *raw == "t130200_1301_0033,002b"),
|
||||
"expected the published JA4S with its raw form, saw {servers:?}"
|
||||
);
|
||||
|
||||
let hellos = client_hellos(&events);
|
||||
assert!(hellos.len() >= 5);
|
||||
assert!(
|
||||
hellos
|
||||
.iter()
|
||||
.any(|(hash, _)| *hash == "t13d1516h2_8daaf6152771_e5627efa2ab1")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn http1_with_cookies_reproduces_pinned_ja4h() {
|
||||
let events = fingerprint("http1-with-cookies.pcapng");
|
||||
let hashes = ja4h_hashes(&events);
|
||||
assert_eq!(
|
||||
hashes,
|
||||
vec!["ge11cr04da00_8ddaef5d77af_280f366eaa04_c2fb0fe53442"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tls12_capture_recomputes_to_pinned_ja4() {
|
||||
let events = fingerprint("tls12.pcap");
|
||||
let hellos = client_hellos(&events);
|
||||
assert_eq!(hellos.len(), 1);
|
||||
assert_eq!(hellos[0].0, "t13d1715h2_5b57614c22b0_3d5424432f57");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_ascii_alpn_follows_the_spec_hex_rule() {
|
||||
let events = fingerprint("tls-non-ascii-alpn.pcapng");
|
||||
let hellos = client_hellos(&events);
|
||||
assert_eq!(hellos.len(), 1);
|
||||
let (hash, _) = hellos[0];
|
||||
let alpn_chars = &hash[8..10];
|
||||
assert_eq!(
|
||||
alpn_chars, "bd",
|
||||
"spec rule prints first and last hex chars of the raw ALPN value"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn browsers_x509_extracts_every_certificate_in_chain_order() {
|
||||
let events = fingerprint("browsers-x509.pcapng");
|
||||
let certs = ja4x_hashes(&events);
|
||||
assert!(
|
||||
certs.len() >= 7,
|
||||
"expected several certificates, saw {}",
|
||||
certs.len()
|
||||
);
|
||||
assert!(certs.iter().all(|c| c.matches('_').count() == 2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tunneled_capture_yields_nothing_without_crashing() {
|
||||
let events = fingerprint("gre-erspan-vxlan.pcap");
|
||||
assert!(events.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn evasion_capture_is_processed_without_panicking() {
|
||||
let events = fingerprint("CVE-2018-6794.pcap");
|
||||
assert!(
|
||||
ja4h_hashes(&events).len() >= 2,
|
||||
"the reassembler should still recover the HTTP requests"
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,114 @@
|
|||
// ©AngelaMos | 2026
|
||||
// reassembly.rs
|
||||
|
||||
//! Property tests for the TCP stream reassembler.
|
||||
//!
|
||||
//! The reassembler is the component an adversary reaches first: it consumes
|
||||
//! attacker controlled sequence numbers and segment boundaries on every
|
||||
//! connection. Two properties pin it down. The first is correctness under
|
||||
//! reordering: any stream cut into any segments and delivered in any order
|
||||
//! must reassemble back to the original bytes, because that is the whole job.
|
||||
//! The second is the absence of panics under fully arbitrary input, because a
|
||||
//! passive sensor that can be crashed by a crafted segment is a denial of
|
||||
//! service waiting to happen.
|
||||
|
||||
use proptest::prelude::*;
|
||||
use tlsfp_core::pipeline::flow::{PushOutcome, ReassemblyLimits, StreamReassembler};
|
||||
|
||||
fn generous_limits() -> ReassemblyLimits {
|
||||
ReassemblyLimits {
|
||||
max_assembled_bytes: 1 << 20,
|
||||
max_pending_bytes: 1 << 20,
|
||||
max_pending_segments: 4096,
|
||||
}
|
||||
}
|
||||
|
||||
/// A stream plus its segment boundaries and a delivery permutation.
|
||||
fn stream_and_delivery() -> impl Strategy<Value = (Vec<u8>, Vec<(usize, usize)>, Vec<usize>)> {
|
||||
prop::collection::vec(any::<u8>(), 1..2048).prop_flat_map(|data| {
|
||||
let len = data.len();
|
||||
let cuts = prop::collection::vec(0..len, 0..16);
|
||||
(Just(data), cuts).prop_flat_map(|(data, mut cuts)| {
|
||||
cuts.sort_unstable();
|
||||
cuts.dedup();
|
||||
let boundaries = boundaries_from_cuts(&cuts, data.len());
|
||||
let permutation = Just((0..boundaries.len()).collect::<Vec<_>>()).prop_shuffle();
|
||||
(Just(data), Just(boundaries), permutation)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn boundaries_from_cuts(cuts: &[usize], len: usize) -> Vec<(usize, usize)> {
|
||||
let mut points = vec![0];
|
||||
points.extend(cuts.iter().copied().filter(|&c| c > 0 && c < len));
|
||||
points.push(len);
|
||||
points.dedup();
|
||||
points.windows(2).map(|w| (w[0], w[1])).collect()
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#![proptest_config(ProptestConfig::with_cases(512))]
|
||||
|
||||
/// Any segmentation delivered in any order reassembles to the original.
|
||||
#[test]
|
||||
fn arbitrary_segmentation_and_reordering_reconstructs_the_stream(
|
||||
(data, boundaries, permutation) in stream_and_delivery(),
|
||||
) {
|
||||
let base_seq = 0x1234_5678u32;
|
||||
let mut reasm = StreamReassembler::new(generous_limits());
|
||||
reasm.anchor(base_seq);
|
||||
|
||||
for &segment_index in &permutation {
|
||||
let (start, end) = boundaries[segment_index];
|
||||
let seq = base_seq.wrapping_add(u32::try_from(start).unwrap());
|
||||
reasm.push(seq, &data[start..end]);
|
||||
}
|
||||
|
||||
prop_assert_eq!(reasm.data(), data.as_slice());
|
||||
}
|
||||
|
||||
/// Duplicates layered on top of a complete stream change nothing.
|
||||
#[test]
|
||||
fn duplicate_delivery_is_idempotent(
|
||||
(data, boundaries, permutation) in stream_and_delivery(),
|
||||
) {
|
||||
let base_seq = 42u32;
|
||||
let mut reasm = StreamReassembler::new(generous_limits());
|
||||
reasm.anchor(base_seq);
|
||||
|
||||
for round in 0..2 {
|
||||
for &segment_index in &permutation {
|
||||
let (start, end) = boundaries[segment_index];
|
||||
let seq = base_seq.wrapping_add(u32::try_from(start).unwrap());
|
||||
let outcome = reasm.push(seq, &data[start..end]);
|
||||
if round == 1 {
|
||||
prop_assert!(matches!(
|
||||
outcome,
|
||||
PushOutcome::Unchanged | PushOutcome::Grew
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
prop_assert_eq!(reasm.data(), data.as_slice());
|
||||
}
|
||||
|
||||
/// Fully arbitrary segments never panic and never exceed the cap.
|
||||
#[test]
|
||||
fn arbitrary_input_never_panics_or_overruns(
|
||||
segments in prop::collection::vec(
|
||||
(any::<u32>(), prop::collection::vec(any::<u8>(), 0..64)),
|
||||
0..256,
|
||||
),
|
||||
) {
|
||||
let limits = ReassemblyLimits {
|
||||
max_assembled_bytes: 512,
|
||||
max_pending_bytes: 512,
|
||||
max_pending_segments: 32,
|
||||
};
|
||||
let mut reasm = StreamReassembler::new(limits);
|
||||
for (seq, payload) in &segments {
|
||||
reasm.push(*seq, payload);
|
||||
prop_assert!(reasm.data().len() <= limits.max_assembled_bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
# ©AngelaMos | 2026
|
||||
# Cargo.toml
|
||||
|
||||
[package]
|
||||
name = "tlsfp"
|
||||
description = "JA3/JA4 TLS fingerprinting tool with live capture, intel matching, and anomaly detection"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
authors.workspace = true
|
||||
repository.workspace = true
|
||||
default-run = "tlsfp"
|
||||
|
||||
[[bin]]
|
||||
name = "tlsfp"
|
||||
path = "src/main.rs"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
tlsfp-core.workspace = true
|
||||
anyhow.workspace = true
|
||||
tracing.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
pcap-parser.workspace = true
|
||||
pcap.workspace = true
|
||||
tokio.workspace = true
|
||||
flume.workspace = true
|
||||
rusqlite.workspace = true
|
||||
axum.workspace = true
|
||||
tower-http.workspace = true
|
||||
tower.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
clap.workspace = true
|
||||
etherparse.workspace = true
|
||||
|
|
@ -0,0 +1,126 @@
|
|||
// ©AngelaMos | 2026
|
||||
// cli.rs
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use clap::{Parser, Subcommand};
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
use tlsfp_core::{PcapFileSource, Pipeline, PipelineConfig};
|
||||
|
||||
/// JA3/JA4 TLS fingerprinting tool.
|
||||
///
|
||||
/// Fingerprints TLS clients and servers from live capture or packet captures,
|
||||
/// matches them against a local intelligence database, and flags anomalies such
|
||||
/// as a fingerprint that disagrees with its own User-Agent.
|
||||
#[derive(Debug, Parser)]
|
||||
#[command(name = "tlsfp", version, about, long_about = None)]
|
||||
pub struct Cli {
|
||||
/// Increase log verbosity (repeat for more detail).
|
||||
#[arg(short, long, global = true, action = clap::ArgAction::Count)]
|
||||
pub verbose: u8,
|
||||
|
||||
#[command(subcommand)]
|
||||
pub command: Command,
|
||||
}
|
||||
|
||||
#[derive(Debug, Subcommand)]
|
||||
pub enum Command {
|
||||
/// Fingerprint every TLS and QUIC handshake in a packet capture file.
|
||||
Pcap {
|
||||
/// Path to a pcap or pcapng file.
|
||||
path: std::path::PathBuf,
|
||||
|
||||
/// Emit one JSON object per event instead of readable lines.
|
||||
#[arg(long)]
|
||||
json: bool,
|
||||
},
|
||||
|
||||
/// Capture live from a network interface and fingerprint in real time.
|
||||
Live {
|
||||
/// Interface name, for example eth0.
|
||||
interface: String,
|
||||
},
|
||||
|
||||
/// Serve the web dashboard and HTTP API.
|
||||
Serve {
|
||||
/// Address to bind, for example 127.0.0.1:8080.
|
||||
#[arg(default_value = "127.0.0.1:8080")]
|
||||
bind: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl Cli {
|
||||
pub fn init_tracing(&self) {
|
||||
let default = match self.verbose {
|
||||
0 => "tlsfp=info",
|
||||
1 => "tlsfp=debug",
|
||||
_ => "tlsfp=trace",
|
||||
};
|
||||
let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(default));
|
||||
tracing_subscriber::fmt().with_env_filter(filter).init();
|
||||
}
|
||||
|
||||
pub fn run(self) -> Result<()> {
|
||||
match self.command {
|
||||
Command::Pcap { path, json } => run_pcap(&path, json),
|
||||
Command::Live { interface } => {
|
||||
anyhow::bail!("live capture on {interface} is not wired up yet")
|
||||
}
|
||||
Command::Serve { bind } => {
|
||||
anyhow::bail!("dashboard on {bind} is not wired up yet")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fingerprints a capture file and prints one event per line on stdout.
|
||||
///
|
||||
/// The summary goes to the log rather than stdout so that piping the output
|
||||
/// into a tool sees only events, while a human still learns how much of the
|
||||
/// capture was readable and whether the file was cut short mid packet.
|
||||
fn run_pcap(path: &std::path::Path, json: bool) -> Result<()> {
|
||||
let mut source = PcapFileSource::open(path)
|
||||
.with_context(|| format!("cannot open capture {}", path.display()))?;
|
||||
let mut pipeline = Pipeline::new(PipelineConfig::default());
|
||||
|
||||
let stdout = std::io::stdout().lock();
|
||||
let mut out = std::io::BufWriter::new(stdout);
|
||||
let mut write_failure = None;
|
||||
pipeline.run(&mut source, |event| {
|
||||
use std::io::Write as _;
|
||||
let result = if json {
|
||||
serde_json::to_writer(&mut out, &event)
|
||||
.map_err(anyhow::Error::from)
|
||||
.and_then(|()| writeln!(out).map_err(anyhow::Error::from))
|
||||
} else {
|
||||
writeln!(out, "{event}").map_err(anyhow::Error::from)
|
||||
};
|
||||
if write_failure.is_none() {
|
||||
if let Err(error) = result {
|
||||
write_failure = Some(error);
|
||||
}
|
||||
}
|
||||
})?;
|
||||
if let Some(error) = write_failure {
|
||||
return Err(error.context("writing events to stdout"));
|
||||
}
|
||||
{
|
||||
use std::io::Write as _;
|
||||
out.flush().context("flushing events to stdout")?;
|
||||
}
|
||||
|
||||
let counters = pipeline.counters();
|
||||
tracing::info!(
|
||||
frames = counters.frames,
|
||||
tcp_segments = counters.tcp_segments,
|
||||
events = counters.events,
|
||||
flows = counters.flows_created,
|
||||
unfinished_tls_streams = counters.unfinished_tls_streams,
|
||||
segments_dropped = counters.segments_dropped,
|
||||
"capture processed"
|
||||
);
|
||||
if source.truncated() {
|
||||
tracing::warn!("capture file ended mid packet; the tail was not read");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
// ©AngelaMos | 2026
|
||||
// main.rs
|
||||
|
||||
mod cli;
|
||||
|
||||
use anyhow::Result;
|
||||
use clap::Parser;
|
||||
|
||||
use crate::cli::Cli;
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let cli = Cli::parse();
|
||||
cli.init_tracing();
|
||||
cli.run()
|
||||
}
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
# ©AngelaMos | 2026
|
||||
# deny.toml
|
||||
|
||||
[graph]
|
||||
targets = ["x86_64-unknown-linux-gnu"]
|
||||
all-features = true
|
||||
|
||||
[advisories]
|
||||
version = 2
|
||||
yanked = "deny"
|
||||
|
||||
[licenses]
|
||||
version = 2
|
||||
confidence-threshold = 0.93
|
||||
allow = [
|
||||
"MIT",
|
||||
"Apache-2.0",
|
||||
"BSD-2-Clause",
|
||||
"BSD-3-Clause",
|
||||
"ISC",
|
||||
"Zlib",
|
||||
"Unicode-3.0",
|
||||
"Unicode-DFS-2016",
|
||||
"MPL-2.0",
|
||||
"CDLA-Permissive-2.0",
|
||||
]
|
||||
|
||||
[bans]
|
||||
multiple-versions = "warn"
|
||||
wildcards = "deny"
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
# ©AngelaMos | 2026
|
||||
# rustfmt.toml
|
||||
|
||||
edition = "2024"
|
||||
BIN
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/CVE-2018-6794.pcap
vendored
Normal file
BIN
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/CVE-2018-6794.pcap
vendored
Normal file
Binary file not shown.
49
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/PROVENANCE.md
vendored
Normal file
49
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/PROVENANCE.md
vendored
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
<!-- ©AngelaMos | 2026 -->
|
||||
<!-- PROVENANCE.md -->
|
||||
|
||||
# Test capture provenance
|
||||
|
||||
Every capture file in this directory is vendored, bit for bit, from the
|
||||
FoxIO JA4 reference repository. None of them were created or modified here,
|
||||
and the SHA-256 digests below let anyone verify that claim offline.
|
||||
|
||||
## Source
|
||||
|
||||
- Repository: <https://github.com/FoxIO-LLC/ja4>
|
||||
- Commit: `4ab8e3f18f4b27e0b896c4ec2c251e20506eb87e` (fetched 2026-06-10)
|
||||
- Path within the repository: `pcap/`
|
||||
- Fetch URL pattern: `https://raw.githubusercontent.com/FoxIO-LLC/ja4/<commit>/pcap/<file>`
|
||||
|
||||
## Licensing
|
||||
|
||||
The FoxIO repository carries two licenses at the commit above: `LICENSE-JA4`
|
||||
(BSD 3-Clause, covering the JA4 TLS client fingerprint) and `LICENSE`
|
||||
(FoxIO License 1.1, covering the JA4+ suite). These capture files are the
|
||||
repository's own test fixtures and are redistributed here unmodified, with
|
||||
attribution, solely as test inputs for this non-commercial educational
|
||||
project. See `NOTICE.md` at the project root for how the licensing split
|
||||
applies to this project as a whole.
|
||||
|
||||
## Files
|
||||
|
||||
| File | SHA-256 | Why it is here |
|
||||
| --- | --- | --- |
|
||||
| `tls-handshake.pcapng` | `5a0c9f3d0f437e16fc68c3ce0d87998edf4f229b335c391361ed301bd22a513e` | TLS 1.3 handshake; anchors the published JA4S vector `t130200_1301_234ea6891581` |
|
||||
| `tls-alpn-h2.pcap` | `8c00fd3e6c370b39dac61ad3a15c693088f74f3dbc836ee4e8f57105b1e84a91` | TLS 1.2 with ALPN h2; anchors the published JA4 vector `t12d4605h2_85626a9a5f7f_aaf95bb78ec9` and the JA4X DigiCert chain vectors |
|
||||
| `tls12.pcap` | `d8c9ae8781c9bbba3a1bf5a95d7a6f309a3edd14c64a7c8adbc673d337fd5af4` | Minimal TLS 1.2 ClientHello |
|
||||
| `tls-non-ascii-alpn.pcapng` | `cf1dd939619b8d65904dfd23b4f21c3255b6f513dc2e12117d5de2633d063f71` | ALPN value with non-ASCII bytes; exercises the spec-vs-reference divergence (FoxIO issue 178) |
|
||||
| `chrome-cloudflare-quic-with-secrets.pcapng` | `b7c9de1238aef44d53dbe1add125a7b9e344e9063b98850c78dec23632b83942` | Chrome to Cloudflare; TCP stream anchors the published JA4 vector `t13d1516h2_8daaf6152771_e5627efa2ab1`, QUIC stream reserved for the QUIC milestone |
|
||||
| `browsers-x509.pcapng` | `e05937fe5f3659f1b94b46305e419b878ba309ebcba8308750acaac704112906` | Browser certificate chains for JA4X over real captures |
|
||||
| `http1-with-cookies.pcapng` | `7083ca41bcd09b21cb92e7f2d5bd09d73f25703be8555bb82642c2495eb15ef9` | Cleartext HTTP/1.1 request with cookies for JA4H over a reassembled stream |
|
||||
| `gre-erspan-vxlan.pcap` | `5bbdb30a0707e21070ece3cd26068f72c34a0750d97c1e7720166cb4d0baf6d6` | Tunneled traffic; proves the decoder skips what it does not understand instead of crashing |
|
||||
| `CVE-2018-6794.pcap` | `d1aa18b493bc68bf7cb367ce5fc1ee493262b47baff886bbe42533e7495d8b1d` | TCP stream evasion capture; torture input for the reassembler |
|
||||
| `quic-with-several-tls-frames.pcapng` | `8d2c8a6787b942091aa63e33bde9f28c214b306dc024a66c552077ad30640e71` | QUIC initial with CRYPTO frames split across packets, reserved for the QUIC milestone |
|
||||
|
||||
## Verifying
|
||||
|
||||
```sh
|
||||
cd testdata/pcap && sha256sum *.pcap *.pcapng
|
||||
```
|
||||
|
||||
Compare the output against the table above. Any mismatch means a file no
|
||||
longer matches what FoxIO published at the pinned commit.
|
||||
BIN
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/browsers-x509.pcapng
vendored
Normal file
BIN
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/browsers-x509.pcapng
vendored
Normal file
Binary file not shown.
Binary file not shown.
BIN
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/gre-erspan-vxlan.pcap
vendored
Normal file
BIN
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/gre-erspan-vxlan.pcap
vendored
Normal file
Binary file not shown.
BIN
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/http1-with-cookies.pcapng
vendored
Normal file
BIN
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/http1-with-cookies.pcapng
vendored
Normal file
Binary file not shown.
BIN
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/quic-with-several-tls-frames.pcapng
vendored
Normal file
BIN
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/quic-with-several-tls-frames.pcapng
vendored
Normal file
Binary file not shown.
BIN
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/tls-alpn-h2.pcap
vendored
Normal file
BIN
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/tls-alpn-h2.pcap
vendored
Normal file
Binary file not shown.
BIN
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/tls-handshake.pcapng
vendored
Normal file
BIN
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/tls-handshake.pcapng
vendored
Normal file
Binary file not shown.
BIN
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/tls-non-ascii-alpn.pcapng
vendored
Normal file
BIN
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/testdata/pcap/tls-non-ascii-alpn.pcapng
vendored
Normal file
Binary file not shown.
Binary file not shown.
Loading…
Reference in New Issue