RomanNum3ral
/
Cybersecurity-Projects
mirror of https://github.com/CarterPerez-dev/Cybersecurity-Projects.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #286 from CarterPerez-dev/project/ja3-ja4-tls-fingerprinting 

Project/ja3 ja4 tls fingerprinting
This commit is contained in:
Carter Perez 2026-06-18 19:38:51 -04:00 committed by GitHub
parent 3347843c61 fb8e8c0ac9
commit 21741e0906
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
203 changed files with 27481 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
.github/workflows/lint.yml vendored
View File

@ -109,6 +109,10 @@ jobs:
- name: monitor-the-situation-dashboard-frontend
type: biome
path: PROJECTS/advanced/monitor-the-situation-dashboard/frontend
# Frontend (biome + stylelint + tsc)
- name: ja3-ja4-tls-fingerprinting-frontend
type: frontend
path: PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/frontend
# Go
- name: simple-vulnerability-scanner
type: go
@ -142,6 +146,9 @@ jobs:
- name: binary-analysis-tool-backend
type: rust
path: PROJECTS/intermediate/binary-analysis-tool/backend
- name: ja3-ja4-tls-fingerprinting
type: rust
path: PROJECTS/intermediate/ja3-ja4-tls-fingerprinting
# Crystal
- name: credential-rotation-enforcer
type: crystal
@ -184,21 +191,21 @@ jobs:
if: matrix.type == 'ruff'
run: pip install ruff
# Biome Setup
# Biome / Frontend Setup
- name: Setup Node.js
if: matrix.type == 'biome'
if: matrix.type == 'biome' || matrix.type == 'frontend'
uses: actions/setup-node@v4
with:
node-version: '22'
- name: Setup pnpm
if: matrix.type == 'biome'
if: matrix.type == 'biome' || matrix.type == 'frontend'
uses: pnpm/action-setup@v4
with:
version: 10
- name: Cache pnpm store
if: matrix.type == 'biome'
if: matrix.type == 'biome' || matrix.type == 'frontend'
uses: actions/cache@v4
with:
path: ~/.local/share/pnpm/store/v10
@ -208,7 +215,7 @@ jobs:
${{ runner.os }}-pnpm-
- name: Install frontend dependencies
if: matrix.type == 'biome'
if: matrix.type == 'biome' || matrix.type == 'frontend'
run: pnpm install --frozen-lockfile
# Go Setup
@ -333,6 +340,41 @@ jobs:
cat biome-output.txt
continue-on-error: true
# Frontend Linting (biome + stylelint + tsc)
- name: Run Biome, Stylelint, and tsc
if: matrix.type == 'frontend'
id: frontend
run: |
FRONTEND_OK=true
echo "Running Biome check..."
if pnpm biome check . > frontend-output.txt 2>&1; then
echo "biome: no issues"
else
FRONTEND_OK=false
echo "biome: issues found"
fi
echo "Running Stylelint..."
if pnpm stylelint "**/*.scss" >> frontend-output.txt 2>&1; then
echo "stylelint: no issues"
else
FRONTEND_OK=false
echo "stylelint: issues found"
fi
echo "Running tsc --noEmit..."
if pnpm tsc --noEmit >> frontend-output.txt 2>&1; then
echo "tsc: no issues"
else
FRONTEND_OK=false
echo "tsc: type errors found"
fi
if [[ "$FRONTEND_OK" == "true" ]]; then
echo "FRONTEND_PASSED=true" >> $GITHUB_ENV
else
echo "FRONTEND_PASSED=false" >> $GITHUB_ENV
fi
cat frontend-output.txt
continue-on-error: true
# Go Linting
- name: Run golangci-lint
if: matrix.type == 'go'
@ -589,6 +631,35 @@ jobs:
fi
} >> $GITHUB_STEP_SUMMARY
# Create Summary for Frontend
- name: Create Frontend Lint Summary
if: matrix.type == 'frontend'
run: |
{
echo "## Lint Results: ${{ matrix.name }}"
echo ''
if [[ "${{ env.FRONTEND_PASSED }}" == "true" ]]; then
echo '### Biome + Stylelint + tsc: **Passed**'
echo 'No frontend issues found.'
else
echo '### Biome + Stylelint + tsc: **Issues Found**'
echo '<details><summary>View output</summary>'
echo ''
echo '```'
head -100 frontend-output.txt
echo '```'
echo '</details>'
fi
echo ''
if [[ "${{ env.FRONTEND_PASSED }}" == "true" ]]; then
echo '---'
echo '### All checks passed!'
else
echo '---'
echo '### Review the issues above'
fi
} >> $GITHUB_STEP_SUMMARY
# Create Summary for Go
- name: Create Go Lint Summary
if: matrix.type == 'go'
@ -809,6 +880,11 @@ jobs:
echo "Biome lint checks failed"
exit 1
fi
elif [[ "${{ matrix.type }}" == "frontend" ]]; then
if [[ "${{ env.FRONTEND_PASSED }}" == "false" ]]; then
echo "Frontend lint checks failed"
exit 1
fi
elif [[ "${{ matrix.type }}" == "go" ]]; then
if [[ "${{ env.GOLANGCI_PASSED }}" == "false" ]]; then
echo "Go lint checks failed"

47
.pre-commit-config.yaml
View File

@ -258,6 +258,33 @@ repos:
files: ^PROJECTS/advanced/monitor-the-situation-dashboard/frontend/src/
pass_filenames: false
- id: biome-ja3-ja4-tls-fingerprinting
name: biome check (ja3-ja4-tls-fingerprinting frontend)
entry: bash -c 'cd PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/frontend && pnpm biome check .'
language: system
files: ^PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/frontend/src/
pass_filenames: false
# Stylelint SCSS Checks
- repo: local
hooks:
- id: stylelint-ja3-ja4-tls-fingerprinting
name: stylelint (ja3-ja4-tls-fingerprinting frontend)
entry: bash -c 'cd PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/frontend && pnpm stylelint "**/*.scss"'
language: system
files: ^PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/frontend/src/
pass_filenames: false
# TypeScript tsc Checks
- repo: local
hooks:
- id: tsc-ja3-ja4-tls-fingerprinting
name: tsc typecheck (ja3-ja4-tls-fingerprinting frontend)
entry: bash -c 'cd PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/frontend && pnpm tsc --noEmit'
language: system
files: ^PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/frontend/src/
pass_filenames: false
# Nim nph Checks
- repo: local
hooks:
@ -287,6 +314,22 @@ repos:
types: [rust]
pass_filenames: false
- id: cargo-fmt-ja3-ja4-tls-fingerprinting
name: cargo fmt (ja3-ja4-tls-fingerprinting)
entry: bash -c 'cd PROJECTS/intermediate/ja3-ja4-tls-fingerprinting && cargo fmt --all -- --check'
language: system
files: ^PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/
types: [rust]
pass_filenames: false
- id: cargo-clippy-ja3-ja4-tls-fingerprinting
name: cargo clippy (ja3-ja4-tls-fingerprinting)
entry: bash -c 'cd PROJECTS/intermediate/ja3-ja4-tls-fingerprinting && cargo clippy --workspace --all-targets -- -D warnings'
language: system
files: ^PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/
types: [rust]
pass_filenames: false
# Crystal Checks (format + ameba + compile)
- repo: local
hooks:
@ -406,9 +449,9 @@ repos:
- id: check-symlinks
exclude: (\.pnpm-store|node_modules)/
- id: end-of-file-fixer
exclude: (\.pnpm-store|node_modules)/|^PROJECTS/advanced/hsm-emulator/vendor/
exclude: (\.pnpm-store|node_modules)/|^PROJECTS/advanced/hsm-emulator/vendor/|^PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-intel/seeds/
- id: trailing-whitespace
exclude: (\.pnpm-store|node_modules)/|^PROJECTS/advanced/hsm-emulator/vendor/
exclude: (\.pnpm-store|node_modules)/|^PROJECTS/advanced/hsm-emulator/vendor/|^PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-intel/seeds/
- id: check-illegal-windows-names
exclude: (\.pnpm-store|node_modules)/
- id: check-executables-have-shebangs

4
PROJECTS/advanced/hsm-emulator/examples/smoke.zig
View File

@ -319,11 +319,15 @@ pub fn main() !void {
var rsapub_tmpl = [_]ck.CK_ATTRIBUTE{
.{ .type = ck.CKA_MODULUS_BITS, .pValue = &rsa_bits, .ulValueLen = @sizeOf(ck.CK_ULONG) },
.{ .type = ck.CKA_VERIFY, .pValue = &ck_yes, .ulValueLen = 1 },
.{ .type = ck.CKA_VERIFY_RECOVER, .pValue = &ck_yes, .ulValueLen = 1 },
.{ .type = ck.CKA_ENCRYPT, .pValue = &ck_yes, .ulValueLen = 1 },
.{ .type = ck.CKA_WRAP, .pValue = &ck_yes, .ulValueLen = 1 },
};
var rsapriv_tmpl = [_]ck.CK_ATTRIBUTE{
.{ .type = ck.CKA_SIGN, .pValue = &ck_yes, .ulValueLen = 1 },
.{ .type = ck.CKA_SIGN_RECOVER, .pValue = &ck_yes, .ulValueLen = 1 },
.{ .type = ck.CKA_DECRYPT, .pValue = &ck_yes, .ulValueLen = 1 },
.{ .type = ck.CKA_UNWRAP, .pValue = &ck_yes, .ulValueLen = 1 },
.{ .type = ck.CKA_PRIVATE, .pValue = &ck_false, .ulValueLen = 1 },
};
var h_rsapub: ck.CK_OBJECT_HANDLE = 0;

15
PROJECTS/advanced/hsm-emulator/src/api/object.zig
View File

@ -86,6 +86,21 @@ pub fn materializeDefaults(obj: *Object, allocator: std.mem.Allocator, class: ck
const def: u8 = if (class == ck.CKO_PRIVATE_KEY) ck.CK_TRUE else ck.CK_FALSE;
try obj.set(allocator, ck.CKA_PRIVATE, &[_]u8{def});
}
switch (class) {
ck.CKO_PUBLIC_KEY => {
const attrs = [_]ck.CK_ATTRIBUTE_TYPE{ ck.CKA_ENCRYPT, ck.CKA_VERIFY, ck.CKA_VERIFY_RECOVER, ck.CKA_WRAP, ck.CKA_DERIVE };
for (attrs) |a| if (!obj.has(a)) try obj.set(allocator, a, &[_]u8{ck.CK_FALSE});
},
ck.CKO_PRIVATE_KEY => {
const attrs = [_]ck.CK_ATTRIBUTE_TYPE{ ck.CKA_DECRYPT, ck.CKA_SIGN, ck.CKA_SIGN_RECOVER, ck.CKA_UNWRAP, ck.CKA_DERIVE, ck.CKA_ALWAYS_AUTHENTICATE };
for (attrs) |a| if (!obj.has(a)) try obj.set(allocator, a, &[_]u8{ck.CK_FALSE});
},
ck.CKO_SECRET_KEY => {
const attrs = [_]ck.CK_ATTRIBUTE_TYPE{ ck.CKA_ENCRYPT, ck.CKA_DECRYPT, ck.CKA_SIGN, ck.CKA_VERIFY, ck.CKA_WRAP, ck.CKA_UNWRAP, ck.CKA_DERIVE };
for (attrs) |a| if (!obj.has(a)) try obj.set(allocator, a, &[_]u8{ck.CK_FALSE});
},
else => {},
}
}
pub fn insertNew(inst: *state.Instance, sess: *session.Session, obj_in: Object, phObject: *ck.CK_OBJECT_HANDLE) ck.CK_RV {

5
PROJECTS/advanced/monitor-the-situation-dashboard/Justfile
View File

@ -221,10 +221,7 @@ prod-restart:
docker compose --env-file .env.prod -f compose.yml -f cloudflared.compose.yml logs -f
[group('tunnel')]
prod-redeploy:
docker compose --env-file .env.prod -f compose.yml -f cloudflared.compose.yml down
docker compose --env-file .env.prod -f compose.yml -f cloudflared.compose.yml up -d --build
docker compose --env-file .env.prod -f compose.yml -f cloudflared.compose.yml logs -f
redeploy: (tunnel-down "--remove-orphans") (tunnel-start "--remove-orphans")
[group('tunnel')]
prod-logs *SERVICE:

2
PROJECTS/beginner/canary-token-generator/frontend/src/pages/landing/copy.ts
View File

@ -12,7 +12,7 @@ export const PAGE_COPY = {
'Field-deployable trip-wires for the quiet hours. Each specimen waits in place, reports when touched, and leaves a forensic note behind.',
STRIP_FIELD_STATION: 'FIELD STATION',
STRIP_ISSUE: 'ISSUE',
STRIP_VOLUME: 'VOL.001',
STRIP_VOLUME: 'SPEC.003',
STRIP_FOLIO: 'FOLIO',
SECTION_01_INDEX: '01 / SPECIES',
SECTION_01_TITLE: 'SELECT SPECIES',

3
PROJECTS/beginner/canary-token-generator/justfile
View File

@ -153,6 +153,9 @@ ps:
# Docker Compose (Production + Cloudflare Tunnel)
# =============================================================================
[group('tunnel')]
redeploy: (tunnel-down "--remove-orphans") build (tunnel-start "--remove-orphans")
[group('tunnel')]
tunnel-up *ARGS:
docker compose --env-file .env -f compose.yml -f cloudflared.compose.yml up {{ARGS}}

4
PROJECTS/intermediate/binary-analysis-tool/frontend/.npmrc Normal file
View File

@ -0,0 +1,4 @@
# ©AngelaMos | 2026
# .npmrc
strict-dep-builds=false
auto-install-peers=true

BIN
PROJECTS/intermediate/binary-analysis-tool/frontend/public/assets/android-chrome-192x192.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 40 KiB

After

Width:  |  Height:  |  Size: 86 KiB

BIN
PROJECTS/intermediate/binary-analysis-tool/frontend/public/assets/android-chrome-512x512.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 221 KiB

After

Width:  |  Height:  |  Size: 524 KiB

BIN
PROJECTS/intermediate/binary-analysis-tool/frontend/public/assets/apple-touch-icon.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 37 KiB

After

Width:  |  Height:  |  Size: 78 KiB

BIN
PROJECTS/intermediate/binary-analysis-tool/frontend/public/assets/favicon-16x16.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 600 B

After

Width:  |  Height:  |  Size: 936 B

BIN
PROJECTS/intermediate/binary-analysis-tool/frontend/public/assets/favicon-32x32.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 1.8 KiB

After

Width:  |  Height:  |  Size: 3.1 KiB

BIN
PROJECTS/intermediate/binary-analysis-tool/frontend/public/assets/favicon.ico
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 15 KiB

After

Width:  |  Height:  |  Size: 15 KiB

4
PROJECTS/intermediate/binary-analysis-tool/frontend/src/pages/analysis/index.tsx
View File

@ -120,7 +120,7 @@ export function Component(): React.ReactElement {
if (isLoading) {
return (
<div className={styles.state}>
<span className={styles.stateLabel}>ANALYZING SPECIMEN\u2026</span>
<span className={styles.stateLabel}>ANALYZING SUBJECT\u2026</span>
</div>
)
}
@ -129,7 +129,7 @@ export function Component(): React.ReactElement {
return (
<div className={styles.state}>
<span className={styles.stateCode}>404</span>
<span className={styles.stateLabel}>SPECIMEN NOT FOUND</span>
<span className={styles.stateLabel}>SUBJECT NOT FOUND</span>
<Link to={ROUTES.HOME} className={styles.stateBack}>
NEW ANALYSIS
</Link>

12
PROJECTS/intermediate/binary-analysis-tool/frontend/src/pages/landing/index.tsx
View File

@ -5,7 +5,7 @@
// Binary upload landing page with drag-and-drop file
// intake and analysis pipeline visualization
//
// Renders the Axumortem specimen intake interface: an
// Renders the Axumortem subject intake interface: an
// animated SVG grain background, hex offset margin
// decoration (16 addresses), corner brackets, meta
// strip header, and format support badges (ELF/PE/
@ -143,7 +143,7 @@ export function Component(): React.ReactElement {
<div className={styles.cornerBR} aria-hidden="true" />
<header className={styles.metaStrip}>
<span>AXM-001</span>
<span>SPECIMEN-005</span>
<span className={styles.metaCenter}>STATIC ANALYSIS SUITE</span>
<span>v0.1.1</span>
</header>
@ -164,7 +164,7 @@ export function Component(): React.ReactElement {
<section className={styles.intake}>
<div className={styles.intakeHeader}>
<span className={styles.intakeLabel}>SPECIMEN INTAKE</span>
<span className={styles.intakeLabel}>SUBJECT INTAKE</span>
{file && (
<button
type="button"
@ -211,7 +211,7 @@ export function Component(): React.ReactElement {
className={styles.dropPrompt}
onClick={() => inputRef.current?.click()}
>
<span className={styles.dropStatus}>AWAITING SPECIMEN</span>
<span className={styles.dropStatus}>AWAITING SUBJECT</span>
<span className={styles.dropInstruction}>
DRAG + DROP BINARY / CLICK TO BROWSE
</span>
@ -226,7 +226,7 @@ export function Component(): React.ReactElement {
onClick={handleSubmit}
disabled={upload.isPending}
>
{upload.isPending ? 'ANALYZING\u2026' : 'SUBMIT SPECIMEN'}
{upload.isPending ? 'ANALYZING\u2026' : 'SUBMIT SUBJECT'}
</button>
)}
</section>
@ -250,7 +250,7 @@ export function Component(): React.ReactElement {
<footer className={styles.footer}>
<span>&copy; ANGELAMOS 2026</span>
<span className={styles.footerDesignation}>
SYS AXM-BDE {'//'} UNIT-001
SYS AXM-BDE {'//'} SPECIMEN-005
</span>
<span>AXUMORTEM</span>
</footer>

4
PROJECTS/intermediate/binary-analysis-tool/infra/docker/vite.prod
View File

@ -11,11 +11,11 @@
# ============================================================================
FROM node:22-slim AS builder
RUN corepack enable && corepack prepare pnpm@latest --activate
RUN corepack enable && corepack prepare pnpm@10.29.1 --activate
WORKDIR /app
COPY frontend/package.json frontend/pnpm-lock.yaml* ./
COPY frontend/package.json frontend/pnpm-lock.yaml* frontend/.npmrc* ./
RUN --mount=type=cache,target=/root/.local/share/pnpm/store \
pnpm install --frozen-lockfile

3
PROJECTS/intermediate/binary-analysis-tool/justfile
View File

@ -80,6 +80,9 @@ ps:
# Docker Compose (Production + Cloudflare Tunnel)
# =============================================================================
[group('tunnel')]
redeploy: (tunnel-down "--remove-orphans") build (tunnel-start "--remove-orphans")
[group('tunnel')]
tunnel-up *ARGS:
docker compose --env-file .env -f compose.yml -f cloudflared.compose.yml up {{ARGS}}

12
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/.dockerignore Normal file
View File

@ -0,0 +1,12 @@
# ©AngelaMos | 2026
# .dockerignore
target/
**/node_modules/
**/dist/
.git/
docs/
*.db
*.sqlite
.env
.env.development

37
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/.env.example Normal file
View File

@ -0,0 +1,37 @@
# ©AngelaMos | 2026
# .env.example
# Copy to .env (production) and .env.development (development), then fill in.
# Both .env and .env.development are gitignored. Run `just init` to set the
# project name and randomize ports, or `just ports` to re-randomize ports.
# ---------------------------------------------------------------------------
# Identity
# ---------------------------------------------------------------------------
APP_NAME=tlsfp
# Frontend build-time vars (baked into the bundle by Vite)
VITE_APP_TITLE=JA3/JA4 TLS Fingerprinting
VITE_API_URL=/api
# Vite standalone dev proxy target (only used when running pnpm dev outside
# the compose network; inside compose, nginx proxies /api to the backend)
VITE_API_TARGET=http://tlsfp:8080
# ---------------------------------------------------------------------------
# Host ports (randomized per-machine to avoid collisions)
# ---------------------------------------------------------------------------
NGINX_HOST_PORT=11790
FRONTEND_HOST_PORT=39755
# ---------------------------------------------------------------------------
# Backend (Rust)
# ---------------------------------------------------------------------------
# Read by the tlsfp binary via EnvFilter. The dashboard backend service is
# defined under the `backend` compose profile and is not started by default
# until the serve command is implemented.
RUST_LOG=tlsfp=info
# ---------------------------------------------------------------------------
# Cloudflare Tunnel (overlay: cloudflared.compose.yml)
# ---------------------------------------------------------------------------
CLOUDFLARE_TUNNEL_TOKEN=

46
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/.github/workflows/ci.yml vendored Normal file
View File

@ -0,0 +1,46 @@
# ©AngelaMos | 2026
# ci.yml
name: ci
on:
push:
paths:
- "PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/**"
pull_request:
paths:
- "PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/**"
defaults:
run:
working-directory: PROJECTS/intermediate/ja3-ja4-tls-fingerprinting
jobs:
test:
runs-on: ubuntu-latest
strategy:
matrix:
rust: [stable, "1.85"]
steps:
- uses: actions/checkout@v4
- name: Install libpcap
run: sudo apt-get update && sudo apt-get install -y libpcap-dev
- name: Install Rust ${{ matrix.rust }}
run: rustup toolchain install ${{ matrix.rust }} --component clippy rustfmt --profile minimal
- name: Build
run: cargo +${{ matrix.rust }} build --workspace --all-targets
- name: Test
run: cargo +${{ matrix.rust }} test --workspace
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install libpcap
run: sudo apt-get update && sudo apt-get install -y libpcap-dev
- name: Install Rust
run: rustup toolchain install stable --component clippy rustfmt --profile minimal
- name: Format
run: cargo fmt --all --check
- name: Clippy
run: cargo clippy --workspace --all-targets -- -D warnings

17
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/.gitignore vendored Normal file
View File

@ -0,0 +1,17 @@
# ©AngelaMos | 2026
# .gitignore
docs/
target/
*.db
*.sqlite
fuzz/corpus/
fuzz/artifacts/
.env
.env.development
node_modules/
**/node_modules/
dist/
**/dist/
.biome_cache/

2310
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/Cargo.lock generated Normal file
View File

File diff suppressed because it is too large Load Diff

69
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/Cargo.toml Normal file
View File

@ -0,0 +1,69 @@
# ©AngelaMos | 2026
# Cargo.toml
[workspace]
resolver = "3"
members = ["crates/tlsfp-core", "crates/tlsfp-intel", "crates/tlsfp"]
[workspace.package]
version = "0.1.0"
edition = "2024"
rust-version = "1.85"
license = "MIT"
authors = ["Carter Perez"]
repository = "https://github.com/CarterPerez-dev/Cybersecurity-Projects"
[workspace.dependencies]
tlsfp-core = { path = "crates/tlsfp-core" }
tlsfp-intel = { path = "crates/tlsfp-intel" }
winnow = "0.7"
md-5 = "0.10"
sha2 = "0.10"
hex = "0.4"
smallvec = "1"
thiserror = "2"
anyhow = "1"
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
etherparse = "0.18"
pcap-parser = "0.16"
pcap = "2"
tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "signal", "time", "net"] }
tokio-stream = { version = "0.1", features = ["sync"] }
flume = "0.11"
rustix = { version = "1", features = ["std", "event"] }
rusqlite = { version = "0.32", features = ["bundled"] }
csv = "1"
axum = "0.8"
tower-http = { version = "0.6", features = ["fs", "trace", "cors", "compression-br", "compression-gzip", "set-header"] }
tower = "0.5"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
clap = { version = "4", features = ["derive"] }
hkdf = "0.12"
aes = "0.8"
aes-gcm = "0.10"
[workspace.lints.rust]
unsafe_code = "forbid"
[workspace.lints.clippy]
pedantic = { level = "warn", priority = -1 }
module_name_repetitions = "allow"
missing_errors_doc = "allow"
missing_panics_doc = "allow"
must_use_candidate = "allow"
doc_markdown = "allow"
[profile.release]
opt-level = 3
lto = "thin"
codegen-units = 1

661
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/LICENSE Normal file
View File

@ -0,0 +1,661 @@
GNU AFFERO GENERAL PUBLIC LICENSE
Version 3, 19 November 2007
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
Preamble
The GNU Affero General Public License is a free, copyleft license for
software and other kinds of works, specifically designed to ensure
cooperation with the community in the case of network server software.
The licenses for most software and other practical works are designed
to take away your freedom to share and change the works. By contrast,
our General Public Licenses are intended to guarantee your freedom to
share and change all versions of a program--to make sure it remains free
software for all its users.
When we speak of free software, we are referring to freedom, not
price. Our General Public Licenses are designed to make sure that you
have the freedom to distribute copies of free software (and charge for
them if you wish), that you receive source code or can get it if you
want it, that you can change the software or use pieces of it in new
free programs, and that you know you can do these things.
Developers that use our General Public Licenses protect your rights
with two steps: (1) assert copyright on the software, and (2) offer
you this License which gives you legal permission to copy, distribute
and/or modify the software.
A secondary benefit of defending all users' freedom is that
improvements made in alternate versions of the program, if they
receive widespread use, become available for other developers to
incorporate. Many developers of free software are heartened and
encouraged by the resulting cooperation. However, in the case of
software used on network servers, this result may fail to come about.
The GNU General Public License permits making a modified version and
letting the public access it on a server without ever releasing its
source code to the public.
The GNU Affero General Public License is designed specifically to
ensure that, in such cases, the modified source code becomes available
to the community. It requires the operator of a network server to
provide the source code of the modified version running there to the
users of that server. Therefore, public use of a modified version, on
a publicly accessible server, gives the public access to the source
code of the modified version.
An older license, called the Affero General Public License and
published by Affero, was designed to accomplish similar goals. This is
a different license, not a version of the Affero GPL, but Affero has
released a new version of the Affero GPL which permits relicensing under
this license.
The precise terms and conditions for copying, distribution and
modification follow.
TERMS AND CONDITIONS
0. Definitions.
"This License" refers to version 3 of the GNU Affero General Public License.
"Copyright" also means copyright-like laws that apply to other kinds of
works, such as semiconductor masks.
"The Program" refers to any copyrightable work licensed under this
License. Each licensee is addressed as "you". "Licensees" and
"recipients" may be individuals or organizations.
To "modify" a work means to copy from or adapt all or part of the work
in a fashion requiring copyright permission, other than the making of an
exact copy. The resulting work is called a "modified version" of the
earlier work or a work "based on" the earlier work.
A "covered work" means either the unmodified Program or a work based
on the Program.
To "propagate" a work means to do anything with it that, without
permission, would make you directly or secondarily liable for
infringement under applicable copyright law, except executing it on a
computer or modifying a private copy. Propagation includes copying,
distribution (with or without modification), making available to the
public, and in some countries other activities as well.
To "convey" a work means any kind of propagation that enables other
parties to make or receive copies. Mere interaction with a user through
a computer network, with no transfer of a copy, is not conveying.
An interactive user interface displays "Appropriate Legal Notices"
to the extent that it includes a convenient and prominently visible
feature that (1) displays an appropriate copyright notice, and (2)
tells the user that there is no warranty for the work (except to the
extent that warranties are provided), that licensees may convey the
work under this License, and how to view a copy of this License. If
the interface presents a list of user commands or options, such as a
menu, a prominent item in the list meets this criterion.
1. Source Code.
The "source code" for a work means the preferred form of the work
for making modifications to it. "Object code" means any non-source
form of a work.
A "Standard Interface" means an interface that either is an official
standard defined by a recognized standards body, or, in the case of
interfaces specified for a particular programming language, one that
is widely used among developers working in that language.
The "System Libraries" of an executable work include anything, other
than the work as a whole, that (a) is included in the normal form of
packaging a Major Component, but which is not part of that Major
Component, and (b) serves only to enable use of the work with that
Major Component, or to implement a Standard Interface for which an
implementation is available to the public in source code form. A
"Major Component", in this context, means a major essential component
(kernel, window system, and so on) of the specific operating system
(if any) on which the executable work runs, or a compiler used to
produce the work, or an object code interpreter used to run it.
The "Corresponding Source" for a work in object code form means all
the source code needed to generate, install, and (for an executable
work) run the object code and to modify the work, including scripts to
control those activities. However, it does not include the work's
System Libraries, or general-purpose tools or generally available free
programs which are used unmodified in performing those activities but
which are not part of the work. For example, Corresponding Source
includes interface definition files associated with source files for
the work, and the source code for shared libraries and dynamically
linked subprograms that the work is specifically designed to require,
such as by intimate data communication or control flow between those
subprograms and other parts of the work.
The Corresponding Source need not include anything that users
can regenerate automatically from other parts of the Corresponding
Source.
The Corresponding Source for a work in source code form is that
same work.
2. Basic Permissions.
All rights granted under this License are granted for the term of
copyright on the Program, and are irrevocable provided the stated
conditions are met. This License explicitly affirms your unlimited
permission to run the unmodified Program. The output from running a
covered work is covered by this License only if the output, given its
content, constitutes a covered work. This License acknowledges your
rights of fair use or other equivalent, as provided by copyright law.
You may make, run and propagate covered works that you do not
convey, without conditions so long as your license otherwise remains
in force. You may convey covered works to others for the sole purpose
of having them make modifications exclusively for you, or provide you
with facilities for running those works, provided that you comply with
the terms of this License in conveying all material for which you do
not control copyright. Those thus making or running the covered works
for you must do so exclusively on your behalf, under your direction
and control, on terms that prohibit them from making any copies of
your copyrighted material outside their relationship with you.
Conveying under any other circumstances is permitted solely under
the conditions stated below. Sublicensing is not allowed; section 10
makes it unnecessary.
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
No covered work shall be deemed part of an effective technological
measure under any applicable law fulfilling obligations under article
11 of the WIPO copyright treaty adopted on 20 December 1996, or
similar laws prohibiting or restricting circumvention of such
measures.
When you convey a covered work, you waive any legal power to forbid
circumvention of technological measures to the extent such circumvention
is effected by exercising rights under this License with respect to
the covered work, and you disclaim any intention to limit operation or
modification of the work as a means of enforcing, against the work's
users, your or third parties' legal rights to forbid circumvention of
technological measures.
4. Conveying Verbatim Copies.
You may convey verbatim copies of the Program's source code as you
receive it, in any medium, provided that you conspicuously and
appropriately publish on each copy an appropriate copyright notice;
keep intact all notices stating that this License and any
non-permissive terms added in accord with section 7 apply to the code;
keep intact all notices of the absence of any warranty; and give all
recipients a copy of this License along with the Program.
You may charge any price or no price for each copy that you convey,
and you may offer support or warranty protection for a fee.
5. Conveying Modified Source Versions.
You may convey a work based on the Program, or the modifications to
produce it from the Program, in the form of source code under the
terms of section 4, provided that you also meet all of these conditions:
a) The work must carry prominent notices stating that you modified
it, and giving a relevant date.
b) The work must carry prominent notices stating that it is
released under this License and any conditions added under section
7. This requirement modifies the requirement in section 4 to
"keep intact all notices".
c) You must license the entire work, as a whole, under this
License to anyone who comes into possession of a copy. This
License will therefore apply, along with any applicable section 7
additional terms, to the whole of the work, and all its parts,
regardless of how they are packaged. This License gives no
permission to license the work in any other way, but it does not
invalidate such permission if you have separately received it.
d) If the work has interactive user interfaces, each must display
Appropriate Legal Notices; however, if the Program has interactive
interfaces that do not display Appropriate Legal Notices, your
work need not make them do so.
A compilation of a covered work with other separate and independent
works, which are not by their nature extensions of the covered work,
and which are not combined with it such as to form a larger program,
in or on a volume of a storage or distribution medium, is called an
"aggregate" if the compilation and its resulting copyright are not
used to limit the access or legal rights of the compilation's users
beyond what the individual works permit. Inclusion of a covered work
in an aggregate does not cause this License to apply to the other
parts of the aggregate.
6. Conveying Non-Source Forms.
You may convey a covered work in object code form under the terms
of sections 4 and 5, provided that you also convey the
machine-readable Corresponding Source under the terms of this License,
in one of these ways:
a) Convey the object code in, or embodied in, a physical product
(including a physical distribution medium), accompanied by the
Corresponding Source fixed on a durable physical medium
customarily used for software interchange.
b) Convey the object code in, or embodied in, a physical product
(including a physical distribution medium), accompanied by a
written offer, valid for at least three years and valid for as
long as you offer spare parts or customer support for that product
model, to give anyone who possesses the object code either (1) a
copy of the Corresponding Source for all the software in the
product that is covered by this License, on a durable physical
medium customarily used for software interchange, for a price no
more than your reasonable cost of physically performing this
conveying of source, or (2) access to copy the
Corresponding Source from a network server at no charge.
c) Convey individual copies of the object code with a copy of the
written offer to provide the Corresponding Source. This
alternative is allowed only occasionally and noncommercially, and
only if you received the object code with such an offer, in accord
with subsection 6b.
d) Convey the object code by offering access from a designated
place (gratis or for a charge), and offer equivalent access to the
Corresponding Source in the same way through the same place at no
further charge. You need not require recipients to copy the
Corresponding Source along with the object code. If the place to
copy the object code is a network server, the Corresponding Source
may be on a different server (operated by you or a third party)
that supports equivalent copying facilities, provided you maintain
clear directions next to the object code saying where to find the
Corresponding Source. Regardless of what server hosts the
Corresponding Source, you remain obligated to ensure that it is
available for as long as needed to satisfy these requirements.
e) Convey the object code using peer-to-peer transmission, provided
you inform other peers where the object code and Corresponding
Source of the work are being offered to the general public at no
charge under subsection 6d.
A separable portion of the object code, whose source code is excluded
from the Corresponding Source as a System Library, need not be
included in conveying the object code work.
A "User Product" is either (1) a "consumer product", which means any
tangible personal property which is normally used for personal, family,
or household purposes, or (2) anything designed or sold for incorporation
into a dwelling. In determining whether a product is a consumer product,
doubtful cases shall be resolved in favor of coverage. For a particular
product received by a particular user, "normally used" refers to a
typical or common use of that class of product, regardless of the status
of the particular user or of the way in which the particular user
actually uses, or expects or is expected to use, the product. A product
is a consumer product regardless of whether the product has substantial
commercial, industrial or non-consumer uses, unless such uses represent
the only significant mode of use of the product.
"Installation Information" for a User Product means any methods,
procedures, authorization keys, or other information required to install
and execute modified versions of a covered work in that User Product from
a modified version of its Corresponding Source. The information must
suffice to ensure that the continued functioning of the modified object
code is in no case prevented or interfered with solely because
modification has been made.
If you convey an object code work under this section in, or with, or
specifically for use in, a User Product, and the conveying occurs as
part of a transaction in which the right of possession and use of the
User Product is transferred to the recipient in perpetuity or for a
fixed term (regardless of how the transaction is characterized), the
Corresponding Source conveyed under this section must be accompanied
by the Installation Information. But this requirement does not apply
if neither you nor any third party retains the ability to install
modified object code on the User Product (for example, the work has
been installed in ROM).
The requirement to provide Installation Information does not include a
requirement to continue to provide support service, warranty, or updates
for a work that has been modified or installed by the recipient, or for
the User Product in which it has been modified or installed. Access to a
network may be denied when the modification itself materially and
adversely affects the operation of the network or violates the rules and
protocols for communication across the network.
Corresponding Source conveyed, and Installation Information provided,
in accord with this section must be in a format that is publicly
documented (and with an implementation available to the public in
source code form), and must require no special password or key for
unpacking, reading or copying.
7. Additional Terms.
"Additional permissions" are terms that supplement the terms of this
License by making exceptions from one or more of its conditions.
Additional permissions that are applicable to the entire Program shall
be treated as though they were included in this License, to the extent
that they are valid under applicable law. If additional permissions
apply only to part of the Program, that part may be used separately
under those permissions, but the entire Program remains governed by
this License without regard to the additional permissions.
When you convey a copy of a covered work, you may at your option
remove any additional permissions from that copy, or from any part of
it. (Additional permissions may be written to require their own
removal in certain cases when you modify the work.) You may place
additional permissions on material, added by you to a covered work,
for which you have or can give appropriate copyright permission.
Notwithstanding any other provision of this License, for material you
add to a covered work, you may (if authorized by the copyright holders of
that material) supplement the terms of this License with terms:
a) Disclaiming warranty or limiting liability differently from the
terms of sections 15 and 16 of this License; or
b) Requiring preservation of specified reasonable legal notices or
author attributions in that material or in the Appropriate Legal
Notices displayed by works containing it; or
c) Prohibiting misrepresentation of the origin of that material, or
requiring that modified versions of such material be marked in
reasonable ways as different from the original version; or
d) Limiting the use for publicity purposes of names of licensors or
authors of the material; or
e) Declining to grant rights under trademark law for use of some
trade names, trademarks, or service marks; or
f) Requiring indemnification of licensors and authors of that
material by anyone who conveys the material (or modified versions of
it) with contractual assumptions of liability to the recipient, for
any liability that these contractual assumptions directly impose on
those licensors and authors.
All other non-permissive additional terms are considered "further
restrictions" within the meaning of section 10. If the Program as you
received it, or any part of it, contains a notice stating that it is
governed by this License along with a term that is a further
restriction, you may remove that term. If a license document contains
a further restriction but permits relicensing or conveying under this
License, you may add to a covered work material governed by the terms
of that license document, provided that the further restriction does
not survive such relicensing or conveying.
If you add terms to a covered work in accord with this section, you
must place, in the relevant source files, a statement of the
additional terms that apply to those files, or a notice indicating
where to find the applicable terms.
Additional terms, permissive or non-permissive, may be stated in the
form of a separately written license, or stated as exceptions;
the above requirements apply either way.
8. Termination.
You may not propagate or modify a covered work except as expressly
provided under this License. Any attempt otherwise to propagate or
modify it is void, and will automatically terminate your rights under
this License (including any patent licenses granted under the third
paragraph of section 11).
However, if you cease all violation of this License, then your
license from a particular copyright holder is reinstated (a)
provisionally, unless and until the copyright holder explicitly and
finally terminates your license, and (b) permanently, if the copyright
holder fails to notify you of the violation by some reasonable means
prior to 60 days after the cessation.
Moreover, your license from a particular copyright holder is
reinstated permanently if the copyright holder notifies you of the
violation by some reasonable means, this is the first time you have
received notice of violation of this License (for any work) from that
copyright holder, and you cure the violation prior to 30 days after
your receipt of the notice.
Termination of your rights under this section does not terminate the
licenses of parties who have received copies or rights from you under
this License. If your rights have been terminated and not permanently
reinstated, you do not qualify to receive new licenses for the same
material under section 10.
9. Acceptance Not Required for Having Copies.
You are not required to accept this License in order to receive or
run a copy of the Program. Ancillary propagation of a covered work
occurring solely as a consequence of using peer-to-peer transmission
to receive a copy likewise does not require acceptance. However,
nothing other than this License grants you permission to propagate or
modify any covered work. These actions infringe copyright if you do
not accept this License. Therefore, by modifying or propagating a
covered work, you indicate your acceptance of this License to do so.
10. Automatic Licensing of Downstream Recipients.
Each time you convey a covered work, the recipient automatically
receives a license from the original licensors, to run, modify and
propagate that work, subject to this License. You are not responsible
for enforcing compliance by third parties with this License.
An "entity transaction" is a transaction transferring control of an
organization, or substantially all assets of one, or subdividing an
organization, or merging organizations. If propagation of a covered
work results from an entity transaction, each party to that
transaction who receives a copy of the work also receives whatever
licenses to the work the party's predecessor in interest had or could
give under the previous paragraph, plus a right to possession of the
Corresponding Source of the work from the predecessor in interest, if
the predecessor has it or can get it with reasonable efforts.
You may not impose any further restrictions on the exercise of the
rights granted or affirmed under this License. For example, you may
not impose a license fee, royalty, or other charge for exercise of
rights granted under this License, and you may not initiate litigation
(including a cross-claim or counterclaim in a lawsuit) alleging that
any patent claim is infringed by making, using, selling, offering for
sale, or importing the Program or any portion of it.
11. Patents.
A "contributor" is a copyright holder who authorizes use under this
License of the Program or a work on which the Program is based. The
work thus licensed is called the contributor's "contributor version".
A contributor's "essential patent claims" are all patent claims
owned or controlled by the contributor, whether already acquired or
hereafter acquired, that would be infringed by some manner, permitted
by this License, of making, using, or selling its contributor version,
but do not include claims that would be infringed only as a
consequence of further modification of the contributor version. For
purposes of this definition, "control" includes the right to grant
patent sublicenses in a manner consistent with the requirements of
this License.
Each contributor grants you a non-exclusive, worldwide, royalty-free
patent license under the contributor's essential patent claims, to
make, use, sell, offer for sale, import and otherwise run, modify and
propagate the contents of its contributor version.
In the following three paragraphs, a "patent license" is any express
agreement or commitment, however denominated, not to enforce a patent
(such as an express permission to practice a patent or covenant not to
sue for patent infringement). To "grant" such a patent license to a
party means to make such an agreement or commitment not to enforce a
patent against the party.
If you convey a covered work, knowingly relying on a patent license,
and the Corresponding Source of the work is not available for anyone
to copy, free of charge and under the terms of this License, through a
publicly available network server or other readily accessible means,
then you must either (1) cause the Corresponding Source to be so
available, or (2) arrange to deprive yourself of the benefit of the
patent license for this particular work, or (3) arrange, in a manner
consistent with the requirements of this License, to extend the patent
license to downstream recipients. "Knowingly relying" means you have
actual knowledge that, but for the patent license, your conveying the
covered work in a country, or your recipient's use of the covered work
in a country, would infringe one or more identifiable patents in that
country that you have reason to believe are valid.
If, pursuant to or in connection with a single transaction or
arrangement, you convey, or propagate by procuring conveyance of, a
covered work, and grant a patent license to some of the parties
receiving the covered work authorizing them to use, propagate, modify
or convey a specific copy of the covered work, then the patent license
you grant is automatically extended to all recipients of the covered
work and works based on it.
A patent license is "discriminatory" if it does not include within
the scope of its coverage, prohibits the exercise of, or is
conditioned on the non-exercise of one or more of the rights that are
specifically granted under this License. You may not convey a covered
work if you are a party to an arrangement with a third party that is
in the business of distributing software, under which you make payment
to the third party based on the extent of your activity of conveying
the work, and under which the third party grants, to any of the
parties who would receive the covered work from you, a discriminatory
patent license (a) in connection with copies of the covered work
conveyed by you (or copies made from those copies), or (b) primarily
for and in connection with specific products or compilations that
contain the covered work, unless you entered into that arrangement,
or that patent license was granted, prior to 28 March 2007.
Nothing in this License shall be construed as excluding or limiting
any implied license or other defenses to infringement that may
otherwise be available to you under applicable patent law.
12. No Surrender of Others' Freedom.
If conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License. If you cannot convey a
covered work so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you may
not convey it at all. For example, if you agree to terms that obligate you
to collect a royalty for further conveying from those to whom you convey
the Program, the only way you could satisfy both those terms and this
License would be to refrain entirely from conveying the Program.
13. Remote Network Interaction; Use with the GNU General Public License.
Notwithstanding any other provision of this License, if you modify the
Program, your modified version must prominently offer all users
interacting with it remotely through a computer network (if your version
supports such interaction) an opportunity to receive the Corresponding
Source of your version by providing access to the Corresponding Source
from a network server at no charge, through some standard or customary
means of facilitating copying of software. This Corresponding Source
shall include the Corresponding Source for any work covered by version 3
of the GNU General Public License that is incorporated pursuant to the
following paragraph.
Notwithstanding any other provision of this License, you have
permission to link or combine any covered work with a work licensed
under version 3 of the GNU General Public License into a single
combined work, and to convey the resulting work. The terms of this
License will continue to apply to the part which is the covered work,
but the work with which it is combined will remain governed by version
3 of the GNU General Public License.
14. Revised Versions of this License.
The Free Software Foundation may publish revised and/or new versions of
the GNU Affero General Public License from time to time. Such new versions
will be similar in spirit to the present version, but may differ in detail to
address new problems or concerns.
Each version is given a distinguishing version number. If the
Program specifies that a certain numbered version of the GNU Affero General
Public License "or any later version" applies to it, you have the
option of following the terms and conditions either of that numbered
version or of any later version published by the Free Software
Foundation. If the Program does not specify a version number of the
GNU Affero General Public License, you may choose any version ever published
by the Free Software Foundation.
If the Program specifies that a proxy can decide which future
versions of the GNU Affero General Public License can be used, that proxy's
public statement of acceptance of a version permanently authorizes you
to choose that version for the Program.
Later license versions may give you additional or different
permissions. However, no additional obligations are imposed on any
author or copyright holder as a result of your choosing to follow a
later version.
15. Disclaimer of Warranty.
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
16. Limitation of Liability.
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
SUCH DAMAGES.
17. Interpretation of Sections 15 and 16.
If the disclaimer of warranty and limitation of liability provided
above cannot be given local legal effect according to their terms,
reviewing courts shall apply local law that most closely approximates
an absolute waiver of all civil liability in connection with the
Program, unless a warranty or assumption of liability accompanies a
copy of the Program in return for a fee.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Programs
If you develop a new program, and you want it to be of the greatest
possible use to the public, the best way to achieve this is to make it
free software which everyone can redistribute and change under these terms.
To do so, attach the following notices to the program. It is safest
to attach them to the start of each source file to most effectively
state the exclusion of warranty; and each file should have at least
the "copyright" line and a pointer to where the full notice is found.
<one line to give the program's name and a brief idea of what it does.>
Copyright (C) <year> <name of author>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published
by the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
Also add information on how to contact you by electronic and paper mail.
If your software can interact with users remotely through a computer
network, you should also make sure that it provides a way for users to
get its source. For example, if your program is a web application, its
interface could display a "Source" link that leads users to an archive
of the code. There are many ways you could offer source, and different
solutions will be better for different programs; see section 13 for the
specific requirements.
You should also get your employer (if you work as a programmer) or school,
if any, to sign a "copyright disclaimer" for the program, if necessary.
For more information on this, and how to apply and follow the GNU AGPL, see
<https://www.gnu.org/licenses/>.

61
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/NOTICE.md Normal file
View File

@ -0,0 +1,61 @@
# Third-party fingerprint algorithm licensing
This project implements several published TLS fingerprinting algorithms. They do
not all carry the same license, and the split matters. This notice records what
applies to what.
## Our own code
Everything under `crates/` is original work licensed under the AGPL 3.0 License (see
`LICENSE`). No source code from any reference implementation was copied. The
algorithms were implemented from their published specifications.
## JA3 and JA3S
JA3 and JA3S were created by John Althouse, Jeff Atkinson, and Josh Atkins at
Salesforce and released under the BSD 3-Clause license at
`https://github.com/salesforce/ja3`. The algorithm is free to implement and use.
That repository was archived in May 2025 and is no longer maintained.
## JA4 (TLS client fingerprint)
JA4, the TLS client fingerprint, is licensed by FoxIO under the BSD 3-Clause
license, separately from the rest of the JA4+ suite. FoxIO has stated it holds
no patents on JA4 TLS client fingerprinting. The canonical license text is at
`https://github.com/FoxIO-LLC/ja4/blob/main/LICENSE-JA4`. JA4 may be implemented
and used without restriction.
## JA4S, JA4H, JA4X, JA4T (the rest of the JA4+ suite)
The remaining JA4+ fingerprints implemented here are licensed under the FoxIO
License 1.1, and the methods are patent pending. The canonical license text is
at `https://github.com/FoxIO-LLC/ja4/blob/main/LICENSE`.
The FoxIO License 1.1 grants the right to use and modify these methods for non
commercial purposes, which it defines to include personal use, academic research
and development, and internal evaluation. It excludes any use for which a fee is
charged and excludes providing the methods as a hosted or managed service to
others.
This project is a free, open source, educational tool. It is not sold, not
monetized, and not offered as a service. It therefore falls within the non
commercial grant of the FoxIO License 1.1. Anyone who forks this project and
intends to monetize it, or to offer it as a hosted service, must obtain an OEM
license from FoxIO for the JA4+ methods first.
This is the same boundary that led the Suricata project to ship only JA4 and not
the rest of the JA4+ suite: their GPL licensing is incompatible with the FoxIO
License 1.1. This project is MIT licensed and non commercial, so both the GPL
incompatibility and the monetization restriction are avoided.
## Seeded fingerprint data
The intelligence database is seeded from public sources with their own terms:
- abuse.ch SSLBL JA3 feed is released under CC0 and a snapshot may be
redistributed. Its known limitation, that the fingerprints have not been
tested against benign traffic and may produce false positives, is surfaced in
the tool output.
- The salesforce/ja3 application lists are MIT licensed.
- The ja4db.com database has no stated redistribution license, so it is fetched
at install time rather than bundled, and entries are validated on import.

211
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/README.md Normal file
View File

@ -0,0 +1,211 @@
<!-- ©AngelaMos | 2026 -->
<!-- README.md -->
```json
████████╗██╗ ███████╗███████╗██████╗
╚══██╔══╝██║ ██╔════╝██╔════╝██╔══██╗
██║ ██║ ███████╗█████╗ ██████╔╝
██║ ██║ ╚════██║██╔══╝ ██╔═══╝
██║ ███████╗███████║██║ ██║
╚═╝ ╚══════╝╚══════╝╚═╝ ╚═╝
```
[![Cybersecurity Projects](https://img.shields.io/badge/Cybersecurity--Projects-Project%20%2334-red?style=flat&logo=github)](https://github.com/CarterPerez-dev/Cybersecurity-Projects/tree/main/PROJECTS/intermediate/ja3-ja4-tls-fingerprinting)
[![Rust](https://img.shields.io/badge/Rust-edition%202024-CE412B?style=flat&logo=rust&logoColor=white)](https://www.rust-lang.org)
[![JA4+](https://img.shields.io/badge/JA4%2B-JA3%20%C2%B7%20JA4%20%C2%B7%20JA4H%20%C2%B7%20JA4X%20%C2%B7%20JA4T-4B7BEC?style=flat)](https://github.com/FoxIO-LLC/ja4)
[![License: AGPLv3](https://img.shields.io/badge/License-AGPL_v3-purple.svg)](https://www.gnu.org/licenses/agpl-3.0)
---
[![Live demo](https://img.shields.io/badge/demo-mkultraalumni.com-9b59b6?style=flat&logo=cloudflare&logoColor=white)](https://mkultraalumni.com)
> A passive TLS fingerprinting sensor in Rust. Point it at a capture file or a live interface and it computes the JA3, JA4, JA4S, JA4H, JA4X, and JA4T fingerprints of every handshake, matches them against a local intelligence database, and flags the things a fingerprint alone cannot hide: a TLS stack that disagrees with its own User-Agent, a brand-new fingerprint, a client that rotates its identity to evade a blocklist. It reads TCP and the TLS hidden inside QUIC initial packets, never sends a byte, and carries half a million fingerprints a second.
## Why fingerprint TLS
When a client opens a TLS connection, the very first message it sends, the ClientHello, is a detailed self-description: which TLS versions it supports, which cipher suites in which order, which extensions, which elliptic curves. A browser, a Go program, a Python script, and a piece of malware each assemble that message differently, because each is built on a different TLS library configured a different way. The ClientHello travels in the clear, before any encryption is negotiated, so a passive observer who never decrypts anything can still read it.
A fingerprint is a hash of those choices. The same software produces the same fingerprint on every connection, so a fingerprint that is on a blocklist today catches the same malware family tomorrow even if its IP, its domain, and its certificate all changed. That is the idea behind JA3, published by Salesforce in 2017, and JA4, its 2023 successor from FoxIO that fixed the one weakness that eventually killed JA3 for browser traffic: when Chrome started shuffling its extension order on every connection, JA3, which hashes extensions in wire order, produced a fresh hash every time. JA4 sorts first, so the shuffle changes nothing.
This project builds the whole sensor around that idea, in a language where a parser bug is a memory-safety bug. The fingerprinting core forbids `unsafe`, the capture path is bounded so an adversarial packet cannot exhaust memory, and every fingerprint is checked byte for byte against the reference implementations.
## What Works Today
This is not a stub. The tool fingerprints real captures, decrypts real QUIC, matches against real public threat feeds, and raises real alerts, and every capability below is exercised by a known-answer test against a published vector, an integration test against a vendored capture, and a run of the actual `tlsfp` binary.
**Fingerprints**
- **JA3 / JA3S** (MD5 of the ClientHello / ServerHello field list), kept because public malware feeds still speak JA3 and because watching it collapse next to JA4 is the clearest way to see why JA4 exists
- **JA4 / JA4S** (the FoxIO TLS client and server fingerprint, sorted cipher and extension lists), the headline fingerprint, stable under extension shuffling
- **JA4H** the HTTP client fingerprint, from a cleartext request's method, version, header order, cookies, and accept-language
- **JA4X** the X.509 fingerprint, from the issuer, subject, and extension object identifiers of a certificate, which clusters certificates minted by one toolchain
- **JA4T / JA4TS** the TCP-stack fingerprint, from the SYN's window size, options, MSS, and window scale, which catches a tool wearing a browser's TLS clothing while its OS speaks with a different TCP accent
- GREASE values stripped from every list, so the deliberate noise a modern client inserts never changes its fingerprint
**Capture**
- Reads `pcap` and `pcapng` files, and captures live from an interface through `libpcap` with the raw-socket capabilities dropped to exactly the two the kernel needs
- A reassembly layer rebuilds each direction of each TCP conversation, surviving reordering, retransmission, and overlap, so a ClientHello split across segments still fingerprints
- Bounded by construction: a flow cap, an idle timeout, and per-stream byte ceilings keep an adversarial capture from turning the flow table into a memory bomb
**QUIC**
- Decrypts QUIC Initial packets to read the ClientHello inside, deriving the client initial keys from the packet's own Destination Connection ID per RFC 9001 (QUIC v1) and RFC 9369 (QUIC v2), with no server-side secret
- Reassembles CRYPTO frames across packets, so a QUIC ClientHello spread over several initials still yields a `q`-transport JA4
**Intelligence**
- A bundled SQLite database seeded from three vendored feeds with no network call: abuse.ch SSLBL, the Salesforce `osx-nix` JA3 list, and a small curated C2 set (**271 fingerprints**)
- An optional install-time pull of ja4db.com, validated record by record on the way in
- Exact lookups plus JA4 fuzzy matching on the capability-and-cipher prefix, scored into a verdict with a threat score and a confidence
**Detection**
- Six rules that run as a capture streams: `known_bad` (a feed hit), `ua_mismatch` (the headline: a JA4 that disagrees with its own User-Agent), `os_mismatch` (a JA4T that disagrees with the OS the User-Agent claims), `first_seen`, `fp_rotation`, and `monoculture`
- A forensic `--report` mode that reads a whole capture and prints one ranked summary, folding in intelligence and detection automatically whenever the database is present
- A web dashboard ([live demo](https://mkultraalumni.com)) that streams events and alerts over Server-Sent Events, fed by a replayed capture, a live interface, or an external sensor tailing the same database
See [`learn/CONFORMANCE.md`](learn/CONFORMANCE.md) for the exact published vector each fingerprint is pinned to, and every deliberate scope boundary.
## Quick Start
```bash
curl -fsSL https://angelamos.com/tlsfp/install.sh | bash
```
The installer builds the release binary, puts `tlsfp` on your PATH, and seeds the intelligence database. Pass `--live` to also grant the raw-socket capabilities that live capture needs. Then point it at a capture:
```bash
# Fingerprint every handshake in a capture, one line each
tlsfp pcap traffic.pcapng
# Seed the threat feeds, then match and run the detection rules
tlsfp intel seed
tlsfp pcap traffic.pcapng --report
# Watch an interface in real time, matching and detecting as it goes
sudo setcap cap_net_raw,cap_net_admin=eip "$(command -v tlsfp)"
tlsfp live eth0 --intel --detect
```
A single fingerprint line looks like this, a Chrome handshake to a Google host:
```
1675707151.805 192.168.1.168:50112 -> 142.251.16.94:443 client_hello ja4=t13d1516h2_8daaf6152771_e5627efa2ab1 ja3=1c258ebef8eee2dfa3df6d8d07285af9 sni=clientservices.googleapis.com alpn=h2
```
> [!TIP]
> This project uses [`just`](https://github.com/casey/just) as a command runner. Type `just` to see every recipe. `just bench` runs the throughput benchmarks; `just dev-up` brings up the dockerized dashboard with hot reload.
>
> Install: `curl -sSf https://just.systems/install.sh | bash -s -- --to ~/.local/bin`
## Learn
This project ships a full teaching track. Read it in order, or jump to what you need.
| Doc | What it covers |
|-----|----------------|
| [`learn/00-OVERVIEW.md`](learn/00-OVERVIEW.md) | What TLS fingerprinting is, why it works, and a 10-minute tour |
| [`learn/01-CONCEPTS.md`](learn/01-CONCEPTS.md) | The ClientHello, JA3 vs JA4, GREASE, evasion, QUIC, passive capture, grounded in real intrusions |
| [`learn/02-ARCHITECTURE.md`](learn/02-ARCHITECTURE.md) | The three-crate split, the capture pipeline, the intelligence store, the threat model |
| [`learn/03-IMPLEMENTATION.md`](learn/03-IMPLEMENTATION.md) | A code walkthrough from a raw frame to a scored alert, and the reassembly and bounding patterns |
| [`learn/ALGORITHMS.md`](learn/ALGORITHMS.md) | How each fingerprint is computed byte by byte, and how a QUIC initial is decrypted |
| [`learn/CONFORMANCE.md`](learn/CONFORMANCE.md) | The published vector each fingerprint is pinned to, and every deliberate scope boundary |
| [`learn/04-CHALLENGES.md`](learn/04-CHALLENGES.md) | Extension ideas from beginner to expert |
## Architecture
Three crates, in a strict dependency line. The engine knows nothing about databases or networks; the intelligence store knows nothing about capture; the binary wires them together.
```
pcap / pcapng file live interface (libpcap) QUIC initial
│ │ │
└─────────────┬───────────┴──────────────────────────┘
│ raw link-layer frames
▼
┌────────────────────────────────────────────────────┐
│ tlsfp-core the engine, no I/O, forbids unsafe │
│ decode → flow reassembly → TLS/HTTP/QUIC → hash │
│ ja3 · ja4 · ja4h · ja4x · ja4t · parse · quic │
└───────────────────────┬────────────────────────────┘
│ FingerprintEvent
┌───────────────────────┴────────────────────────────┐
│ tlsfp-intel the judgement, a bundled SQLite DB │
│ match (exact + JA4 fuzzy) → score → detection rules │
│ matcher · seed · import · detect · signal · schema │
└───────────────────────┬────────────────────────────┘
│ MatchReport + Alert
┌───────────────────────┴────────────────────────────┐
│ tlsfp the binary: CLI + web dashboard │
│ pcap · live · serve (axum + SSE) · intel · report │
└────────────────────────────────────────────────────┘
```
**Design decisions:** the engine forbids `unsafe` outright, so a malformed packet can never be more than a parse error. The store is deliberately synchronous, because a lookup is one indexed query and a capture is a plain loop; the async runtime lives only in the web server, where concurrent readers actually need it. JA3 uses MD5 because that is what the original definition and every public JA3 feed use, and reproducing those feed hashes is the whole point of keeping it. The QUIC decryption uses no server secret because the client initial keys are derived from a Connection ID that travels in the clear.
## Build and Test
```bash
cargo build --release # the shipped binary → target/release/tlsfp
cargo test --workspace # 204 unit + integration tests, 1 ignored
cargo bench -p tlsfp-core # criterion throughput benchmarks
just clippy # clippy::pedantic, warnings as errors
just fmt-check # rustfmt
```
Every fingerprint is pinned to a published vector. The JA3 tests reproduce the original Salesforce blog vectors through MD5; the JA4 tests reproduce the FoxIO cipher, extension, and TCP section vectors; the QUIC tests derive the client initial keys and match RFC 9001 Appendix A (v1) and RFC 9369 Appendix A (v2) byte for byte. The reassembly tests rebuild a ClientHello from out-of-order and overlapping segments. The JA4X parser has a property-test fuzz harness because it walks attacker-controlled certificate DER.
The benchmarks replay vendored captures frame by frame through the whole pipeline. On a modern laptop the pipeline sustains roughly **380,000 to 500,000 fingerprints per second**, comfortably past the project target of 10,000.
## Run in Docker
No Rust toolchain on the host? The dashboard runs entirely in containers.
```bash
just up # production stack: built dashboard + backend
just dev-up # development stack: vite hot reload
```
The production image is a multi-stage build that compiles the release binary in a Rust builder and ships only the binary plus the built dashboard assets behind nginx. The development stack bind-mounts the frontend and runs `pnpm install` on startup, so an added package is always present after a restart.
## Project Structure
```
ja3-ja4-tls-fingerprinting/
├── Cargo.toml # the 3-crate virtual workspace
├── crates/
│ ├── tlsfp-core/ # the engine: no I/O, forbids unsafe
│ │ ├── src/
│ │ │ ├── parse/ # TLS record, ClientHello, ServerHello, certificate readers
│ │ │ ├── pipeline/ # decode → flow reassembly → TLS/HTTP → event
│ │ │ ├── ja3.rs # JA3 / JA3S (the dead-but-still-fed MD5 fingerprint)
│ │ │ ├── ja4.rs # JA4 / JA4S (the headline sorted fingerprint)
│ │ │ ├── ja4h.rs # JA4H (the HTTP request fingerprint)
│ │ │ ├── ja4x.rs # JA4X (the X.509 certificate fingerprint)
│ │ │ ├── ja4t.rs # JA4T / JA4TS (the TCP-stack fingerprint)
│ │ │ ├── quic.rs # QUIC initial decryption (RFC 9001 + RFC 9369)
│ │ │ ├── grease.rs # the GREASE value table and the strip
│ │ │ ├── der.rs # the minimal DER reader JA4X needs
│ │ │ └── registry.rs # version codes and extension constants
│ │ ├── benches/fingerprint.rs# criterion throughput benchmarks
│ │ └── tests/ # KAT + integration: ja3, ja4, ja4x, parse, reassembly
│ ├── tlsfp-intel/ # the judgement: a bundled SQLite store
│ │ ├── src/
│ │ │ ├── schema.rs # the migrations
│ │ │ ├── seed.rs # the three vendored feeds, compiled in
│ │ │ ├── import.rs # the validated ja4db.com importer
│ │ │ ├── matcher.rs # exact + JA4 fuzzy lookup, scored into a verdict
│ │ │ ├── detect.rs # the six detection rules
│ │ │ ├── signal.rs # the User-Agent / OS heuristics the rules read
│ │ │ └── model.rs # FpKind, Category, Verdict, the report types
│ │ └── seeds/ # the vendored CSV feeds
│ └── tlsfp/ # the binary
│ └── src/
│ ├── cli.rs # the clap command tree
│ ├── live.rs # the libpcap capture thread and the async bridge
│ ├── report.rs # the forensic --report builder
│ └── serve.rs # the axum dashboard + SSE stream
├── frontend/ # the anti-design dashboard (Vite + React 19)
├── testdata/pcap/ # vendored captures, the integration fixtures
├── install.sh # the one-shot curl-able installer
└── justfile # every recipe
```
## License
[AGPL 3.0](LICENSE). The vendored threat feeds under `crates/tlsfp-intel/seeds/` keep their original licenses, recorded per feed in [`NOTICE.md`](NOTICE.md) and in the `intel_source` table.

27
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/cloudflared.compose.yml Normal file
View File

@ -0,0 +1,27 @@
# =============================================================================
# ©AngelaMos | 2026
# cloudflared.compose.yml
# =============================================================================
# Cloudflare Tunnel for production remote access
# Usage: docker compose -f compose.yml -f cloudflared.compose.yml up -d
# =============================================================================
services:
cloudflared:
image: cloudflare/cloudflared:latest
container_name: ${APP_NAME:-tlsfp}-tunnel
command: tunnel run --token ${CLOUDFLARE_TUNNEL_TOKEN}
networks:
- app
depends_on:
nginx:
condition: service_started
deploy:
resources:
limits:
cpus: '0.5'
memory: 128M
reservations:
cpus: '0.1'
memory: 32M
restart: unless-stopped

75
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/compose.yml Normal file
View File

@ -0,0 +1,75 @@
# =============================================================================
# ©AngelaMos | 2026
# compose.yml
# =============================================================================
# Production compose: nginx serves the built React app and proxies /api to the
# tlsfp backend. The backend lives under the `backend` profile and is not
# started by default until the serve command is implemented.
# docker compose --env-file .env up
# docker compose --env-file .env --profile backend up
# docker compose -f compose.yml -f cloudflared.compose.yml up
# =============================================================================
name: ${APP_NAME:-tlsfp}
services:
nginx:
build:
context: .
dockerfile: infra/docker/vite.prod
args:
- VITE_API_URL=${VITE_API_URL:-/api}
- VITE_APP_TITLE=${VITE_APP_TITLE:-JA3/JA4 TLS Fingerprinting}
container_name: ${APP_NAME:-tlsfp}-nginx
ports:
- "${NGINX_HOST_PORT:-11790}:80"
networks:
- app
deploy:
resources:
limits:
cpus: '1.0'
memory: 256M
reservations:
cpus: '0.25'
memory: 64M
restart: unless-stopped
tlsfp:
build:
context: .
dockerfile: infra/docker/tlsfp.prod
container_name: ${APP_NAME:-tlsfp}-backend
# Demo posture: seed the bundled corpus, then serve while replaying a
# vendored capture on a loop so the dashboard is alive the moment it boots.
# For a real sensor instead, set command back to ["serve", "0.0.0.0:8080"]
# and feed it from `tlsfp live --detect` against the shared /data volume.
entrypoint: ["/bin/sh", "-c"]
command:
- |
tlsfp intel seed --db /data/intel.db || true
exec tlsfp serve 0.0.0.0:8080 --db /data/intel.db --replay /demo/demo.pcap --loop --interval-ms 600
profiles:
- backend
environment:
- RUST_LOG=${RUST_LOG:-tlsfp=info}
volumes:
- tlsfp_data:/data
networks:
- app
deploy:
resources:
limits:
cpus: '1.0'
memory: 256M
reservations:
cpus: '0.25'
memory: 64M
restart: unless-stopped
networks:
app:
driver: bridge
volumes:
tlsfp_data:

38
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/Cargo.toml Normal file
View File

@ -0,0 +1,38 @@
# ©AngelaMos | 2026
# Cargo.toml
[package]
name = "tlsfp-core"
description = "TLS handshake parsing and JA3/JA4 fingerprint computation"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
authors.workspace = true
repository.workspace = true
[lints]
workspace = true
[dependencies]
md-5.workspace = true
sha2.workspace = true
hex.workspace = true
smallvec.workspace = true
thiserror.workspace = true
serde.workspace = true
etherparse.workspace = true
pcap-parser.workspace = true
hkdf.workspace = true
aes.workspace = true
aes-gcm.workspace = true
[dev-dependencies]
hex.workspace = true
proptest = "1"
serde_json.workspace = true
criterion = { version = "0.5", default-features = false, features = ["cargo_bench_support"] }
[[bench]]
name = "fingerprint"
harness = false

215
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/benches/fingerprint.rs Normal file
View File

@ -0,0 +1,215 @@
// ©AngelaMos | 2026
// fingerprint.rs
//! Throughput benchmarks for the fingerprinting hot path.
//!
//! Two questions matter for a passive sensor: how fast it parses one handshake,
//! and how fast it carries a whole capture through the pipeline. The first
//! benchmark isolates parsing and hashing a single ClientHello; the second
//! replays a vendored capture frame by frame, reporting fingerprints per second
//! against the project target of ten thousand. The capture is read into memory
//! once at setup so the file system never enters the measured loop.
use std::hint::black_box;
use std::path::Path;
use criterion::{Criterion, Throughput, criterion_group, criterion_main};
use tlsfp_core::ja3::ja3;
use tlsfp_core::ja4::{Transport, ja4};
use tlsfp_core::parse::parse_client_hello;
use tlsfp_core::pipeline::source::{PacketSource, PcapFileSource, RawFrame};
use tlsfp_core::pipeline::{Pipeline, PipelineConfig};
/// Captures replayed by the pipeline benchmark, named for the report.
const PCAPS: [(&str, &str); 2] = [
(
"tls-handshake",
concat!(
env!("CARGO_MANIFEST_DIR"),
"/../../testdata/pcap/tls-handshake.pcapng"
),
),
(
"browsers-x509",
concat!(
env!("CARGO_MANIFEST_DIR"),
"/../../testdata/pcap/browsers-x509.pcapng"
),
),
];
/// One captured frame copied into an owned buffer for replay.
struct OwnedFrame {
ts_nanos: u64,
link_type: i32,
data: Vec<u8>,
}
/// Reads every frame of a capture into memory so the benchmark loop measures
/// the pipeline rather than the file system.
fn load_frames(path: &str) -> Vec<OwnedFrame> {
let mut source = PcapFileSource::open(Path::new(path)).expect("open capture");
let mut frames = Vec::new();
while let Some(frame) = source.next_frame().expect("read frame") {
frames.push(OwnedFrame {
ts_nanos: frame.ts_nanos,
link_type: frame.link_type,
data: frame.data.to_vec(),
});
}
frames
}
/// Replays preloaded frames through a fresh pipeline, returning the event count.
fn replay(frames: &[OwnedFrame]) -> u64 {
let mut pipeline = Pipeline::new(PipelineConfig::default());
for frame in frames {
let raw = RawFrame {
ts_nanos: frame.ts_nanos,
link_type: frame.link_type,
data: &frame.data,
};
pipeline.feed(&raw, &mut |event| {
black_box(&event);
});
}
pipeline.finish();
pipeline.counters().events
}
fn bench_pipeline(c: &mut Criterion) {
let mut group = c.benchmark_group("pipeline");
for (name, path) in PCAPS {
let frames = load_frames(path);
let events = replay(&frames).max(1);
group.throughput(Throughput::Elements(events));
group.bench_function(name, |b| {
b.iter(|| black_box(replay(black_box(&frames))));
});
}
group.finish();
}
fn bench_fingerprint(c: &mut Criterion) {
let body = client_hello_body();
let hello = parse_client_hello(&body).expect("the benchmark hello parses");
let mut group = c.benchmark_group("fingerprint");
group.bench_function("parse_client_hello", |b| {
b.iter(|| black_box(parse_client_hello(black_box(&body)).expect("parse")));
});
group.bench_function("ja3", |b| b.iter(|| black_box(ja3(black_box(&hello)))));
group.bench_function("ja4", |b| {
b.iter(|| black_box(ja4(black_box(&hello), Transport::Tcp)));
});
group.finish();
}
/// Assembles a representative TLS 1.3 ClientHello body, the input the parser and
/// the hash functions take. The exact bytes are not pinned to any published
/// vector here: this is a throughput fixture, and the conformance vectors live
/// in the integration tests.
fn client_hello_body() -> Vec<u8> {
const CIPHERS: [u16; 15] = [
0x1301, 0x1302, 0x1303, 0xc02b, 0xc02f, 0xc02c, 0xc030, 0xcca9, 0xcca8, 0xc013, 0xc014,
0x009c, 0x009d, 0x002f, 0x0035,
];
const GROUPS: [u16; 4] = [0x001d, 0x0017, 0x0018, 0x0019];
const SIG_ALGS: [u16; 8] = [
0x0403, 0x0804, 0x0401, 0x0503, 0x0805, 0x0501, 0x0806, 0x0601,
];
const VERSIONS: [u16; 2] = [0x0304, 0x0303];
const OPAQUE: [u16; 4] = [0x0005, 0x0017, 0x0023, 0xff01];
let mut body = Vec::new();
body.extend_from_slice(&0x0303u16.to_be_bytes());
body.extend_from_slice(&[0u8; 32]);
body.push(0);
let mut ciphers = Vec::new();
for suite in CIPHERS {
ciphers.extend_from_slice(&suite.to_be_bytes());
}
push_u16(&mut body, &ciphers);
push_u8(&mut body, &[0]);
let mut exts = Vec::new();
add_ext(&mut exts, 0x0000, &sni("example.com"));
add_ext(&mut exts, 0x000a, &u16_list_u16(&GROUPS));
add_ext(&mut exts, 0x000b, &u8_list(&[0]));
add_ext(&mut exts, 0x000d, &u16_list_u16(&SIG_ALGS));
add_ext(&mut exts, 0x0010, &alpn(&[b"h2", b"http/1.1"]));
add_ext(&mut exts, 0x002b, &u8_list_u16(&VERSIONS));
for ext in OPAQUE {
add_ext(&mut exts, ext, &[]);
}
push_u16(&mut body, &exts);
body
}
fn push_u8(out: &mut Vec<u8>, data: &[u8]) {
out.push(u8::try_from(data.len()).expect("length fits a byte"));
out.extend_from_slice(data);
}
fn push_u16(out: &mut Vec<u8>, data: &[u8]) {
out.extend_from_slice(
&u16::try_from(data.len())
.expect("length fits two bytes")
.to_be_bytes(),
);
out.extend_from_slice(data);
}
fn add_ext(out: &mut Vec<u8>, ext_type: u16, data: &[u8]) {
out.extend_from_slice(&ext_type.to_be_bytes());
push_u16(out, data);
}
fn sni(host: &str) -> Vec<u8> {
let mut entry = vec![0u8];
push_u16(&mut entry, host.as_bytes());
let mut data = Vec::new();
push_u16(&mut data, &entry);
data
}
fn alpn(protocols: &[&[u8]]) -> Vec<u8> {
let mut list = Vec::new();
for protocol in protocols {
push_u8(&mut list, protocol);
}
let mut data = Vec::new();
push_u16(&mut data, &list);
data
}
fn u16_list_u16(values: &[u16]) -> Vec<u8> {
let mut list = Vec::new();
for value in values {
list.extend_from_slice(&value.to_be_bytes());
}
let mut data = Vec::new();
push_u16(&mut data, &list);
data
}
fn u8_list_u16(values: &[u16]) -> Vec<u8> {
let mut list = Vec::new();
for value in values {
list.extend_from_slice(&value.to_be_bytes());
}
let mut data = Vec::new();
push_u8(&mut data, &list);
data
}
fn u8_list(values: &[u8]) -> Vec<u8> {
let mut data = Vec::new();
push_u8(&mut data, values);
data
}
criterion_group!(benches, bench_pipeline, bench_fingerprint);
criterion_main!(benches);

111
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/der.rs Normal file
View File

@ -0,0 +1,111 @@
// ©AngelaMos | 2026
// der.rs
use crate::error::{ParseError, Result};
/// DER tag bytes that the certificate walk needs to recognize.
pub mod tag {
pub const OBJECT_IDENTIFIER: u8 = 0x06;
pub const SEQUENCE: u8 = 0x30;
pub const SET: u8 = 0x31;
pub const CONTEXT_0: u8 = 0xa0;
pub const CONTEXT_3: u8 = 0xa3;
}
/// A minimal reader for the subset of DER that X.509 certificates use.
///
/// It decodes one tag length value triple at a time and hands back the content
/// as a borrowed slice. It deliberately does not understand the meaning of any
/// structure; the JA4X walk drives it, deciding which fields to descend into and
/// which to skip. Keeping the reader this small keeps it auditable, which
/// matters because certificate parsers are a classic source of memory safety
/// bugs and this one has no `unsafe` and cannot read out of bounds.
pub struct Der<'a> {
buf: &'a [u8],
pos: usize,
}
impl<'a> Der<'a> {
#[must_use]
pub const fn new(buf: &'a [u8]) -> Self {
Self { buf, pos: 0 }
}
#[must_use]
pub const fn is_empty(&self) -> bool {
self.pos >= self.buf.len()
}
fn byte(&mut self) -> Result<u8> {
if self.pos >= self.buf.len() {
return Err(ParseError::Truncated { needed: 1, have: 0 });
}
let b = self.buf[self.pos];
self.pos += 1;
Ok(b)
}
/// Reads one tag length value triple and returns the tag and the content
/// slice, advancing past the whole triple.
pub fn read_tlv(&mut self) -> Result<(u8, &'a [u8])> {
let tag = self.byte()?;
let len = self.read_length()?;
if self.buf.len() - self.pos < len {
return Err(ParseError::LengthOverrun {
field: "der",
declared: len,
available: self.buf.len() - self.pos,
});
}
let content = &self.buf[self.pos..self.pos + len];
self.pos += len;
Ok((tag, content))
}
fn read_length(&mut self) -> Result<usize> {
let first = self.byte()?;
if first & 0x80 == 0 {
return Ok(first as usize);
}
let count = (first & 0x7f) as usize;
if count == 0 || count > core::mem::size_of::<usize>() {
return Err(ParseError::Misaligned(count));
}
let mut len = 0usize;
for _ in 0..count {
len = (len << 8) | self.byte()? as usize;
}
Ok(len)
}
}
#[cfg(test)]
mod tests {
use super::{Der, tag};
#[test]
fn reads_short_form_sequence() {
let data = [0x30, 0x03, 0x06, 0x01, 0x55];
let mut der = Der::new(&data);
let (t, content) = der.read_tlv().unwrap();
assert_eq!(t, tag::SEQUENCE);
assert_eq!(content, &[0x06, 0x01, 0x55]);
assert!(der.is_empty());
}
#[test]
fn reads_long_form_length() {
let mut data = vec![0x04, 0x82, 0x01, 0x00];
data.extend(std::iter::repeat_n(0xaa, 256));
let mut der = Der::new(&data);
let (_t, content) = der.read_tlv().unwrap();
assert_eq!(content.len(), 256);
}
#[test]
fn rejects_length_overrun() {
let data = [0x06, 0x05, 0x55, 0x04];
let mut der = Der::new(&data);
assert!(der.read_tlv().is_err());
}
}

54
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/error.rs Normal file
View File

@ -0,0 +1,54 @@
// ©AngelaMos | 2026
// error.rs
use thiserror::Error;
/// Errors produced while parsing TLS records, handshake messages, or QUIC
/// initial packets.
///
/// Every variant is stack only. No variant carries a heap allocation, so the
/// malformed packet path never touches the allocator. This matters because a
/// fingerprinting engine spends most of its time rejecting traffic that is not
/// a clean handshake, and allocating on each rejection adds jitter at line rate.
#[derive(Debug, Error, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum ParseError {
#[error("buffer too short: needed {needed} bytes, had {have}")]
Truncated { needed: usize, have: usize },
#[error("not a TLS handshake record (content type {0:#04x})")]
NotHandshake(u8),
#[error("unexpected handshake message type {0:#04x}")]
UnexpectedHandshake(u8),
#[error("length field {field} declares {declared} bytes but {available} remain")]
LengthOverrun {
field: &'static str,
declared: usize,
available: usize,
},
#[error("vector length {0} is not a whole number of elements")]
Misaligned(usize),
#[error("trailing {0} bytes after a complete message")]
Trailing(usize),
#[error("handshake message spans more bytes than the reassembly cap allows")]
OversizedHandshake,
#[error("malformed extension {ext_type:#06x}")]
BadExtension { ext_type: u16 },
#[error("not a QUIC long header initial packet")]
NotQuicInitial,
#[error("unsupported QUIC version {0:#010x}")]
UnsupportedQuicVersion(u32),
#[error("QUIC header protection or AEAD removal failed")]
QuicCryptoFailure,
}
pub type Result<T> = core::result::Result<T, ParseError>;

100
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/fingerprint.rs Normal file
View File

@ -0,0 +1,100 @@
// ©AngelaMos | 2026
// fingerprint.rs
use core::fmt;
use serde::Serialize;
/// A JA3 or JA3S fingerprint: the MD5 digest of the pre hash string.
///
/// JA3 is carried as the raw sixteen byte digest rather than a hex string so
/// that equality, hashing, and database keys operate on the compact binary form
/// and the hex rendering happens only at display boundaries.
#[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize)]
#[serde(into = "String")]
pub struct Ja3([u8; 16]);
impl Ja3 {
#[must_use]
pub const fn from_digest(digest: [u8; 16]) -> Self {
Self(digest)
}
#[must_use]
pub const fn bytes(&self) -> &[u8; 16] {
&self.0
}
}
impl fmt::Display for Ja3 {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
for byte in self.0 {
write!(f, "{byte:02x}")?;
}
Ok(())
}
}
impl fmt::Debug for Ja3 {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "Ja3({self})")
}
}
impl From<Ja3> for String {
fn from(value: Ja3) -> Self {
value.to_string()
}
}
/// A fingerprint from the JA4 family, carried in both its hashed canonical form
/// and its raw pre hash form.
///
/// The raw form is the unhashed list of cipher and extension values. It is kept
/// alongside the hash because it is the form an analyst reads when explaining
/// why two clients differ, and because clustering on the raw lists is more
/// informative than clustering on opaque digests.
#[derive(Clone, PartialEq, Eq, Hash, Serialize)]
pub struct Ja4Family {
pub hash: String,
pub raw: String,
}
impl Ja4Family {
#[must_use]
pub fn new(hash: String, raw: String) -> Self {
Self { hash, raw }
}
}
impl fmt::Display for Ja4Family {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(&self.hash)
}
}
impl fmt::Debug for Ja4Family {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "Ja4Family(hash={}, raw={})", self.hash, self.raw)
}
}
#[cfg(test)]
mod tests {
use super::{Ja3, Ja4Family};
#[test]
fn ja3_display_is_lowercase_hex() {
let fp = Ja3::from_digest([
0xad, 0xa7, 0x02, 0x06, 0xe4, 0x06, 0x42, 0xa3, 0xe4, 0x46, 0x1f, 0x35, 0x50, 0x32,
0x41, 0xd5,
]);
assert_eq!(fp.to_string(), "ada70206e40642a3e4461f35503241d5");
}
#[test]
fn ja4_family_displays_hash() {
let fp = Ja4Family::new("t13d1516h2_8daaf6152771_e5627efa2ab1".into(), "raw".into());
assert_eq!(fp.to_string(), "t13d1516h2_8daaf6152771_e5627efa2ab1");
}
}

52
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/grease.rs Normal file
View File

@ -0,0 +1,52 @@
// ©AngelaMos | 2026
// grease.rs
/// The sixteen GREASE values reserved by RFC 8701.
///
/// GREASE (Generate Random Extensions And Sustain Extensibility) values are
/// inserted by clients into cipher lists, extension lists, supported groups,
/// supported versions, and signature algorithms. They are deliberate noise
/// whose only purpose is to keep middleboxes tolerant of unknown values. Both
/// JA3 and JA4 strip them before hashing so that the same client produces a
/// stable fingerprint regardless of which GREASE values it happened to pick.
///
/// Every value has the form `0xZaZa` where the two bytes are equal and the low
/// nibble of each is `a`.
pub const GREASE_VALUES: [u16; 16] = [
0x0a0a, 0x1a1a, 0x2a2a, 0x3a3a, 0x4a4a, 0x5a5a, 0x6a6a, 0x7a7a, 0x8a8a, 0x9a9a, 0xaaaa, 0xbaba,
0xcaca, 0xdada, 0xeaea, 0xfafa,
];
/// Returns true when `value` is one of the sixteen RFC 8701 GREASE values.
///
/// The check exploits the structural regularity of the GREASE set rather than
/// scanning the table: both bytes must be equal and the low nibble of each must
/// be `0xa`. This is a single pair of comparisons rather than a sixteen way
/// branch, which keeps it cheap on the per packet path.
#[inline]
#[must_use]
pub const fn is_grease(value: u16) -> bool {
let high = (value >> 8) as u8;
let low = (value & 0x00ff) as u8;
high == low && (low & 0x0f) == 0x0a
}
#[cfg(test)]
mod tests {
use super::{GREASE_VALUES, is_grease};
#[test]
fn table_matches_structural_check() {
for v in 0..=u16::MAX {
let in_table = GREASE_VALUES.contains(&v);
assert_eq!(in_table, is_grease(v), "mismatch for {v:#06x}");
}
}
#[test]
fn known_non_grease_values() {
for v in [0x0000_u16, 0x0010, 0x1301, 0x00ff, 0x5600, 0xc02f] {
assert!(!is_grease(v), "{v:#06x} should not be grease");
}
}
}

26
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/hash.rs Normal file
View File

@ -0,0 +1,26 @@
// ©AngelaMos | 2026
// hash.rs
use sha2::{Digest, Sha256};
/// Returns the first twelve hex characters of the SHA256 digest of `bytes`.
///
/// The whole JA4 family truncates SHA256 to twelve hex characters, which is six
/// bytes of digest. Twelve characters is enough to make accidental collisions
/// vanishingly unlikely across the fingerprint space while keeping the
/// fingerprint short enough to read and to paste into a search box.
#[must_use]
pub fn sha256_hex12(bytes: &[u8]) -> String {
let digest = Sha256::digest(bytes);
hex::encode(&digest[..6])
}
#[cfg(test)]
mod tests {
use super::sha256_hex12;
#[test]
fn known_digest_prefix() {
assert_eq!(sha256_hex12(b""), "e3b0c44298fc");
}
}

125
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/ja3.rs Normal file
View File

@ -0,0 +1,125 @@
// ©AngelaMos | 2026
// ja3.rs
use std::fmt::Write as _;
use md5::{Digest, Md5};
use smallvec::SmallVec;
use crate::fingerprint::Ja3;
use crate::grease::is_grease;
use crate::parse::{ClientHello, ServerHello};
/// Computes the JA3 string and digest for a ClientHello.
///
/// JA3 concatenates five fields, in order: the legacy version, the cipher
/// suites, the extension types, the supported groups, and the elliptic curve
/// point formats. Within a field the values are decimal and joined with
/// hyphens; the fields themselves are joined with commas. GREASE values are
/// removed from every list field so that the deliberate noise a client inserts
/// does not change its fingerprint. The MD5 of that string is the JA3 hash.
///
/// JA3 is kept here despite being effectively dead for modern browser traffic,
/// because malware fingerprints in public feeds are still expressed as JA3 and
/// because watching JA3 collapse next to JA4 is the clearest way to understand
/// why JA4 exists.
#[must_use]
pub fn ja3_string(ch: &ClientHello) -> String {
let ext_types: SmallVec<[u16; 16]> = ch.extensions.iter().map(|e| e.ext_type).collect();
let groups = ch.supported_groups();
let formats = ch.ec_point_formats();
let mut s = String::new();
let _ = write!(s, "{}", ch.legacy_version);
s.push(',');
append_u16_hyphenated(&mut s, &ch.cipher_suites);
s.push(',');
append_u16_hyphenated(&mut s, &ext_types);
s.push(',');
append_u16_hyphenated(&mut s, &groups);
s.push(',');
append_u8_hyphenated(&mut s, &formats);
s
}
/// Computes the JA3 digest for a ClientHello.
#[must_use]
pub fn ja3(ch: &ClientHello) -> Ja3 {
digest(&ja3_string(ch))
}
/// Computes the JA3S string for a ServerHello.
///
/// JA3S mirrors JA3 on the server side with three fields: the version, the
/// single chosen cipher suite, and the extension types. A server and the exact
/// ClientHello it answered together identify a deployment more tightly than
/// either side alone.
#[must_use]
pub fn ja3s_string(sh: &ServerHello) -> String {
let ext_types: SmallVec<[u16; 16]> = sh.extensions.iter().map(|e| e.ext_type).collect();
let mut s = String::new();
let _ = write!(s, "{},{},", sh.legacy_version, sh.cipher_suite);
append_u16_hyphenated(&mut s, &ext_types);
s
}
/// Computes the JA3S digest for a ServerHello.
#[must_use]
pub fn ja3s(sh: &ServerHello) -> Ja3 {
digest(&ja3s_string(sh))
}
fn digest(pre_hash: &str) -> Ja3 {
let out = Md5::digest(pre_hash.as_bytes());
let mut bytes = [0u8; 16];
bytes.copy_from_slice(&out);
Ja3::from_digest(bytes)
}
fn append_u16_hyphenated(out: &mut String, values: &[u16]) {
let mut first = true;
for &v in values {
if is_grease(v) {
continue;
}
if !first {
out.push('-');
}
first = false;
let _ = write!(out, "{v}");
}
}
fn append_u8_hyphenated(out: &mut String, values: &[u8]) {
let mut first = true;
for &v in values {
if !first {
out.push('-');
}
first = false;
let _ = write!(out, "{v}");
}
}
#[cfg(test)]
mod tests {
use super::digest;
#[test]
fn salesforce_client_vector_one() {
let pre = "769,47-53-5-10-49161-49162-49171-49172-50-56-19-4,0-10-11,23-24-25,0";
assert_eq!(digest(pre).to_string(), "ada70206e40642a3e4461f35503241d5");
}
#[test]
fn salesforce_client_vector_two_empty_fields() {
let pre = "769,4-5-10-9-100-98-3-6-19-18-99,,,";
assert_eq!(digest(pre).to_string(), "de350869b8c85de67a350c8d186f11e6");
}
#[test]
fn server_vector_round_trips_through_md5() {
let pre = "769,47,65281-0-11-35-5-16";
assert_eq!(digest(pre).to_string(), "836ce314215654b5b1f85f97c73e506f");
}
}

263
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/ja4.rs Normal file
View File

@ -0,0 +1,263 @@
// ©AngelaMos | 2026
// ja4.rs
use std::fmt::Write as _;
use smallvec::SmallVec;
use crate::fingerprint::Ja4Family;
use crate::grease::is_grease;
use crate::hash::sha256_hex12;
use crate::parse::{ClientHello, ServerHello};
use crate::registry::{extension, ja4_version_code};
/// The transport that carried the handshake.
///
/// JA4 records the transport in its first character because the same TLS stack
/// produces a recognizably different ClientHello over QUIC than over TCP, and an
/// analyst wants to see that difference at a glance rather than infer it.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Transport {
Tcp,
Quic,
Dtls,
}
impl Transport {
const fn marker(self) -> char {
match self {
Transport::Tcp => 't',
Transport::Quic => 'q',
Transport::Dtls => 'd',
}
}
}
const EMPTY_HASH: &str = "000000000000";
/// Computes the JA4 fingerprint of a ClientHello in both hashed and raw forms.
///
/// The fingerprint has three underscore separated sections. The first is ten
/// human readable characters: transport, version, whether SNI was present,
/// cipher count, extension count, and two characters derived from the first
/// ALPN value. The second is a truncated SHA256 of the cipher list sorted by
/// value. The third is a truncated SHA256 of the extension list sorted by value,
/// with SNI and ALPN removed and the signature algorithms appended in their
/// original order.
///
/// Sorting the cipher and extension lists before hashing is the whole reason
/// JA4 survived what killed JA3. When Chrome began shuffling its extension order
/// on every connection, JA3, which hashes extensions in wire order, produced a
/// fresh hash for every Chrome request. JA4 sorts first, so order shuffling
/// leaves the fingerprint unchanged.
#[must_use]
pub fn ja4(ch: &ClientHello, transport: Transport) -> Ja4Family {
let prefix = ja4_prefix(ch, transport);
let (cipher_csv, _) = sorted_hex_csv(&ch.cipher_suites, &[]);
let cipher_hash = truncated_sha256(&cipher_csv);
let ext_raw = ja4_extension_raw(ch);
let ext_hash = truncated_sha256(&ext_raw);
let hash = format!("{prefix}_{cipher_hash}_{ext_hash}");
let raw = format!("{prefix}_{cipher_csv}_{ext_raw}");
Ja4Family::new(hash, raw)
}
/// Computes the JA4S fingerprint of a ServerHello.
///
/// JA4S mirrors JA4 on the server side, with three differences that follow from
/// what a server controls. The server picks exactly one cipher suite, so the
/// cipher section is that single value in hex rather than a hash of a list. The
/// extensions are hashed in the order the server sent them, not sorted, because
/// a server does not shuffle its own extensions. And there is no SNI field,
/// because the server is not the party naming a host.
#[must_use]
pub fn ja4s(sh: &ServerHello, transport: Transport) -> Ja4Family {
let version = ja4_version_code(sh.selected_version());
let alpn = alpn_chars(sh.alpn_protocol());
let ext_count = sh.extensions.len().min(99);
let mut prefix = String::with_capacity(7);
prefix.push(transport.marker());
prefix.push_str(version);
let _ = write!(prefix, "{ext_count:02}");
prefix.push_str(&alpn);
let cipher_hex = format!("{:04x}", sh.cipher_suite);
let ext_csv = wire_order_hex_csv(
&sh.extensions
.iter()
.map(|e| e.ext_type)
.collect::<SmallVec<[u16; 16]>>(),
);
let ext_hash = truncated_sha256(&ext_csv);
let hash = format!("{prefix}_{cipher_hex}_{ext_hash}");
let raw = format!("{prefix}_{cipher_hex}_{ext_csv}");
Ja4Family::new(hash, raw)
}
fn wire_order_hex_csv(values: &[u16]) -> String {
let hexed: SmallVec<[String; 16]> = values.iter().map(|v| format!("{v:04x}")).collect();
hexed.join(",")
}
fn ja4_prefix(ch: &ClientHello, transport: Transport) -> String {
let version = select_version(ch);
let sni = if ch.has_extension(extension::SERVER_NAME) {
'd'
} else {
'i'
};
let cipher_count = capped_count(ch.cipher_suites.iter().copied());
let ext_count = capped_count(ch.extensions.iter().map(|e| e.ext_type));
let alpn = ja4_alpn(ch);
let mut prefix = String::with_capacity(10);
prefix.push(transport.marker());
prefix.push_str(ja4_version_code(version));
prefix.push(sni);
let _ = write!(prefix, "{cipher_count:02}{ext_count:02}");
prefix.push_str(&alpn);
prefix
}
/// Selects the JA4 version word: the highest non GREASE value from the supported
/// versions extension when present, otherwise the legacy record version.
fn select_version(ch: &ClientHello) -> u16 {
ch.supported_versions()
.iter()
.copied()
.filter(|v| !is_grease(*v))
.max()
.unwrap_or(ch.legacy_version)
}
fn capped_count(values: impl Iterator<Item = u16>) -> usize {
values.filter(|v| !is_grease(*v)).count().min(99)
}
/// Derives the two ALPN characters.
///
/// The implementation follows the published JA4 specification rather than the
/// FoxIO Python reference. The two diverge for ALPN values whose first or last
/// byte is not an ASCII alphanumeric: the specification says to print the first
/// and last characters of the hex encoding of the value, while the Python
/// reference emits a fixed fallback. The specification is the more informative
/// and more portable choice, so it is what this code does.
fn ja4_alpn(ch: &ClientHello) -> String {
alpn_chars(ch.alpn_protocols().first().copied())
}
fn alpn_chars(first: Option<&[u8]>) -> String {
let Some(first) = first else {
return "00".to_string();
};
if first.is_empty() {
return "00".to_string();
}
let first_byte = first[0];
let last_byte = first[first.len() - 1];
if is_ascii_alphanumeric(first_byte) && is_ascii_alphanumeric(last_byte) {
let mut out = String::with_capacity(2);
out.push(first_byte as char);
out.push(last_byte as char);
out
} else {
let encoded = hex::encode(first);
let mut chars = encoded.chars();
let first_char = chars.next().unwrap_or('0');
let last_char = encoded.chars().last().unwrap_or('0');
let mut out = String::with_capacity(2);
out.push(first_char);
out.push(last_char);
out
}
}
const fn is_ascii_alphanumeric(byte: u8) -> bool {
byte.is_ascii_digit() || byte.is_ascii_uppercase() || byte.is_ascii_lowercase()
}
/// Builds the raw, pre hash extension string for section three.
///
/// The extension types are sorted by hex value after removing GREASE, SNI, and
/// ALPN. If a signature algorithms extension is present, its values are appended
/// after an underscore, in their original order. If it is absent, the string
/// ends without a trailing underscore, which is the behavior the specification
/// requires and which changes the resulting hash.
fn ja4_extension_raw(ch: &ClientHello) -> String {
let excluded = [extension::SERVER_NAME, extension::ALPN];
let ext_types: SmallVec<[u16; 16]> = ch.extensions.iter().map(|e| e.ext_type).collect();
let (ext_csv, _) = sorted_hex_csv(&ext_types, &excluded);
let sig_algs = ch.signature_algorithms();
if ch.has_extension(extension::SIGNATURE_ALGORITHMS) {
let sig_csv = unsorted_hex_csv(&sig_algs);
format!("{ext_csv}_{sig_csv}")
} else {
ext_csv
}
}
fn sorted_hex_csv(values: &[u16], excluded: &[u16]) -> (String, usize) {
let mut hexed: SmallVec<[String; 32]> = values
.iter()
.copied()
.filter(|v| !is_grease(*v) && !excluded.contains(v))
.map(|v| format!("{v:04x}"))
.collect();
hexed.sort_unstable();
(hexed.join(","), hexed.len())
}
fn unsorted_hex_csv(values: &[u16]) -> String {
let hexed: SmallVec<[String; 16]> = values
.iter()
.copied()
.filter(|v| !is_grease(*v))
.map(|v| format!("{v:04x}"))
.collect();
hexed.join(",")
}
fn truncated_sha256(input: &str) -> String {
if input.is_empty() {
return EMPTY_HASH.to_string();
}
sha256_hex12(input.as_bytes())
}
#[cfg(test)]
mod tests {
use super::{EMPTY_HASH, truncated_sha256};
#[test]
fn empty_input_is_the_zero_hash() {
assert_eq!(truncated_sha256(""), EMPTY_HASH);
}
#[test]
fn truncation_is_twelve_hex_chars() {
let h = truncated_sha256("1301,1302,1303");
assert_eq!(h.len(), 12);
assert!(h.chars().all(|c| c.is_ascii_hexdigit()));
}
#[test]
fn foxio_cipher_section_vector() {
let ciphers = "002f,0035,009c,009d,1301,1302,1303,c013,c014,c02b,c02c,c02f,c030,cca8,cca9";
assert_eq!(truncated_sha256(ciphers), "8daaf6152771");
}
#[test]
fn foxio_extension_section_vector() {
let exts = "0005,000a,000b,000d,0012,0015,0017,001b,0023,002b,002d,0033,4469,ff01_0403,0804,0401,0503,0805,0501,0806,0601";
assert_eq!(truncated_sha256(exts), "e5627efa2ab1");
}
}

195
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/ja4h.rs Normal file
View File

@ -0,0 +1,195 @@
// ©AngelaMos | 2026
// ja4h.rs
use crate::fingerprint::Ja4Family;
use crate::hash::sha256_hex12;
/// A parsed HTTP request, holding what JA4H reads.
///
/// JA4H fingerprints an HTTP client from one request: its method, version,
/// whether it carries cookies and a referer, the names of its other headers in
/// the order they were sent, its accept language, and its cookie names and
/// values. A request that omits an accept language and sends no cookies is far
/// more likely to be a script than a person, and JA4H makes that visible in the
/// first few characters.
///
/// This applies to cleartext HTTP only. Over HTTPS the request is encrypted and
/// invisible to a passive observer, and HTTP/2 carries its headers in HPACK,
/// which a passive tool cannot decode without following the whole connection.
#[derive(Debug, Clone)]
pub struct HttpRequest {
pub method: String,
pub version: String,
pub headers: Vec<(String, String)>,
}
/// Parses an HTTP/1.x request from the start of a reassembled byte stream.
///
/// Returns `None` when the bytes are not a well formed request line followed by
/// headers. Header values keep their original bytes; header names keep their
/// original case because JA4H hashes them as sent.
#[must_use]
pub fn parse_http_request(bytes: &[u8]) -> Option<HttpRequest> {
let text = std::str::from_utf8(bytes).ok()?;
let mut lines = text.split("\r\n");
let request_line = lines.next()?;
let mut parts = request_line.split(' ');
let method = parts.next()?.to_string();
let _target = parts.next()?;
let http_token = parts.next()?;
if !http_token.starts_with("HTTP/") {
return None;
}
let version = http_token.trim_start_matches("HTTP/").replace('.', "");
let mut headers = Vec::new();
for line in lines {
if line.is_empty() {
break;
}
let (name, value) = line.split_once(':')?;
headers.push((name.to_string(), value.trim_start().to_string()));
}
Some(HttpRequest {
method,
version,
headers,
})
}
/// Computes the JA4H fingerprint for a parsed HTTP request.
#[must_use]
pub fn ja4h(req: &HttpRequest) -> Ja4Family {
let method = method_code(&req.method);
let version = version_code(&req.version);
let cookie_flag = if has_header(req, "cookie") { 'c' } else { 'n' };
let referer_flag = if has_named_header(req, "referer") {
'r'
} else {
'n'
};
let counted: Vec<&str> = req
.headers
.iter()
.map(|(name, _)| name.as_str())
.filter(|name| is_counted_header(name))
.collect();
let header_len = counted.len().min(99);
let lang = accept_language(req);
let prefix = format!("{method}{version}{cookie_flag}{referer_flag}{header_len:02}{lang}");
let header_hash = sha12(&counted.join(","));
let cookies = cookie_pairs(req);
let (cookie_hash, value_hash, raw_cookie_tail) = if let Some(mut pairs) = cookies {
pairs.sort_by(|a, b| a.0.cmp(&b.0));
let names: Vec<&str> = pairs.iter().map(|p| p.0.as_str()).collect();
let entries: Vec<&str> = pairs.iter().map(|p| p.1.as_str()).collect();
let tail = format!("{}_{}", names.join(","), entries.join(","));
(sha12(&names.join(",")), sha12(&entries.join(",")), tail)
} else {
(ZERO_HASH.to_string(), ZERO_HASH.to_string(), String::new())
};
let hash = format!("{prefix}_{header_hash}_{cookie_hash}_{value_hash}");
let raw = format!("{prefix}_{}_{raw_cookie_tail}", counted.join(","));
Ja4Family::new(hash, raw)
}
const ZERO_HASH: &str = "000000000000";
fn method_code(method: &str) -> String {
method.to_lowercase().chars().take(2).collect()
}
fn version_code(version: &str) -> String {
match version {
"2" | "20" => "20".to_string(),
"3" | "30" => "30".to_string(),
"10" => "10".to_string(),
_ => "11".to_string(),
}
}
fn has_header(req: &HttpRequest, prefix_lower: &str) -> bool {
req.headers
.iter()
.any(|(name, _)| name.to_lowercase().starts_with(prefix_lower))
}
fn has_named_header(req: &HttpRequest, name_lower: &str) -> bool {
req.headers
.iter()
.any(|(name, _)| name.to_lowercase() == name_lower)
}
fn is_counted_header(name: &str) -> bool {
let lower = name.to_lowercase();
!name.starts_with(':') && !lower.starts_with("cookie") && lower != "referer" && !name.is_empty()
}
fn accept_language(req: &HttpRequest) -> String {
let Some((_, value)) = req
.headers
.iter()
.find(|(name, _)| name.to_lowercase() == "accept-language")
else {
return "0000".to_string();
};
let normalized = value.replace('-', "").replace(';', ",").to_lowercase();
let first = normalized.split(',').next().unwrap_or("");
let mut code: String = first.chars().take(4).collect();
while code.len() < 4 {
code.push('0');
}
code
}
fn cookie_pairs(req: &HttpRequest) -> Option<Vec<(String, String)>> {
let (_, value) = req
.headers
.iter()
.find(|(name, _)| name.to_lowercase() == "cookie")?;
let pairs = value
.split(';')
.map(str::trim)
.filter(|p| !p.is_empty())
.map(|p| {
let name = p.split('=').next().unwrap_or(p).trim().to_string();
(name, p.to_string())
})
.collect();
Some(pairs)
}
fn sha12(joined: &str) -> String {
sha256_hex12(joined.as_bytes())
}
#[cfg(test)]
mod tests {
use super::{parse_http_request, version_code};
#[test]
fn parses_request_line_and_headers() {
let raw = b"GET /index.html HTTP/1.1\r\nHost: example.com\r\nAccept: */*\r\n\r\n";
let req = parse_http_request(raw).unwrap();
assert_eq!(req.method, "GET");
assert_eq!(req.version, "11");
assert_eq!(req.headers.len(), 2);
assert_eq!(
req.headers[0],
("Host".to_string(), "example.com".to_string())
);
}
#[test]
fn version_codes() {
assert_eq!(version_code("11"), "11");
assert_eq!(version_code("10"), "10");
assert_eq!(version_code("2"), "20");
}
}

81
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/ja4t.rs Normal file
View File

@ -0,0 +1,81 @@
// ©AngelaMos | 2026
// ja4t.rs
use std::fmt::Write as _;
use smallvec::SmallVec;
/// The TCP layer inputs to a JA4T fingerprint, read from a SYN or SYN ACK.
///
/// JA4T fingerprints the TCP stack rather than the TLS stack. It is computed
/// from fields that an operating system sets the same way on every connection
/// but that differ between operating systems: the advertised window size, the
/// kinds and order of TCP options, the maximum segment size, and the window
/// scale factor. Pairing it with JA4 exposes a class of evasion that TLS only
/// fingerprinting misses, where a tool wears a browser's TLS clothing while its
/// host operating system speaks with a different TCP accent.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TcpFingerprintInput {
pub window_size: u16,
pub option_kinds: SmallVec<[u8; 8]>,
pub mss: u16,
pub window_scale: u8,
}
/// Computes the JA4T or JA4TS string.
///
/// The format is the window size, then the option kinds joined with hyphens,
/// then the maximum segment size, then the window scale, with the four parts
/// separated by underscores. A missing MSS or window scale option is reported as
/// zero. The same function serves both the client SYN, which yields JA4T, and
/// the server SYN ACK, which yields JA4TS.
#[must_use]
pub fn ja4t(input: &TcpFingerprintInput) -> String {
let mut options = String::new();
let mut first = true;
for kind in &input.option_kinds {
if !first {
options.push('-');
}
first = false;
let _ = write!(options, "{kind}");
}
format!(
"{}_{}_{}_{}",
input.window_size, options, input.mss, input.window_scale
)
}
#[cfg(test)]
mod tests {
use super::{TcpFingerprintInput, ja4t};
use smallvec::SmallVec;
fn input(window: u16, kinds: &[u8], mss: u16, scale: u8) -> TcpFingerprintInput {
TcpFingerprintInput {
window_size: window,
option_kinds: SmallVec::from_slice(kinds),
mss,
window_scale: scale,
}
}
#[test]
fn foxio_windows_default_vector() {
let i = input(64240, &[2, 1, 3, 1, 1, 4], 1460, 8);
assert_eq!(ja4t(&i), "64240_2-1-3-1-1-4_1460_8");
}
#[test]
fn foxio_windowed_vector() {
let i = input(65535, &[2, 1, 3, 1, 1, 8, 4, 0, 0], 1346, 6);
assert_eq!(ja4t(&i), "65535_2-1-3-1-1-8-4-0-0_1346_6");
}
#[test]
fn missing_mss_and_scale_report_zero() {
let i = input(8192, &[2, 4], 0, 0);
assert_eq!(ja4t(&i), "8192_2-4_0_0");
}
}

214
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/ja4x.rs Normal file
View File

@ -0,0 +1,214 @@
// ©AngelaMos | 2026
// ja4x.rs
use smallvec::SmallVec;
use crate::der::{Der, tag};
use crate::error::{ParseError, Result};
use crate::hash::sha256_hex12;
/// Computes the JA4X fingerprint of one DER encoded X.509 certificate.
///
/// JA4X does not fingerprint the contents of a certificate. It fingerprints how
/// the certificate was built: which object identifiers appear in the issuer
/// name, which appear in the subject name, and which appear among the
/// extensions, each in the order they were written. Two certificates minted by
/// the same software with the same template share a JA4X even when every name
/// and serial differs, which is what makes it useful for clustering certificates
/// from one toolchain or one malware family.
///
/// Passively this only works on TLS 1.2 and earlier, where the certificate
/// travels in the clear. TLS 1.3 encrypts the certificate message, so a passive
/// observer never sees it.
pub fn ja4x(cert_der: &[u8]) -> Result<String> {
let (issuer_oids, subject_oids, ext_oids) = certificate_oids(cert_der)?;
let issuer_hash = sha256_hex12(issuer_oids.join(",").as_bytes());
let subject_hash = sha256_hex12(subject_oids.join(",").as_bytes());
let ext_hash = sha256_hex12(ext_oids.join(",").as_bytes());
Ok(format!("{issuer_hash}_{subject_hash}_{ext_hash}"))
}
type OidList = SmallVec<[String; 8]>;
/// Walks a certificate down to the issuer Name, the subject Name, and the
/// extension block, the three field groups JA4X reads.
///
/// The TBSCertificate fields sit in a fixed order (RFC 5280): an optional
/// explicit version tagged context zero, then the serial number, signature
/// algorithm, issuer, validity, subject, and public key, with the extensions
/// tagged context three at the end. The walk names every field it steps over and
/// bounds checks each one, so a certificate truncated before the public key or
/// the extensions is an error rather than a panic.
fn certificate_oids(cert_der: &[u8]) -> Result<(OidList, OidList, OidList)> {
let mut outer = Der::new(cert_der);
let (_, certificate) = outer.read_tlv()?;
let mut cert = Der::new(certificate);
let (_, tbs) = cert.read_tlv()?;
let mut fields: SmallVec<[(u8, &[u8]); 10]> = SmallVec::new();
let mut walker = Der::new(tbs);
while !walker.is_empty() {
fields.push(walker.read_tlv()?);
}
let mut cursor = 0usize;
if fields.first().is_some_and(|(t, _)| *t == tag::CONTEXT_0) {
cursor += 1;
}
let _serial = field_content(&fields, cursor)?;
cursor += 1;
let _signature = field_content(&fields, cursor)?;
cursor += 1;
let issuer = field_content(&fields, cursor)?;
cursor += 1;
let _validity = field_content(&fields, cursor)?;
cursor += 1;
let subject = field_content(&fields, cursor)?;
cursor += 1;
let extensions = fields
.get(cursor..)
.unwrap_or(&[])
.iter()
.find(|(t, _)| *t == tag::CONTEXT_3)
.map(|(_, c)| *c);
let issuer_oids = name_oids(issuer)?;
let subject_oids = name_oids(subject)?;
let ext_oids = match extensions {
Some(content) => extension_oids(content)?,
None => OidList::new(),
};
Ok((issuer_oids, subject_oids, ext_oids))
}
fn field_content<'a>(fields: &[(u8, &'a [u8])], idx: usize) -> Result<&'a [u8]> {
fields
.get(idx)
.map(|(_, c)| *c)
.ok_or(ParseError::Truncated { needed: 1, have: 0 })
}
/// Collects the attribute type object identifiers from a Name, in order.
///
/// A Name is a sequence of relative distinguished names, each a set of attribute
/// type and value pairs. The fingerprint reads only the attribute type, the
/// leading object identifier of each pair, and renders it as the hex of its DER
/// content bytes, which is exactly the form JA4X hashes.
fn name_oids(name: &[u8]) -> Result<OidList> {
let mut oids = OidList::new();
let mut rdns = Der::new(name);
while !rdns.is_empty() {
let (_, rdn) = rdns.read_tlv()?;
let mut set = Der::new(rdn);
while !set.is_empty() {
let (_, atv) = set.read_tlv()?;
let mut pair = Der::new(atv);
let (oid_tag, oid) = pair.read_tlv()?;
if oid_tag == tag::OBJECT_IDENTIFIER {
oids.push(hex::encode(oid));
}
}
}
Ok(oids)
}
/// Collects the extension object identifiers, in order.
fn extension_oids(context: &[u8]) -> Result<OidList> {
let mut oids = OidList::new();
let mut wrapper = Der::new(context);
let (_, sequence) = wrapper.read_tlv()?;
let mut exts = Der::new(sequence);
while !exts.is_empty() {
let (_, ext) = exts.read_tlv()?;
let mut fields = Der::new(ext);
let (oid_tag, oid) = fields.read_tlv()?;
if oid_tag == tag::OBJECT_IDENTIFIER {
oids.push(hex::encode(oid));
}
}
Ok(oids)
}
#[cfg(test)]
mod tests {
use super::{extension_oids, name_oids};
fn oid(content: &[u8]) -> Vec<u8> {
let mut v = vec![0x06, u8::try_from(content.len()).unwrap()];
v.extend_from_slice(content);
v
}
fn tlv(tag: u8, content: &[u8]) -> Vec<u8> {
let mut v = vec![tag, u8::try_from(content.len()).unwrap()];
v.extend_from_slice(content);
v
}
#[test]
fn extracts_issuer_oids_in_order() {
let mut name = Vec::new();
for content in [
&[0x55u8, 0x04, 0x06][..],
&[0x55, 0x04, 0x0a],
&[0x55, 0x04, 0x0b],
&[0x55, 0x04, 0x03],
] {
let atv = tlv(0x30, &{
let mut a = oid(content);
a.extend(tlv(0x13, b"x"));
a
});
name.extend(tlv(0x31, &atv));
}
let oids = name_oids(&name).unwrap();
assert_eq!(oids.as_slice(), &["550406", "55040a", "55040b", "550403"]);
}
#[test]
fn certificate_ending_after_subject_is_handled_without_panic() {
let rdn = tlv(
0x31,
&tlv(0x30, &{
let mut atv = oid(&[0x55, 0x04, 0x03]);
atv.extend(tlv(0x13, b"x"));
atv
}),
);
let tbs = tlv(
0x30,
&[
tlv(0x02, &[0x01]),
tlv(0x30, &[]),
tlv(0x30, &rdn),
tlv(0x30, &[]),
tlv(0x30, &rdn),
]
.concat(),
);
let cert = tlv(0x30, &tbs);
assert!(super::ja4x(&cert).is_ok());
}
#[test]
fn extracts_extension_oids() {
let mut sequence = Vec::new();
for content in [&[0x55u8, 0x1d, 0x23][..], &[0x55, 0x1d, 0x0e]] {
let ext = tlv(0x30, &{
let mut e = oid(content);
e.extend(tlv(0x04, b"v"));
e
});
sequence.extend(ext);
}
let context = tlv(0x30, &sequence);
let oids = extension_oids(&context).unwrap();
assert_eq!(oids.as_slice(), &["551d23", "551d0e"]);
}
}

38
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/lib.rs Normal file
View File

@ -0,0 +1,38 @@
// ©AngelaMos | 2026
// lib.rs
//! TLS handshake parsing and JA3/JA4 family fingerprint computation.
//!
//! This crate is the engine. It parses TLS records and handshake messages, the
//! TLS carried inside QUIC initial packets, and computes the JA3, JA3S, JA4,
//! JA4S, JA4H, JA4X, and JA4T fingerprints. It depends on nothing that touches
//! a network interface, a database, or an async runtime, so it can be embedded,
//! fuzzed, and unit tested in isolation.
pub mod der;
pub mod error;
pub mod fingerprint;
pub mod grease;
pub mod hash;
pub mod ja3;
pub mod ja4;
pub mod ja4h;
pub mod ja4t;
pub mod ja4x;
pub mod parse;
pub mod pipeline;
pub mod quic;
pub mod registry;
pub use error::{ParseError, Result};
pub use fingerprint::{Ja3, Ja4Family};
pub use grease::{GREASE_VALUES, is_grease};
pub use ja3::{ja3, ja3_string, ja3s, ja3s_string};
pub use ja4::{Transport, ja4, ja4s};
pub use ja4h::{HttpRequest, ja4h, parse_http_request};
pub use ja4t::{TcpFingerprintInput, ja4t};
pub use ja4x::ja4x;
pub use parse::{ClientHello, Extension, ServerHello};
pub use pipeline::event::{FingerprintEvent, StreamEvent};
pub use pipeline::source::{PacketSource, PcapFileSource, RawFrame, SourceError};
pub use pipeline::{Counters, Pipeline, PipelineConfig};

70
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/parse/cert.rs Normal file
View File

@ -0,0 +1,70 @@
// ©AngelaMos | 2026
// cert.rs
use smallvec::SmallVec;
use crate::error::Result;
use crate::parse::reader::Reader;
/// The DER certificates carried by one TLS Certificate handshake message.
pub type CertificateList<'pkt> = SmallVec<[&'pkt [u8]; 4]>;
/// Extracts the DER encoded certificates from a Certificate message body.
///
/// The body is a three byte total length followed by entries that are each a
/// three byte length and the raw DER bytes. This is the TLS 1.2 framing; it is
/// the only framing a passive observer ever parses, because TLS 1.3 moved the
/// Certificate message inside the encrypted part of the handshake. The
/// certificates come back in wire order, leaf first, which is the order JA4X
/// reports them in.
pub fn certificate_der_list(body: &[u8]) -> Result<CertificateList<'_>> {
let mut r = Reader::new(body);
let mut list = r.sub_u24_vec()?;
let mut certs = CertificateList::new();
while !list.is_empty() {
certs.push(list.take_u24_vec()?);
}
Ok(certs)
}
#[cfg(test)]
mod tests {
use super::certificate_der_list;
use crate::error::ParseError;
fn message(certs: &[&[u8]]) -> Vec<u8> {
let total: usize = certs.iter().map(|c| c.len() + 3).sum();
let mut v = Vec::new();
v.extend_from_slice(&u32::try_from(total).unwrap().to_be_bytes()[1..]);
for cert in certs {
v.extend_from_slice(&u32::try_from(cert.len()).unwrap().to_be_bytes()[1..]);
v.extend_from_slice(cert);
}
v
}
#[test]
fn splits_a_two_certificate_chain() {
let body = message(&[&[0x30, 0x01, 0xaa], &[0x30, 0x02, 0xbb, 0xcc]]);
let certs = certificate_der_list(&body).unwrap();
assert_eq!(certs.len(), 2);
assert_eq!(certs[0], &[0x30, 0x01, 0xaa]);
assert_eq!(certs[1], &[0x30, 0x02, 0xbb, 0xcc]);
}
#[test]
fn truncated_entry_is_an_error_not_a_panic() {
let mut body = message(&[&[0x30, 0x01, 0xaa]]);
body.truncate(body.len() - 1);
assert!(matches!(
certificate_der_list(&body),
Err(ParseError::Truncated { .. })
));
}
#[test]
fn empty_chain_is_empty() {
let certs = certificate_der_list(&[0, 0, 0]).unwrap();
assert!(certs.is_empty());
}
}

246
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/parse/hello.rs Normal file
View File

@ -0,0 +1,246 @@
// ©AngelaMos | 2026
// hello.rs
use smallvec::SmallVec;
use crate::error::Result;
use crate::parse::reader::Reader;
use crate::registry::extension;
/// A single TLS extension as it appeared on the wire.
///
/// The extension body is borrowed from the packet, not copied. Specific
/// extensions are decoded on demand through the accessor methods on
/// [`ClientHello`] so that the common parse path does no work for extensions a
/// given fingerprint does not read.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Extension<'pkt> {
pub ext_type: u16,
pub data: &'pkt [u8],
}
/// A parsed ClientHello, holding exactly what the fingerprint algorithms read.
///
/// Cipher suites and extensions are stored in wire order. Order is preserved
/// because JA3 hashes extensions in their original order, and because the
/// difference between wire order and sorted order is itself a signal: a client
/// that permutes its extension order on every connection is doing something a
/// fixed order client is not.
#[derive(Debug, Clone)]
pub struct ClientHello<'pkt> {
pub legacy_version: u16,
pub cipher_suites: SmallVec<[u16; 32]>,
pub extensions: SmallVec<[Extension<'pkt>; 16]>,
pub is_sslv2: bool,
}
/// A parsed ServerHello.
///
/// The server selects exactly one cipher suite, so `cipher_suite` is a single
/// value rather than a list. Extensions are again kept in wire order.
#[derive(Debug, Clone)]
pub struct ServerHello<'pkt> {
pub legacy_version: u16,
pub cipher_suite: u16,
pub extensions: SmallVec<[Extension<'pkt>; 16]>,
}
fn parse_extensions<'pkt>(r: &mut Reader<'pkt>) -> Result<SmallVec<[Extension<'pkt>; 16]>> {
let mut exts = SmallVec::new();
if r.is_empty() {
return Ok(exts);
}
let mut block = r.sub_u16_vec()?;
while !block.is_empty() {
let ext_type = block.u16()?;
let data = block.take_u16_vec()?;
exts.push(Extension { ext_type, data });
}
Ok(exts)
}
/// Parses the body of a ClientHello handshake message.
///
/// The input is the message body, meaning the four byte handshake header has
/// already been stripped. The session id, which fingerprinting ignores, is
/// skipped, as are the compression methods; getting those two skips right is a
/// classic source of bugs because a parser that forgets them reads the wrong
/// bytes as cipher suites.
pub fn parse_client_hello(body: &[u8]) -> Result<ClientHello<'_>> {
let mut r = Reader::new(body);
let legacy_version = r.u16()?;
let _random = r.take(32)?;
let _session_id = r.take_u8_vec()?;
let mut cipher_reader = r.sub_u16_vec()?;
let mut cipher_suites = SmallVec::new();
while !cipher_reader.is_empty() {
cipher_suites.push(cipher_reader.u16()?);
}
let _compression = r.take_u8_vec()?;
let extensions = parse_extensions(&mut r)?;
Ok(ClientHello {
legacy_version,
cipher_suites,
extensions,
is_sslv2: false,
})
}
/// Parses the body of a ServerHello handshake message.
pub fn parse_server_hello(body: &[u8]) -> Result<ServerHello<'_>> {
let mut r = Reader::new(body);
let legacy_version = r.u16()?;
let _random = r.take(32)?;
let _session_id = r.take_u8_vec()?;
let cipher_suite = r.u16()?;
let _compression = r.u8()?;
let extensions = parse_extensions(&mut r)?;
Ok(ServerHello {
legacy_version,
cipher_suite,
extensions,
})
}
impl<'pkt> ServerHello<'pkt> {
fn extension(&self, ext_type: u16) -> Option<&Extension<'pkt>> {
self.extensions.iter().find(|e| e.ext_type == ext_type)
}
/// Returns the version the server selected.
///
/// In TLS 1.3 the negotiated version lives in the supported versions
/// extension as a single value rather than in the legacy version word, which
/// the server pins to TLS 1.2 for compatibility. JA4S reads the real version
/// from the extension when it is present.
#[must_use]
pub fn selected_version(&self) -> u16 {
self.extension(extension::SUPPORTED_VERSIONS)
.and_then(|ext| {
let mut r = Reader::new(ext.data);
r.u16().ok()
})
.unwrap_or(self.legacy_version)
}
/// Returns the ALPN protocol the server chose, if any.
#[must_use]
pub fn alpn_protocol(&self) -> Option<&'pkt [u8]> {
let ext = self.extension(extension::ALPN)?;
let mut r = Reader::new(ext.data);
let mut list = r.sub_u16_vec().ok()?;
list.take_u8_vec().ok().filter(|p| !p.is_empty())
}
}
impl<'pkt> ClientHello<'pkt> {
fn extension(&self, ext_type: u16) -> Option<&Extension<'pkt>> {
self.extensions.iter().find(|e| e.ext_type == ext_type)
}
#[must_use]
pub fn has_extension(&self, ext_type: u16) -> bool {
self.extension(ext_type).is_some()
}
/// Returns the server name from the SNI extension, if a host name entry is
/// present.
#[must_use]
pub fn server_name(&self) -> Option<&'pkt str> {
let ext = self.extension(extension::SERVER_NAME)?;
let mut list = Reader::new(ext.data).sub_u16_vec().ok()?;
while !list.is_empty() {
let name_type = list.u8().ok()?;
let name = list.take_u16_vec().ok()?;
if name_type == 0 {
return core::str::from_utf8(name).ok();
}
}
None
}
/// Returns the supported groups, the field JA3 calls elliptic curves.
#[must_use]
pub fn supported_groups(&self) -> SmallVec<[u16; 16]> {
self.u16_list(extension::SUPPORTED_GROUPS)
}
/// Returns the elliptic curve point formats.
#[must_use]
pub fn ec_point_formats(&self) -> SmallVec<[u8; 4]> {
let mut out = SmallVec::new();
let Some(ext) = self.extension(extension::EC_POINT_FORMATS) else {
return out;
};
let mut r = Reader::new(ext.data);
let Ok(list) = r.take_u8_vec() else {
return out;
};
out.extend_from_slice(list);
out
}
/// Returns the protocol versions the client offers in the supported versions
/// extension. JA4 selects its version field from the highest non GREASE
/// value here when the extension is present.
#[must_use]
pub fn supported_versions(&self) -> SmallVec<[u16; 8]> {
let mut out = SmallVec::new();
let Some(ext) = self.extension(extension::SUPPORTED_VERSIONS) else {
return out;
};
let mut r = Reader::new(ext.data);
let Ok(list) = r.take_u8_vec() else {
return out;
};
let mut inner = Reader::new(list);
while let Ok(v) = inner.u16() {
out.push(v);
}
out
}
/// Returns the signature algorithms in their original order. JA4 appends
/// these, unsorted, to the extension hash input.
#[must_use]
pub fn signature_algorithms(&self) -> SmallVec<[u16; 16]> {
self.u16_list(extension::SIGNATURE_ALGORITHMS)
}
/// Returns the ALPN protocol identifiers in order. JA4 uses the first one.
#[must_use]
pub fn alpn_protocols(&self) -> SmallVec<[&'pkt [u8]; 4]> {
let mut out = SmallVec::new();
let Some(ext) = self.extension(extension::ALPN) else {
return out;
};
let mut r = Reader::new(ext.data);
let Ok(mut list) = r.sub_u16_vec() else {
return out;
};
while let Ok(proto) = list.take_u8_vec() {
out.push(proto);
}
out
}
fn u16_list(&self, ext_type: u16) -> SmallVec<[u16; 16]> {
let mut out = SmallVec::new();
let Some(ext) = self.extension(ext_type) else {
return out;
};
let mut r = Reader::new(ext.data);
let Ok(list) = r.take_u16_vec() else {
return out;
};
let mut inner = Reader::new(list);
while let Ok(v) = inner.u16() {
out.push(v);
}
out
}
}

23
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/parse/mod.rs Normal file
View File

@ -0,0 +1,23 @@
// ©AngelaMos | 2026
// mod.rs
//! Hand written, bounds checked TLS parsing.
//!
//! The parser is deliberately not built on a parser combinator framework. It is
//! the security critical core of the tool, and the byte by byte reader here maps
//! directly onto the wire format described in the TLS RFCs, which makes it easy
//! to audit against the specification. Reassembly of fragmented handshakes is
//! handled before parsing, so every parse function sees a complete message and
//! never has to model partial input.
pub mod cert;
pub mod hello;
pub mod reader;
pub mod record;
pub use cert::{CertificateList, certificate_der_list};
pub use hello::{ClientHello, Extension, ServerHello, parse_client_hello, parse_server_hello};
pub use reader::Reader;
pub use record::{
first_handshake_message, handshake_bytes, is_sslv2_client_hello, parse_sslv2_client_hello,
};

173
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/parse/reader.rs Normal file
View File

@ -0,0 +1,173 @@
// ©AngelaMos | 2026
// reader.rs
use crate::error::{ParseError, Result};
/// A forward only cursor over a borrowed byte slice with bounds checked reads.
///
/// Every read advances the cursor and returns an error rather than panicking
/// when the buffer is too short. This is the foundation the whole parser stands
/// on: because the cursor can never read past the end of the slice, the parser
/// has no `unsafe`, cannot index out of bounds, and treats a truncated or
/// hostile packet as an ordinary error instead of a crash. The slices it hands
/// back borrow from the original packet buffer, so parsing copies nothing.
pub struct Reader<'pkt> {
buf: &'pkt [u8],
pos: usize,
}
impl<'pkt> Reader<'pkt> {
#[must_use]
pub const fn new(buf: &'pkt [u8]) -> Self {
Self { buf, pos: 0 }
}
#[must_use]
pub const fn remaining(&self) -> usize {
self.buf.len() - self.pos
}
#[must_use]
pub const fn is_empty(&self) -> bool {
self.remaining() == 0
}
fn need(&self, n: usize) -> Result<()> {
if self.remaining() < n {
return Err(ParseError::Truncated {
needed: n,
have: self.remaining(),
});
}
Ok(())
}
pub fn u8(&mut self) -> Result<u8> {
self.need(1)?;
let v = self.buf[self.pos];
self.pos += 1;
Ok(v)
}
pub fn u16(&mut self) -> Result<u16> {
self.need(2)?;
let v = u16::from_be_bytes([self.buf[self.pos], self.buf[self.pos + 1]]);
self.pos += 2;
Ok(v)
}
/// Reads a 24 bit big endian length, the width TLS uses for handshake
/// message bodies and certificate entries.
pub fn u24(&mut self) -> Result<u32> {
self.need(3)?;
let v = u32::from_be_bytes([
0,
self.buf[self.pos],
self.buf[self.pos + 1],
self.buf[self.pos + 2],
]);
self.pos += 3;
Ok(v)
}
pub fn u32(&mut self) -> Result<u32> {
self.need(4)?;
let v = u32::from_be_bytes([
self.buf[self.pos],
self.buf[self.pos + 1],
self.buf[self.pos + 2],
self.buf[self.pos + 3],
]);
self.pos += 4;
Ok(v)
}
/// Borrows the next `n` bytes and advances past them.
pub fn take(&mut self, n: usize) -> Result<&'pkt [u8]> {
self.need(n)?;
let slice = &self.buf[self.pos..self.pos + n];
self.pos += n;
Ok(slice)
}
/// Reads a one byte length prefix, then borrows that many bytes.
pub fn take_u8_vec(&mut self) -> Result<&'pkt [u8]> {
let len = self.u8()? as usize;
self.take(len)
}
/// Reads a two byte length prefix, then borrows that many bytes.
pub fn take_u16_vec(&mut self) -> Result<&'pkt [u8]> {
let len = self.u16()? as usize;
self.take(len)
}
/// Reads a three byte length prefix, then borrows that many bytes.
pub fn take_u24_vec(&mut self) -> Result<&'pkt [u8]> {
let len = self.u24()? as usize;
self.take(len)
}
/// Returns a sub reader over a two byte length prefixed region.
///
/// This is the workhorse for nested vectors such as the extensions block,
/// where an outer length governs a region that itself contains a sequence of
/// length prefixed elements.
pub fn sub_u16_vec(&mut self) -> Result<Reader<'pkt>> {
let len = self.u16()? as usize;
let slice = self.take(len)?;
Ok(Reader::new(slice))
}
/// Returns a sub reader over a three byte length prefixed region, the width
/// the Certificate message uses for its certificate list.
pub fn sub_u24_vec(&mut self) -> Result<Reader<'pkt>> {
let len = self.u24()? as usize;
let slice = self.take(len)?;
Ok(Reader::new(slice))
}
}
#[cfg(test)]
mod tests {
use super::Reader;
use crate::error::ParseError;
#[test]
fn reads_widths_in_order() {
let data = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06];
let mut r = Reader::new(&data);
assert_eq!(r.u8().unwrap(), 0x01);
assert_eq!(r.u16().unwrap(), 0x0203);
assert_eq!(r.u24().unwrap(), 0x0004_0506);
assert!(r.is_empty());
}
#[test]
fn short_read_is_an_error_not_a_panic() {
let data = [0x01];
let mut r = Reader::new(&data);
assert_eq!(
r.u16().unwrap_err(),
ParseError::Truncated { needed: 2, have: 1 }
);
}
#[test]
fn length_prefixed_take_respects_bounds() {
let data = [0x03, 0xaa, 0xbb, 0xcc, 0xff];
let mut r = Reader::new(&data);
assert_eq!(r.take_u8_vec().unwrap(), &[0xaa, 0xbb, 0xcc]);
assert_eq!(r.u8().unwrap(), 0xff);
}
#[test]
fn sub_vector_isolates_a_region() {
let data = [0x00, 0x02, 0x11, 0x22, 0x33];
let mut r = Reader::new(&data);
let mut sub = r.sub_u16_vec().unwrap();
assert_eq!(sub.remaining(), 2);
assert_eq!(sub.u16().unwrap(), 0x1122);
assert_eq!(r.u8().unwrap(), 0x33);
}
}

179
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/parse/record.rs Normal file
View File

@ -0,0 +1,179 @@
// ©AngelaMos | 2026
// record.rs
use std::borrow::Cow;
use smallvec::SmallVec;
use crate::error::{ParseError, Result};
use crate::parse::hello::ClientHello;
use crate::parse::reader::Reader;
use crate::registry::content_type;
/// Reassembles the cleartext handshake flight from a TLS record stream.
///
/// A handshake message can be split across several TLS records, and several
/// short messages can share one record. This walks the record framing and
/// concatenates the payloads of the handshake records so the caller sees one
/// contiguous handshake byte stream. The common case is a single record holding
/// a single ClientHello, and that case borrows the original bytes with no copy.
/// Only genuinely fragmented flights allocate.
///
/// Records carrying anything other than handshake data are ignored. In TLS 1.3
/// the later handshake messages travel inside records typed as application data
/// and are encrypted, so they never reach this function, which is correct: the
/// only handshake bytes we can read in the clear are the first flight.
pub fn handshake_bytes(stream: &[u8]) -> Result<Cow<'_, [u8]>> {
let mut segments: SmallVec<[&[u8]; 4]> = SmallVec::new();
let mut r = Reader::new(stream);
while r.remaining() >= 5 {
let ctype = r.u8()?;
let _version = r.u16()?;
let payload = r.take_u16_vec()?;
if ctype == content_type::HANDSHAKE {
segments.push(payload);
} else if !segments.is_empty() {
break;
}
}
match segments.as_slice() {
[] => Err(ParseError::Truncated {
needed: 5,
have: stream.len(),
}),
[only] => Ok(Cow::Borrowed(*only)),
many => {
let mut joined = Vec::with_capacity(many.iter().map(|s| s.len()).sum());
for seg in many {
joined.extend_from_slice(seg);
}
Ok(Cow::Owned(joined))
}
}
}
/// Returns the body of the first handshake message of the requested type.
///
/// The handshake header is a one byte type and a three byte length. This walks
/// the messages in the reassembled flight and returns the body slice of the
/// first one whose type matches, so the caller never has to reason about the
/// header widths.
pub fn first_handshake_message(handshake: &[u8], want_type: u8) -> Result<&[u8]> {
let mut r = Reader::new(handshake);
while r.remaining() >= 4 {
let msg_type = r.u8()?;
let len = r.u24()? as usize;
let body = r.take(len)?;
if msg_type == want_type {
return Ok(body);
}
}
Err(ParseError::UnexpectedHandshake(want_type))
}
/// Returns true when the stream begins with an SSLv2 style ClientHello.
///
/// SSLv2 framing sets the high bit of the first length byte and places the
/// message type in the first body byte. Type 1 is CLIENT-HELLO. Some old
/// malware opens with this backward compatible hello even when it intends to
/// negotiate TLS, so detecting it keeps the TLS parser from misreading the
/// SSLv2 header as a TLS record.
#[must_use]
pub fn is_sslv2_client_hello(stream: &[u8]) -> bool {
stream.len() >= 3 && (stream[0] & 0x80) != 0 && stream[2] == 1
}
/// Parses an SSLv2 style ClientHello into the common ClientHello shape.
///
/// SSLv2 carries no extensions, supported groups, or point formats, so those
/// stay empty, which matches the community consensus for fingerprinting an
/// SSLv2 hello. Cipher specs are three bytes each. Specs that begin with a zero
/// byte are SSLv3 and TLS cipher suites carried in the backward compatible
/// hello, and those are the values a fingerprint cares about, so they are
/// extracted as their two byte suite numbers. True SSLv2 only specs are
/// counted but cannot be expressed as two byte suites and are skipped.
pub fn parse_sslv2_client_hello(stream: &[u8]) -> Result<ClientHello<'static>> {
let mut r = Reader::new(stream);
let len_hi = r.u8()? & 0x7f;
let len_lo = r.u8()?;
let _record_len = (u16::from(len_hi) << 8) | u16::from(len_lo);
let msg_type = r.u8()?;
if msg_type != 1 {
return Err(ParseError::UnexpectedHandshake(msg_type));
}
let legacy_version = r.u16()?;
let cipher_spec_len = r.u16()? as usize;
let session_id_len = r.u16()? as usize;
let challenge_len = r.u16()? as usize;
let cipher_specs = r.take(cipher_spec_len)?;
let _session_id = r.take(session_id_len)?;
let _challenge = r.take(challenge_len)?;
let mut cipher_suites = SmallVec::new();
let mut specs = Reader::new(cipher_specs);
while specs.remaining() >= 3 {
let kind = specs.u8()?;
let suite = specs.u16()?;
if kind == 0 {
cipher_suites.push(suite);
}
}
Ok(ClientHello {
legacy_version,
cipher_suites,
extensions: SmallVec::new(),
is_sslv2: true,
})
}
#[cfg(test)]
mod tests {
use super::{first_handshake_message, handshake_bytes, is_sslv2_client_hello};
use crate::registry::handshake_type;
fn record(ctype: u8, payload: &[u8]) -> Vec<u8> {
let mut v = vec![ctype, 0x03, 0x03];
let len = u16::try_from(payload.len()).unwrap();
v.extend_from_slice(&len.to_be_bytes());
v.extend_from_slice(payload);
v
}
#[test]
fn single_record_borrows() {
let stream = record(22, &[0x01, 0x00, 0x00, 0x00]);
let hs = handshake_bytes(&stream).unwrap();
assert!(matches!(hs, std::borrow::Cow::Borrowed(_)));
assert_eq!(hs.as_ref(), &[0x01, 0x00, 0x00, 0x00]);
}
#[test]
fn fragmented_records_join() {
let mut stream = record(22, &[0x01, 0x00, 0x00, 0x06, 0xaa, 0xbb]);
stream.extend(record(22, &[0xcc, 0xdd, 0xee, 0xff]));
let hs = handshake_bytes(&stream).unwrap();
assert!(matches!(hs, std::borrow::Cow::Owned(_)));
assert_eq!(hs.as_ref().len(), 10);
}
#[test]
fn finds_the_requested_message() {
let hs = [
0x02, 0x00, 0x00, 0x01, 0x99, 0x01, 0x00, 0x00, 0x02, 0xaa, 0xbb,
];
let body = first_handshake_message(&hs, handshake_type::CLIENT_HELLO).unwrap();
assert_eq!(body, &[0xaa, 0xbb]);
}
#[test]
fn sslv2_detection() {
assert!(is_sslv2_client_hello(&[0x80, 0x2e, 0x01, 0x00, 0x02]));
assert!(!is_sslv2_client_hello(&[0x16, 0x03, 0x01, 0x00]));
}
}

402
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/pipeline/decode.rs Normal file
View File

@ -0,0 +1,402 @@
// ©AngelaMos | 2026
// decode.rs
use std::net::{IpAddr, SocketAddr};
use etherparse::{EtherType, NetSlice, SlicedPacket, TransportSlice};
use smallvec::SmallVec;
use crate::ja4t::TcpFingerprintInput;
/// LINKTYPE registry numbers this decoder understands.
///
/// The values come from the tcpdump link layer header type registry. They are
/// redeclared here as plain constants because pcap file readers and live
/// captures both report them as bare integers, and the decoder is the single
/// place that interprets them.
pub mod link_type {
pub const NULL: i32 = 0;
pub const ETHERNET: i32 = 1;
pub const RAW: i32 = 101;
pub const LOOP: i32 = 108;
pub const LINUX_SLL: i32 = 113;
pub const IPV4: i32 = 228;
pub const IPV6: i32 = 229;
pub const LINUX_SLL2: i32 = 276;
}
/// The BSD null and loopback link headers are four bytes of address family.
const NULL_HEADER_LEN: usize = 4;
/// The Linux cooked capture v2 header is twenty bytes with the protocol in the
/// first two.
const SLL2_HEADER_LEN: usize = 20;
/// The TCP kind numbers the JA4T walk extracts values from.
const TCP_OPT_END: u8 = 0;
const TCP_OPT_NOP: u8 = 1;
const TCP_OPT_MSS: u8 = 2;
const TCP_OPT_WSCALE: u8 = 3;
const TCP_OPT_MSS_LEN: u8 = 4;
const TCP_OPT_WSCALE_LEN: u8 = 3;
/// The TCP flag bits, exactly as byte thirteen of the header carries them.
///
/// Keeping the flags as the wire bitfield instead of a fistful of bools means
/// the struct mirrors the protocol and reads the byte the packet already
/// holds, rather than rebuilding it from a handful of accessor calls. Adding
/// a flag later is then a constant, not a field.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct TcpFlags(u8);
impl TcpFlags {
pub const FIN: u8 = 0x01;
pub const SYN: u8 = 0x02;
pub const RST: u8 = 0x04;
pub const ACK: u8 = 0x10;
/// The offset of the flags byte within a TCP header.
const FLAGS_BYTE: usize = 13;
#[must_use]
pub const fn new(bits: u8) -> Self {
Self(bits)
}
/// Reads the flags byte from a TCP header slice.
///
/// A missing byte cannot happen for a slice the decoder hands in, since
/// the transport layer is only present when a full header parsed, but the
/// bounds checked read keeps this honest under direct unit testing.
#[must_use]
fn from_header(header: &[u8]) -> Self {
Self(header.get(Self::FLAGS_BYTE).copied().unwrap_or(0))
}
#[must_use]
pub const fn syn(self) -> bool {
self.0 & Self::SYN != 0
}
#[must_use]
pub const fn ack(self) -> bool {
self.0 & Self::ACK != 0
}
#[must_use]
pub const fn fin(self) -> bool {
self.0 & Self::FIN != 0
}
#[must_use]
pub const fn rst(self) -> bool {
self.0 & Self::RST != 0
}
}
/// The TCP level facts about one decoded segment.
#[derive(Debug, Clone, Copy)]
pub struct TcpMeta {
pub seq: u32,
pub flags: TcpFlags,
pub window_size: u16,
}
/// One TCP segment decoded out of a captured frame.
///
/// Addresses are directional: `src` sent this segment to `dst`. The JA4T
/// input is walked eagerly, but only for SYN packets, because those are the
/// only packets whose options JA4T reads and the walk needs the option bytes
/// that do not outlive the decode.
#[derive(Debug)]
pub struct DecodedSegment<'pkt> {
pub src: SocketAddr,
pub dst: SocketAddr,
pub tcp: TcpMeta,
pub syn_fingerprint: Option<TcpFingerprintInput>,
pub payload: &'pkt [u8],
}
/// One UDP datagram decoded out of a captured frame.
///
/// Addresses are directional like a TCP segment's. The payload is the UDP
/// data, which the pipeline hands to the QUIC layer; a datagram that turns
/// out not to be QUIC is simply ignored there, since UDP carries far more
/// than QUIC and the fingerprinter only reads the handshake it understands.
#[derive(Debug)]
pub struct DecodedDatagram<'pkt> {
pub src: SocketAddr,
pub dst: SocketAddr,
pub payload: &'pkt [u8],
}
/// One transport payload decoded out of a captured frame.
///
/// The decoder surfaces the two transports the pipeline fingerprints: TCP,
/// which carries TLS and HTTP, and UDP, which carries QUIC. Everything else
/// is a [`Skip`].
#[derive(Debug)]
pub enum Decoded<'pkt> {
Tcp(DecodedSegment<'pkt>),
Udp(DecodedDatagram<'pkt>),
}
/// Why a frame produced no transport payload. The distinction only feeds
/// counters, but the counters are how an operator learns what a capture was
/// made of.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Skip {
UnsupportedLinkType,
NotIp,
NotTransport,
Malformed,
}
/// Decodes a captured frame down to its transport payload, if it has one the
/// pipeline fingerprints.
///
/// VLAN tags, including stacked QinQ, are stepped over by etherparse. Frames
/// the decoder does not understand are skipped with a reason rather than
/// failing the capture: a fingerprinting pipeline must shrug off the GRE
/// tunnel, the ARP chatter, and the malformed frame that share every real
/// network with the TLS and QUIC it cares about.
pub fn decode_frame(link: i32, data: &[u8]) -> Result<Decoded<'_>, Skip> {
let sliced = match link {
link_type::ETHERNET => SlicedPacket::from_ethernet(data),
link_type::LINUX_SLL => SlicedPacket::from_linux_sll(data),
link_type::RAW | link_type::IPV4 | link_type::IPV6 => SlicedPacket::from_ip(data),
link_type::NULL | link_type::LOOP => {
let Some(inner) = data.get(NULL_HEADER_LEN..) else {
return Err(Skip::Malformed);
};
SlicedPacket::from_ip(inner)
}
link_type::LINUX_SLL2 => {
let Some(proto) = data.first_chunk::<2>() else {
return Err(Skip::Malformed);
};
let Some(inner) = data.get(SLL2_HEADER_LEN..) else {
return Err(Skip::Malformed);
};
SlicedPacket::from_ether_type(EtherType(u16::from_be_bytes(*proto)), inner)
}
_ => return Err(Skip::UnsupportedLinkType),
};
let sliced = sliced.map_err(|_| Skip::Malformed)?;
let (src_ip, dst_ip): (IpAddr, IpAddr) = match &sliced.net {
Some(NetSlice::Ipv4(v4)) => (
IpAddr::V4(v4.header().source_addr()),
IpAddr::V4(v4.header().destination_addr()),
),
Some(NetSlice::Ipv6(v6)) => (
IpAddr::V6(v6.header().source_addr()),
IpAddr::V6(v6.header().destination_addr()),
),
Some(NetSlice::Arp(_)) | None => return Err(Skip::NotIp),
};
match &sliced.transport {
Some(TransportSlice::Tcp(tcp)) => {
let flags = TcpFlags::from_header(tcp.slice());
let syn_fingerprint = flags
.syn()
.then(|| tcp_fingerprint_input(tcp.window_size(), tcp.options()));
Ok(Decoded::Tcp(DecodedSegment {
src: SocketAddr::new(src_ip, tcp.source_port()),
dst: SocketAddr::new(dst_ip, tcp.destination_port()),
tcp: TcpMeta {
seq: tcp.sequence_number(),
flags,
window_size: tcp.window_size(),
},
syn_fingerprint,
payload: tcp.payload(),
}))
}
Some(TransportSlice::Udp(udp)) => Ok(Decoded::Udp(DecodedDatagram {
src: SocketAddr::new(src_ip, udp.source_port()),
dst: SocketAddr::new(dst_ip, udp.destination_port()),
payload: udp.payload(),
})),
_ => Err(Skip::NotTransport),
}
}
/// Walks raw TCP options into the JA4T input.
///
/// JA4T records every option kind in order, including each NOP and each
/// trailing end of list byte, because the padding pattern is part of how an
/// operating system's stack writes a SYN. The walk is deliberately tolerant:
/// a truncated or nonsense length byte ends the walk after recording the kind
/// it was found on, so a hostile SYN cannot push the parser out of bounds.
pub fn tcp_fingerprint_input(window_size: u16, options: &[u8]) -> TcpFingerprintInput {
let mut kinds: SmallVec<[u8; 8]> = SmallVec::new();
let mut mss = 0u16;
let mut window_scale = 0u8;
let mut i = 0;
while i < options.len() {
let kind = options[i];
kinds.push(kind);
if kind == TCP_OPT_END || kind == TCP_OPT_NOP {
i += 1;
continue;
}
let Some(&len) = options.get(i + 1) else {
break;
};
if len < 2 {
break;
}
let Some(body) = options.get(i + 2..i + usize::from(len)) else {
break;
};
if kind == TCP_OPT_MSS && len == TCP_OPT_MSS_LEN {
if let Some(value) = body.first_chunk::<2>() {
mss = u16::from_be_bytes(*value);
}
}
if kind == TCP_OPT_WSCALE && len == TCP_OPT_WSCALE_LEN {
if let Some(&value) = body.first() {
window_scale = value;
}
}
i += usize::from(len);
}
TcpFingerprintInput {
window_size,
option_kinds: kinds,
mss,
window_scale,
}
}
#[cfg(test)]
mod tests {
use super::{Decoded, Skip, TcpFlags, decode_frame, link_type, tcp_fingerprint_input};
use crate::ja4t::ja4t;
use etherparse::PacketBuilder;
fn tcp_segment(link: i32, data: &[u8]) -> super::DecodedSegment<'_> {
match decode_frame(link, data) {
Ok(Decoded::Tcp(seg)) => seg,
other => panic!("expected a TCP segment, got {other:?}"),
}
}
fn tcp_frame(payload: &[u8]) -> Vec<u8> {
let builder = PacketBuilder::ethernet2([1; 6], [2; 6])
.ipv4([10, 0, 0, 1], [10, 0, 0, 2], 64)
.tcp(40000, 443, 1000, 64240);
let mut out = Vec::with_capacity(builder.size(payload.len()));
builder.write(&mut out, payload).unwrap();
out
}
#[test]
fn flags_byte_decodes_to_the_right_bits() {
let syn = TcpFlags::from_header(&[0u8; 14]);
assert!(!syn.syn());
let mut header = [0u8; 20];
header[13] = TcpFlags::SYN | TcpFlags::ACK;
let flags = TcpFlags::from_header(&header);
assert!(flags.syn() && flags.ack());
assert!(!flags.fin() && !flags.rst());
assert!(!TcpFlags::from_header(&[]).syn());
}
#[test]
fn decodes_an_ethernet_tcp_frame() {
let frame = tcp_frame(b"hello");
let seg = tcp_segment(link_type::ETHERNET, &frame);
assert_eq!(seg.src.to_string(), "10.0.0.1:40000");
assert_eq!(seg.dst.to_string(), "10.0.0.2:443");
assert_eq!(seg.tcp.seq, 1000);
assert_eq!(seg.payload, b"hello");
}
#[test]
fn decodes_a_vlan_tagged_frame() {
let builder = PacketBuilder::ethernet2([1; 6], [2; 6])
.single_vlan(etherparse::VlanId::try_new(7).unwrap())
.ipv4([10, 0, 0, 1], [10, 0, 0, 2], 64)
.tcp(40000, 443, 1, 64240);
let mut frame = Vec::with_capacity(builder.size(0));
builder.write(&mut frame, &[]).unwrap();
let seg = tcp_segment(link_type::ETHERNET, &frame);
assert_eq!(seg.dst.port(), 443);
}
#[test]
fn decodes_a_udp_datagram() {
let builder = PacketBuilder::ethernet2([1; 6], [2; 6])
.ipv4([10, 0, 0, 1], [10, 0, 0, 2], 64)
.udp(50000, 443);
let mut udp = Vec::with_capacity(builder.size(4));
builder.write(&mut udp, &[0xde, 0xad, 0xbe, 0xef]).unwrap();
let Ok(Decoded::Udp(datagram)) = decode_frame(link_type::ETHERNET, &udp) else {
panic!("expected a UDP datagram");
};
assert_eq!(datagram.src.to_string(), "10.0.0.1:50000");
assert_eq!(datagram.dst.port(), 443);
assert_eq!(datagram.payload, &[0xde, 0xad, 0xbe, 0xef]);
}
#[test]
fn non_transport_and_garbage_are_skips_not_panics() {
let builder = PacketBuilder::ethernet2([1; 6], [2; 6])
.ipv4([10, 0, 0, 1], [10, 0, 0, 2], 64)
.icmpv4_echo_request(1, 1);
let mut icmp = Vec::with_capacity(builder.size(0));
builder.write(&mut icmp, &[]).unwrap();
assert!(matches!(
decode_frame(link_type::ETHERNET, &icmp),
Err(Skip::NotTransport)
));
assert!(matches!(
decode_frame(link_type::ETHERNET, &[0x01, 0x02]),
Err(Skip::Malformed)
));
assert!(matches!(
decode_frame(147, &icmp),
Err(Skip::UnsupportedLinkType)
));
}
#[test]
fn ja4t_walk_reproduces_the_windows_default_vector() {
let options = [
0x02, 0x04, 0x05, 0xb4, 0x01, 0x03, 0x03, 0x08, 0x01, 0x01, 0x04, 0x02,
];
let input = tcp_fingerprint_input(64240, &options);
assert_eq!(ja4t(&input), "64240_2-1-3-1-1-4_1460_8");
}
#[test]
fn ja4t_walk_counts_trailing_end_of_list_padding() {
let options = [
0x02, 0x04, 0x05, 0x42, 0x01, 0x03, 0x03, 0x06, 0x01, 0x01, 0x08, 0x0a, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x02, 0x00, 0x00,
];
let input = tcp_fingerprint_input(65535, &options);
assert_eq!(ja4t(&input), "65535_2-1-3-1-1-8-4-0-0_1346_6");
}
#[test]
fn ja4t_walk_survives_truncated_options() {
let input = tcp_fingerprint_input(1024, &[0x02, 0x04, 0x05]);
assert_eq!(input.option_kinds.as_slice(), &[0x02]);
assert_eq!(input.mss, 0);
let zero_len = tcp_fingerprint_input(1024, &[0x05, 0x00, 0x02]);
assert_eq!(zero_len.option_kinds.as_slice(), &[0x05]);
}
}

145
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/pipeline/event.rs Normal file
View File

@ -0,0 +1,145 @@
// ©AngelaMos | 2026
// event.rs
use std::fmt;
use std::net::SocketAddr;
use serde::Serialize;
use crate::fingerprint::{Ja3, Ja4Family};
/// A fingerprint produced by one direction of one flow, without addressing.
///
/// The protocol layer emits these; the pipeline wraps them with the flow's
/// addresses and timestamp to make a [`FingerprintEvent`]. Keeping the two
/// layers apart means the protocol extractor can be tested with bare byte
/// streams, no packets required.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum StreamEvent {
ClientHello {
ja3: Ja3,
ja3_raw: String,
ja4: Ja4Family,
sni: Option<String>,
alpn: Option<String>,
},
ServerHello {
ja3s: Ja3,
ja3s_raw: String,
ja4s: Ja4Family,
},
Certificate {
ja4x: String,
},
HttpRequest {
ja4h: Ja4Family,
method: String,
host: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
user_agent: Option<String>,
},
TcpSyn {
ja4t: String,
},
TcpSynAck {
ja4ts: String,
},
}
/// One fingerprint observation, addressed and timestamped.
///
/// `src` is always the party that sent the fingerprinted bytes: the client
/// for a ClientHello or SYN, the server for a ServerHello or certificate.
#[derive(Debug, Clone, Serialize)]
pub struct FingerprintEvent {
pub ts_nanos: u64,
pub src: SocketAddr,
pub dst: SocketAddr,
#[serde(flatten)]
pub event: StreamEvent,
}
impl fmt::Display for FingerprintEvent {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let secs = self.ts_nanos / 1_000_000_000;
let millis = self.ts_nanos % 1_000_000_000 / 1_000_000;
write!(f, "{secs}.{millis:03} {} -> {} ", self.src, self.dst)?;
match &self.event {
StreamEvent::ClientHello {
ja3,
ja4,
sni,
alpn,
..
} => {
write!(f, "client_hello ja4={} ja3={ja3}", ja4.hash)?;
if let Some(sni) = sni {
write!(f, " sni={sni}")?;
}
if let Some(alpn) = alpn {
write!(f, " alpn={alpn}")?;
}
Ok(())
}
StreamEvent::ServerHello { ja3s, ja4s, .. } => {
write!(f, "server_hello ja4s={} ja3s={ja3s}", ja4s.hash)
}
StreamEvent::Certificate { ja4x } => write!(f, "certificate ja4x={ja4x}"),
StreamEvent::HttpRequest {
ja4h,
method,
host,
user_agent,
} => {
write!(f, "http_request ja4h={} method={method}", ja4h.hash)?;
if let Some(host) = host {
write!(f, " host={host}")?;
}
if let Some(user_agent) = user_agent {
write!(f, " ua={user_agent}")?;
}
Ok(())
}
StreamEvent::TcpSyn { ja4t } => write!(f, "tcp_syn ja4t={ja4t}"),
StreamEvent::TcpSynAck { ja4ts } => write!(f, "tcp_syn_ack ja4ts={ja4ts}"),
}
}
}
#[cfg(test)]
mod tests {
use super::{FingerprintEvent, StreamEvent};
#[test]
fn json_shape_is_tagged_and_flat() {
let event = FingerprintEvent {
ts_nanos: 1_500_000_000,
src: "10.0.0.1:40000".parse().unwrap(),
dst: "10.0.0.2:443".parse().unwrap(),
event: StreamEvent::Certificate {
ja4x: "7d5dbb3783b4_ba7ce0880c07_7bf9a7bf7029".into(),
},
};
let json = serde_json::to_value(&event).unwrap();
assert_eq!(json["kind"], "certificate");
assert_eq!(json["ja4x"], "7d5dbb3783b4_ba7ce0880c07_7bf9a7bf7029");
assert_eq!(json["src"], "10.0.0.1:40000");
}
#[test]
fn display_is_one_greppable_line() {
let event = FingerprintEvent {
ts_nanos: 1_234_000_000,
src: "10.0.0.1:40000".parse().unwrap(),
dst: "10.0.0.2:443".parse().unwrap(),
event: StreamEvent::TcpSyn {
ja4t: "64240_2-1-3-1-1-4_1460_8".into(),
},
};
assert_eq!(
event.to_string(),
"1.234 10.0.0.1:40000 -> 10.0.0.2:443 tcp_syn ja4t=64240_2-1-3-1-1-4_1460_8"
);
}
}

430
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/pipeline/flow.rs Normal file
View File

@ -0,0 +1,430 @@
// ©AngelaMos | 2026
// flow.rs
use std::collections::BTreeMap;
use std::net::SocketAddr;
/// A bidirectional flow identity.
///
/// The two endpoints are stored in sorted order so that a packet and its reply
/// hash to the same key. Which endpoint is the client is a separate question,
/// answered by who sent the SYN or, failing that, who spoke a ClientHello, and
/// it is deliberately not baked into the key: captures routinely start in the
/// middle of connections, and a key that guessed wrong would split one
/// conversation into two.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct FlowKey {
pub lo: SocketAddr,
pub hi: SocketAddr,
}
/// Which endpoint of a [`FlowKey`] sent a given segment.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Direction {
FromLo,
FromHi,
}
impl Direction {
#[must_use]
pub const fn index(self) -> usize {
match self {
Direction::FromLo => 0,
Direction::FromHi => 1,
}
}
/// The source and destination addresses of traffic flowing this way.
#[must_use]
pub const fn addresses(self, key: &FlowKey) -> (SocketAddr, SocketAddr) {
match self {
Direction::FromLo => (key.lo, key.hi),
Direction::FromHi => (key.hi, key.lo),
}
}
}
impl FlowKey {
/// Normalizes a directional (source, destination) pair into a key plus the
/// direction the packet travelled.
#[must_use]
pub fn from_pair(src: SocketAddr, dst: SocketAddr) -> (Self, Direction) {
if src <= dst {
(Self { lo: src, hi: dst }, Direction::FromLo)
} else {
(Self { lo: dst, hi: src }, Direction::FromHi)
}
}
}
/// The midpoint of the sequence space. In TCP serial arithmetic an offset at
/// or beyond this point is read as the segment sitting behind the anchor, not
/// absurdly far ahead of it.
const HALF_SERIAL_SPACE: u32 = 0x8000_0000;
/// Resource limits for one reassembled direction of one flow.
#[derive(Debug, Clone, Copy)]
pub struct ReassemblyLimits {
/// Most contiguous bytes kept. Everything a passive fingerprinter reads
/// sits in the first kilobytes of a stream, so this is a cap on patience,
/// not on correctness.
pub max_assembled_bytes: usize,
/// Most bytes parked in the out of order buffer.
pub max_pending_bytes: usize,
/// Most segments parked in the out of order buffer.
pub max_pending_segments: usize,
}
/// What [`StreamReassembler::push`] did with a segment.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PushOutcome {
/// The contiguous stream grew; the protocol layer should look again.
Grew,
/// Nothing new: a duplicate, pure overlap, or empty segment.
Unchanged,
/// The segment was parked out of order for later.
Parked,
/// The segment fell outside the window this reassembler is willing to
/// track, or a buffer limit was hit, and it was dropped.
Dropped,
}
/// Reassembles one direction of a TCP stream into contiguous bytes.
///
/// This is the piece most toy fingerprinting tools skip, and skipping it is
/// why they miss handshakes: a ClientHello, and even more so a certificate
/// chain, regularly spans several segments, and those segments arrive
/// reordered on any path with packet loss. The reassembler anchors at the
/// sequence number the SYN names or, on a flow whose start the capture
/// missed, at the first segment it sees. Everything else is a relative
/// offset from that anchor in wrapping serial arithmetic: in order segments
/// append to one contiguous buffer, out of order segments park in a map
/// keyed by offset until the gap before them fills.
///
/// Data from before the anchor on a SYN-less flow is gone; a streaming
/// engine cannot retroactively prepend, and accepting that loss explicitly
/// is what Suricata does for midstream pickup too. A segment that straddles
/// the anchor is trimmed to its useful part rather than discarded.
///
/// Overlaps resolve first write wins: bytes already accepted are never
/// rewritten by a later segment. A passive observer cannot know which copy
/// the receiver kept, and the CVE-2018-6794 capture in the test corpus exists
/// precisely because inconsistent overlap handling let attackers show an IDS
/// a different stream than the one the victim read. First write wins is one
/// deterministic, documented answer.
#[derive(Debug)]
pub struct StreamReassembler {
limits: ReassemblyLimits,
anchor: Option<u32>,
assembled: Vec<u8>,
pending: BTreeMap<u32, Vec<u8>>,
pending_bytes: usize,
released: bool,
capped: bool,
}
impl StreamReassembler {
#[must_use]
pub fn new(limits: ReassemblyLimits) -> Self {
Self {
limits,
anchor: None,
assembled: Vec::new(),
pending: BTreeMap::new(),
pending_bytes: 0,
released: false,
capped: false,
}
}
/// The contiguous bytes assembled so far, from the anchor onward.
#[must_use]
pub fn data(&self) -> &[u8] {
&self.assembled
}
/// Pins the stream start, used when a SYN reveals the true initial
/// sequence number before any data arrives. Later anchors are ignored.
pub fn anchor(&mut self, seq: u32) {
if self.anchor.is_none() {
self.anchor = Some(seq);
}
}
/// Drops every buffer and refuses all future data.
///
/// Called once the protocol layer has what it needs, or knows it never
/// will. This is what keeps memory flat when a capture contains long
/// lived flows: the flow entry stays, the payload buffers do not.
pub fn release(&mut self) {
self.assembled = Vec::new();
self.pending = BTreeMap::new();
self.pending_bytes = 0;
self.released = true;
}
#[must_use]
pub fn released(&self) -> bool {
self.released
}
/// True when the assembled cap was hit and the tail of the stream is gone.
#[must_use]
pub fn capped(&self) -> bool {
self.capped
}
/// Offers one segment to the stream.
pub fn push(&mut self, seq: u32, payload: &[u8]) -> PushOutcome {
if self.released || payload.is_empty() {
return PushOutcome::Unchanged;
}
if self.capped {
return PushOutcome::Dropped;
}
let anchor = *self.anchor.get_or_insert(seq);
let offset = seq.wrapping_sub(anchor);
if offset >= HALF_SERIAL_SPACE {
let stale = offset.wrapping_neg() as usize;
if stale >= payload.len() {
return PushOutcome::Unchanged;
}
return self.push(anchor, &payload[stale..]);
}
let window_end = self
.limits
.max_assembled_bytes
.saturating_add(self.limits.max_pending_bytes);
if offset as usize > window_end {
return PushOutcome::Dropped;
}
let assembled_len = self.assembled.len();
if (offset as usize) < assembled_len {
let overlap = assembled_len - offset as usize;
if overlap >= payload.len() {
return PushOutcome::Unchanged;
}
return self.append_in_order(&payload[overlap..]);
}
if offset as usize == assembled_len {
return self.append_in_order(payload);
}
if self.pending.len() >= self.limits.max_pending_segments
|| self.pending_bytes.saturating_add(payload.len()) > self.limits.max_pending_bytes
{
return PushOutcome::Dropped;
}
match self.pending.entry(offset) {
std::collections::btree_map::Entry::Occupied(existing) => {
if existing.get().len() >= payload.len() {
return PushOutcome::Unchanged;
}
self.pending_bytes += payload.len() - existing.get().len();
*existing.into_mut() = payload.to_vec();
}
std::collections::btree_map::Entry::Vacant(slot) => {
self.pending_bytes += payload.len();
slot.insert(payload.to_vec());
}
}
PushOutcome::Parked
}
fn append_in_order(&mut self, payload: &[u8]) -> PushOutcome {
let room = self
.limits
.max_assembled_bytes
.saturating_sub(self.assembled.len());
if room == 0 {
self.mark_capped();
return PushOutcome::Dropped;
}
let take = payload.len().min(room);
self.assembled.extend_from_slice(&payload[..take]);
if take < payload.len() {
self.mark_capped();
} else {
self.drain_pending();
}
PushOutcome::Grew
}
/// Splices parked segments onto the contiguous buffer while they touch it.
fn drain_pending(&mut self) {
while let Some(entry) = self.pending.first_entry() {
let offset = *entry.key() as usize;
if offset > self.assembled.len() {
break;
}
let segment = entry.remove();
self.pending_bytes -= segment.len();
let overlap = self.assembled.len() - offset;
if overlap >= segment.len() {
continue;
}
let room = self
.limits
.max_assembled_bytes
.saturating_sub(self.assembled.len());
let take = (segment.len() - overlap).min(room);
self.assembled
.extend_from_slice(&segment[overlap..overlap + take]);
if take < segment.len() - overlap {
self.mark_capped();
return;
}
}
}
/// Once the assembled cap is hit nothing later can ever become contiguous,
/// so the parked segments are garbage. Drop them and refuse new data, but
/// keep the assembled prefix: it is still a valid stream head and whatever
/// the protocol layer already read from it stands.
fn mark_capped(&mut self) {
self.capped = true;
self.pending = BTreeMap::new();
self.pending_bytes = 0;
}
}
#[cfg(test)]
mod tests {
use super::{FlowKey, PushOutcome, ReassemblyLimits, StreamReassembler};
fn limits() -> ReassemblyLimits {
ReassemblyLimits {
max_assembled_bytes: 64,
max_pending_bytes: 64,
max_pending_segments: 4,
}
}
fn reasm() -> StreamReassembler {
StreamReassembler::new(limits())
}
#[test]
fn both_directions_share_one_key() {
let a: std::net::SocketAddr = "10.0.0.1:40000".parse().unwrap();
let b: std::net::SocketAddr = "10.0.0.2:443".parse().unwrap();
let (forward, fwd_dir) = FlowKey::from_pair(a, b);
let (reverse, rev_dir) = FlowKey::from_pair(b, a);
assert_eq!(forward, reverse);
assert_ne!(fwd_dir, rev_dir);
assert_eq!(fwd_dir.addresses(&forward), (a, b));
assert_eq!(rev_dir.addresses(&reverse), (b, a));
}
#[test]
fn in_order_segments_concatenate() {
let mut r = reasm();
assert_eq!(r.push(100, b"hell"), PushOutcome::Grew);
assert_eq!(r.push(104, b"o"), PushOutcome::Grew);
assert_eq!(r.data(), b"hello");
}
#[test]
fn out_of_order_segments_wait_for_the_gap() {
let mut r = reasm();
assert_eq!(r.push(100, b"hell"), PushOutcome::Grew);
assert_eq!(r.push(107, b"orld"), PushOutcome::Parked);
assert_eq!(r.data(), b"hell");
assert_eq!(r.push(104, b"o w"), PushOutcome::Grew);
assert_eq!(r.data(), b"hello world");
}
#[test]
fn anchor_from_syn_orders_data_arriving_backwards() {
let mut r = reasm();
r.anchor(1000);
assert_eq!(r.push(1004, b"data"), PushOutcome::Parked);
assert_eq!(r.push(1000, b"more"), PushOutcome::Grew);
assert_eq!(r.data(), b"moredata");
}
#[test]
fn retransmissions_change_nothing() {
let mut r = reasm();
r.push(100, b"abcdef");
assert_eq!(r.push(100, b"abcdef"), PushOutcome::Unchanged);
assert_eq!(r.push(102, b"cd"), PushOutcome::Unchanged);
assert_eq!(r.data(), b"abcdef");
}
#[test]
fn overlapping_segment_keeps_the_first_write() {
let mut r = reasm();
r.push(100, b"abcdef");
assert_eq!(r.push(103, b"XXXghi"), PushOutcome::Grew);
assert_eq!(r.data(), b"abcdefghi");
}
#[test]
fn parked_overlap_keeps_the_first_write_too() {
let mut r = reasm();
r.anchor(100);
assert_eq!(r.push(104, b"efgh"), PushOutcome::Parked);
assert_eq!(r.push(100, b"abcdEFG"), PushOutcome::Grew);
assert_eq!(r.data(), b"abcdEFGh");
}
#[test]
fn sequence_numbers_wrap_around_zero() {
let mut r = reasm();
let anchor = u32::MAX - 1;
assert_eq!(r.push(anchor, b"ab"), PushOutcome::Grew);
assert_eq!(r.push(0, b"cd"), PushOutcome::Grew);
assert_eq!(r.push(2, b"ef"), PushOutcome::Grew);
assert_eq!(r.data(), b"abcdef");
}
#[test]
fn stale_pre_anchor_data_is_ignored_and_far_future_dropped() {
let mut r = reasm();
r.push(1000, b"ab");
assert_eq!(r.push(990, b"old"), PushOutcome::Unchanged);
assert_eq!(r.push(100_000, b"far"), PushOutcome::Dropped);
assert_eq!(r.data(), b"ab");
}
#[test]
fn segment_straddling_the_anchor_is_trimmed_not_lost() {
let mut r = reasm();
r.anchor(1000);
assert_eq!(r.push(996, b"oldNEW"), PushOutcome::Grew);
assert_eq!(r.data(), b"EW");
}
#[test]
fn assembled_cap_truncates_but_keeps_the_prefix() {
let mut r = reasm();
let big = vec![0x41u8; 100];
assert_eq!(r.push(0, &big), PushOutcome::Grew);
assert_eq!(r.data().len(), limits().max_assembled_bytes);
assert_eq!(r.push(100, b"more"), PushOutcome::Dropped);
}
#[test]
fn pending_limits_drop_excess_segments() {
let mut r = reasm();
r.anchor(0);
assert_eq!(r.push(10, b"a"), PushOutcome::Parked);
assert_eq!(r.push(20, b"b"), PushOutcome::Parked);
assert_eq!(r.push(30, b"c"), PushOutcome::Parked);
assert_eq!(r.push(40, b"d"), PushOutcome::Parked);
assert_eq!(r.push(50, b"e"), PushOutcome::Dropped);
}
#[test]
fn release_drops_buffers_and_refuses_data() {
let mut r = reasm();
r.push(0, b"abc");
r.release();
assert!(r.released());
assert_eq!(r.data(), b"");
assert_eq!(r.push(3, b"def"), PushOutcome::Unchanged);
}
}

696
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/pipeline/mod.rs Normal file
View File

@ -0,0 +1,696 @@
// ©AngelaMos | 2026
// mod.rs
//! The passive fingerprinting pipeline: frames in, fingerprint events out.
//!
//! The stages are deliberately separable. A [`PacketSource`] yields raw link
//! layer frames from a capture file or, later, a live interface. The decoder
//! strips the frame down to a TCP segment. The flow table reassembles each
//! direction of each conversation into a contiguous byte stream, surviving
//! reordering, retransmission, and overlap. The protocol layer watches each
//! stream until it recognizes a TLS flight or an HTTP request head and emits
//! fingerprints. Nothing in here touches a network interface, so the whole
//! pipeline runs byte exact in tests against vendored captures.
pub mod decode;
pub mod event;
pub mod flow;
pub mod source;
pub mod tls;
use std::collections::HashMap;
use serde::Serialize;
use crate::ja3::{ja3, ja3_string};
use crate::ja4::{Transport, ja4};
use crate::ja4t::ja4t;
use crate::parse::parse_client_hello;
use crate::pipeline::decode::{Decoded, DecodedDatagram, DecodedSegment, Skip, decode_frame};
use crate::pipeline::event::{FingerprintEvent, StreamEvent};
use crate::pipeline::flow::{FlowKey, PushOutcome, ReassemblyLimits, StreamReassembler};
use crate::pipeline::source::{PacketSource, RawFrame, SourceError};
use crate::pipeline::tls::StreamProtocol;
use crate::quic::{
ClientHelloState, CryptoAssembler, InitialKeys, InitialPacket, walk_crypto_frames,
};
/// Tuning knobs for the pipeline.
///
/// The defaults are sized for handshake harvesting: generous enough that a
/// fat certificate chain or a multi segment ClientHello always fits, small
/// enough that an adversarial capture cannot turn the flow table into a
/// memory bomb.
#[derive(Debug, Clone, Copy)]
pub struct PipelineConfig {
/// Flows tracked at once before the table sheds its oldest entries.
pub max_flows: usize,
/// A flow untouched for this long is eligible for eviction.
pub idle_timeout_nanos: u64,
/// Contiguous bytes kept per direction.
pub max_assembled_bytes: usize,
/// Out of order bytes parked per direction.
pub max_pending_bytes: usize,
/// Out of order segments parked per direction.
pub max_pending_segments: usize,
}
impl PipelineConfig {
pub const DEFAULT_MAX_FLOWS: usize = 65_536;
pub const DEFAULT_IDLE_TIMEOUT_NANOS: u64 = 60 * 1_000_000_000;
pub const DEFAULT_MAX_ASSEMBLED_BYTES: usize = 256 * 1024;
pub const DEFAULT_MAX_PENDING_BYTES: usize = 256 * 1024;
pub const DEFAULT_MAX_PENDING_SEGMENTS: usize = 128;
fn limits(&self) -> ReassemblyLimits {
ReassemblyLimits {
max_assembled_bytes: self.max_assembled_bytes,
max_pending_bytes: self.max_pending_bytes,
max_pending_segments: self.max_pending_segments,
}
}
}
impl Default for PipelineConfig {
fn default() -> Self {
Self {
max_flows: Self::DEFAULT_MAX_FLOWS,
idle_timeout_nanos: Self::DEFAULT_IDLE_TIMEOUT_NANOS,
max_assembled_bytes: Self::DEFAULT_MAX_ASSEMBLED_BYTES,
max_pending_bytes: Self::DEFAULT_MAX_PENDING_BYTES,
max_pending_segments: Self::DEFAULT_MAX_PENDING_SEGMENTS,
}
}
}
/// What the pipeline saw, for the operator and for the miss rate honesty
/// check: a fingerprinting tool that cannot say what it failed to read is a
/// tool whose silence gets mistaken for absence.
#[derive(Debug, Default, Clone, Copy, Serialize)]
pub struct Counters {
pub frames: u64,
pub bytes: u64,
pub tcp_segments: u64,
pub udp_datagrams: u64,
pub skipped_unsupported_link_type: u64,
pub skipped_not_ip: u64,
pub skipped_not_transport: u64,
pub skipped_malformed: u64,
pub flows_created: u64,
pub flows_evicted_idle: u64,
pub flows_evicted_pressure: u64,
pub segments_dropped: u64,
pub events: u64,
pub streams_capped: u64,
/// TCP streams recognized as TLS that yielded a complete ClientHello or
/// ServerHello. The denominator, with `unfinished_tls_streams`, of the miss
/// rate: how many handshakes the tool actually read.
pub tls_handshakes_fingerprinted: u64,
pub unfinished_tls_streams: u64,
/// QUIC long header Initial packets observed, both directions.
pub quic_initials: u64,
/// Client Initials whose protection was removed and payload decrypted.
/// The gap to `quic_initials` is mostly server Initials, which a passive
/// observer cannot open, and is exactly the honesty an operator needs.
pub quic_decrypted: u64,
/// Initials carrying a QUIC version this build has no salt for.
pub quic_version_unsupported: u64,
}
impl Counters {
/// The share of recognized TLS streams the capture clipped before a
/// complete handshake message could be read.
///
/// A fingerprinting tool that cannot say what it failed to read is one
/// whose silence gets mistaken for absence. A truncated or multi segment
/// ClientHello whose later segments never arrived counts as a miss here, so
/// an operator can tell a clean link from a clipped capture before trusting
/// the count of fingerprints. Zero recognized streams reports a zero rate
/// rather than dividing by nothing.
#[must_use]
#[allow(clippy::cast_precision_loss)]
pub fn tls_miss_rate(&self) -> f64 {
let recognized = self.tls_handshakes_fingerprinted + self.unfinished_tls_streams;
if recognized == 0 {
0.0
} else {
self.unfinished_tls_streams as f64 / recognized as f64
}
}
}
/// One direction of one tracked flow.
struct StreamHalf {
reassembler: StreamReassembler,
protocol: StreamProtocol,
syn_fingerprint_emitted: bool,
}
impl StreamHalf {
fn new(limits: ReassemblyLimits) -> Self {
Self {
reassembler: StreamReassembler::new(limits),
protocol: StreamProtocol::Undecided,
syn_fingerprint_emitted: false,
}
}
}
struct FlowState {
halves: [StreamHalf; 2],
last_seen_nanos: u64,
}
impl FlowState {
fn new(limits: ReassemblyLimits) -> Self {
Self {
halves: [StreamHalf::new(limits), StreamHalf::new(limits)],
last_seen_nanos: 0,
}
}
fn finished(&self) -> bool {
self.halves.iter().all(|h| h.protocol.finished())
}
}
/// One tracked QUIC conversation.
///
/// QUIC needs far less per flow state than TCP. There is one cryptographic
/// stream to reassemble, not two byte streams, and the only message this
/// pipeline reads from it is the ClientHello, which lives entirely in the
/// client's first flight of Initial packets. Once the client keys are locked
/// from the first Initial that authenticates, every later Initial on the flow
/// reuses them, and `done` retires the flow the moment the ClientHello is in
/// hand or the stream proves it will never hold one.
struct QuicFlow {
keys: Option<InitialKeys>,
crypto: CryptoAssembler,
largest_pn: Option<u64>,
done: bool,
last_seen_nanos: u64,
}
impl QuicFlow {
fn new(max_crypto_bytes: usize) -> Self {
Self {
keys: None,
crypto: CryptoAssembler::new(max_crypto_bytes),
largest_pn: None,
done: false,
last_seen_nanos: 0,
}
}
}
/// The passive fingerprinting engine.
///
/// Feed it frames, take events out through the sink closure. The pipeline is
/// synchronous and single threaded by design: one pipeline owns its flow
/// table outright, and running one per worker beats sharing a locked table
/// between workers.
pub struct Pipeline {
config: PipelineConfig,
flows: HashMap<FlowKey, FlowState>,
quic_flows: HashMap<FlowKey, QuicFlow>,
counters: Counters,
}
impl Pipeline {
#[must_use]
pub fn new(config: PipelineConfig) -> Self {
Self {
config,
flows: HashMap::new(),
quic_flows: HashMap::new(),
counters: Counters::default(),
}
}
#[must_use]
pub fn counters(&self) -> &Counters {
&self.counters
}
/// Drains a source through the pipeline, sending every event to `sink`.
pub fn run<S: PacketSource>(
&mut self,
source: &mut S,
mut sink: impl FnMut(FingerprintEvent),
) -> Result<(), SourceError> {
while let Some(frame) = source.next_frame()? {
self.feed(&frame, &mut sink);
}
self.finish();
Ok(())
}
/// Processes one captured frame, dispatching to the TCP or QUIC path.
pub fn feed(&mut self, frame: &RawFrame<'_>, sink: &mut impl FnMut(FingerprintEvent)) {
self.counters.frames += 1;
self.counters.bytes += frame.data.len() as u64;
match decode_frame(frame.link_type, frame.data) {
Ok(Decoded::Tcp(segment)) => {
self.counters.tcp_segments += 1;
self.feed_tcp(&segment, frame, sink);
}
Ok(Decoded::Udp(datagram)) => {
self.counters.udp_datagrams += 1;
self.feed_quic(&datagram, frame, sink);
}
Err(skip) => match skip {
Skip::UnsupportedLinkType => self.counters.skipped_unsupported_link_type += 1,
Skip::NotIp => self.counters.skipped_not_ip += 1,
Skip::NotTransport => self.counters.skipped_not_transport += 1,
Skip::Malformed => self.counters.skipped_malformed += 1,
},
}
}
/// Feeds one TCP segment into its flow's reassembler and protocol layer.
fn feed_tcp(
&mut self,
segment: &DecodedSegment<'_>,
frame: &RawFrame<'_>,
sink: &mut impl FnMut(FingerprintEvent),
) {
let (key, direction) = FlowKey::from_pair(segment.src, segment.dst);
if !self.flows.contains_key(&key) {
if self.flows.len() >= self.config.max_flows {
self.evict(frame.ts_nanos);
}
self.flows.insert(key, FlowState::new(self.config.limits()));
self.counters.flows_created += 1;
}
let Some(flow) = self.flows.get_mut(&key) else {
return;
};
flow.last_seen_nanos = flow.last_seen_nanos.max(frame.ts_nanos);
let (src, dst) = direction.addresses(&key);
let half = &mut flow.halves[direction.index()];
if !half.syn_fingerprint_emitted {
if let Some(input) = &segment.syn_fingerprint {
half.syn_fingerprint_emitted = true;
let fingerprint = ja4t(input);
let event = if segment.tcp.flags.ack() {
StreamEvent::TcpSynAck { ja4ts: fingerprint }
} else {
StreamEvent::TcpSyn { ja4t: fingerprint }
};
self.counters.events += 1;
sink(FingerprintEvent {
ts_nanos: frame.ts_nanos,
src,
dst,
event,
});
}
}
if segment.tcp.flags.syn() {
half.reassembler.anchor(segment.tcp.seq.wrapping_add(1));
}
let payload_seq = if segment.tcp.flags.syn() {
segment.tcp.seq.wrapping_add(1)
} else {
segment.tcp.seq
};
let outcome = half.reassembler.push(payload_seq, segment.payload);
if outcome == PushOutcome::Dropped {
self.counters.segments_dropped += 1;
}
if outcome == PushOutcome::Grew {
let mut emitted = 0u64;
let mut tls_fingerprinted = 0u64;
tls::advance(&mut half.protocol, half.reassembler.data(), &mut |event| {
emitted += 1;
if matches!(
event,
StreamEvent::ClientHello { .. } | StreamEvent::ServerHello { .. }
) {
tls_fingerprinted += 1;
}
sink(FingerprintEvent {
ts_nanos: frame.ts_nanos,
src,
dst,
event,
});
});
self.counters.events += emitted;
self.counters.tls_handshakes_fingerprinted += tls_fingerprinted;
if half.protocol.finished() && !half.reassembler.released() {
half.reassembler.release();
}
}
}
/// Feeds one UDP datagram into its QUIC flow, decrypting any client
/// Initial packets and reassembling the ClientHello they carry.
///
/// A datagram may coalesce several QUIC packets, so the parse walks them
/// in turn. Each Initial is opened with the flow's locked client keys, or,
/// before any are locked, with keys derived from that packet's own
/// Destination Connection ID; the AEAD tag is what confirms a packet was
/// a client Initial rather than a server one this observer cannot read.
/// Once the contiguous CRYPTO stream holds a complete ClientHello it is
/// fingerprinted exactly like a TCP one, only carrying the QUIC transport
/// marker, and the flow is retired.
fn feed_quic(
&mut self,
datagram: &DecodedDatagram<'_>,
frame: &RawFrame<'_>,
sink: &mut impl FnMut(FingerprintEvent),
) {
let (key, direction) = FlowKey::from_pair(datagram.src, datagram.dst);
if !self.quic_flows.contains_key(&key) {
if self.quic_flows.len() >= self.config.max_flows {
self.evict_quic(frame.ts_nanos);
}
self.quic_flows
.insert(key, QuicFlow::new(self.config.max_assembled_bytes));
}
let Some(flow) = self.quic_flows.get_mut(&key) else {
return;
};
flow.last_seen_nanos = flow.last_seen_nanos.max(frame.ts_nanos);
if flow.done {
return;
}
let mut offset = 0usize;
while offset < datagram.payload.len() {
let packet = match InitialPacket::parse(datagram.payload, offset) {
Ok(packet) => packet,
Err(crate::error::ParseError::UnsupportedQuicVersion(_)) => {
self.counters.quic_version_unsupported += 1;
break;
}
Err(_) => break,
};
self.counters.quic_initials += 1;
offset = packet.next_offset;
let opened = if let Some(keys) = flow.keys.as_ref() {
packet.open(keys, flow.largest_pn).ok()
} else {
let candidate = packet.client_keys();
match packet.open(&candidate, None) {
Ok(opened) => {
flow.keys = Some(candidate);
Some(opened)
}
Err(_) => None,
}
};
let Some(opened) = opened else {
continue;
};
self.counters.quic_decrypted += 1;
flow.largest_pn = Some(
flow.largest_pn
.map_or(opened.packet_number, |seen| seen.max(opened.packet_number)),
);
let crypto = &mut flow.crypto;
let _ = walk_crypto_frames(&opened.frames, |off, data| crypto.push(off, data));
}
match flow.crypto.client_hello() {
ClientHelloState::Ready(body) => {
if let Ok(hello) = parse_client_hello(body) {
let (src, dst) = direction.addresses(&key);
sink(FingerprintEvent {
ts_nanos: frame.ts_nanos,
src,
dst,
event: StreamEvent::ClientHello {
ja3: ja3(&hello),
ja3_raw: ja3_string(&hello),
ja4: ja4(&hello, Transport::Quic),
sni: hello.server_name().map(str::to_owned),
alpn: hello
.alpn_protocols()
.first()
.map(|p| String::from_utf8_lossy(p).into_owned()),
},
});
self.counters.events += 1;
}
flow.done = true;
}
ClientHelloState::NotClientHello | ClientHelloState::Abandoned => flow.done = true,
ClientHelloState::Incomplete => {}
}
}
/// Settles the books at end of capture.
///
/// Streams that were recognized as TLS but never produced a complete
/// handshake message are counted: each one is a handshake the capture
/// clipped, which is exactly the number an operator needs before trusting
/// an absence of fingerprints.
pub fn finish(&mut self) {
for flow in self.flows.values() {
for half in &flow.halves {
if half.protocol.unfinished_tls() {
self.counters.unfinished_tls_streams += 1;
}
if half.reassembler.capped() {
self.counters.streams_capped += 1;
}
}
}
self.flows.clear();
self.quic_flows.clear();
}
/// Sheds flows when the table is full: everything idle past the timeout
/// or fully harvested goes, and if nothing qualifies, the single stalest
/// flow goes, so the table never refuses a brand new conversation in
/// favor of a dead one.
fn evict(&mut self, now_nanos: u64) {
let timeout = self.config.idle_timeout_nanos;
let idle: Vec<FlowKey> = self
.flows
.iter()
.filter(|(_, flow)| {
now_nanos.saturating_sub(flow.last_seen_nanos) > timeout || flow.finished()
})
.map(|(key, _)| *key)
.collect();
if idle.is_empty() {
let stalest = self
.flows
.iter()
.min_by_key(|(_, flow)| flow.last_seen_nanos)
.map(|(key, _)| *key);
if let Some(key) = stalest {
self.drop_flow(key);
self.counters.flows_evicted_pressure += 1;
}
return;
}
for key in idle {
self.drop_flow(key);
self.counters.flows_evicted_idle += 1;
}
}
fn drop_flow(&mut self, key: FlowKey) {
if let Some(flow) = self.flows.remove(&key) {
for half in &flow.halves {
if half.protocol.unfinished_tls() {
self.counters.unfinished_tls_streams += 1;
}
if half.reassembler.capped() {
self.counters.streams_capped += 1;
}
}
}
}
/// Sheds QUIC flows under the same policy as the TCP table: retire the
/// idle and the already harvested first, and if none qualify, the single
/// stalest flow, so a fresh handshake is never turned away for a dead one.
fn evict_quic(&mut self, now_nanos: u64) {
let timeout = self.config.idle_timeout_nanos;
let idle: Vec<FlowKey> = self
.quic_flows
.iter()
.filter(|(_, flow)| {
now_nanos.saturating_sub(flow.last_seen_nanos) > timeout || flow.done
})
.map(|(key, _)| *key)
.collect();
if idle.is_empty() {
let stalest = self
.quic_flows
.iter()
.min_by_key(|(_, flow)| flow.last_seen_nanos)
.map(|(key, _)| *key);
if let Some(key) = stalest {
self.quic_flows.remove(&key);
self.counters.flows_evicted_pressure += 1;
}
return;
}
for key in idle {
self.quic_flows.remove(&key);
self.counters.flows_evicted_idle += 1;
}
}
}
#[cfg(test)]
mod tests {
use super::{Pipeline, PipelineConfig};
use crate::pipeline::event::FingerprintEvent;
use crate::pipeline::source::RawFrame;
use etherparse::PacketBuilder;
fn tcp_frame(src: ([u8; 4], u16), dst: ([u8; 4], u16), seq: u32, payload: &[u8]) -> Vec<u8> {
let builder = PacketBuilder::ethernet2([1; 6], [2; 6])
.ipv4(src.0, dst.0, 64)
.tcp(src.1, dst.1, seq, 64240);
let mut out = Vec::with_capacity(builder.size(payload.len()));
builder.write(&mut out, payload).unwrap();
out
}
fn feed_all(pipeline: &mut Pipeline, frames: &[Vec<u8>]) -> Vec<FingerprintEvent> {
let mut events = Vec::new();
for (i, data) in frames.iter().enumerate() {
let frame = RawFrame {
ts_nanos: u64::try_from(i).unwrap() * 1_000_000,
link_type: 1,
data,
};
pipeline.feed(&frame, &mut |e| events.push(e));
}
events
}
#[test]
fn http_request_split_across_segments_fingerprints_once() {
let client = ([10, 0, 0, 1], 40000);
let server = ([10, 0, 0, 2], 80);
let request = b"GET / HTTP/1.1\r\nHost: example.com\r\nAccept: */*\r\n\r\n";
let (a, b) = request.split_at(20);
let frames = vec![
tcp_frame(client, server, 1000, a),
tcp_frame(client, server, 1000 + u32::try_from(a.len()).unwrap(), b),
];
let mut pipeline = Pipeline::new(PipelineConfig::default());
let events = feed_all(&mut pipeline, &frames);
assert_eq!(events.len(), 1);
assert_eq!(events[0].src.to_string(), "10.0.0.1:40000");
assert_eq!(pipeline.counters().tcp_segments, 2);
}
#[test]
fn out_of_order_delivery_after_a_syn_still_fingerprints() {
let client = ([10, 0, 0, 1], 40001);
let server = ([10, 0, 0, 2], 80);
let request = b"GET / HTTP/1.1\r\nHost: example.com\r\nAccept: */*\r\n\r\n";
let (a, b) = request.split_at(20);
let syn = {
let builder = PacketBuilder::ethernet2([1; 6], [2; 6])
.ipv4(client.0, server.0, 64)
.tcp(client.1, server.1, 999, 64240)
.syn();
let mut out = Vec::with_capacity(builder.size(0));
builder.write(&mut out, &[]).unwrap();
out
};
let frames = vec![
syn,
tcp_frame(client, server, 1000 + u32::try_from(a.len()).unwrap(), b),
tcp_frame(client, server, 1000, a),
];
let mut pipeline = Pipeline::new(PipelineConfig::default());
let events = feed_all(&mut pipeline, &frames);
assert_eq!(events.len(), 2);
assert!(events[0].to_string().contains("tcp_syn ja4t="));
assert!(events[1].to_string().contains("http_request"));
}
#[test]
fn miss_rate_arithmetic_is_misses_over_recognized() {
use super::Counters;
let none = Counters::default();
assert!(none.tls_miss_rate().abs() < 1e-9);
let clean = Counters {
tls_handshakes_fingerprinted: 4,
unfinished_tls_streams: 0,
..Counters::default()
};
assert!(clean.tls_miss_rate().abs() < 1e-9);
let clipped = Counters {
tls_handshakes_fingerprinted: 3,
unfinished_tls_streams: 1,
..Counters::default()
};
assert!((clipped.tls_miss_rate() - 0.25).abs() < 1e-9);
}
#[test]
fn clipped_tls_handshake_counts_as_a_miss() {
let client = ([10, 0, 0, 1], 40002);
let server = ([10, 0, 0, 2], 443);
let clipped_client_hello = [
0x16, 0x03, 0x01, 0x01, 0x00, 0x01, 0x00, 0x00, 0xfa, 0x03, 0x03,
];
let frames = vec![tcp_frame(client, server, 1000, &clipped_client_hello)];
let mut pipeline = Pipeline::new(PipelineConfig::default());
let events = feed_all(&mut pipeline, &frames);
pipeline.finish();
assert!(events.is_empty());
let counters = pipeline.counters();
assert_eq!(counters.tls_handshakes_fingerprinted, 0);
assert_eq!(counters.unfinished_tls_streams, 1);
assert!((counters.tls_miss_rate() - 1.0).abs() < 1e-9);
}
#[test]
fn pressure_eviction_keeps_the_table_bounded() {
let config = PipelineConfig {
max_flows: 4,
..PipelineConfig::default()
};
let mut pipeline = Pipeline::new(config);
let mut frames = Vec::new();
for i in 0..8u16 {
let port = 40000 + i;
frames.push(tcp_frame(
([10, 0, 0, 1], port),
([10, 0, 0, 2], 80),
1,
b"x",
));
}
feed_all(&mut pipeline, &frames);
assert_eq!(pipeline.counters().flows_created, 8);
assert!(pipeline.counters().flows_evicted_pressure >= 4);
}
}

365
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/pipeline/source.rs Normal file
View File

@ -0,0 +1,365 @@
// ©AngelaMos | 2026
// source.rs
use std::fs::File;
use std::io::Read;
use std::path::Path;
use pcap_parser::traits::PcapReaderIterator;
use pcap_parser::{Block, PcapBlockOwned, PcapError, PcapHeader, create_reader};
use thiserror::Error;
/// How many bytes of buffer the file reader starts with.
///
/// A single capture block must fit in the buffer. Offload features such as TSO
/// can put frames far larger than an MTU into a capture, so the buffer starts
/// generous and can still grow up to [`MAX_BUFFER_CAPACITY`] if a bigger block
/// appears.
const INITIAL_BUFFER_CAPACITY: usize = 1024 * 1024;
/// The ceiling for buffer growth. A block larger than this is treated as a
/// malformed capture rather than a reason to exhaust memory.
const MAX_BUFFER_CAPACITY: usize = 64 * 1024 * 1024;
/// Timestamp units per second when a capture does not say otherwise.
///
/// Both the legacy pcap format and the pcapng default are microsecond
/// resolution.
const DEFAULT_UNITS_PER_SECOND: u64 = 1_000_000;
const NANOS_PER_SECOND: u64 = 1_000_000_000;
const NANOS_PER_MICRO: u64 = 1_000;
/// Errors produced while reading frames from a capture source.
#[derive(Debug, Error)]
#[non_exhaustive]
pub enum SourceError {
#[error("failed to read capture: {0}")]
Io(#[from] std::io::Error),
#[error("not a pcap or pcapng capture")]
NotACapture,
#[error("capture block exceeds the {MAX_BUFFER_CAPACITY} byte buffer ceiling")]
BlockTooLarge,
#[error("malformed capture: {0}")]
Malformed(String),
#[error("capture source failed: {0}")]
Capture(String),
}
/// One link layer frame as captured, with the metadata needed to decode it.
#[derive(Debug, Clone, Copy)]
pub struct RawFrame<'src> {
/// Capture timestamp in nanoseconds since the epoch. Zero when the capture
/// format carries no timestamp for this frame.
pub ts_nanos: u64,
/// The link layer type, using the tcpdump LINKTYPE registry numbers.
pub link_type: i32,
pub data: &'src [u8],
}
/// A source of captured frames.
///
/// The trait is a lending iterator: each frame borrows from the source and is
/// only valid until the next call. That shape fits both file readers, which
/// hand out windows into an internal buffer, and live captures, which hand out
/// the kernel's buffer. A consumer that needs to keep a frame longer copies
/// it, and that decision stays visible at the call site.
pub trait PacketSource {
/// Returns the next frame, or `None` when the source is exhausted.
fn next_frame(&mut self) -> Result<Option<RawFrame<'_>>, SourceError>;
}
/// Per interface metadata from a pcapng interface description block.
#[derive(Debug, Clone, Copy)]
struct InterfaceInfo {
link_type: i32,
units_per_second: u64,
ts_offset_seconds: i64,
}
/// Everything in the source except the parser, split out so the borrow of the
/// parser's buffer held by a block and the mutable borrow needed to stage a
/// frame land on different fields.
#[derive(Default)]
struct SourceState {
interfaces: Vec<InterfaceInfo>,
legacy: Option<InterfaceInfo>,
legacy_nanos: bool,
frame: Vec<u8>,
frame_ts_nanos: u64,
frame_link_type: i32,
}
impl SourceState {
/// Copies a frame out of the parser's buffer so the borrow on the parser
/// can end before the block is consumed.
fn stage(&mut self, ts_nanos: u64, link_type: i32, data: &[u8]) {
self.frame.clear();
self.frame.extend_from_slice(data);
self.frame_ts_nanos = ts_nanos;
self.frame_link_type = link_type;
}
fn handle_legacy_header(&mut self, header: &PcapHeader) {
self.legacy = Some(InterfaceInfo {
link_type: header.network.0,
units_per_second: DEFAULT_UNITS_PER_SECOND,
ts_offset_seconds: 0,
});
self.legacy_nanos = header.is_nanosecond_precision();
}
/// Stages a packet block. Returns false for metadata blocks.
fn handle_block(&mut self, block: &PcapBlockOwned<'_>) -> bool {
match block {
PcapBlockOwned::LegacyHeader(header) => {
self.handle_legacy_header(header);
false
}
PcapBlockOwned::Legacy(frame) => {
let Some(meta) = self.legacy else {
return false;
};
let fraction = if self.legacy_nanos {
u64::from(frame.ts_usec)
} else {
u64::from(frame.ts_usec) * NANOS_PER_MICRO
};
let ts = u64::from(frame.ts_sec)
.saturating_mul(NANOS_PER_SECOND)
.saturating_add(fraction);
let len = frame.data.len().min(frame.caplen as usize);
self.stage(ts, meta.link_type, &frame.data[..len]);
true
}
PcapBlockOwned::NG(Block::SectionHeader(_)) => {
self.interfaces.clear();
false
}
PcapBlockOwned::NG(Block::InterfaceDescription(idb)) => {
self.interfaces.push(InterfaceInfo {
link_type: idb.linktype.0,
units_per_second: idb.ts_resolution().unwrap_or(DEFAULT_UNITS_PER_SECOND),
ts_offset_seconds: idb.ts_offset(),
});
false
}
PcapBlockOwned::NG(Block::EnhancedPacket(epb)) => {
let Some(meta) = self.interfaces.get(epb.if_id as usize).copied() else {
return false;
};
let units = (u64::from(epb.ts_high) << 32) | u64::from(epb.ts_low);
let ts = scale_to_nanos(units, meta.units_per_second, meta.ts_offset_seconds);
let len = epb.data.len().min(epb.caplen as usize);
self.stage(ts, meta.link_type, &epb.data[..len]);
true
}
PcapBlockOwned::NG(Block::SimplePacket(spb)) => {
let Some(meta) = self.interfaces.first().copied() else {
return false;
};
let len = spb.data.len().min(spb.origlen as usize);
self.stage(0, meta.link_type, &spb.data[..len]);
true
}
PcapBlockOwned::NG(_) => false,
}
}
}
/// Reads frames from a pcap or pcapng file.
///
/// The two formats are probed automatically. pcapng is handled with its full
/// generality: every interface carries its own link type and timestamp
/// resolution, multiple sections reset the interface list, and metadata blocks
/// such as name resolution and decryption secrets are skipped rather than
/// treated as packets. A truncated final packet, the signature of a capture
/// that was stopped rather than closed, ends iteration cleanly and is reported
/// through [`PcapFileSource::truncated`] instead of failing the whole file.
pub struct PcapFileSource {
reader: Box<dyn PcapReaderIterator>,
state: SourceState,
buffer_capacity: usize,
truncated: bool,
finished: bool,
}
impl PcapFileSource {
/// Opens a capture file from a path.
pub fn open(path: impl AsRef<Path>) -> Result<Self, SourceError> {
Self::from_reader(File::open(path)?)
}
/// Builds a source from any byte reader holding pcap or pcapng data.
pub fn from_reader(reader: impl Read + 'static) -> Result<Self, SourceError> {
let reader = create_reader(INITIAL_BUFFER_CAPACITY, reader).map_err(|e| match e {
PcapError::HeaderNotRecognized | PcapError::Eof => SourceError::NotACapture,
PcapError::ReadError => SourceError::Io(std::io::Error::other("read failed")),
other => SourceError::Malformed(other.to_string()),
})?;
Ok(Self {
reader,
state: SourceState::default(),
buffer_capacity: INITIAL_BUFFER_CAPACITY,
truncated: false,
finished: false,
})
}
/// Returns true when the file ended in the middle of a block.
pub fn truncated(&self) -> bool {
self.truncated
}
}
impl PacketSource for PcapFileSource {
fn next_frame(&mut self) -> Result<Option<RawFrame<'_>>, SourceError> {
if self.finished {
return Ok(None);
}
loop {
let staged = match self.reader.next() {
Ok((offset, block)) => {
let staged = self.state.handle_block(&block);
self.reader.consume(offset);
staged
}
Err(PcapError::Eof) => {
self.finished = true;
return Ok(None);
}
Err(PcapError::UnexpectedEof) => {
self.finished = true;
self.truncated = true;
return Ok(None);
}
Err(PcapError::Incomplete(_)) => {
self.reader
.refill()
.map_err(|e| SourceError::Malformed(e.to_string()))?;
continue;
}
Err(PcapError::BufferTooSmall) => {
let grown = self.buffer_capacity.saturating_mul(2);
if grown > MAX_BUFFER_CAPACITY || !self.reader.grow(grown) {
self.finished = true;
return Err(SourceError::BlockTooLarge);
}
self.buffer_capacity = grown;
continue;
}
Err(PcapError::ReadError) => {
self.finished = true;
return Err(SourceError::Io(std::io::Error::other("read failed")));
}
Err(
e @ (PcapError::HeaderNotRecognized
| PcapError::NomError(..)
| PcapError::OwnedNomError(..)),
) => {
self.finished = true;
return Err(SourceError::Malformed(e.to_string()));
}
};
if staged {
return Ok(Some(RawFrame {
ts_nanos: self.state.frame_ts_nanos,
link_type: self.state.frame_link_type,
data: &self.state.frame,
}));
}
}
}
}
/// Converts a timestamp in interface units to nanoseconds since the epoch.
///
/// The arithmetic runs in 128 bits so the conversion stays exact for every
/// resolution pcapng can express, including nanosecond counts that already
/// fill most of a u64.
fn scale_to_nanos(units: u64, units_per_second: u64, offset_seconds: i64) -> u64 {
if units_per_second == 0 {
return 0;
}
let nanos = u128::from(units) * u128::from(NANOS_PER_SECOND) / u128::from(units_per_second);
let offset = i128::from(offset_seconds) * i128::from(NANOS_PER_SECOND);
u64::try_from(
i128::try_from(nanos)
.unwrap_or(i128::MAX)
.saturating_add(offset),
)
.unwrap_or_default()
}
#[cfg(test)]
mod tests {
use super::{PacketSource, PcapFileSource, SourceError, scale_to_nanos};
fn legacy_pcap(frames: &[&[u8]]) -> Vec<u8> {
let mut v = Vec::new();
v.extend_from_slice(&0xa1b2_c3d4_u32.to_le_bytes());
v.extend_from_slice(&2u16.to_le_bytes());
v.extend_from_slice(&4u16.to_le_bytes());
v.extend_from_slice(&0i32.to_le_bytes());
v.extend_from_slice(&0u32.to_le_bytes());
v.extend_from_slice(&65535u32.to_le_bytes());
v.extend_from_slice(&1u32.to_le_bytes());
for (i, frame) in frames.iter().enumerate() {
let len = u32::try_from(frame.len()).unwrap();
v.extend_from_slice(&u32::try_from(i + 1).unwrap().to_le_bytes());
v.extend_from_slice(&500_000u32.to_le_bytes());
v.extend_from_slice(&len.to_le_bytes());
v.extend_from_slice(&len.to_le_bytes());
v.extend_from_slice(frame);
}
v
}
#[test]
fn reads_legacy_frames_with_timestamps() {
let data = legacy_pcap(&[&[0xaa; 14], &[0xbb; 20]]);
let mut source = PcapFileSource::from_reader(std::io::Cursor::new(data)).unwrap();
let one = source.next_frame().unwrap().unwrap();
assert_eq!(one.link_type, 1);
assert_eq!(one.ts_nanos, 1_500_000_000);
assert_eq!(one.data.len(), 14);
let two = source.next_frame().unwrap().unwrap();
assert_eq!(two.data, &[0xbb; 20]);
assert!(source.next_frame().unwrap().is_none());
assert!(!source.truncated());
}
#[test]
fn truncated_final_frame_ends_cleanly() {
let mut data = legacy_pcap(&[&[0xaa; 14], &[0xbb; 20]]);
data.truncate(data.len() - 5);
let mut source = PcapFileSource::from_reader(std::io::Cursor::new(data)).unwrap();
assert!(source.next_frame().unwrap().is_some());
assert!(source.next_frame().unwrap().is_none());
assert!(source.truncated());
assert!(source.next_frame().unwrap().is_none());
}
#[test]
fn garbage_is_not_a_capture() {
let err = PcapFileSource::from_reader(std::io::Cursor::new(vec![0x55; 64]))
.err()
.unwrap();
assert!(matches!(err, SourceError::NotACapture));
}
#[test]
fn timestamp_scaling_is_exact_for_common_resolutions() {
assert_eq!(scale_to_nanos(1_500_000, 1_000_000, 0), 1_500_000_000);
assert_eq!(scale_to_nanos(7, 1_000_000_000, 0), 7);
assert_eq!(scale_to_nanos(1, 1, 1), 2_000_000_000);
}
}

407
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/pipeline/tls.rs Normal file
View File

@ -0,0 +1,407 @@
// ©AngelaMos | 2026
// tls.rs
use std::borrow::Cow;
use crate::ja3::{ja3, ja3_string, ja3s, ja3s_string};
use crate::ja4::{Transport, ja4, ja4s};
use crate::ja4h::{ja4h, parse_http_request};
use crate::ja4x::ja4x;
use crate::parse::reader::Reader;
use crate::parse::{
certificate_der_list, is_sslv2_client_hello, parse_client_hello, parse_server_hello,
parse_sslv2_client_hello,
};
use crate::pipeline::event::StreamEvent;
use crate::registry::{content_type, handshake_type, version};
/// How many bytes the sniffer needs before it gives up on classifying a
/// stream. Every protocol this pipeline recognizes shows its hand within the
/// first few bytes; eight covers the longest HTTP method prefix.
const SNIFF_DECISION_LEN: usize = 8;
/// The HTTP methods the sniffer accepts as the start of a cleartext request.
const HTTP_METHOD_PREFIXES: [&[u8]; 9] = [
b"GET ",
b"POST ",
b"PUT ",
b"HEAD ",
b"DELETE ",
b"OPTIONS ",
b"PATCH ",
b"TRACE ",
b"CONNECT ",
];
/// A stream whose HTTP request head has not finished inside this many bytes
/// is not worth waiting on.
const HTTP_HEAD_CAP: usize = 8 * 1024;
const HTTP_HEAD_TERMINATOR: &[u8] = b"\r\n\r\n";
/// A TLS record payload cannot exceed 2^14 plus expansion; RFC 8446 allows
/// 255 bytes of expansion on top of the 16384 byte plaintext limit. A length
/// beyond that means the stream is not actually TLS record framing.
const MAX_TLS_RECORD_LEN: usize = 16384 + 255;
/// What one direction of a flow is, as far as the protocol layer can tell.
#[derive(Debug)]
pub enum StreamProtocol {
/// Not enough bytes yet to say.
Undecided,
/// TLS record framing; the cleartext first flight is being extracted.
Tls(TlsFlight),
/// A cleartext HTTP/1.x request head is being accumulated.
Http,
/// Recognized and fully harvested; the stream needs no more buffering.
Done,
/// Unrecognized or unparseable; the stream is ignored.
Ignored,
}
impl StreamProtocol {
/// True when this direction will never produce another event, which is
/// the signal to drop its reassembly buffers.
#[must_use]
pub fn finished(&self) -> bool {
matches!(self, StreamProtocol::Done | StreamProtocol::Ignored)
}
/// True when the stream was recognized as TLS but the capture ended
/// before a complete hello could be read. Feeds the miss rate counter
/// that tells an operator their capture is clipping handshakes.
#[must_use]
pub fn unfinished_tls(&self) -> bool {
match self {
StreamProtocol::Tls(flight) => !flight.saw_any_message,
_ => false,
}
}
}
/// Incremental extraction state for one direction's cleartext TLS flight.
///
/// Only what must survive between walks lives here. A ClientHello ends its
/// direction immediately, so it needs no flag; a ServerHello does not, since
/// a TLS 1.2 Certificate may still be in flight behind it, so the emission
/// guard for it persists.
#[derive(Debug, Default)]
pub struct TlsFlight {
emitted_server_hello: bool,
saw_any_message: bool,
}
/// Drives protocol detection and extraction over one direction of a stream.
///
/// `stream` is always the full contiguous bytes from the start of the
/// direction; the extractor re-walks them on each call. That sounds wasteful
/// and is not: the walk is linear over at most the reassembly cap, the
/// interesting messages sit in the first packets, and re-walking from the
/// start is what makes a message that arrives split across three segments
/// parse correctly with no incremental parser state to get wrong.
///
/// Returns events through `sink` and updates `proto` in place.
pub fn advance(proto: &mut StreamProtocol, stream: &[u8], sink: &mut impl FnMut(StreamEvent)) {
if matches!(proto, StreamProtocol::Undecided) {
*proto = sniff(stream);
}
match proto {
StreamProtocol::Undecided | StreamProtocol::Done | StreamProtocol::Ignored => {}
StreamProtocol::Tls(_) => advance_tls(proto, stream, sink),
StreamProtocol::Http => advance_http(proto, stream, sink),
}
}
/// Classifies the first bytes of a stream.
fn sniff(stream: &[u8]) -> StreamProtocol {
if stream.len() >= 3 {
if stream[0] == content_type::HANDSHAKE && stream[1] == 0x03 && stream[2] <= 0x04 {
return StreamProtocol::Tls(TlsFlight::default());
}
if is_sslv2_client_hello(stream) {
return StreamProtocol::Tls(TlsFlight::default());
}
}
if stream.len() >= SNIFF_DECISION_LEN {
if HTTP_METHOD_PREFIXES.iter().any(|m| stream.starts_with(m)) {
return StreamProtocol::Http;
}
return StreamProtocol::Ignored;
}
StreamProtocol::Undecided
}
fn advance_http(proto: &mut StreamProtocol, stream: &[u8], sink: &mut impl FnMut(StreamEvent)) {
let head_end = stream
.windows(HTTP_HEAD_TERMINATOR.len())
.position(|w| w == HTTP_HEAD_TERMINATOR);
let Some(head_end) = head_end else {
if stream.len() > HTTP_HEAD_CAP {
*proto = StreamProtocol::Ignored;
}
return;
};
let head = &stream[..head_end + HTTP_HEAD_TERMINATOR.len()];
if let Some(request) = parse_http_request(head) {
let host = request
.headers
.iter()
.find(|(name, _)| name.eq_ignore_ascii_case("host"))
.map(|(_, value)| value.clone());
let user_agent = request
.headers
.iter()
.find(|(name, _)| name.eq_ignore_ascii_case("user-agent"))
.map(|(_, value)| value.clone());
sink(StreamEvent::HttpRequest {
ja4h: ja4h(&request),
method: request.method.clone(),
host,
user_agent,
});
*proto = StreamProtocol::Done;
} else {
*proto = StreamProtocol::Ignored;
}
}
fn advance_tls(proto: &mut StreamProtocol, stream: &[u8], sink: &mut impl FnMut(StreamEvent)) {
if is_sslv2_client_hello(stream) {
advance_sslv2(proto, stream, sink);
return;
}
let Some(flight_bytes) = collect_flight(stream) else {
*proto = StreamProtocol::Ignored;
return;
};
let StreamProtocol::Tls(flight) = proto else {
return;
};
let mut done = walk_messages(flight, flight_bytes.handshake.as_ref(), sink);
if flight_bytes.flight_closed && !done {
done = true;
}
if done {
*proto = StreamProtocol::Done;
}
}
fn advance_sslv2(proto: &mut StreamProtocol, stream: &[u8], sink: &mut impl FnMut(StreamEvent)) {
match parse_sslv2_client_hello(stream) {
Ok(hello) => {
sink(StreamEvent::ClientHello {
ja3: ja3(&hello),
ja3_raw: ja3_string(&hello),
ja4: ja4(&hello, Transport::Tcp),
sni: None,
alpn: None,
});
*proto = StreamProtocol::Done;
}
Err(crate::error::ParseError::Truncated { .. }) => {}
Err(_) => *proto = StreamProtocol::Ignored,
}
}
struct FlightBytes<'stream> {
handshake: Cow<'stream, [u8]>,
/// True when a non handshake record followed the handshake records, which
/// in cleartext TLS means the readable part of the flight is over.
flight_closed: bool,
}
/// Collects the payloads of the leading complete handshake records.
///
/// Unlike the strict reassembled flight walker in the parse module, this
/// tolerates a trailing partial record, because the stream is still growing.
/// Returns `None` when the bytes stop looking like TLS record framing at all.
fn collect_flight(stream: &[u8]) -> Option<FlightBytes<'_>> {
let mut segments: Vec<&[u8]> = Vec::new();
let mut r = Reader::new(stream);
let mut flight_closed = false;
while r.remaining() >= 5 {
let Ok(ctype) = r.u8() else { break };
let Ok(record_version) = r.u16() else { break };
if !matches!(
ctype,
content_type::CHANGE_CIPHER_SPEC
| content_type::ALERT
| content_type::HANDSHAKE
| content_type::APPLICATION_DATA
) {
return None;
}
if (record_version & 0xff00) != 0x0300 || (record_version & 0x00ff) > 0x04 {
return None;
}
let Ok(declared) = r.u16() else { break };
if declared as usize > MAX_TLS_RECORD_LEN {
return None;
}
let Ok(payload) = r.take(declared as usize) else {
break;
};
if ctype == content_type::HANDSHAKE {
segments.push(payload);
} else if !segments.is_empty() {
flight_closed = true;
break;
}
}
let handshake = match segments.as_slice() {
[] => Cow::Borrowed(&[][..]),
[only] => Cow::Borrowed(*only),
many => {
let mut joined = Vec::with_capacity(many.iter().map(|s| s.len()).sum());
for seg in many {
joined.extend_from_slice(seg);
}
Cow::Owned(joined)
}
};
Some(FlightBytes {
handshake,
flight_closed,
})
}
/// Walks the complete handshake messages in the flight, emitting fingerprints
/// for the ones that carry them. Returns true when this direction has yielded
/// everything it ever will.
fn walk_messages(
flight: &mut TlsFlight,
handshake: &[u8],
sink: &mut impl FnMut(StreamEvent),
) -> bool {
let mut r = Reader::new(handshake);
while r.remaining() >= 4 {
let Ok(msg_type) = r.u8() else { break };
let Ok(len) = r.u24() else { break };
let Ok(body) = r.take(len as usize) else {
break;
};
flight.saw_any_message = true;
match msg_type {
handshake_type::CLIENT_HELLO => {
if let Ok(hello) = parse_client_hello(body) {
sink(StreamEvent::ClientHello {
ja3: ja3(&hello),
ja3_raw: ja3_string(&hello),
ja4: ja4(&hello, Transport::Tcp),
sni: hello.server_name().map(str::to_owned),
alpn: hello
.alpn_protocols()
.first()
.map(|p| String::from_utf8_lossy(p).into_owned()),
});
}
return true;
}
handshake_type::SERVER_HELLO => {
if !flight.emitted_server_hello {
if let Ok(hello) = parse_server_hello(body) {
flight.emitted_server_hello = true;
let negotiated_tls13 = hello.selected_version() == version::TLS_1_3;
sink(StreamEvent::ServerHello {
ja3s: ja3s(&hello),
ja3s_raw: ja3s_string(&hello),
ja4s: ja4s(&hello, Transport::Tcp),
});
if negotiated_tls13 {
return true;
}
}
}
}
handshake_type::CERTIFICATE => {
if let Ok(certs) = certificate_der_list(body) {
for cert in certs {
if let Ok(fingerprint) = ja4x(cert) {
sink(StreamEvent::Certificate { ja4x: fingerprint });
}
}
}
return true;
}
_ => {}
}
}
false
}
#[cfg(test)]
mod tests {
use super::{StreamProtocol, advance, sniff};
use crate::pipeline::event::StreamEvent;
fn record(payload: &[u8]) -> Vec<u8> {
let mut v = vec![0x16, 0x03, 0x01];
v.extend_from_slice(&u16::try_from(payload.len()).unwrap().to_be_bytes());
v.extend_from_slice(payload);
v
}
#[test]
fn sniffs_tls_http_and_garbage() {
assert!(matches!(
sniff(&[0x16, 0x03, 0x01, 0x00, 0x05]),
StreamProtocol::Tls(_)
));
assert!(matches!(sniff(b"GET / HTTP/1.1\r\n"), StreamProtocol::Http));
assert!(matches!(
sniff(b"SSH-2.0-OpenSSH_9.7"),
StreamProtocol::Ignored
));
assert!(matches!(sniff(b"GE"), StreamProtocol::Undecided));
}
#[test]
fn http_request_yields_ja4h_once_head_completes() {
let mut proto = StreamProtocol::Undecided;
let mut events = Vec::new();
let partial = b"GET / HTTP/1.1\r\nHost: example.com\r\n";
advance(&mut proto, partial, &mut |e| events.push(e));
assert!(events.is_empty());
assert!(matches!(proto, StreamProtocol::Http));
let full = b"GET / HTTP/1.1\r\nHost: example.com\r\nAccept: */*\r\n\r\n";
advance(&mut proto, full, &mut |e| events.push(e));
assert_eq!(events.len(), 1);
assert!(matches!(
&events[0],
StreamEvent::HttpRequest { method, host: Some(h), .. }
if method == "GET" && h == "example.com"
));
assert!(proto.finished());
}
#[test]
fn partial_tls_record_waits_for_more_bytes() {
let mut proto = StreamProtocol::Undecided;
let mut events = Vec::new();
let full = record(&[0x01, 0x00, 0x00, 0x02, 0xaa, 0xbb]);
advance(&mut proto, &full[..7], &mut |e| events.push(e));
assert!(events.is_empty());
assert!(matches!(proto, StreamProtocol::Tls(_)));
assert!(!proto.finished());
}
#[test]
fn nonsense_record_length_poisons_the_stream() {
let mut proto = StreamProtocol::Undecided;
let mut events = Vec::new();
let stream = [0x16, 0x03, 0x01, 0xff, 0xff, 0x00, 0x00, 0x00];
advance(&mut proto, &stream, &mut |e| events.push(e));
assert!(matches!(proto, StreamProtocol::Ignored));
assert!(events.is_empty());
}
}

886
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/quic.rs Normal file
View File

@ -0,0 +1,886 @@
// ©AngelaMos | 2026
// quic.rs
//! QUIC version 1 Initial packet decryption, from RFC 9001 and RFC 9000.
//!
//! A QUIC ClientHello is not sent in the clear. It travels inside CRYPTO
//! frames in one or more Initial packets, and those packets are encrypted,
//! even though the key is derivable by anyone who sees the connection start.
//! The point of that encryption is not secrecy. It is ossification defence:
//! by protecting the Initial under a key derived from a connection ID that
//! every observer can read, QUIC forces middleboxes to either implement the
//! whole scheme or leave the packet alone, which keeps the wire format free
//! to evolve. A passive fingerprinter is the rare observer that genuinely
//! wants to read the handshake, so it implements the whole scheme.
//!
//! The recipe, from RFC 9001 Section 5.2 and Section 5.4, is:
//!
//! 1. The initial secret is `HKDF-Extract(initial_salt, dcid)`, where `dcid`
//! is the Destination Connection ID of the client's first Initial packet
//! and `initial_salt` is a constant fixed by the QUIC version.
//! 2. The client traffic secret is `HKDF-Expand-Label(initial_secret,
//! "client in", "", 32)`, and from it come the AEAD key, the AEAD IV, and
//! the header protection key, each its own `HKDF-Expand-Label`.
//! 3. Header protection masks the low bits of the first byte and the whole
//! packet number with `AES-ECB(hp, sample)`, where `sample` is sixteen
//! bytes of ciphertext taken four bytes past the start of the packet
//! number field. Removing it reveals the packet number length and value.
//! 4. The payload is `AEAD_AES_128_GCM`, with the unprotected header as
//! associated data and a nonce built by XORing the packet number into the
//! IV.
//!
//! Everything here is bounds checked and allocation light. A malformed or
//! truncated Initial is an ordinary error, never a panic, because this code
//! runs against whatever a network hands it.
use std::collections::BTreeMap;
use aes::Aes128;
use aes::cipher::{BlockEncrypt, KeyInit as _, generic_array::GenericArray};
use aes_gcm::Aes128Gcm;
use aes_gcm::aead::AeadInPlace;
use hkdf::Hkdf;
use sha2::Sha256;
use smallvec::SmallVec;
use crate::error::{ParseError, Result};
use crate::registry::handshake_type;
/// The QUIC version 1 code point, from RFC 9000.
pub const VERSION_1: u32 = 0x0000_0001;
/// The QUIC version 2 code point, from RFC 9369.
pub const VERSION_2: u32 = 0x6b33_43cf;
/// The version 1 Initial salt, from RFC 9001 Section 5.2. This constant is
/// what binds a set of Initial keys to one QUIC version: a different version
/// uses a different salt, so keys derived under the wrong salt simply fail
/// the AEAD tag.
const INITIAL_SALT_V1: [u8; 20] = [
0x38, 0x76, 0x2c, 0xf7, 0xf5, 0x59, 0x34, 0xb3, 0x4d, 0x17, 0x9a, 0xe6, 0xa4, 0xc8, 0x0c, 0xad,
0xcc, 0xbb, 0x7f, 0x0a,
];
/// The version 2 Initial salt, from RFC 9369 Section 3.3.1. QUIC v2 deliberately
/// changes the salt, the key derivation labels, and the long header type code
/// for Initial away from their v1 values, so an observer must implement the
/// version on purpose rather than assume v1 framing.
const INITIAL_SALT_V2: [u8; 20] = [
0x0d, 0xed, 0xe3, 0xde, 0xf7, 0x00, 0xa6, 0xdb, 0x81, 0x93, 0x81, 0xbe, 0x6e, 0x26, 0x9d, 0xcb,
0xf9, 0xbd, 0x2e, 0xd9,
];
const KEY_LEN: usize = 16;
const IV_LEN: usize = 12;
const HP_LEN: usize = 16;
/// The header protection sample is sixteen bytes, the AES block size.
const SAMPLE_LEN: usize = 16;
/// The AEAD authentication tag occupies the last sixteen bytes of the
/// protected payload.
const TAG_LEN: usize = 16;
/// Header protection assumes a four byte packet number when locating the
/// sample, because the true length is not known until protection is removed.
const PN_SAMPLE_OFFSET: usize = 4;
/// A connection ID is at most twenty bytes in QUIC version 1 (RFC 9000
/// Section 17.2). A longer length field marks the packet as not version 1
/// framing and is rejected rather than trusted.
const MAX_CID_LEN: usize = 20;
/// The high bit of the first byte marks a long header; the next bit is the
/// version independent fixed bit. The two type bits identify an Initial, but
/// their value differs by version, `0b00` in v1 and `0b01` in v2, so the type
/// code is carried per version rather than as one constant.
const LONG_HEADER_FORM: u8 = 0x80;
const PACKET_TYPE_MASK: u8 = 0x30;
/// Header protection masks the low four bits of a long header's first byte.
const LONG_HEADER_PN_MASK: u8 = 0x0f;
/// The two low bits of the unprotected first byte hold the packet number
/// length, biased by one.
const PN_LEN_MASK: u8 = 0x03;
/// QUIC frame type codes that may appear in an Initial packet, from RFC 9000
/// Section 12.4. Anything outside this set means the decrypted bytes are not
/// a well formed Initial payload, which is treated as a decryption failure
/// rather than guessed past.
mod frame {
pub const PADDING: u8 = 0x00;
pub const PING: u8 = 0x01;
pub const ACK: u8 = 0x02;
pub const ACK_ECN: u8 = 0x03;
pub const CRYPTO: u8 = 0x06;
pub const CONNECTION_CLOSE: u8 = 0x1c;
}
/// Reads a QUIC variable length integer, returning the value and the number
/// of bytes it consumed.
///
/// The length lives in the top two bits of the first byte, giving a one, two,
/// four, or eight byte encoding. A buffer too short for the indicated length
/// is an error, not a truncated read.
fn read_varint(buf: &[u8]) -> Result<(u64, usize)> {
let first = *buf.first().ok_or(short(1, 0))?;
let len = 1usize << (first >> 6);
let bytes = buf.get(..len).ok_or(short(len, buf.len()))?;
let mut value = u64::from(first & 0x3f);
for &b in &bytes[1..] {
value = (value << 8) | u64::from(b);
}
Ok((value, len))
}
const fn short(needed: usize, have: usize) -> ParseError {
ParseError::Truncated { needed, have }
}
/// Decodes a truncated packet number into the full value, from RFC 9000
/// Appendix A.3.
///
/// The wire carries only the low bits of the packet number; the rest is
/// inferred from the largest packet number already seen so that the value
/// lands in the window around what was expected next. The very first Initial
/// on a flow has no prior packet, so its truncated value is taken as is,
/// which is correct because a connection's first packet numbers start at zero
/// and never need the window arithmetic.
fn decode_packet_number(largest: Option<u64>, truncated: u64, pn_nbits: u32) -> u64 {
let Some(largest) = largest else {
return truncated;
};
let expected = largest + 1;
let win = 1u64 << pn_nbits;
let hwin = win / 2;
let mask = win - 1;
let candidate = (expected & !mask) | truncated;
if candidate + hwin <= expected && candidate < (1u64 << 62) - win {
return candidate + win;
}
if candidate > expected + hwin && candidate >= win {
return candidate - win;
}
candidate
}
/// Builds the `HkdfLabel` structure that QUIC and TLS 1.3 feed to
/// HKDF-Expand, from RFC 8446 Section 7.1.
///
/// The structure is the output length as a sixteen bit integer, then the
/// label prefixed by its own one byte length, then an empty context prefixed
/// by a zero length byte. QUIC always uses the `tls13 ` prefix and an empty
/// context for Initial keys.
fn hkdf_expand_label(secret: &[u8], label: &[u8], out: &mut [u8]) {
const PREFIX: &[u8] = b"tls13 ";
let mut info: SmallVec<[u8; 32]> = SmallVec::new();
let out_len = u16::try_from(out.len()).expect("Initial key material is never that long");
info.extend_from_slice(&out_len.to_be_bytes());
info.push(u8::try_from(PREFIX.len() + label.len()).expect("Initial labels are short"));
info.extend_from_slice(PREFIX);
info.extend_from_slice(label);
info.push(0);
let hk = Hkdf::<Sha256>::from_prk(secret).expect("the secret is one SHA-256 block");
hk.expand(&info, out)
.expect("Initial output never exceeds the HKDF limit");
}
/// The version specific inputs to Initial key derivation: the HKDF salt, the
/// key, IV, and header protection labels, and the long header type code that
/// marks an Initial packet.
///
/// QUIC v1 (RFC 9001) and v2 (RFC 9369) differ in all of these. Binding them to
/// the version is what makes keys derived under the wrong version fail the AEAD
/// tag rather than silently misread a packet, and the `client in` secret label
/// is shared because it comes from the TLS 1.3 derivation, which v2 left alone.
#[derive(Clone, Copy)]
struct QuicVersion {
salt: &'static [u8; 20],
key_label: &'static [u8],
iv_label: &'static [u8],
hp_label: &'static [u8],
initial_type: u8,
}
impl QuicVersion {
const V1: QuicVersion = QuicVersion {
salt: &INITIAL_SALT_V1,
key_label: b"quic key",
iv_label: b"quic iv",
hp_label: b"quic hp",
initial_type: 0x00,
};
const V2: QuicVersion = QuicVersion {
salt: &INITIAL_SALT_V2,
key_label: b"quicv2 key",
iv_label: b"quicv2 iv",
hp_label: b"quicv2 hp",
initial_type: 0x10,
};
/// Returns the parameters for a known QUIC version code, or `None` for a
/// version this build has no Initial salt for.
fn from_code(code: u32) -> Option<QuicVersion> {
match code {
VERSION_1 => Some(Self::V1),
VERSION_2 => Some(Self::V2),
_ => None,
}
}
}
/// The AEAD and header protection keys for one direction of one QUIC Initial.
pub struct InitialKeys {
key: [u8; KEY_LEN],
iv: [u8; IV_LEN],
hp: [u8; HP_LEN],
}
impl InitialKeys {
/// Derives the client's Initial keys from a Destination Connection ID, under
/// the salt and labels of the given QUIC version.
///
/// A passive observer derives these from the connection ID alone, with no
/// secret input, which is exactly why the AEAD tag rather than secrecy is
/// what tells a client Initial apart from a server one: only a packet the
/// client actually protected under these keys will verify.
#[must_use]
fn client(dcid: &[u8], version: QuicVersion) -> Self {
let (initial_secret, _) = Hkdf::<Sha256>::extract(Some(version.salt), dcid);
let mut client_secret = [0u8; 32];
hkdf_expand_label(&initial_secret, b"client in", &mut client_secret);
let mut keys = Self {
key: [0u8; KEY_LEN],
iv: [0u8; IV_LEN],
hp: [0u8; HP_LEN],
};
hkdf_expand_label(&client_secret, version.key_label, &mut keys.key);
hkdf_expand_label(&client_secret, version.iv_label, &mut keys.iv);
hkdf_expand_label(&client_secret, version.hp_label, &mut keys.hp);
keys
}
}
/// One QUIC Initial packet located within a UDP datagram, still protected.
///
/// The fields outside header protection, the connection IDs, the token, and
/// the length, are read on construction; the first byte and the packet number
/// stay protected until [`InitialPacket::open`] removes protection and
/// decrypts the payload. The datagram may carry several coalesced packets, so
/// `next_offset` says where the following packet begins.
pub struct InitialPacket<'pkt> {
datagram: &'pkt [u8],
/// The Destination Connection ID, which seeds key derivation.
pub dcid: &'pkt [u8],
/// The QUIC version this packet declared, selecting the salt and labels its
/// client keys derive under.
version: QuicVersion,
/// Absolute offset of this packet's first byte within the datagram.
start: usize,
/// Absolute offset of the packet number field within the datagram.
pn_offset: usize,
/// Absolute offset one past this packet's protected payload.
end: usize,
/// Absolute offset where the next coalesced packet would begin.
pub next_offset: usize,
}
/// A successfully opened Initial: its decrypted frame bytes and packet number.
pub struct OpenedInitial {
pub packet_number: u64,
pub frames: Vec<u8>,
}
impl<'pkt> InitialPacket<'pkt> {
/// Parses one long header Initial packet starting at `start` in a
/// datagram, reading only the unprotected header fields.
///
/// Returns [`ParseError::NotQuicInitial`] when the bytes at `start` are
/// not a long header Initial, and [`ParseError::UnsupportedQuicVersion`]
/// when the packet is a long header of a QUIC version this code has no
/// salt for. Both are routine on real traffic and feed counters rather
/// than failing a capture.
pub fn parse(datagram: &'pkt [u8], start: usize) -> Result<Self> {
let buf = datagram.get(start..).ok_or(ParseError::NotQuicInitial)?;
let first = *buf.first().ok_or(ParseError::NotQuicInitial)?;
if first & LONG_HEADER_FORM == 0 {
return Err(ParseError::NotQuicInitial);
}
let version_code = u32::from_be_bytes(
buf.get(1..5)
.ok_or(ParseError::NotQuicInitial)?
.try_into()
.expect("a four byte slice is four bytes"),
);
let Some(version) = QuicVersion::from_code(version_code) else {
return Err(ParseError::UnsupportedQuicVersion(version_code));
};
if first & PACKET_TYPE_MASK != version.initial_type {
return Err(ParseError::NotQuicInitial);
}
let mut pos = 5usize;
let dcid = read_cid(buf, &mut pos)?;
let _scid = read_cid(buf, &mut pos)?;
let (token_len, n) = read_varint(buf.get(pos..).ok_or(ParseError::NotQuicInitial)?)
.map_err(|_| ParseError::NotQuicInitial)?;
pos += n;
let token_len = usize::try_from(token_len).map_err(|_| ParseError::NotQuicInitial)?;
pos = pos
.checked_add(token_len)
.ok_or(ParseError::NotQuicInitial)?;
let (length, n) = read_varint(buf.get(pos..).ok_or(ParseError::NotQuicInitial)?)
.map_err(|_| ParseError::NotQuicInitial)?;
pos += n;
let length = usize::try_from(length).map_err(|_| ParseError::NotQuicInitial)?;
let pn_offset = start + pos;
let end = pn_offset
.checked_add(length)
.ok_or(ParseError::NotQuicInitial)?;
if end > datagram.len() {
return Err(ParseError::NotQuicInitial);
}
Ok(Self {
datagram,
dcid,
version,
start,
pn_offset,
end,
next_offset: end,
})
}
/// Derives the client Initial keys for this packet, from its own Destination
/// Connection ID under the salt and labels of the version it declared.
#[must_use]
pub fn client_keys(&self) -> InitialKeys {
InitialKeys::client(self.dcid, self.version)
}
/// Removes header protection and decrypts the payload, returning the
/// cleartext QUIC frames.
///
/// `largest_pn` is the largest packet number already decrypted on this
/// flow, used to reconstruct the full packet number for the AEAD nonce.
/// A failure to authenticate is reported as
/// [`ParseError::QuicCryptoFailure`]; for a passive observer this most
/// often means the packet was a server Initial, protected under keys the
/// observer did not derive, rather than corruption.
pub fn open(&self, keys: &InitialKeys, largest_pn: Option<u64>) -> Result<OpenedInitial> {
let sample_start = self.pn_offset + PN_SAMPLE_OFFSET;
let sample = self
.datagram
.get(sample_start..sample_start + SAMPLE_LEN)
.ok_or(ParseError::QuicCryptoFailure)?;
let mask = header_protection_mask(&keys.hp, sample);
let protected_first = self.datagram[self.start];
let first = protected_first ^ (mask[0] & LONG_HEADER_PN_MASK);
let pn_len = usize::from(first & PN_LEN_MASK) + 1;
let pn_end = self.pn_offset + pn_len;
if pn_end > self.end {
return Err(ParseError::QuicCryptoFailure);
}
let mut pn_bytes = [0u8; 4];
let mut truncated = 0u64;
for i in 0..pn_len {
let clear = self.datagram[self.pn_offset + i] ^ mask[1 + i];
pn_bytes[i] = clear;
truncated = (truncated << 8) | u64::from(clear);
}
let pn_nbits = u32::try_from(pn_len * 8).expect("the packet number is at most four bytes");
let packet_number = decode_packet_number(largest_pn, truncated, pn_nbits);
let mut header: SmallVec<[u8; 64]> = SmallVec::new();
header.push(first);
header.extend_from_slice(&self.datagram[self.start + 1..self.pn_offset]);
header.extend_from_slice(&pn_bytes[..pn_len]);
let ciphertext = self
.datagram
.get(pn_end..self.end)
.ok_or(ParseError::QuicCryptoFailure)?;
if ciphertext.len() < TAG_LEN {
return Err(ParseError::QuicCryptoFailure);
}
let split = ciphertext.len() - TAG_LEN;
let mut frames = ciphertext[..split].to_vec();
let tag = GenericArray::from_slice(&ciphertext[split..]);
let nonce = aead_nonce(&keys.iv, packet_number);
let cipher =
Aes128Gcm::new_from_slice(&keys.key).map_err(|_| ParseError::QuicCryptoFailure)?;
cipher
.decrypt_in_place_detached(GenericArray::from_slice(&nonce), &header, &mut frames, tag)
.map_err(|_| ParseError::QuicCryptoFailure)?;
Ok(OpenedInitial {
packet_number,
frames,
})
}
}
/// Reads a one byte length prefixed connection ID, advancing the cursor.
fn read_cid<'pkt>(buf: &'pkt [u8], pos: &mut usize) -> Result<&'pkt [u8]> {
let len = usize::from(*buf.get(*pos).ok_or(ParseError::NotQuicInitial)?);
if len > MAX_CID_LEN {
return Err(ParseError::NotQuicInitial);
}
let start = *pos + 1;
let cid = buf
.get(start..start + len)
.ok_or(ParseError::NotQuicInitial)?;
*pos = start + len;
Ok(cid)
}
/// Computes the five byte header protection mask, `AES-ECB(hp, sample)`.
fn header_protection_mask(hp: &[u8; HP_LEN], sample: &[u8]) -> [u8; 5] {
let cipher = Aes128::new_from_slice(hp).expect("the header protection key is sixteen bytes");
let mut block = GenericArray::clone_from_slice(sample);
cipher.encrypt_block(&mut block);
let mut mask = [0u8; 5];
mask.copy_from_slice(&block[..5]);
mask
}
/// Builds the AEAD nonce by XORing the packet number into the static IV, from
/// RFC 9001 Section 5.3.
fn aead_nonce(iv: &[u8; IV_LEN], packet_number: u64) -> [u8; IV_LEN] {
let mut nonce = *iv;
let pn = packet_number.to_be_bytes();
for i in 0..8 {
nonce[IV_LEN - 8 + i] ^= pn[i];
}
nonce
}
/// Walks the frames in a decrypted Initial payload, handing each CRYPTO
/// frame's offset and bytes to `on_crypto`.
///
/// Only the frame types RFC 9000 Section 12.4 permits in an Initial packet
/// are recognized. An unrecognized type means the bytes are not a valid
/// Initial payload after all, reported as [`ParseError::QuicCryptoFailure`];
/// since the AEAD tag already authenticated these bytes, that points at a
/// version or framing this code does not model rather than at an attacker.
pub fn walk_crypto_frames(frames: &[u8], mut on_crypto: impl FnMut(u64, &[u8])) -> Result<()> {
let mut pos = 0usize;
while pos < frames.len() {
let ty = frames[pos];
pos += 1;
match ty {
frame::PADDING | frame::PING => {}
frame::ACK | frame::ACK_ECN => {
let (_largest, n) = read_varint(&frames[pos..])?;
pos += n;
let (_delay, n) = read_varint(&frames[pos..])?;
pos += n;
let (range_count, n) = read_varint(&frames[pos..])?;
pos += n;
let (_first_range, n) = read_varint(&frames[pos..])?;
pos += n;
for _ in 0..range_count {
let (_gap, n) = read_varint(&frames[pos..])?;
pos += n;
let (_len, n) = read_varint(&frames[pos..])?;
pos += n;
}
if ty == frame::ACK_ECN {
for _ in 0..3 {
let (_ecn, n) = read_varint(&frames[pos..])?;
pos += n;
}
}
}
frame::CRYPTO => {
let (offset, n) = read_varint(&frames[pos..])?;
pos += n;
let (len, n) = read_varint(&frames[pos..])?;
pos += n;
let len = usize::try_from(len).map_err(|_| ParseError::QuicCryptoFailure)?;
let end = pos.checked_add(len).ok_or(ParseError::QuicCryptoFailure)?;
let data = frames.get(pos..end).ok_or(ParseError::QuicCryptoFailure)?;
on_crypto(offset, data);
pos = end;
}
frame::CONNECTION_CLOSE => {
let (_code, n) = read_varint(&frames[pos..])?;
pos += n;
let (_frame_type, n) = read_varint(&frames[pos..])?;
pos += n;
let (reason_len, n) = read_varint(&frames[pos..])?;
pos += n;
let reason_len =
usize::try_from(reason_len).map_err(|_| ParseError::QuicCryptoFailure)?;
pos = pos
.checked_add(reason_len)
.filter(|&p| p <= frames.len())
.ok_or(ParseError::QuicCryptoFailure)?;
}
_ => return Err(ParseError::QuicCryptoFailure),
}
}
Ok(())
}
/// Reassembles the cleartext handshake stream that CRYPTO frames carry.
///
/// CRYPTO frames are the QUIC analogue of a TCP byte stream: each carries an
/// absolute offset, frames can arrive out of order and span packets, and a
/// ClientHello routinely splits across several. This is the QUIC counterpart
/// of the TCP reassembler, kept separate because CRYPTO offsets are sixty
/// four bit and start at zero per stream rather than at a negotiated sequence
/// number. Contiguous bytes from offset zero accumulate in one buffer;
/// everything ahead of the write cursor parks until the gap before it fills.
/// Both buffers are capped so a hostile sender cannot turn the assembler into
/// a memory bomb.
pub struct CryptoAssembler {
assembled: Vec<u8>,
pending: BTreeMap<u64, Vec<u8>>,
pending_bytes: usize,
max_bytes: usize,
overflowed: bool,
}
impl CryptoAssembler {
#[must_use]
pub fn new(max_bytes: usize) -> Self {
Self {
assembled: Vec::new(),
pending: BTreeMap::new(),
pending_bytes: 0,
max_bytes,
overflowed: false,
}
}
/// Returns the contiguous handshake bytes assembled from offset zero.
#[must_use]
pub fn data(&self) -> &[u8] {
&self.assembled
}
/// Adds one CRYPTO frame's bytes at their stream offset.
///
/// Bytes wholly before the write cursor are duplicates and ignored; bytes
/// straddling it extend the contiguous run and then pull in any parked
/// segments the new bytes made contiguous; bytes wholly ahead of it park.
/// Anything that would push either buffer past its cap is dropped and the
/// assembler latches an overflow so the flow can be abandoned.
pub fn push(&mut self, offset: u64, data: &[u8]) {
if data.is_empty() || self.overflowed {
return;
}
let Ok(cursor) = u64::try_from(self.assembled.len()) else {
self.overflowed = true;
return;
};
if offset > cursor {
self.park(offset, data);
return;
}
let skip = usize::try_from(cursor - offset).unwrap_or(usize::MAX);
if skip >= data.len() {
return;
}
if !self.extend(&data[skip..]) {
return;
}
self.drain_pending();
}
fn extend(&mut self, bytes: &[u8]) -> bool {
if self.assembled.len() + bytes.len() > self.max_bytes {
self.overflowed = true;
return false;
}
self.assembled.extend_from_slice(bytes);
true
}
fn park(&mut self, offset: u64, data: &[u8]) {
if self.pending_bytes + data.len() > self.max_bytes {
self.overflowed = true;
return;
}
if let Some(existing) = self.pending.get(&offset) {
if existing.len() >= data.len() {
return;
}
self.pending_bytes -= existing.len();
}
self.pending_bytes += data.len();
self.pending.insert(offset, data.to_vec());
}
fn drain_pending(&mut self) {
while let Some((&offset, _)) = self.pending.iter().next() {
let Ok(cursor) = u64::try_from(self.assembled.len()) else {
self.overflowed = true;
return;
};
if offset > cursor {
break;
}
let segment = self.pending.remove(&offset).expect("offset just observed");
self.pending_bytes -= segment.len();
let skip = usize::try_from(cursor - offset).unwrap_or(usize::MAX);
if skip < segment.len() && !self.extend(&segment[skip..]) {
return;
}
}
}
/// Returns the body of the leading ClientHello once it is fully present.
///
/// The QUIC Initial cryptographic stream begins with the ClientHello
/// handshake message: a one byte type, a three byte length, then the
/// body. This reports the body once the contiguous bytes hold all of it,
/// reports that the stream does not begin with a ClientHello, or reports
/// that more bytes are still needed.
#[must_use]
pub fn client_hello(&self) -> ClientHelloState<'_> {
if self.assembled.len() < 4 {
return if self.overflowed {
ClientHelloState::Abandoned
} else {
ClientHelloState::Incomplete
};
}
if self.assembled[0] != handshake_type::CLIENT_HELLO {
return ClientHelloState::NotClientHello;
}
let len = usize::from(self.assembled[1]) << 16
| usize::from(self.assembled[2]) << 8
| usize::from(self.assembled[3]);
match self.assembled.get(4..4 + len) {
Some(body) => ClientHelloState::Ready(body),
None if self.overflowed => ClientHelloState::Abandoned,
None => ClientHelloState::Incomplete,
}
}
}
/// What [`CryptoAssembler::client_hello`] found in the assembled bytes.
#[derive(Debug, PartialEq, Eq)]
pub enum ClientHelloState<'a> {
/// The full ClientHello body is present.
Ready(&'a [u8]),
/// More CRYPTO bytes are needed before the ClientHello is complete.
Incomplete,
/// The stream does not begin with a ClientHello and never will.
NotClientHello,
/// A buffer cap was hit; the flow should be given up.
Abandoned,
}
#[cfg(test)]
mod tests {
use super::{
ClientHelloState, CryptoAssembler, InitialKeys, InitialPacket, QuicVersion,
decode_packet_number, hkdf_expand_label, read_varint, walk_crypto_frames,
};
/// RFC 9000 Appendix A.1 sample variable length integer decodings.
#[test]
fn varint_matches_rfc9000_appendix_a1() {
assert_eq!(
read_varint(&hex("c2197c5eff14e88c")).unwrap(),
(151_288_809_941_952_652, 8)
);
assert_eq!(read_varint(&hex("9d7f3e7d")).unwrap(), (494_878_333, 4));
assert_eq!(read_varint(&hex("7bbd")).unwrap(), (15_293, 2));
assert_eq!(read_varint(&hex("25")).unwrap(), (37, 1));
assert_eq!(read_varint(&hex("4025")).unwrap(), (37, 2));
}
/// RFC 9000 Appendix A.3 sample packet number decoding.
#[test]
fn packet_number_decode_matches_rfc9000_appendix_a3() {
assert_eq!(
decode_packet_number(Some(0xa82f_30ea), 0x9b32, 16),
0xa82f_9b32
);
assert_eq!(decode_packet_number(None, 2, 8), 2);
}
/// RFC 9001 Appendix A.1 derives a known set of client Initial keys from
/// the sample Destination Connection ID.
#[test]
fn client_initial_keys_match_rfc9001_appendix_a1() {
let dcid = hex("8394c8f03e515708");
let keys = InitialKeys::client(&dcid, QuicVersion::V1);
assert_eq!(keys.key.to_vec(), hex("1f369613dd76d5467730efcbe3b1a22d"));
assert_eq!(keys.iv.to_vec(), hex("fa044b2f42a3fd3b46fb255c"));
assert_eq!(keys.hp.to_vec(), hex("9f50449e04a0e810283a1e9933adedd2"));
}
/// RFC 9369 Appendix A derives the QUIC v2 client Initial keys from the same
/// sample Destination Connection ID, under the v2 salt and labels.
#[test]
fn client_initial_keys_match_rfc9369_appendix_a() {
let dcid = hex("8394c8f03e515708");
let keys = InitialKeys::client(&dcid, QuicVersion::V2);
assert_eq!(keys.key.to_vec(), hex("8b1a0bc121284290a29e0971b5cd045d"));
assert_eq!(keys.iv.to_vec(), hex("91f73e2351d8fa91660e909f"));
assert_eq!(keys.hp.to_vec(), hex("45b95e15235d6f45a6b19cbcb0294ba9"));
}
/// RFC 9001 Appendix A.1 also pins the intermediate expand label output.
#[test]
fn expand_label_reproduces_rfc9001_client_secret() {
let dcid = hex("8394c8f03e515708");
let (initial_secret, _) =
hkdf::Hkdf::<sha2::Sha256>::extract(Some(&super::INITIAL_SALT_V1), &dcid);
let mut client_secret = [0u8; 32];
hkdf_expand_label(&initial_secret, b"client in", &mut client_secret);
assert_eq!(
client_secret.to_vec(),
hex("c00cf151ca5be075ed0ebfb5c80323c42d6b7db67881289af4008f1f6c357aea")
);
}
/// RFC 9001 Appendix A.2 carries the full protected client Initial. Its
/// decryption must recover the documented CRYPTO frame and a parseable
/// ClientHello.
#[test]
fn opens_rfc9001_appendix_a2_client_initial() {
let datagram = rfc9001_a2_protected_packet();
let packet = InitialPacket::parse(&datagram, 0).unwrap();
assert_eq!(packet.dcid, &hex("8394c8f03e515708")[..]);
let keys = packet.client_keys();
let opened = packet.open(&keys, None).unwrap();
assert_eq!(opened.packet_number, 2);
let expected_head = hex("060040f1010000ed0303ebf8fa56f12939b9584a3896472ec40bb863cfd3e868");
assert_eq!(&opened.frames[..expected_head.len()], &expected_head[..]);
let mut assembler = CryptoAssembler::new(1 << 16);
walk_crypto_frames(&opened.frames, |offset, data| assembler.push(offset, data)).unwrap();
let ClientHelloState::Ready(body) = assembler.client_hello() else {
panic!("expected a complete ClientHello");
};
let hello = crate::parse::parse_client_hello(body).unwrap();
assert!(!hello.cipher_suites.is_empty());
}
#[test]
fn assembler_orders_out_of_order_crypto_frames() {
let mut a = CryptoAssembler::new(1 << 16);
a.push(5, b"world");
assert_eq!(a.client_hello(), ClientHelloState::Incomplete);
a.push(0, b"hello");
assert_eq!(a.data(), b"helloworld");
}
#[test]
fn assembler_ignores_pure_duplicates_and_overlap() {
let mut a = CryptoAssembler::new(1 << 16);
a.push(0, b"hello");
a.push(0, b"hel");
a.push(2, b"llo world");
assert_eq!(a.data(), b"hello world");
}
#[test]
fn assembler_latches_overflow_past_the_cap() {
let mut a = CryptoAssembler::new(8);
a.push(0, b"12345678");
a.push(8, b"9");
assert_eq!(a.data(), b"12345678");
assert_eq!(a.client_hello(), ClientHelloState::NotClientHello);
}
#[test]
fn non_quic_bytes_are_not_an_initial() {
let udp = b"GET / HTTP/1.1\r\n";
assert!(InitialPacket::parse(udp, 0).is_err());
}
/// A QUIC v2 long header Initial is recognized by its version code and its
/// remapped Initial type bits, `0b01`, where the v1 Initial code `0b00` is
/// not an Initial in v2. This proves the version specific parse logic; the v2
/// key schedule is pinned by the keys KAT above, and the AEAD decryption path
/// is the one the RFC 9001 Appendix A.2 test already exercises unchanged.
#[test]
fn parses_a_quic_v2_initial_header() {
let mut packet = vec![0xd3];
packet.extend_from_slice(&super::VERSION_2.to_be_bytes());
packet.push(8);
packet.extend_from_slice(&hex("8394c8f03e515708"));
packet.push(0);
packet.push(0);
packet.push(0x10);
packet.extend_from_slice(&[0u8; 16]);
let parsed = InitialPacket::parse(&packet, 0).unwrap();
assert_eq!(parsed.dcid, &hex("8394c8f03e515708")[..]);
let mut v1_type = packet.clone();
v1_type[0] = 0xc3;
assert!(InitialPacket::parse(&v1_type, 0).is_err());
}
fn hex(s: &str) -> Vec<u8> {
::hex::decode(s).unwrap()
}
/// The protected client Initial packet from RFC 9001 Appendix A.2.
fn rfc9001_a2_protected_packet() -> Vec<u8> {
hex(concat!(
"c000000001088394c8f03e5157080000449e7b9aec34d1b1c98dd7689fb8ec11",
"d242b123dc9bd8bab936b47d92ec356c0bab7df5976d27cd449f63300099f399",
"1c260ec4c60d17b31f8429157bb35a1282a643a8d2262cad67500cadb8e7378c",
"8eb7539ec4d4905fed1bee1fc8aafba17c750e2c7ace01e6005f80fcb7df6212",
"30c83711b39343fa028cea7f7fb5ff89eac2308249a02252155e2347b63d58c5",
"457afd84d05dfffdb20392844ae812154682e9cf012f9021a6f0be17ddd0c208",
"4dce25ff9b06cde535d0f920a2db1bf362c23e596d11a4f5a6cf3948838a3aec",
"4e15daf8500a6ef69ec4e3feb6b1d98e610ac8b7ec3faf6ad760b7bad1db4ba3",
"485e8a94dc250ae3fdb41ed15fb6a8e5eba0fc3dd60bc8e30c5c4287e53805db",
"059ae0648db2f64264ed5e39be2e20d82df566da8dd5998ccabdae053060ae6c",
"7b4378e846d29f37ed7b4ea9ec5d82e7961b7f25a9323851f681d582363aa5f8",
"9937f5a67258bf63ad6f1a0b1d96dbd4faddfcefc5266ba6611722395c906556",
"be52afe3f565636ad1b17d508b73d8743eeb524be22b3dcbc2c7468d54119c74",
"68449a13d8e3b95811a198f3491de3e7fe942b330407abf82a4ed7c1b311663a",
"c69890f4157015853d91e923037c227a33cdd5ec281ca3f79c44546b9d90ca00",
"f064c99e3dd97911d39fe9c5d0b23a229a234cb36186c4819e8b9c5927726632",
"291d6a418211cc2962e20fe47feb3edf330f2c603a9d48c0fcb5699dbfe58964",
"25c5bac4aee82e57a85aaf4e2513e4f05796b07ba2ee47d80506f8d2c25e50fd",
"14de71e6c418559302f939b0e1abd576f279c4b2e0feb85c1f28ff18f58891ff",
"ef132eef2fa09346aee33c28eb130ff28f5b766953334113211996d20011a198",
"e3fc433f9f2541010ae17c1bf202580f6047472fb36857fe843b19f5984009dd",
"c324044e847a4f4a0ab34f719595de37252d6235365e9b84392b061085349d73",
"203a4a13e96f5432ec0fd4a1ee65accdd5e3904df54c1da510b0ff20dcc0c77f",
"cb2c0e0eb605cb0504db87632cf3d8b4dae6e705769d1de354270123cb11450e",
"fc60ac47683d7b8d0f811365565fd98c4c8eb936bcab8d069fc33bd801b03ade",
"a2e1fbc5aa463d08ca19896d2bf59a071b851e6c239052172f296bfb5e724047",
"90a2181014f3b94a4e97d117b438130368cc39dbb2d198065ae3986547926cd2",
"162f40a29f0c3c8745c0f50fba3852e566d44575c29d39a03f0cda721984b6f4",
"40591f355e12d439ff150aab7613499dbd49adabc8676eef023b15b65bfc5ca0",
"6948109f23f350db82123535eb8a7433bdabcb909271a6ecbcb58b936a88cd4e",
"8f2e6ff5800175f113253d8fa9ca8885c2f552e657dc603f252e1a8e308f76f0",
"be79e2fb8f5d5fbbe2e30ecadd220723c8c0aea8078cdfcb3868263ff8f09400",
"54da48781893a7e49ad5aff4af300cd804a6b6279ab3ff3afb64491c85194aab",
"760d58a606654f9f4400e8b38591356fbf6425aca26dc85244259ff2b19c41b9",
"f96f3ca9ec1dde434da7d2d392b905ddf3d1f9af93d1af5950bd493f5aa731b4",
"056df31bd267b6b90a079831aaf579be0a39013137aac6d404f518cfd4684064",
"7e78bfe706ca4cf5e9c5453e9f7cfd2b8b4c8d169a44e55c88d4a9a7f9474241",
"e221af44860018ab0856972e194cd934",
))
}
}

91
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/src/registry.rs Normal file
View File

@ -0,0 +1,91 @@
// ©AngelaMos | 2026
// registry.rs
//! Named constants for the slices of the TLS and DTLS registries that the
//! fingerprint algorithms reference by number.
//!
//! Wire values are kept as raw `u16` everywhere in this crate rather than being
//! decoded into a closed enum. A fingerprinting engine must preserve cipher and
//! extension values it has never seen, because a future RFC value or a vendor
//! specific extension is exactly the kind of detail that makes one client
//! distinguishable from another. Decoding into a closed enum would collapse all
//! unknown values into a single bucket and corrupt the fingerprint.
/// TLS and DTLS record content types (the first byte of a record).
pub mod content_type {
pub const CHANGE_CIPHER_SPEC: u8 = 20;
pub const ALERT: u8 = 21;
pub const HANDSHAKE: u8 = 22;
pub const APPLICATION_DATA: u8 = 23;
}
/// Handshake message types (the first byte of a handshake message body).
pub mod handshake_type {
pub const CLIENT_HELLO: u8 = 1;
pub const SERVER_HELLO: u8 = 2;
pub const CERTIFICATE: u8 = 11;
}
/// Extension type numbers that the fingerprint algorithms treat specially.
pub mod extension {
pub const SERVER_NAME: u16 = 0x0000;
pub const SUPPORTED_GROUPS: u16 = 0x000a;
pub const EC_POINT_FORMATS: u16 = 0x000b;
pub const SIGNATURE_ALGORITHMS: u16 = 0x000d;
pub const ALPN: u16 = 0x0010;
pub const SUPPORTED_VERSIONS: u16 = 0x002b;
}
/// Legacy and negotiated protocol version words.
pub mod version {
pub const SSL_2_0: u16 = 0x0002;
pub const SSL_3_0: u16 = 0x0300;
pub const TLS_1_0: u16 = 0x0301;
pub const TLS_1_1: u16 = 0x0302;
pub const TLS_1_2: u16 = 0x0303;
pub const TLS_1_3: u16 = 0x0304;
pub const DTLS_1_0: u16 = 0xfeff;
pub const DTLS_1_2: u16 = 0xfefd;
pub const DTLS_1_3: u16 = 0xfefc;
}
/// Maps a protocol version word to the two character JA4 version code.
///
/// The mapping is taken verbatim from the FoxIO JA4 specification. Unknown
/// words collapse to `00`, which is the specified fallback. DTLS words are
/// included even though the Python reference omits them, because the published
/// specification lists them and the Wireshark and Rust references honor them.
#[must_use]
pub fn ja4_version_code(word: u16) -> &'static str {
match word {
version::TLS_1_3 => "13",
version::TLS_1_2 => "12",
version::TLS_1_1 => "11",
version::TLS_1_0 => "10",
version::SSL_3_0 => "s3",
version::SSL_2_0 => "s2",
version::DTLS_1_0 => "d1",
version::DTLS_1_2 => "d2",
version::DTLS_1_3 => "d3",
_ => "00",
}
}
#[cfg(test)]
mod tests {
use super::ja4_version_code;
use super::version;
#[test]
fn known_versions_map() {
assert_eq!(ja4_version_code(version::TLS_1_3), "13");
assert_eq!(ja4_version_code(version::TLS_1_2), "12");
assert_eq!(ja4_version_code(version::SSL_3_0), "s3");
assert_eq!(ja4_version_code(version::DTLS_1_2), "d2");
}
#[test]
fn unknown_version_falls_back() {
assert_eq!(ja4_version_code(0x7f1d), "00");
}
}

161
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/tests/common/mod.rs Normal file
View File

@ -0,0 +1,161 @@
// ©AngelaMos | 2026
// mod.rs
//! Builders that assemble TLS handshake bytes for tests.
//!
//! Hand assembling wire bytes keeps the fingerprint tests honest. Rather than
//! trusting a higher level library to produce a ClientHello, the tests state the
//! exact cipher list, extension list, and extension bodies, then assert that the
//! parser and the fingerprint algorithms read them back the way the
//! specifications require.
//!
//! Each integration test binary links this module independently, so a builder
//! method that one binary does not call reads as dead code there even though
//! another binary uses it. The allow keeps that shared infrastructure honest
//! without scattering per method annotations.
#![allow(dead_code)]
/// Assembles a ClientHello handshake message body.
#[derive(Default)]
pub struct ClientHelloBuilder {
legacy_version: u16,
cipher_suites: Vec<u16>,
extensions: Vec<(u16, Vec<u8>)>,
}
impl ClientHelloBuilder {
pub fn new() -> Self {
Self {
legacy_version: 0x0303,
..Self::default()
}
}
pub fn legacy_version(mut self, v: u16) -> Self {
self.legacy_version = v;
self
}
pub fn ciphers(mut self, suites: &[u16]) -> Self {
self.cipher_suites = suites.to_vec();
self
}
pub fn extension(mut self, ext_type: u16, data: Vec<u8>) -> Self {
self.extensions.push((ext_type, data));
self
}
/// Adds a server name indication extension for the given host.
pub fn sni(self, host: &str) -> Self {
let host = host.as_bytes();
let mut entry = vec![0u8];
push_u16_vec(&mut entry, host);
let mut data = Vec::new();
push_u16_vec(&mut data, &entry);
self.extension(0x0000, data)
}
/// Adds a supported groups extension.
pub fn supported_groups(self, groups: &[u16]) -> Self {
let mut list = Vec::new();
for g in groups {
list.extend_from_slice(&g.to_be_bytes());
}
let mut data = Vec::new();
push_u16_vec(&mut data, &list);
self.extension(0x000a, data)
}
/// Adds an elliptic curve point formats extension.
pub fn ec_point_formats(self, formats: &[u8]) -> Self {
let mut data = Vec::new();
push_u8_vec(&mut data, formats);
self.extension(0x000b, data)
}
/// Adds a signature algorithms extension.
pub fn signature_algorithms(self, algs: &[u16]) -> Self {
let mut list = Vec::new();
for a in algs {
list.extend_from_slice(&a.to_be_bytes());
}
let mut data = Vec::new();
push_u16_vec(&mut data, &list);
self.extension(0x000d, data)
}
/// Adds an ALPN extension advertising the given protocols.
pub fn alpn(self, protos: &[&[u8]]) -> Self {
let mut list = Vec::new();
for p in protos {
push_u8_vec(&mut list, p);
}
let mut data = Vec::new();
push_u16_vec(&mut data, &list);
self.extension(0x0010, data)
}
/// Adds a supported versions extension.
pub fn supported_versions(self, versions: &[u16]) -> Self {
let mut list = Vec::new();
for v in versions {
list.extend_from_slice(&v.to_be_bytes());
}
let mut data = Vec::new();
push_u8_vec(&mut data, &list);
self.extension(0x002b, data)
}
/// Returns the handshake message body, the input to `parse_client_hello`.
pub fn build_body(&self) -> Vec<u8> {
let mut body = Vec::new();
body.extend_from_slice(&self.legacy_version.to_be_bytes());
body.extend_from_slice(&[0u8; 32]);
body.push(0);
let mut ciphers = Vec::new();
for c in &self.cipher_suites {
ciphers.extend_from_slice(&c.to_be_bytes());
}
push_u16_vec(&mut body, &ciphers);
push_u8_vec(&mut body, &[0]);
let mut exts = Vec::new();
for (ext_type, data) in &self.extensions {
exts.extend_from_slice(&ext_type.to_be_bytes());
push_u16_vec(&mut exts, data);
}
push_u16_vec(&mut body, &exts);
body
}
/// Wraps the handshake body in a handshake header and a TLS record so the
/// result is a complete record stream.
pub fn build_record(&self) -> Vec<u8> {
let body = self.build_body();
let mut msg = vec![1u8];
push_u24_vec(&mut msg, &body);
let mut record = vec![22u8, 0x03, 0x01];
push_u16_vec(&mut record, &msg);
record
}
}
fn push_u8_vec(out: &mut Vec<u8>, data: &[u8]) {
out.push(u8::try_from(data.len()).unwrap());
out.extend_from_slice(data);
}
fn push_u16_vec(out: &mut Vec<u8>, data: &[u8]) {
out.extend_from_slice(&u16::try_from(data.len()).unwrap().to_be_bytes());
out.extend_from_slice(data);
}
fn push_u24_vec(out: &mut Vec<u8>, data: &[u8]) {
let len = u32::try_from(data.len()).unwrap();
out.extend_from_slice(&len.to_be_bytes()[1..]);
out.extend_from_slice(data);
}

55
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/tests/ja3.rs Normal file
View File

@ -0,0 +1,55 @@
// ©AngelaMos | 2026
// ja3.rs
mod common;
use common::ClientHelloBuilder;
use tlsfp_core::ja3::{ja3, ja3_string};
use tlsfp_core::parse::parse_client_hello;
/// Rebuilds the exact ClientHello that produces the first published Salesforce
/// JA3 vector, then checks both the pre hash string and the digest end to end.
/// The SNI body content is arbitrary because JA3 records only that extension
/// type zero was present, not its value.
#[test]
fn end_to_end_matches_salesforce_vector_one() {
let body = ClientHelloBuilder::new()
.legacy_version(0x0301)
.ciphers(&[47, 53, 5, 10, 49161, 49162, 49171, 49172, 50, 56, 19, 4])
.extension(0x0000, vec![0x00, 0x00])
.supported_groups(&[23, 24, 25])
.ec_point_formats(&[0])
.build_body();
let ch = parse_client_hello(&body).unwrap();
assert_eq!(
ja3_string(&ch),
"769,47-53-5-10-49161-49162-49171-49172-50-56-19-4,0-10-11,23-24-25,0"
);
assert_eq!(ja3(&ch).to_string(), "ada70206e40642a3e4461f35503241d5");
}
/// A ClientHello with GREASE in ciphers, extensions, and supported groups must
/// produce the same JA3 as the same hello without GREASE, because GREASE is
/// stripped from every list field before hashing.
#[test]
fn grease_does_not_change_the_fingerprint() {
let clean = ClientHelloBuilder::new()
.legacy_version(0x0303)
.ciphers(&[0x1301, 0x1302])
.supported_groups(&[0x001d, 0x0017])
.ec_point_formats(&[0])
.build_body();
let greasy = ClientHelloBuilder::new()
.legacy_version(0x0303)
.ciphers(&[0x0a0a, 0x1301, 0x1302])
.extension(0x1a1a, vec![])
.supported_groups(&[0x2a2a, 0x001d, 0x0017])
.ec_point_formats(&[0])
.build_body();
let clean = parse_client_hello(&clean).unwrap();
let greasy = parse_client_hello(&greasy).unwrap();
assert_eq!(ja3(&clean), ja3(&greasy));
}

88
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/tests/ja4.rs Normal file
View File

@ -0,0 +1,88 @@
// ©AngelaMos | 2026
// ja4.rs
mod common;
use common::ClientHelloBuilder;
use tlsfp_core::ja4::{Transport, ja4};
use tlsfp_core::parse::parse_client_hello;
const FOXIO_HASH: &str = "t13d1516h2_8daaf6152771_e5627efa2ab1";
const FOXIO_RAW: &str = "t13d1516h2_002f,0035,009c,009d,1301,1302,1303,c013,c014,c02b,c02c,c02f,c030,cca8,cca9_0005,000a,000b,000d,0012,0015,0017,001b,0023,002b,002d,0033,4469,ff01_0403,0804,0401,0503,0805,0501,0806,0601";
const CIPHERS_ORIGINAL_ORDER: [u16; 15] = [
0x1301, 0x1302, 0x1303, 0xc02b, 0xc02f, 0xc02c, 0xc030, 0xcca9, 0xcca8, 0xc013, 0xc014, 0x009c,
0x009d, 0x002f, 0x0035,
];
const OPAQUE_EXTENSIONS: [u16; 10] = [
0x0005, 0x0012, 0x0015, 0x0017, 0x001b, 0x0023, 0x002d, 0x0033, 0x4469, 0xff01,
];
const SIG_ALGS: [u16; 8] = [
0x0403, 0x0804, 0x0401, 0x0503, 0x0805, 0x0501, 0x0806, 0x0601,
];
/// Assembles the handshake body for the canonical FoxIO example.
///
/// The cipher suites are supplied out of order with an injected GREASE value, to
/// prove that sorting and GREASE removal happen before hashing. When `reversed`
/// is set the opaque extensions are added in reverse, which must not change the
/// fingerprint because JA4 sorts the extension list.
fn foxio_body(reversed: bool) -> Vec<u8> {
let mut ciphers = vec![0x0a0a];
ciphers.extend_from_slice(&CIPHERS_ORIGINAL_ORDER);
let mut builder = ClientHelloBuilder::new()
.ciphers(&ciphers)
.sni("example.com")
.supported_groups(&[0x001d, 0x0017])
.ec_point_formats(&[0x00])
.signature_algorithms(&SIG_ALGS)
.alpn(&[b"h2"])
.supported_versions(&[0x2a2a, 0x0304, 0x0303]);
if reversed {
for ext in OPAQUE_EXTENSIONS.iter().rev() {
builder = builder.extension(*ext, vec![]);
}
} else {
for ext in OPAQUE_EXTENSIONS {
builder = builder.extension(ext, vec![]);
}
}
builder.build_body()
}
#[test]
fn reproduces_foxio_canonical_example() {
let body = foxio_body(false);
let ch = parse_client_hello(&body).unwrap();
let fp = ja4(&ch, Transport::Tcp);
assert_eq!(fp.raw, FOXIO_RAW);
assert_eq!(fp.hash, FOXIO_HASH);
}
#[test]
fn extension_order_does_not_change_the_hash() {
let normal = foxio_body(false);
let reversed = foxio_body(true);
let normal = parse_client_hello(&normal).unwrap();
let reversed = parse_client_hello(&reversed).unwrap();
assert_eq!(
ja4(&normal, Transport::Tcp).hash,
ja4(&reversed, Transport::Tcp).hash
);
}
#[test]
fn transport_marker_switches_first_character() {
let body = foxio_body(false);
let ch = parse_client_hello(&body).unwrap();
let tcp = ja4(&ch, Transport::Tcp);
let quic = ja4(&ch, Transport::Quic);
assert!(tcp.hash.starts_with('t'));
assert!(quic.hash.starts_with('q'));
assert_eq!(&tcp.hash[1..], &quic.hash[1..]);
}

64
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/tests/ja4_suite.rs Normal file
View File

@ -0,0 +1,64 @@
// ©AngelaMos | 2026
// ja4_suite.rs
use tlsfp_core::ja4::{Transport, ja4s};
use tlsfp_core::ja4h::{ja4h, parse_http_request};
use tlsfp_core::parse::parse_server_hello;
/// Builds the ServerHello behind the FoxIO JA4S example: TLS 1.3 selected via the
/// supported versions extension, the cipher 0x1301, and two extensions in the
/// order key share then supported versions.
fn foxio_server_hello() -> Vec<u8> {
let mut body = Vec::new();
body.extend_from_slice(&0x0303u16.to_be_bytes());
body.extend_from_slice(&[0u8; 32]);
body.push(0);
body.extend_from_slice(&0x1301u16.to_be_bytes());
body.push(0);
let mut exts = Vec::new();
exts.extend_from_slice(&0x0033u16.to_be_bytes());
exts.extend_from_slice(&0u16.to_be_bytes());
exts.extend_from_slice(&0x002bu16.to_be_bytes());
exts.extend_from_slice(&2u16.to_be_bytes());
exts.extend_from_slice(&0x0304u16.to_be_bytes());
body.extend_from_slice(&u16::try_from(exts.len()).unwrap().to_be_bytes());
body.extend_from_slice(&exts);
body
}
#[test]
fn ja4s_reproduces_foxio_example() {
let body = foxio_server_hello();
let sh = parse_server_hello(&body).unwrap();
let fp = ja4s(&sh, Transport::Tcp);
assert_eq!(fp.hash, "t130200_1301_234ea6891581");
assert_eq!(fp.raw, "t130200_1301_0033,002b");
}
/// The published JA4H example for a request with four uppercase headers, no
/// cookies, no referer, and no accept language. The header hash is the SHA256 of
/// the comma joined header names in wire order.
#[test]
fn ja4h_reproduces_published_example() {
let raw = b"GET * HTTP/1.1\r\nHOST: a\r\nMAN: b\r\nMX: c\r\nST: d\r\n\r\n";
let req = parse_http_request(raw).unwrap();
let fp = ja4h(&req);
assert_eq!(
fp.hash,
"ge11nn040000_a3c882e23515_000000000000_000000000000"
);
}
/// Cookies and a referer must flip their flags and stop being counted as plain
/// headers, while the accept language must populate the language characters.
#[test]
fn ja4h_flags_cookies_referer_and_language() {
let raw = b"GET / HTTP/1.1\r\nHost: x\r\nAccept-Language: en-US,en;q=0.9\r\nReferer: http://x\r\nCookie: a=1; b=2\r\n\r\n";
let req = parse_http_request(raw).unwrap();
let fp = ja4h(&req);
let prefix = fp.hash.split('_').next().unwrap();
assert_eq!(prefix, "ge11cr02enus");
assert!(!fp.hash.contains("_000000000000_000000000000"));
}

21
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/tests/ja4x.rs Normal file
View File

@ -0,0 +1,21 @@
// ©AngelaMos | 2026
// ja4x.rs
//! Fuzz harness proving the X.509 certificate walk behind JA4X never panics.
//!
//! JA4X reads certificate bytes a passive observer takes straight off a cleartext
//! TLS 1.2 handshake, so a malformed or hostile certificate must be an ordinary
//! error rather than a crash, the same contract the TLS parser and the stream
//! reassembler hold and fuzz. This pairs with the constructed truncated
//! certificate unit test: arbitrary bytes in, never a panic out.
use proptest::prelude::*;
proptest! {
#[test]
fn ja4x_never_panics_on_arbitrary_bytes(
bytes in proptest::collection::vec(any::<u8>(), 0..4096),
) {
let _ = tlsfp_core::ja4x(&bytes);
}
}

105
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/tests/parse.rs Normal file
View File

@ -0,0 +1,105 @@
// ©AngelaMos | 2026
// parse.rs
mod common;
use common::ClientHelloBuilder;
use proptest::prelude::*;
use tlsfp_core::parse::{
first_handshake_message, handshake_bytes, parse_client_hello, parse_server_hello,
};
use tlsfp_core::registry::handshake_type;
fn sample() -> ClientHelloBuilder {
ClientHelloBuilder::new()
.ciphers(&[0x0a0a, 0x1301, 0x1302, 0x1303, 0xc02b])
.sni("example.com")
.supported_groups(&[0x1a1a, 0x001d, 0x0017])
.ec_point_formats(&[0x00])
.signature_algorithms(&[0x0403, 0x0804, 0x0401])
.alpn(&[b"h2", b"http/1.1"])
.supported_versions(&[0x2a2a, 0x0304, 0x0303])
}
#[test]
fn parses_every_field_a_fingerprint_reads() {
let body = sample().build_body();
let ch = parse_client_hello(&body).unwrap();
assert_eq!(ch.legacy_version, 0x0303);
assert_eq!(
ch.cipher_suites.as_slice(),
&[0x0a0a, 0x1301, 0x1302, 0x1303, 0xc02b]
);
assert_eq!(ch.extensions.len(), 6);
assert_eq!(ch.server_name(), Some("example.com"));
assert_eq!(ch.supported_groups().as_slice(), &[0x1a1a, 0x001d, 0x0017]);
assert_eq!(ch.ec_point_formats().as_slice(), &[0x00]);
assert_eq!(
ch.signature_algorithms().as_slice(),
&[0x0403, 0x0804, 0x0401]
);
assert_eq!(ch.alpn_protocols().first().copied(), Some(b"h2".as_slice()));
assert_eq!(
ch.supported_versions().as_slice(),
&[0x2a2a, 0x0304, 0x0303]
);
assert!(!ch.is_sslv2);
}
#[test]
fn reads_client_hello_through_the_record_layer() {
let stream = sample().build_record();
let hs = handshake_bytes(&stream).unwrap();
let body = first_handshake_message(&hs, handshake_type::CLIENT_HELLO).unwrap();
let ch = parse_client_hello(body).unwrap();
assert_eq!(ch.cipher_suites.len(), 5);
}
#[test]
fn server_hello_parses_single_cipher() {
let mut body = Vec::new();
body.extend_from_slice(&0x0303u16.to_be_bytes());
body.extend_from_slice(&[0u8; 32]);
body.push(0);
body.extend_from_slice(&0x1301u16.to_be_bytes());
body.push(0);
body.extend_from_slice(&0u16.to_be_bytes());
let sh = parse_server_hello(&body).unwrap();
assert_eq!(sh.cipher_suite, 0x1301);
assert_eq!(sh.legacy_version, 0x0303);
}
#[test]
fn client_hello_without_extensions_parses() {
let body = ClientHelloBuilder::new()
.ciphers(&[0x002f, 0x0035])
.build_body();
let body = &body[..body.len() - 2];
let ch = parse_client_hello(body).unwrap();
assert_eq!(ch.cipher_suites.as_slice(), &[0x002f, 0x0035]);
assert!(ch.extensions.is_empty());
}
proptest! {
#[test]
fn parser_never_panics_on_arbitrary_bytes(bytes in proptest::collection::vec(any::<u8>(), 0..2048)) {
let _ = parse_client_hello(&bytes);
let _ = parse_server_hello(&bytes);
if let Ok(hs) = handshake_bytes(&bytes) {
let _ = first_handshake_message(&hs, handshake_type::CLIENT_HELLO);
}
}
#[test]
fn well_formed_prefix_with_random_extension_tail_is_stable(
tail in proptest::collection::vec(any::<u8>(), 0..256)
) {
let mut ch = ClientHelloBuilder::new().ciphers(&[0x1301, 0x1302]);
ch = ch.extension(0xabcd, tail);
let body = ch.build_body();
let parsed = parse_client_hello(&body).unwrap();
prop_assert_eq!(parsed.cipher_suites.as_slice(), &[0x1301, 0x1302]);
}
}

247
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/tests/pipeline.rs Normal file
View File

@ -0,0 +1,247 @@
// ©AngelaMos | 2026
// pipeline.rs
//! Full pipeline known answer tests over the vendored FoxIO captures.
//!
//! These read a real capture file off disk, run it through the entire stack
//! the binary uses, and assert the fingerprints against the values FoxIO
//! published for the same files. A unit test proves an algorithm in isolation;
//! these prove that the file reader, the link layer decoder, the TCP
//! reassembler, the protocol sniffer, and the algorithms together still land
//! on the published answer. Every hash asserted here was independently
//! recomputed from its raw pre hash string with a shell tool before being
//! pinned, so a regression in any stage shows up as a mismatch rather than a
//! silently wrong but self consistent number.
use std::path::PathBuf;
use tlsfp_core::pipeline::event::StreamEvent;
use tlsfp_core::{FingerprintEvent, PcapFileSource, Pipeline, PipelineConfig};
fn pcap_path(name: &str) -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("../../testdata/pcap")
.join(name)
}
/// Runs one capture all the way through the pipeline and returns its events.
fn fingerprint(name: &str) -> Vec<FingerprintEvent> {
let mut source =
PcapFileSource::open(pcap_path(name)).unwrap_or_else(|e| panic!("opening {name}: {e}"));
let mut pipeline = Pipeline::new(PipelineConfig::default());
let mut events = Vec::new();
pipeline
.run(&mut source, |event| events.push(event))
.unwrap_or_else(|e| panic!("running {name}: {e}"));
events
}
fn client_hellos(events: &[FingerprintEvent]) -> Vec<(&str, &str)> {
events
.iter()
.filter_map(|e| match &e.event {
StreamEvent::ClientHello { ja4, .. } => Some((ja4.hash.as_str(), ja4.raw.as_str())),
_ => None,
})
.collect()
}
/// Returns the ClientHello events whose JA4 transport marker is QUIC.
fn quic_client_hellos(events: &[FingerprintEvent]) -> Vec<&FingerprintEvent> {
events
.iter()
.filter(|e| match &e.event {
StreamEvent::ClientHello { ja4, .. } => ja4.hash.starts_with('q'),
_ => false,
})
.collect()
}
fn ja4s_hashes(events: &[FingerprintEvent]) -> Vec<(&str, &str)> {
events
.iter()
.filter_map(|e| match &e.event {
StreamEvent::ServerHello { ja4s, .. } => Some((ja4s.hash.as_str(), ja4s.raw.as_str())),
_ => None,
})
.collect()
}
fn ja4x_hashes(events: &[FingerprintEvent]) -> Vec<&str> {
events
.iter()
.filter_map(|e| match &e.event {
StreamEvent::Certificate { ja4x } => Some(ja4x.as_str()),
_ => None,
})
.collect()
}
fn ja4h_hashes(events: &[FingerprintEvent]) -> Vec<&str> {
events
.iter()
.filter_map(|e| match &e.event {
StreamEvent::HttpRequest { ja4h, .. } => Some(ja4h.hash.as_str()),
_ => None,
})
.collect()
}
#[test]
fn tls_alpn_h2_reproduces_published_ja4_and_ja4x() {
let events = fingerprint("tls-alpn-h2.pcap");
let hellos = client_hellos(&events);
assert_eq!(hellos.len(), 1);
assert_eq!(hellos[0].0, "t12d4605h2_85626a9a5f7f_aaf95bb78ec9");
let certs = ja4x_hashes(&events);
assert_eq!(
certs.first().copied(),
Some("7d5dbb3783b4_ba7ce0880c07_7bf9a7bf7029")
);
}
#[test]
fn chrome_cloudflare_tcp_handshake_reproduces_published_ja4() {
let events = fingerprint("chrome-cloudflare-quic-with-secrets.pcapng");
let hellos = client_hellos(&events);
assert!(
hellos
.iter()
.any(|(hash, _)| *hash == "t13d1516h2_8daaf6152771_e5627efa2ab1"),
"expected the published chrome JA4, saw {hellos:?}"
);
let servers = ja4s_hashes(&events);
assert!(
servers
.iter()
.any(|(hash, _)| *hash == "t130200_1301_234ea6891581"),
"expected the published cloudflare JA4S, saw {servers:?}"
);
}
#[test]
fn chrome_cloudflare_quic_initial_decrypts_to_published_q_ja4() {
// The same capture carries the same browser over both transports. The TCP
// ClientHello above and this QUIC one share a JA4_a prefix family but
// differ where the transport and ALPN do, which is the whole reason JA4
// records the transport. The QUIC value below was reproduced from the
// decrypted Initial by an independent decryptor and recomputed from its
// raw string with a shell digest before being pinned.
let events = fingerprint("chrome-cloudflare-quic-with-secrets.pcapng");
let quic = quic_client_hellos(&events);
assert_eq!(quic.len(), 1, "expected exactly one QUIC ClientHello");
let StreamEvent::ClientHello { ja4, sni, alpn, .. } = &quic[0].event else {
unreachable!("filtered to ClientHello");
};
assert_eq!(ja4.hash, "q13d0310h3_55b375c5d22e_cd85d2d88918");
assert_eq!(sni.as_deref(), Some("cloudflare-quic.com"));
assert_eq!(alpn.as_deref(), Some("h3"));
}
#[test]
fn quic_several_tls_frames_reassembles_split_client_hello() {
// This capture's ClientHello is delivered across several CRYPTO frames in
// one Initial, exercising the offset based reassembler. It lands on the
// identical JA4 as the chrome capture despite being a different client,
// because JA4 sorts the cipher and extension lists before hashing.
let events = fingerprint("quic-with-several-tls-frames.pcapng");
let quic = quic_client_hellos(&events);
assert_eq!(quic.len(), 1);
let StreamEvent::ClientHello { ja4, .. } = &quic[0].event else {
unreachable!("filtered to ClientHello");
};
assert_eq!(ja4.hash, "q13d0310h3_55b375c5d22e_cd85d2d88918");
assert_eq!(
ja4.raw,
"q13d0310h3_1301,1302,1303_000a,000d,001b,002b,002d,0033,0039,4469_0403,0804,0401,0503,0805,0501,0806,0601,0201"
);
}
#[test]
fn tls_handshake_reproduces_published_ja4s_raw_and_hash() {
let events = fingerprint("tls-handshake.pcapng");
// The capture holds many flows to many Google and Cloudflare endpoints, so
// several distinct JA4S values appear. The published vector is the one for
// the first flow, and it must carry both its published hash and raw form.
let servers = ja4s_hashes(&events);
assert!(
servers
.iter()
.any(|(hash, raw)| *hash == "t130200_1301_234ea6891581"
&& *raw == "t130200_1301_0033,002b"),
"expected the published JA4S with its raw form, saw {servers:?}"
);
let hellos = client_hellos(&events);
assert!(hellos.len() >= 5);
assert!(
hellos
.iter()
.any(|(hash, _)| *hash == "t13d1516h2_8daaf6152771_e5627efa2ab1")
);
}
#[test]
fn http1_with_cookies_reproduces_pinned_ja4h() {
let events = fingerprint("http1-with-cookies.pcapng");
let hashes = ja4h_hashes(&events);
assert_eq!(
hashes,
vec!["ge11cr04da00_8ddaef5d77af_280f366eaa04_c2fb0fe53442"]
);
}
#[test]
fn tls12_capture_recomputes_to_pinned_ja4() {
let events = fingerprint("tls12.pcap");
let hellos = client_hellos(&events);
assert_eq!(hellos.len(), 1);
assert_eq!(hellos[0].0, "t13d1715h2_5b57614c22b0_3d5424432f57");
}
#[test]
fn non_ascii_alpn_follows_the_spec_hex_rule() {
let events = fingerprint("tls-non-ascii-alpn.pcapng");
let hellos = client_hellos(&events);
assert_eq!(hellos.len(), 1);
let (hash, _) = hellos[0];
let alpn_chars = &hash[8..10];
assert_eq!(
alpn_chars, "bd",
"spec rule prints first and last hex chars of the raw ALPN value"
);
}
#[test]
fn browsers_x509_extracts_every_certificate_in_chain_order() {
let events = fingerprint("browsers-x509.pcapng");
let certs = ja4x_hashes(&events);
assert!(
certs.len() >= 7,
"expected several certificates, saw {}",
certs.len()
);
assert!(certs.iter().all(|c| c.matches('_').count() == 2));
}
#[test]
fn tunneled_capture_yields_nothing_without_crashing() {
let events = fingerprint("gre-erspan-vxlan.pcap");
assert!(events.is_empty());
}
#[test]
fn evasion_capture_is_processed_without_panicking() {
let events = fingerprint("CVE-2018-6794.pcap");
assert!(
ja4h_hashes(&events).len() >= 2,
"the reassembler should still recover the HTTP requests"
);
}

114
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-core/tests/reassembly.rs Normal file
View File

@ -0,0 +1,114 @@
// ©AngelaMos | 2026
// reassembly.rs
//! Property tests for the TCP stream reassembler.
//!
//! The reassembler is the component an adversary reaches first: it consumes
//! attacker controlled sequence numbers and segment boundaries on every
//! connection. Two properties pin it down. The first is correctness under
//! reordering: any stream cut into any segments and delivered in any order
//! must reassemble back to the original bytes, because that is the whole job.
//! The second is the absence of panics under fully arbitrary input, because a
//! passive sensor that can be crashed by a crafted segment is a denial of
//! service waiting to happen.
use proptest::prelude::*;
use tlsfp_core::pipeline::flow::{PushOutcome, ReassemblyLimits, StreamReassembler};
fn generous_limits() -> ReassemblyLimits {
ReassemblyLimits {
max_assembled_bytes: 1 << 20,
max_pending_bytes: 1 << 20,
max_pending_segments: 4096,
}
}
/// A stream plus its segment boundaries and a delivery permutation.
fn stream_and_delivery() -> impl Strategy<Value = (Vec<u8>, Vec<(usize, usize)>, Vec<usize>)> {
prop::collection::vec(any::<u8>(), 1..2048).prop_flat_map(|data| {
let len = data.len();
let cuts = prop::collection::vec(0..len, 0..16);
(Just(data), cuts).prop_flat_map(|(data, mut cuts)| {
cuts.sort_unstable();
cuts.dedup();
let boundaries = boundaries_from_cuts(&cuts, data.len());
let permutation = Just((0..boundaries.len()).collect::<Vec<_>>()).prop_shuffle();
(Just(data), Just(boundaries), permutation)
})
})
}
fn boundaries_from_cuts(cuts: &[usize], len: usize) -> Vec<(usize, usize)> {
let mut points = vec![0];
points.extend(cuts.iter().copied().filter(|&c| c > 0 && c < len));
points.push(len);
points.dedup();
points.windows(2).map(|w| (w[0], w[1])).collect()
}
proptest! {
#![proptest_config(ProptestConfig::with_cases(512))]
/// Any segmentation delivered in any order reassembles to the original.
#[test]
fn arbitrary_segmentation_and_reordering_reconstructs_the_stream(
(data, boundaries, permutation) in stream_and_delivery(),
) {
let base_seq = 0x1234_5678u32;
let mut reasm = StreamReassembler::new(generous_limits());
reasm.anchor(base_seq);
for &segment_index in &permutation {
let (start, end) = boundaries[segment_index];
let seq = base_seq.wrapping_add(u32::try_from(start).unwrap());
reasm.push(seq, &data[start..end]);
}
prop_assert_eq!(reasm.data(), data.as_slice());
}
/// Duplicates layered on top of a complete stream change nothing.
#[test]
fn duplicate_delivery_is_idempotent(
(data, boundaries, permutation) in stream_and_delivery(),
) {
let base_seq = 42u32;
let mut reasm = StreamReassembler::new(generous_limits());
reasm.anchor(base_seq);
for round in 0..2 {
for &segment_index in &permutation {
let (start, end) = boundaries[segment_index];
let seq = base_seq.wrapping_add(u32::try_from(start).unwrap());
let outcome = reasm.push(seq, &data[start..end]);
if round == 1 {
prop_assert!(matches!(
outcome,
PushOutcome::Unchanged | PushOutcome::Grew
));
}
}
}
prop_assert_eq!(reasm.data(), data.as_slice());
}
/// Fully arbitrary segments never panic and never exceed the cap.
#[test]
fn arbitrary_input_never_panics_or_overruns(
segments in prop::collection::vec(
(any::<u32>(), prop::collection::vec(any::<u8>(), 0..64)),
0..256,
),
) {
let limits = ReassemblyLimits {
max_assembled_bytes: 512,
max_pending_bytes: 512,
max_pending_segments: 32,
};
let mut reasm = StreamReassembler::new(limits);
for (seq, payload) in &segments {
reasm.push(*seq, payload);
prop_assert!(reasm.data().len() <= limits.max_assembled_bytes);
}
}
}

26
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-intel/Cargo.toml Normal file
View File

@ -0,0 +1,26 @@
# ©AngelaMos | 2026
# Cargo.toml
[package]
name = "tlsfp-intel"
description = "Local threat intelligence store and fingerprint matching engine"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
authors.workspace = true
repository.workspace = true
[lints]
workspace = true
[dependencies]
tlsfp-core.workspace = true
rusqlite.workspace = true
csv.workspace = true
serde.workspace = true
serde_json.workspace = true
anyhow.workspace = true
[dev-dependencies]
tempfile = "3"

64
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-intel/seeds/PROVENANCE.md Normal file
View File

@ -0,0 +1,64 @@
<!-- ©AngelaMos | 2026 -->
<!-- PROVENANCE.md -->
# Seed provenance
These three files are the intelligence the tool ships with. They are compiled
into the binary with `include_str!`, so a freshly built `tlsfp` can seed its
database with no network access. The large external source, ja4db.com, is
fetched at install time instead of bundled, because its license is unspecified
and its records are known to contain dirty entries that the importer validates
and rejects on the way in.
Each row is a fingerprint plus a label. None of these files is edited after
import beyond skipping comment lines and rows whose first field is not a
fingerprint, so the sha256 below is the exact bundled content.
## sslbl-ja3.csv
- Source: https://sslbl.abuse.ch/blacklist/ja3_fingerprints.csv
- Retrieved: 2026-06-14
- Snapshot date in the file header: 2021-08-03 (abuse.ch froze the JA3 list)
- License: CC0 1.0 (public domain dedication), so the snapshot is committable
- Records: 97 malicious JA3 hashes with a family label and first and last seen dates
- sha256: 6974182489b6e5f5f64079454b81a7219a2d82ae950f108567fa371220eb09e9
The abuse.ch SSL Blacklist carries a false positive disclaimer: a JA3 is a hash
of a TLS client library, so a benign program that links the same library lands
on the same hash. That disclaimer is the reason the matching engine scores
prevalence across sources instead of treating any single feed hit as proof.
## salesforce-osx-nix-ja3.csv
- Source: https://raw.githubusercontent.com/salesforce/ja3/master/lists/osx-nix-ja3.csv
- Retrieved: 2026-06-14
- License: BSD 3-Clause, recorded verbatim in the first record of the file
- Records: 157 benign JA3 hashes for common macOS and nix applications
- sha256: 2865b3f73a68e603dd3fa7fb56565dca70a269cda70f460ed5db03f5d724c4e1
The upstream salesforce/ja3 repository was archived on 2025-05-01. The file is
the benign half of the picture: it is what makes a collision visible. Several
hashes in this list also appear in sslbl-ja3.csv under a malware label, which
is the cleanest real demonstration that a JA3 alone cannot decide intent. The
field that names the application is quoted and can hold several comma separated
names, so the loader parses it as RFC 4180 CSV rather than splitting on commas.
Note on licensing: the research notes that seeded this project called this list
MIT. The file header itself says BSD 3-Clause, and the file is the ground truth,
so it is attributed as BSD 3-Clause here.
## curated-c2-intel.csv
- Source: hand curated for this project from named primary sources
- License: same as this repository
- Records: 17 rows covering C2 frameworks, malware families, dual-use tooling,
and two benign browser baselines, each with a primary source in the reference
column
- sha256: 202c0a1edc495a5149d7f6161d21adc152b1b71efdde2f5052d4767ebb0f7b66
This file fills the gaps the public feeds leave: server side JA3S for Cobalt
Strike, TrickBot, and Emotet, a JA4 for RedLine Stealer, and JA4 baselines for
Chrome over TCP and QUIC that let the engine recognise a tool that copies a
browser cipher list but not its full extension order. Where a public feed labels
one of these hashes with a different family, the divergence is written into the
reference column rather than silently reconciled.

28
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-intel/seeds/curated-c2-intel.csv Normal file
View File

@ -0,0 +1,28 @@
# ©AngelaMos | 2026
# curated-c2-intel.csv
#
# Hand-curated fingerprints for C2 frameworks, malware families, and benign
# baselines. Every row is traced to a named primary source in the reference
# column. Categories: c2 and malware are malicious; tool is dual-use offensive
# or privacy tooling that is noteworthy but not conclusive on its own; benign
# is a known-good baseline. Where a public feed labels the same hash with a
# different family name, that divergence is recorded in the reference rather
# than hidden, because a single JA3 can be shared by unrelated software.
fp_kind,value,label,category,reference
ja3,72a589da586844d7f0818ce684948eea,Cobalt Strike,c2,"Cobalt Strike default malleable profile over an IP; the same TLS stack is used by Emotet so this JA3 collides between the two"
ja3,a0e9f5d64349fb13191bc781f81f42e1,Cobalt Strike,c2,"Cobalt Strike default profile over a domain"
ja3,19e29534fd49dd27d09234e639c4057e,Sliver,c2,"Sliver implant; observed during Ivanti gateway exploitation in 2024 (Darktrace)"
ja3,c12f54a3f91dc7bafd92cb59fe009a35,PoshC2,c2,"PoshC2 proxy-aware command and control framework"
ja3,8916410db85077a5460817142dcbc8de,Metasploit,tool,"Metasploit Meterpreter HTTPS stager; abuse.ch SSLBL lists the identical JA3 as TrickBot, a feed label divergence"
ja3,f5e62b5a2ed9467df09fae7a8a54dda6,BazarLoader,malware,"BazarLoader and BazarBackdoor, part of the TrickBot and Conti ecosystem"
ja3,b386946a5a44d1ddcc843bc75336dfce,Dyre,malware,"Dyre and Dyreza banking trojan; the original JA3 pivot that surfaced 112 related samples; abuse.ch SSLBL lists the identical JA3 as Dridex"
ja3,e7d705a3286e19ea42f587b344ee6865,Tor,tool,"Tor client TLS; WannaCry tunnelled its command and control over Tor, but most Tor traffic is legitimate"
ja3,a85be79f7b569f1df5e6087b69deb493,Remcos,malware,"Remcos remote access trojan"
ja3,4d7a28d6f2263ed61de88ca66eb011e3,Emotet,malware,"Emotet loader; abuse.ch SSLBL lists the identical JA3 as Tofsee, a feed label divergence"
ja3,6734f37431670b3ab4292b8f60f29984,TrickBot,malware,"TrickBot banking trojan and loader"
ja3s,b742b407517bac9536a77a7b0fee28e9,Cobalt Strike,c2,"Cobalt Strike team server ServerHello"
ja3s,623de93db17d313345d7ea481e7443cf,TrickBot,c2,"TrickBot command and control server ServerHello"
ja3s,80b3a14bccc8598a1f3bbe83e71f735f,Emotet,c2,"Emotet command and control server ServerHello"
ja4,t10d070600_c50f5591e341_1a3805c3aa63,RedLine Stealer,malware,"RedLine Stealer and SnakeLogger infostealer (VirusTotal 2024)"
ja4,t13d1516h2_8daaf6152771_e5627efa2ab1,Google Chrome,benign,"Chrome over TCP; FoxIO reference fingerprint from the chrome-cloudflare capture"
ja4,q13d0310h3_55b375c5d22e_cd85d2d88918,Google Chrome,benign,"Chrome over QUIC; FoxIO reference fingerprint from the chrome-cloudflare capture"
Can't render this file because it has a wrong number of fields in line 4.

161
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-intel/seeds/salesforce-osx-nix-ja3.csv Normal file
View File

@ -0,0 +1,161 @@
"Copyright (c) 2017 salesforce.com inc.
All rights reserved.
Licensed under the BSD 3-Clause license.
For full license text see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause",
61d50e7771aee7f2f4b89a7200b4d45e,"AcroCEF"
49a6cf42956937669a01438f26e7c609,"AIM"
561145462cfc7de1d6a97e93d3264786,"Airmail 3"
f6fd83a21f9f3c5f9ff7b5c63bbc179d,"Alation Compose"
6003b52942a2e1e1ea72d802d153ec08,"Amazon Music"
eb149984fc9c44d85ed7f12c90d818be,"Amazon Music,Dreamweaver,Spotify"
8e3f1bf87bc652a20de63bfd4952b16a,"AnypointStudio"
5507277945374659a5b4572e1b6d9b9f,"apple.geod"
f753495f2eab5155c61b760c838018f8,"apple.geod"
ba40fea2b2638908a3b3b482ac78d729,"apple.geod,parsecd,apple.photomoments"
474e73aea21d1e0910f25c3e6c178535,"apple.WebKit.Networking"
eeeb5e7485f5e10cbc39db4cfb69b264,"apple.WebKit.Networking"
d4693422c5ce1565377aca25940ad80c,"apple.WebKit.Networking,CalendarAgent,Go for Gmail"
63de2b6188d5694e79b678f585b13264,"apple.WebKit.Networking,Chatter,FieldServiceApp,socialstudio"
3e4e87dda5a3162306609b7e330441d2,"apple.WebKit.Networking,itunesstored"
7b343af1092863fdd822d6f10645abfb,"apple.WebKit.Networking,itunesstored"
a312f9162a08eeedf7feb7a13cd7e9bb,"apple.WebKit.Networking,Spotify,WhatsApp,Skype,iTunes"
c5c11e6105c56fd29cc72c3ac7a2b78b,"AT&T Connect"
fa030dbcb2e3c7141d3c2803780ee8db,"Battle.net,Dropbox"
0ef9ca1c10d3f186f5786e1ef3461a46,"bitgo,ShapeShift"
cdec81515ccc75a5aa41eb3db22226e6,"BlueJeans,CEPHtmlEngine"
83e04bc58d402f9633983cbf22724b02,"Charles,Google Play Music Desktop Player,Postman,Slack,and other desktop programs"
424008725394c634a4616b8b1f2828a5,"Charles,java,eclipse"
be9f1360cf52dc1f61ae025252f192a3,"Chromium"
def8761e4bcaaf91d99801a22ac6f6d4,"Chromium"
fc5cb0985a5f5e295163cc8ffff8a6e1,"Chromium"
e7d46c98b078477c4324031e0d3b22f5,"Cisco AnyConnect Secure Mobility Client"
ed36017db541879619c399c95e22067d,"Cisco AnyConnect Secure Mobility Client"
5ee1a653fb824db7182714897fd3b5df,"Citrix Viewer"
a9d17f74e55dd53fcf7c234f8a240919,"Covenant Eyes"
c882d9444412c00e71b643f3f54145ff,"Creative Cloud"
bc0608d33dc64506b42f7f5f87958f37,"cscan"
ccaa60f6ccc701bde536ef409be3cf63,"curl no-SNI"
fe048fe8faf797796e278f2b4f1e9c24,"curl SNI"
4fcd1770545298cc119865aeba81daba,"Deezer"
4c40bf8baa7c301c5dba8a20bc4119e2,"Dynalist,Postman,Google Chrome,Franz,GOG Galaxy"
0411bbb5ff27ad46e1874a7a8beedacb,"eclipse"
4990c9da08f44a01ecd7ddc3837caf25,"eclipse"
fa106fe5beec443af7e211ef8902e7e0,"eclipse"
d74778f454e2b047e030b291b94dd698,"eclipse,java"
187dfde7edc8ceddccd3deeccc21daeb,"eclipse,java,studio,STS"
8c5a50f1e833ed581e9cfc690814719a,"eclipse,JavaApplicationStub,idea"
1fbe5382f9d8430fe921df747c46d95f,"FieldServiceApp,socialstudio"
0a81538cf247c104edb677bdb8902ed5,"firefox"
0b6592fd91d4843c823b75e49b43838d,"firefox"
0ffee3ba8e615ad22535e7f771690a28,"firefox"
1c15aca4a38bad90f9c40678f6aface9,"firefox"
5163bc7c08f57077bc652ec370459c2f,"firefox"
a88f1426c4603f2a8cd8bb41e875cb75,"firefox"
b03910cc6de801d2fcfa0c3b9f397df4,"firefox"
bfcc1a3891601edb4f137ab7ab25b840,"firefox"
ce694315cbb81ce95e6ae4ae8cbafde6,"firefox"
f15797a734d0b4f171a86fd35c9a5e43,"firefox"
07b4162d4db57554961824a21c4a0fde,"firefox,thunderbird"
61d0d709fe7ac199ef4b2c52bc8cef75,"firefox,thunderbird"
8498fe4268764dbf926a38283e9d3d8f,"Franz,Google Chrome,Kiwi,Spotify,nwjs,Slack"
900c1fa84b4ea86537e1d148ee16eae8,"Fuze"
107144b88827da5da9ed42d8776ccdc5,"geod"
c46941d4de99445aef6b497679474cf4,"geod"
002205d0f96c37c5e660b9f041363c11,"Google Chrome"
073eede15b2a5a0302d823ecbd5ad15b,"Google Chrome"
0b61c673ee71fe9ee725bd687c455809,"Google Chrome"
6cd1b944f5885e2cfbe98a840b75eeb8,"Google Chrome"
94c485bca29d5392be53f2b8cf7f4304,"Google Chrome"
b4f4e6164f938870486578536fc1ffce,"Google Chrome"
b8f81673c0e1d29908346f3bab892b9b,"Google Chrome"
baaac9b6bf25ad098115c71c59d29e51,"Google Chrome"
bc6c386f480ee97b9d9e52d472b772d8,"Google Chrome"
da949afd9bd6df820730f8f171584a71,"Google Chrome"
f58966d34ff9488a83797b55c804724d,"Google Chrome"
fd6314b03413399e4f23d1524d206692,"Google Chrome"
0e46737668fe75092919ee047a0b5945,"Google Chrome Helper"
39fa85654105398ee7ef6a3a1c81d685,"Google Chrome Helper"
4ba7b7022f5f5e1e500bb19199d8b1a4,"Google Chrome Helper"
5498cef2cca704eb01cf2041cc1089c1,"Google Chrome,Slack"
d27fb8deca6e3b9739db3fda2b229fe3,"Google Drive File Stream"
ae340571b4fd0755c4a0821b18d8fa93,"Google Earth"
f059212ce3de94b1e8253a7522cb1b44,"Google Photos Backup"
fd10cc8cce9493a966c57249e074755f,"gramblr"
3e860202fc555b939e83e7a7ab518c38,"hola_svc"
56ac3a0bef0824c49e4b569941937088,"hola_svc"
5c1c89f930122bccc7a97d52f73bea2c,"hola_svc"
77310efe11f1943306ee317cf02150b7,"hola_svc"
8bd59c4b7f3193db80fd64318429bcec,"hola_svc"
d1f9f9b224387d2597f02095fcec96d7,"hola_svc"
ff1040ba1e3d235855ef0d7cd9237fdc,"hola_svc"
5af143afdbf58ec11ab3b3d53dd4e5e3,"IDSyncDaemon"
d06acbe8ac31e753f40600a9d6717cba,"Inbox OSX"
093081b45872912be9a1f2a8163fe041,"java"
2080bf56cb87e64303e27fcd781e7efd,"java"
225a24b45f0f1adbc2e245d4624c6e08,"java"
3afe1fb5976d0999abe833b14b7d6485,"java"
3b844830bfbb12eb5d2f8dc281d349a9,"java"
51a7ad14509fd614c7bb3a50c4982b8c,"java"
550628650380ff418de25d3d890e836e,"java"
5b270b309ad8c6478586a15dece20a88,"java"
5d7abe53ae15b4272a34f10431e06bf3,"java"
7c7a68b96d2aab15d678497a12119f4f,"java"
88afa0dea1608e28f50acbad32d7f195,"java"
8ce6933b8c12ce931ca238e9420cc5dd,"java"
a61299f9b501adcf680b9275d79d4ac6,"java"
a9fead344bf3ac09f62df3cd9b22c268,"java"
4056657a50a8a4e5cfac40ba48becfa2,"java,eclipse"
f22bdd57e3a52de86cda40da2d84e83b,"java,eclipse,Cyberduck"
028563cffc7a3a2e32090aee0294d636,"java,eclipse,STS"
5f9b53f0d39dc9d940a3b5568fe5f0bb,"java,JavaApplicationStub"
2db6873021f2a95daa7de0d93a1d1bf2,"java,studio,eclipse"
c376061f96329e1020865a1dc726927d,"JavaApplicationStub"
e516ad69a423f8e0407307aa7bfd6344,"Kindle,stack,nextcloud"
3959d0a1344896e9fb5c0564ca0a2956,"LeagueClientUx"
0fe51fa93812c2ebb50a655222a57bf2,"LINE Messaging"
2e094913d88f0ad8dc69447cb7d2ce65,"LINE Messaging"
193349d34561d1d5d1a270172eb2d97e,"LogMeIn Client"
d732ca39155f38942f90e9fc2b0f97f7,"Maxthon"
c9dbeed362a32f9a50a26f4d9b32bbd8,"Messenger,Jumpshare"
6acb250ada693067812c3335705dae79,"mono-sgen,Syncplicity,Axure RP 8,Amazon Drive"
3ee4aaac7147ff2b80ada31686db660c,"node-webkit,Kindle"
641df9d6dbe7fdb74f70c8ad93def8cc,"node.js"
9811c1bb9f0f6835d5c13a831cca4173,"node.js"
106ecbd3d14b4dc6e413494263720afe,"node.js,Postman,WhatsApp"
49de9b1c7e60bd3b8e1d4f7a49ba362e,"nwjs,Chromium"
38cbe70b308f42da7c9980c0e1c89656,"p4v,owncloud"
62448833d8230241227c03b7d441e31b,"parsecd,apple.geod,apple.photomoments,photoanalysisd,FreedomProxy"
e846898acc767ebeb2b4388e58a968d4,"postbox-bin"
a7823092705a5e91ce2b7f561b6e5b98,"Qsync Client"
c048d9f26a79e11ca7276499ef24daf3,"RescueTime,Plantronics Hub"
d219efd07cbb8fbe547e6a5335843f0f,"ruby"
c36fb08942cf19508c08d96af22d4ffc,"Safari"
844166382cc98d98595e6778c470f5d5,"Salesforce Files"
49a341a21f4fd4ac63b027ff2b1a331f,"Skype"
a5aa6e939e4770e3b8ac38ce414fd0d5,"Slack"
116ffc8889873efad60457cd55eaf543,"Spark"
8db4b0f8e9dd8f2fff38ee7c5a1e4496,"SpotlightNetHelper,Safari"
39cf5b7a13a764494de562add874f016,"Steam OSX"
2d3854d1cbcdceece83eabd85bdcc056,"Tableau"
a585c632a2b49be1256881fb0c16c864,"Tableau"
cd7c06b9459c9cfd4af2dba5696ea930,"Tableau"
df65746370dcabc9b4f370c6e14a8156,"True Key"
84071ea96fc8a60c55fc8a405e214c0f,"Used by many desktop apps,Quip,Spotify,GitHub Desktop"
40fd0a5e81ebdcf0ec82a4710a12dec1,"Used by many programs on OSX,apple.WebKit.Networking"
618ee2509ef52bf0b8216e1564eea909,"Used by many programs on OSX,apple.WebKit.Networking"
799135475da362592a4be9199d258726,"Used by many programs on OSX,apple.WebKit.Networking"
7b530a25af9016a9d12de5abc54d9e74,"Used by many programs on OSX,apple.WebKit.Networking"
7e72698146290dd68239f788a452e7d8,"Used by many programs on OSX,apple.WebKit.Networking"
a9aecaa66ad9c6cfe1c361da31768506,"Used by many programs on OSX,apple.WebKit.Networking"
c05de18b01a054f2f6900ffe96b3da7a,"Used by many programs on OSX,apple.WebKit.Networking"
c07cb55f88702033a8f52c046d23e0b2,"Used by many programs on OSX,apple.WebKit.Networking"
e4d448cdfe06dc1243c1eb026c74ac9a,"Used by many programs on OSX,apple.WebKit.Networking"
f1c5cf087b959cec31bd6285407f689a,"Used by many programs on OSX,apple.WebKit.Networking"
488b6b601cb141b062d4da7f524b4b22,"Used by many programs,Python,PHP,Git,dotnet,Adobe"
f28d34ce9e732f644de2350027d74c3f,"Used by many programs,Quip,Aura,Spotify,Chatty"
190dfb280fe3b541acc6a2e5f00690e6,"Used by many programs,Quip,Spotify,Dropbox,GitHub Desktop,etc"
20dd18bdd3209ea718989030a6f93364,"Used by many programs,Slack,Postman,Spotify,Google Chrome"
e0224fc1c33658f2d3d963bfb0a76a85,"Viber"
01319090aea981dde6fc8d6ae71ead54,"vpnkit"
84607748f3887541dd60fe974a042c71,"wineserver"
c2b4710c6888a5d47befe865c8e6fb19,"ZwiftApp"
1 Copyright (c) 2017 salesforce.com inc. All rights reserved. Licensed under the BSD 3-Clause license. For full license text see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
2 61d50e7771aee7f2f4b89a7200b4d45e AcroCEF
3 49a6cf42956937669a01438f26e7c609 AIM
4 561145462cfc7de1d6a97e93d3264786 Airmail 3
5 f6fd83a21f9f3c5f9ff7b5c63bbc179d Alation Compose
6 6003b52942a2e1e1ea72d802d153ec08 Amazon Music
7 eb149984fc9c44d85ed7f12c90d818be Amazon Music,Dreamweaver,Spotify
8 8e3f1bf87bc652a20de63bfd4952b16a AnypointStudio
9 5507277945374659a5b4572e1b6d9b9f apple.geod
10 f753495f2eab5155c61b760c838018f8 apple.geod
11 ba40fea2b2638908a3b3b482ac78d729 apple.geod,parsecd,apple.photomoments
12 474e73aea21d1e0910f25c3e6c178535 apple.WebKit.Networking
13 eeeb5e7485f5e10cbc39db4cfb69b264 apple.WebKit.Networking
14 d4693422c5ce1565377aca25940ad80c apple.WebKit.Networking,CalendarAgent,Go for Gmail
15 63de2b6188d5694e79b678f585b13264 apple.WebKit.Networking,Chatter,FieldServiceApp,socialstudio
16 3e4e87dda5a3162306609b7e330441d2 apple.WebKit.Networking,itunesstored
17 7b343af1092863fdd822d6f10645abfb apple.WebKit.Networking,itunesstored
18 a312f9162a08eeedf7feb7a13cd7e9bb apple.WebKit.Networking,Spotify,WhatsApp,Skype,iTunes
19 c5c11e6105c56fd29cc72c3ac7a2b78b AT&T Connect
20 fa030dbcb2e3c7141d3c2803780ee8db Battle.net,Dropbox
21 0ef9ca1c10d3f186f5786e1ef3461a46 bitgo,ShapeShift
22 cdec81515ccc75a5aa41eb3db22226e6 BlueJeans,CEPHtmlEngine
23 83e04bc58d402f9633983cbf22724b02 Charles,Google Play Music Desktop Player,Postman,Slack,and other desktop programs
24 424008725394c634a4616b8b1f2828a5 Charles,java,eclipse
25 be9f1360cf52dc1f61ae025252f192a3 Chromium
26 def8761e4bcaaf91d99801a22ac6f6d4 Chromium
27 fc5cb0985a5f5e295163cc8ffff8a6e1 Chromium
28 e7d46c98b078477c4324031e0d3b22f5 Cisco AnyConnect Secure Mobility Client
29 ed36017db541879619c399c95e22067d Cisco AnyConnect Secure Mobility Client
30 5ee1a653fb824db7182714897fd3b5df Citrix Viewer
31 a9d17f74e55dd53fcf7c234f8a240919 Covenant Eyes
32 c882d9444412c00e71b643f3f54145ff Creative Cloud
33 bc0608d33dc64506b42f7f5f87958f37 cscan
34 ccaa60f6ccc701bde536ef409be3cf63 curl no-SNI
35 fe048fe8faf797796e278f2b4f1e9c24 curl SNI
36 4fcd1770545298cc119865aeba81daba Deezer
37 4c40bf8baa7c301c5dba8a20bc4119e2 Dynalist,Postman,Google Chrome,Franz,GOG Galaxy
38 0411bbb5ff27ad46e1874a7a8beedacb eclipse
39 4990c9da08f44a01ecd7ddc3837caf25 eclipse
40 fa106fe5beec443af7e211ef8902e7e0 eclipse
41 d74778f454e2b047e030b291b94dd698 eclipse,java
42 187dfde7edc8ceddccd3deeccc21daeb eclipse,java,studio,STS
43 8c5a50f1e833ed581e9cfc690814719a eclipse,JavaApplicationStub,idea
44 1fbe5382f9d8430fe921df747c46d95f FieldServiceApp,socialstudio
45 0a81538cf247c104edb677bdb8902ed5 firefox
46 0b6592fd91d4843c823b75e49b43838d firefox
47 0ffee3ba8e615ad22535e7f771690a28 firefox
48 1c15aca4a38bad90f9c40678f6aface9 firefox
49 5163bc7c08f57077bc652ec370459c2f firefox
50 a88f1426c4603f2a8cd8bb41e875cb75 firefox
51 b03910cc6de801d2fcfa0c3b9f397df4 firefox
52 bfcc1a3891601edb4f137ab7ab25b840 firefox
53 ce694315cbb81ce95e6ae4ae8cbafde6 firefox
54 f15797a734d0b4f171a86fd35c9a5e43 firefox
55 07b4162d4db57554961824a21c4a0fde firefox,thunderbird
56 61d0d709fe7ac199ef4b2c52bc8cef75 firefox,thunderbird
57 8498fe4268764dbf926a38283e9d3d8f Franz,Google Chrome,Kiwi,Spotify,nwjs,Slack
58 900c1fa84b4ea86537e1d148ee16eae8 Fuze
59 107144b88827da5da9ed42d8776ccdc5 geod
60 c46941d4de99445aef6b497679474cf4 geod
61 002205d0f96c37c5e660b9f041363c11 Google Chrome
62 073eede15b2a5a0302d823ecbd5ad15b Google Chrome
63 0b61c673ee71fe9ee725bd687c455809 Google Chrome
64 6cd1b944f5885e2cfbe98a840b75eeb8 Google Chrome
65 94c485bca29d5392be53f2b8cf7f4304 Google Chrome
66 b4f4e6164f938870486578536fc1ffce Google Chrome
67 b8f81673c0e1d29908346f3bab892b9b Google Chrome
68 baaac9b6bf25ad098115c71c59d29e51 Google Chrome
69 bc6c386f480ee97b9d9e52d472b772d8 Google Chrome
70 da949afd9bd6df820730f8f171584a71 Google Chrome
71 f58966d34ff9488a83797b55c804724d Google Chrome
72 fd6314b03413399e4f23d1524d206692 Google Chrome
73 0e46737668fe75092919ee047a0b5945 Google Chrome Helper
74 39fa85654105398ee7ef6a3a1c81d685 Google Chrome Helper
75 4ba7b7022f5f5e1e500bb19199d8b1a4 Google Chrome Helper
76 5498cef2cca704eb01cf2041cc1089c1 Google Chrome,Slack
77 d27fb8deca6e3b9739db3fda2b229fe3 Google Drive File Stream
78 ae340571b4fd0755c4a0821b18d8fa93 Google Earth
79 f059212ce3de94b1e8253a7522cb1b44 Google Photos Backup
80 fd10cc8cce9493a966c57249e074755f gramblr
81 3e860202fc555b939e83e7a7ab518c38 hola_svc
82 56ac3a0bef0824c49e4b569941937088 hola_svc
83 5c1c89f930122bccc7a97d52f73bea2c hola_svc
84 77310efe11f1943306ee317cf02150b7 hola_svc
85 8bd59c4b7f3193db80fd64318429bcec hola_svc
86 d1f9f9b224387d2597f02095fcec96d7 hola_svc
87 ff1040ba1e3d235855ef0d7cd9237fdc hola_svc
88 5af143afdbf58ec11ab3b3d53dd4e5e3 IDSyncDaemon
89 d06acbe8ac31e753f40600a9d6717cba Inbox OSX
90 093081b45872912be9a1f2a8163fe041 java
91 2080bf56cb87e64303e27fcd781e7efd java
92 225a24b45f0f1adbc2e245d4624c6e08 java
93 3afe1fb5976d0999abe833b14b7d6485 java
94 3b844830bfbb12eb5d2f8dc281d349a9 java
95 51a7ad14509fd614c7bb3a50c4982b8c java
96 550628650380ff418de25d3d890e836e java
97 5b270b309ad8c6478586a15dece20a88 java
98 5d7abe53ae15b4272a34f10431e06bf3 java
99 7c7a68b96d2aab15d678497a12119f4f java
100 88afa0dea1608e28f50acbad32d7f195 java
101 8ce6933b8c12ce931ca238e9420cc5dd java
102 a61299f9b501adcf680b9275d79d4ac6 java
103 a9fead344bf3ac09f62df3cd9b22c268 java
104 4056657a50a8a4e5cfac40ba48becfa2 java,eclipse
105 f22bdd57e3a52de86cda40da2d84e83b java,eclipse,Cyberduck
106 028563cffc7a3a2e32090aee0294d636 java,eclipse,STS
107 5f9b53f0d39dc9d940a3b5568fe5f0bb java,JavaApplicationStub
108 2db6873021f2a95daa7de0d93a1d1bf2 java,studio,eclipse
109 c376061f96329e1020865a1dc726927d JavaApplicationStub
110 e516ad69a423f8e0407307aa7bfd6344 Kindle,stack,nextcloud
111 3959d0a1344896e9fb5c0564ca0a2956 LeagueClientUx
112 0fe51fa93812c2ebb50a655222a57bf2 LINE Messaging
113 2e094913d88f0ad8dc69447cb7d2ce65 LINE Messaging
114 193349d34561d1d5d1a270172eb2d97e LogMeIn Client
115 d732ca39155f38942f90e9fc2b0f97f7 Maxthon
116 c9dbeed362a32f9a50a26f4d9b32bbd8 Messenger,Jumpshare
117 6acb250ada693067812c3335705dae79 mono-sgen,Syncplicity,Axure RP 8,Amazon Drive
118 3ee4aaac7147ff2b80ada31686db660c node-webkit,Kindle
119 641df9d6dbe7fdb74f70c8ad93def8cc node.js
120 9811c1bb9f0f6835d5c13a831cca4173 node.js
121 106ecbd3d14b4dc6e413494263720afe node.js,Postman,WhatsApp
122 49de9b1c7e60bd3b8e1d4f7a49ba362e nwjs,Chromium
123 38cbe70b308f42da7c9980c0e1c89656 p4v,owncloud
124 62448833d8230241227c03b7d441e31b parsecd,apple.geod,apple.photomoments,photoanalysisd,FreedomProxy
125 e846898acc767ebeb2b4388e58a968d4 postbox-bin
126 a7823092705a5e91ce2b7f561b6e5b98 Qsync Client
127 c048d9f26a79e11ca7276499ef24daf3 RescueTime,Plantronics Hub
128 d219efd07cbb8fbe547e6a5335843f0f ruby
129 c36fb08942cf19508c08d96af22d4ffc Safari
130 844166382cc98d98595e6778c470f5d5 Salesforce Files
131 49a341a21f4fd4ac63b027ff2b1a331f Skype
132 a5aa6e939e4770e3b8ac38ce414fd0d5 Slack
133 116ffc8889873efad60457cd55eaf543 Spark
134 8db4b0f8e9dd8f2fff38ee7c5a1e4496 SpotlightNetHelper,Safari
135 39cf5b7a13a764494de562add874f016 Steam OSX
136 2d3854d1cbcdceece83eabd85bdcc056 Tableau
137 a585c632a2b49be1256881fb0c16c864 Tableau
138 cd7c06b9459c9cfd4af2dba5696ea930 Tableau
139 df65746370dcabc9b4f370c6e14a8156 True Key
140 84071ea96fc8a60c55fc8a405e214c0f Used by many desktop apps,Quip,Spotify,GitHub Desktop
141 40fd0a5e81ebdcf0ec82a4710a12dec1 Used by many programs on OSX,apple.WebKit.Networking
142 618ee2509ef52bf0b8216e1564eea909 Used by many programs on OSX,apple.WebKit.Networking
143 799135475da362592a4be9199d258726 Used by many programs on OSX,apple.WebKit.Networking
144 7b530a25af9016a9d12de5abc54d9e74 Used by many programs on OSX,apple.WebKit.Networking
145 7e72698146290dd68239f788a452e7d8 Used by many programs on OSX,apple.WebKit.Networking
146 a9aecaa66ad9c6cfe1c361da31768506 Used by many programs on OSX,apple.WebKit.Networking
147 c05de18b01a054f2f6900ffe96b3da7a Used by many programs on OSX,apple.WebKit.Networking
148 c07cb55f88702033a8f52c046d23e0b2 Used by many programs on OSX,apple.WebKit.Networking
149 e4d448cdfe06dc1243c1eb026c74ac9a Used by many programs on OSX,apple.WebKit.Networking
150 f1c5cf087b959cec31bd6285407f689a Used by many programs on OSX,apple.WebKit.Networking
151 488b6b601cb141b062d4da7f524b4b22 Used by many programs,Python,PHP,Git,dotnet,Adobe
152 f28d34ce9e732f644de2350027d74c3f Used by many programs,Quip,Aura,Spotify,Chatty
153 190dfb280fe3b541acc6a2e5f00690e6 Used by many programs,Quip,Spotify,Dropbox,GitHub Desktop,etc
154 20dd18bdd3209ea718989030a6f93364 Used by many programs,Slack,Postman,Spotify,Google Chrome
155 e0224fc1c33658f2d3d963bfb0a76a85 Viber
156 01319090aea981dde6fc8d6ae71ead54 vpnkit
157 84607748f3887541dd60fe974a042c71 wineserver
158 c2b4710c6888a5d47befe865c8e6fb19 ZwiftApp

108
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-intel/seeds/sslbl-ja3.csv Normal file
View File

@ -0,0 +1,108 @@
################################################################
# abuse.ch Suricata JA3 Fingerprint Blacklist (CSV) #
# For Suricata 4.1.0 or newer #
# Last updated: 2021-08-03 14:33:44 UTC #
# #
# Terms Of Use: https://sslbl.abuse.ch/blacklist/ #
# For questions please contact sslbl [at] abuse.ch #
################################################################
#
# ja3_md5,Firstseen,Lastseen,Listingreason
b386946a5a44d1ddcc843bc75336dfce,2017-07-14 18:08:15,2019-07-27 20:42:54,Dridex
8991a387e4cc841740f25d6f5139f92d,2017-07-14 19:02:03,2019-07-28 00:34:38,Adware
cb98a24ee4b9134448ffb5714fd870ac,2017-07-14 19:48:28,2019-05-22 03:22:38,Dridex
1aa7bf8b97e540ca5edd75f7b8384bfa,2017-07-14 20:23:38,2019-07-28 01:38:22,TrickBot
3d89c0dfb1fa44911b8fa7523ef8dedb,2017-07-15 04:23:45,2021-02-01 18:23:25,Adware
bc6c386f480ee97b9d9e52d472b772d8,2017-07-15 10:57:38,2021-03-13 07:33:39,Adware
8f52d1ce303fb4a6515836aec3cc16b1,2017-07-15 19:05:11,2019-07-27 20:00:57,TrickBot
d6f04b5a910115f4b50ecec09d40a1df,2017-07-15 19:42:24,2018-10-14 08:12:51,Dridex
35c0a31c481927f022a3b530255ac080,2017-07-15 19:43:19,2021-04-10 12:54:04,Tofsee
e330bca99c8a5256ae126a55c4c725c5,2017-07-15 19:59:29,2021-01-13 00:29:37,Adware
d551fafc4f40f1dec2bb45980bfa9492,2017-07-15 19:59:29,2020-11-16 13:06:20,Adware
83e04bc58d402f9633983cbf22724b02,2017-07-16 01:32:03,2021-03-02 04:07:43,Adware
b8f81673c0e1d29908346f3bab892b9b,2017-07-16 01:32:03,2021-03-02 04:07:36,Adware
70722097d1fe1d78d8c2164640ab6df4,2017-07-16 02:39:08,2021-05-04 09:52:20,Tofsee
9c2589e1c0e9f533a022c6205f9719e1,2017-07-16 08:37:17,2021-07-25 08:33:18,Adware
849b04bdbd1d2b983f6e8a457e0632a8,2017-07-16 08:37:17,2021-07-25 08:33:18,Adware
16efcf0e00504ddfedde13bfea997952,2017-07-16 19:45:45,2020-12-23 15:10:32,Adware
4d7a28d6f2263ed61de88ca66eb011e3,2017-07-16 21:20:29,2020-12-08 18:10:55,Tofsee
550dce18de1bb143e69d6dd9413b8355,2017-07-16 22:17:20,2018-12-21 07:04:50,Adware
c50f6a8b9173676b47ba6085bd0c6cee,2017-07-16 22:38:41,2019-05-21 09:42:17,TrickBot
590a232d04d56409fab72e752a8a2634,2017-07-18 18:53:24,2020-10-11 20:48:33,Tofsee
51a7ad14509fd614c7bb3a50c4982b8c,2017-07-19 07:28:19,2019-07-14 11:58:32,JBifrost
96eba628dcb2b47607192ba74a3b55ba,2017-07-19 18:53:48,2021-07-31 01:48:32,Tofsee
df5c30e670dba99f9270ed36060cf054,2017-07-20 17:44:07,2018-04-11 15:57:59,Tofsee
098f55e27d8c4b0a590102cbdb3a5f3a,2017-07-21 09:52:01,2019-04-08 01:09:54,Adware
29085f03f8e8a03f0b399c5c7cf0b0b8,2017-07-22 14:07:36,2021-04-11 06:42:45,Adware
46efd49abcca8ea9baa932da68fdb529,2017-07-22 14:07:36,2021-04-11 05:54:57,Adware
d7150af4514b868defb854db0f62a441,2017-07-23 09:39:24,2018-07-24 01:04:58,Tofsee
03e186a7f83285e93341de478334006e,2017-07-24 18:17:14,2021-03-20 07:45:40,Tofsee
3cda52da4ade09f1f781ad2e82dcfa20,2017-07-30 18:41:36,2019-05-21 17:34:18,Quakbot
b13d01846ad7a14a70bf030a16775c78,2017-08-08 07:12:49,2021-04-10 00:03:33,Adware
1543a7c46633acf71e8401baccbd0568,2017-08-08 21:32:28,2021-04-08 19:50:46,Tofsee
1d095e68489d3c535297cd8dffb06cb9,2017-08-12 19:56:28,2020-10-28 11:06:23,Tofsee
698e36219f3979420fa2581b21dac7ec,2017-08-28 12:20:47,2020-12-31 02:06:31,Adware
93d056782d649deb51cda44ecb714bb0,2017-08-28 12:20:47,2019-04-15 23:47:27,Adware
1712287800ac91b34cadd5884ce85568,2017-08-28 16:01:59,2021-07-28 14:16:00,TorrentLocker
5e573c9c9f8ba720ef9b18e9fce2e2f7,2017-08-30 13:44:56,2021-03-13 07:33:38,Adware
f6fd83a21f9f3c5f9ff7b5c63bbc179d,2017-10-20 08:03:21,2018-11-06 06:42:12,Adware
92579701f145605e9edc0b01a901c6d5,2017-10-23 00:10:48,2021-07-25 17:07:34,Adware
a61299f9b501adcf680b9275d79d4ac6,2017-11-04 18:03:59,2020-04-21 17:08:24,Tofsee
b2b61db7b9490a60d270ccb20b462826,2017-11-14 20:12:03,2021-06-06 20:27:10,Adware
7dcce5b76c8b17472d024758970a406b,2017-11-22 12:42:46,2021-03-16 12:53:35,Tofsee
534ce2dbc413c68e908363b5df0ae5e0,2017-12-22 09:36:21,2019-07-27 15:22:33,TrickBot
fb00055a1196aeea8d1bc609885ba953,2018-01-01 22:49:25,2019-04-09 06:58:58,TrickBot
a50a861119aceb0ccc74902e8fddb618,2018-01-02 08:16:23,2018-07-05 02:33:08,Tofsee
e7643725fcff971e3051fe0e47fc2c71,2018-01-31 08:06:13,2020-03-25 16:19:48,Tofsee
7c410ce832e848a3321432c9a82e972b,2018-01-31 20:04:25,2021-08-01 06:13:14,Tofsee
da949afd9bd6df820730f8f171584a71,2018-02-03 05:19:37,2021-03-08 22:10:10,Tofsee
906004246f3ba5e755b043c057254a29,2018-03-11 08:25:38,2018-04-14 00:59:16,Tofsee
fd80fa9c6120cdeea8520510f3c644ac,2018-03-11 09:34:30,2021-08-11 12:34:00,Tofsee
b90bdbe961a648f0427db21aaa6ccb59,2018-03-11 10:37:43,2020-05-29 23:39:01,Tofsee
1fe4c7a3544eb27afec2adfb3a3dbf60,2018-03-11 19:23:08,2021-08-09 11:42:58,Tofsee
c201b92f8b483fa388be174d6689f534,2018-03-12 13:43:52,2021-01-28 06:17:06,Gozi
9f62c4f26b90d3d757bea609e82f2eaf,2018-03-13 06:23:41,2021-05-20 23:01:06,Tofsee
1be3ecebe5aa9d3654e6e703d81f6928,2018-03-13 11:50:02,2021-08-11 13:02:35,Ransomware.Troldesh
e3b2ab1f9a56f2fb4c9248f2f41631fa,2018-03-15 01:06:34,2021-07-02 21:51:49,Tofsee
dff8a0aa1c904aaea76c5bf624e88333,2018-03-18 09:41:15,2020-10-27 09:50:24,Tofsee
17fd49722f8d11f3d76dce84f8e099a7,2018-03-19 23:02:27,2021-08-01 00:40:52,Tofsee
911479ac8a0813ed1241b3686ccdade9,2018-03-19 23:24:59,2020-03-30 04:09:18,Tofsee
c5deb9465d47232dd48772f9c4d14679,2018-03-22 15:42:48,2021-03-23 00:34:25,Tofsee
f22bdd57e3a52de86cda40da2d84e83b,2018-03-27 13:40:19,2019-01-20 14:31:39,Tofsee
d18a4da84af59e1108862a39bae7c9d4,2018-04-03 00:40:51,2021-02-06 01:53:12,Tofsee
2d8794cb7b52b777bee2695e79c15760,2018-04-04 06:56:37,2021-07-26 08:07:00,Ransomware
40adfd923eb82b89d8836ba37a19bca1,2018-04-15 15:49:08,2021-04-11 04:42:47,CoinMiner
1aee0238942d453d679fc1e37a303387,2018-05-13 01:59:49,2021-07-30 12:27:07,Tofsee
2092e1fffb45d7e4a19a57f9bc5e203a,2018-05-16 21:59:36,2018-09-05 01:58:33,Adware
bffa4501966196d3d6e90cee1f88fc89,2018-06-07 15:08:04,2020-03-16 00:03:44,Tofsee
807fca46d9d0cf63adf4e5e80e414bbe,2018-06-07 16:51:03,2021-08-07 03:15:42,Tofsee
fb58831f892190644fe44e25bc830b45,2018-06-08 12:07:59,2021-07-20 01:39:05,Adware
0cc1e84568e471aa1d62ad4158ade6b5,2018-06-24 10:50:47,2021-06-21 02:35:57,Tofsee
d2935c58fe676744fecc8614ee5356c7,2018-08-14 21:48:41,2021-08-11 11:54:42,Adwind
8916410db85077a5460817142dcbc8de,2018-08-21 12:32:28,2021-08-11 15:00:50,TrickBot
c5235d3a8b9934b7fbbd204d50bc058d,2018-08-23 17:36:08,2019-10-13 05:11:09,Gootkit
57f3642b4e37e28f5cbe3020c9331b4c,2018-08-28 15:54:53,2021-08-11 13:05:18,Gozi
e62a5f4d538cbf169c2af71bec2399b4,2018-08-30 15:45:40,2021-08-11 09:48:52,TrickBot
51c64c77e60f3980eea90869b68c58a8,2018-08-30 21:04:57,2021-08-11 08:13:08,Dridex
7691297bcb20a41233fd0a0baa0a3628,2018-09-17 02:50:05,2021-08-11 12:20:33,Adware
7dd50e112cd23734a310b90f6f44a7cd,2018-09-17 17:54:58,2021-08-01 11:28:46,Quakbot
52c7396a501e4fecbdfa99c5408334ac,2018-09-18 00:29:04,2019-12-03 17:24:02,Tofsee
fc54e0d16d9764783542f0146a98b300,2018-09-24 12:33:44,2021-08-11 12:51:10,AsyncRAT
f735bbc6b69723b9df7b0e7ef27872af,2018-10-02 18:04:16,2021-08-11 07:25:14,TrickBot
49ed2ef3f1321e5f044f1e71b0e6fdd5,2018-10-02 18:04:17,2021-08-08 22:08:01,TrickBot
d76ee64fb7273733cbe455ac81c292e6,2018-11-16 13:26:39,2018-11-18 19:19:36,Tofsee
8f6c918dcb585ebbea05e2cc94530e3d,2018-11-16 13:26:41,2020-05-06 15:45:21,Tofsee
34f14a69ad7009ca5863379218af17f3,2018-11-17 05:17:22,2021-01-28 08:19:18,Tofsee
c2b4710c6888a5d47befe865c8e6fb19,2018-11-29 20:46:04,2021-08-03 23:37:22,Tofsee
decfb48a53789ebe081b88aabb58ee34,2018-12-21 09:06:16,2021-06-14 05:27:16,Adwind
08a8a4e85b25ac42e1490bc85cfdb5ce,2019-01-30 02:48:34,2020-10-27 09:50:19,Tofsee
c0220cd64849a629397a9cb68f78a0ea,2019-03-24 00:12:32,2021-07-31 00:26:06,Tofsee
7a29c223fb122ec64d10f0a159e07996,2019-06-09 22:55:29,2020-10-27 09:50:26,None
70a04365be5bbd4653698bebeb43ce68,2019-07-02 06:26:56,2020-05-30 04:19:00,Tofsee
d81d654effb94714a4086734fa0adad9,2019-07-16 23:29:02,2020-10-27 09:50:21,Tofsee
25d74b7b4b779eb1efd4b31d26d651c6,2019-08-03 20:15:33,2020-07-14 21:43:25,Tofsee
fc2299d5b2964cd242c5a2c8c531a5f0,2019-08-09 23:56:32,2021-08-11 02:20:22,Tofsee
32926ca3e59f0413d0b98725454594f5,2019-09-12 06:56:10,2021-04-08 19:50:43,Tofsee
ffefafdb86336d057eda5fdf02b3d5ce,2019-10-26 07:31:49,2020-07-25 00:14:09,Tofsee
8515076cbbca9dce33151b798f782456,2020-12-27 16:53:04,2021-08-11 15:06:36,BitRAT
# END (97) entries
1 ################################################################
2 # abuse.ch Suricata JA3 Fingerprint Blacklist (CSV) #
3 # For Suricata 4.1.0 or newer #
4 # Last updated: 2021-08-03 14:33:44 UTC #
5 # #
6 # Terms Of Use: https://sslbl.abuse.ch/blacklist/ #
7 # For questions please contact sslbl [at] abuse.ch #
8 ################################################################
9 #
10 # ja3_md5,Firstseen,Lastseen,Listingreason
11 b386946a5a44d1ddcc843bc75336dfce,2017-07-14 18:08:15,2019-07-27 20:42:54,Dridex
12 8991a387e4cc841740f25d6f5139f92d,2017-07-14 19:02:03,2019-07-28 00:34:38,Adware
13 cb98a24ee4b9134448ffb5714fd870ac,2017-07-14 19:48:28,2019-05-22 03:22:38,Dridex
14 1aa7bf8b97e540ca5edd75f7b8384bfa,2017-07-14 20:23:38,2019-07-28 01:38:22,TrickBot
15 3d89c0dfb1fa44911b8fa7523ef8dedb,2017-07-15 04:23:45,2021-02-01 18:23:25,Adware
16 bc6c386f480ee97b9d9e52d472b772d8,2017-07-15 10:57:38,2021-03-13 07:33:39,Adware
17 8f52d1ce303fb4a6515836aec3cc16b1,2017-07-15 19:05:11,2019-07-27 20:00:57,TrickBot
18 d6f04b5a910115f4b50ecec09d40a1df,2017-07-15 19:42:24,2018-10-14 08:12:51,Dridex
19 35c0a31c481927f022a3b530255ac080,2017-07-15 19:43:19,2021-04-10 12:54:04,Tofsee
20 e330bca99c8a5256ae126a55c4c725c5,2017-07-15 19:59:29,2021-01-13 00:29:37,Adware
21 d551fafc4f40f1dec2bb45980bfa9492,2017-07-15 19:59:29,2020-11-16 13:06:20,Adware
22 83e04bc58d402f9633983cbf22724b02,2017-07-16 01:32:03,2021-03-02 04:07:43,Adware
23 b8f81673c0e1d29908346f3bab892b9b,2017-07-16 01:32:03,2021-03-02 04:07:36,Adware
24 70722097d1fe1d78d8c2164640ab6df4,2017-07-16 02:39:08,2021-05-04 09:52:20,Tofsee
25 9c2589e1c0e9f533a022c6205f9719e1,2017-07-16 08:37:17,2021-07-25 08:33:18,Adware
26 849b04bdbd1d2b983f6e8a457e0632a8,2017-07-16 08:37:17,2021-07-25 08:33:18,Adware
27 16efcf0e00504ddfedde13bfea997952,2017-07-16 19:45:45,2020-12-23 15:10:32,Adware
28 4d7a28d6f2263ed61de88ca66eb011e3,2017-07-16 21:20:29,2020-12-08 18:10:55,Tofsee
29 550dce18de1bb143e69d6dd9413b8355,2017-07-16 22:17:20,2018-12-21 07:04:50,Adware
30 c50f6a8b9173676b47ba6085bd0c6cee,2017-07-16 22:38:41,2019-05-21 09:42:17,TrickBot
31 590a232d04d56409fab72e752a8a2634,2017-07-18 18:53:24,2020-10-11 20:48:33,Tofsee
32 51a7ad14509fd614c7bb3a50c4982b8c,2017-07-19 07:28:19,2019-07-14 11:58:32,JBifrost
33 96eba628dcb2b47607192ba74a3b55ba,2017-07-19 18:53:48,2021-07-31 01:48:32,Tofsee
34 df5c30e670dba99f9270ed36060cf054,2017-07-20 17:44:07,2018-04-11 15:57:59,Tofsee
35 098f55e27d8c4b0a590102cbdb3a5f3a,2017-07-21 09:52:01,2019-04-08 01:09:54,Adware
36 29085f03f8e8a03f0b399c5c7cf0b0b8,2017-07-22 14:07:36,2021-04-11 06:42:45,Adware
37 46efd49abcca8ea9baa932da68fdb529,2017-07-22 14:07:36,2021-04-11 05:54:57,Adware
38 d7150af4514b868defb854db0f62a441,2017-07-23 09:39:24,2018-07-24 01:04:58,Tofsee
39 03e186a7f83285e93341de478334006e,2017-07-24 18:17:14,2021-03-20 07:45:40,Tofsee
40 3cda52da4ade09f1f781ad2e82dcfa20,2017-07-30 18:41:36,2019-05-21 17:34:18,Quakbot
41 b13d01846ad7a14a70bf030a16775c78,2017-08-08 07:12:49,2021-04-10 00:03:33,Adware
42 1543a7c46633acf71e8401baccbd0568,2017-08-08 21:32:28,2021-04-08 19:50:46,Tofsee
43 1d095e68489d3c535297cd8dffb06cb9,2017-08-12 19:56:28,2020-10-28 11:06:23,Tofsee
44 698e36219f3979420fa2581b21dac7ec,2017-08-28 12:20:47,2020-12-31 02:06:31,Adware
45 93d056782d649deb51cda44ecb714bb0,2017-08-28 12:20:47,2019-04-15 23:47:27,Adware
46 1712287800ac91b34cadd5884ce85568,2017-08-28 16:01:59,2021-07-28 14:16:00,TorrentLocker
47 5e573c9c9f8ba720ef9b18e9fce2e2f7,2017-08-30 13:44:56,2021-03-13 07:33:38,Adware
48 f6fd83a21f9f3c5f9ff7b5c63bbc179d,2017-10-20 08:03:21,2018-11-06 06:42:12,Adware
49 92579701f145605e9edc0b01a901c6d5,2017-10-23 00:10:48,2021-07-25 17:07:34,Adware
50 a61299f9b501adcf680b9275d79d4ac6,2017-11-04 18:03:59,2020-04-21 17:08:24,Tofsee
51 b2b61db7b9490a60d270ccb20b462826,2017-11-14 20:12:03,2021-06-06 20:27:10,Adware
52 7dcce5b76c8b17472d024758970a406b,2017-11-22 12:42:46,2021-03-16 12:53:35,Tofsee
53 534ce2dbc413c68e908363b5df0ae5e0,2017-12-22 09:36:21,2019-07-27 15:22:33,TrickBot
54 fb00055a1196aeea8d1bc609885ba953,2018-01-01 22:49:25,2019-04-09 06:58:58,TrickBot
55 a50a861119aceb0ccc74902e8fddb618,2018-01-02 08:16:23,2018-07-05 02:33:08,Tofsee
56 e7643725fcff971e3051fe0e47fc2c71,2018-01-31 08:06:13,2020-03-25 16:19:48,Tofsee
57 7c410ce832e848a3321432c9a82e972b,2018-01-31 20:04:25,2021-08-01 06:13:14,Tofsee
58 da949afd9bd6df820730f8f171584a71,2018-02-03 05:19:37,2021-03-08 22:10:10,Tofsee
59 906004246f3ba5e755b043c057254a29,2018-03-11 08:25:38,2018-04-14 00:59:16,Tofsee
60 fd80fa9c6120cdeea8520510f3c644ac,2018-03-11 09:34:30,2021-08-11 12:34:00,Tofsee
61 b90bdbe961a648f0427db21aaa6ccb59,2018-03-11 10:37:43,2020-05-29 23:39:01,Tofsee
62 1fe4c7a3544eb27afec2adfb3a3dbf60,2018-03-11 19:23:08,2021-08-09 11:42:58,Tofsee
63 c201b92f8b483fa388be174d6689f534,2018-03-12 13:43:52,2021-01-28 06:17:06,Gozi
64 9f62c4f26b90d3d757bea609e82f2eaf,2018-03-13 06:23:41,2021-05-20 23:01:06,Tofsee
65 1be3ecebe5aa9d3654e6e703d81f6928,2018-03-13 11:50:02,2021-08-11 13:02:35,Ransomware.Troldesh
66 e3b2ab1f9a56f2fb4c9248f2f41631fa,2018-03-15 01:06:34,2021-07-02 21:51:49,Tofsee
67 dff8a0aa1c904aaea76c5bf624e88333,2018-03-18 09:41:15,2020-10-27 09:50:24,Tofsee
68 17fd49722f8d11f3d76dce84f8e099a7,2018-03-19 23:02:27,2021-08-01 00:40:52,Tofsee
69 911479ac8a0813ed1241b3686ccdade9,2018-03-19 23:24:59,2020-03-30 04:09:18,Tofsee
70 c5deb9465d47232dd48772f9c4d14679,2018-03-22 15:42:48,2021-03-23 00:34:25,Tofsee
71 f22bdd57e3a52de86cda40da2d84e83b,2018-03-27 13:40:19,2019-01-20 14:31:39,Tofsee
72 d18a4da84af59e1108862a39bae7c9d4,2018-04-03 00:40:51,2021-02-06 01:53:12,Tofsee
73 2d8794cb7b52b777bee2695e79c15760,2018-04-04 06:56:37,2021-07-26 08:07:00,Ransomware
74 40adfd923eb82b89d8836ba37a19bca1,2018-04-15 15:49:08,2021-04-11 04:42:47,CoinMiner
75 1aee0238942d453d679fc1e37a303387,2018-05-13 01:59:49,2021-07-30 12:27:07,Tofsee
76 2092e1fffb45d7e4a19a57f9bc5e203a,2018-05-16 21:59:36,2018-09-05 01:58:33,Adware
77 bffa4501966196d3d6e90cee1f88fc89,2018-06-07 15:08:04,2020-03-16 00:03:44,Tofsee
78 807fca46d9d0cf63adf4e5e80e414bbe,2018-06-07 16:51:03,2021-08-07 03:15:42,Tofsee
79 fb58831f892190644fe44e25bc830b45,2018-06-08 12:07:59,2021-07-20 01:39:05,Adware
80 0cc1e84568e471aa1d62ad4158ade6b5,2018-06-24 10:50:47,2021-06-21 02:35:57,Tofsee
81 d2935c58fe676744fecc8614ee5356c7,2018-08-14 21:48:41,2021-08-11 11:54:42,Adwind
82 8916410db85077a5460817142dcbc8de,2018-08-21 12:32:28,2021-08-11 15:00:50,TrickBot
83 c5235d3a8b9934b7fbbd204d50bc058d,2018-08-23 17:36:08,2019-10-13 05:11:09,Gootkit
84 57f3642b4e37e28f5cbe3020c9331b4c,2018-08-28 15:54:53,2021-08-11 13:05:18,Gozi
85 e62a5f4d538cbf169c2af71bec2399b4,2018-08-30 15:45:40,2021-08-11 09:48:52,TrickBot
86 51c64c77e60f3980eea90869b68c58a8,2018-08-30 21:04:57,2021-08-11 08:13:08,Dridex
87 7691297bcb20a41233fd0a0baa0a3628,2018-09-17 02:50:05,2021-08-11 12:20:33,Adware
88 7dd50e112cd23734a310b90f6f44a7cd,2018-09-17 17:54:58,2021-08-01 11:28:46,Quakbot
89 52c7396a501e4fecbdfa99c5408334ac,2018-09-18 00:29:04,2019-12-03 17:24:02,Tofsee
90 fc54e0d16d9764783542f0146a98b300,2018-09-24 12:33:44,2021-08-11 12:51:10,AsyncRAT
91 f735bbc6b69723b9df7b0e7ef27872af,2018-10-02 18:04:16,2021-08-11 07:25:14,TrickBot
92 49ed2ef3f1321e5f044f1e71b0e6fdd5,2018-10-02 18:04:17,2021-08-08 22:08:01,TrickBot
93 d76ee64fb7273733cbe455ac81c292e6,2018-11-16 13:26:39,2018-11-18 19:19:36,Tofsee
94 8f6c918dcb585ebbea05e2cc94530e3d,2018-11-16 13:26:41,2020-05-06 15:45:21,Tofsee
95 34f14a69ad7009ca5863379218af17f3,2018-11-17 05:17:22,2021-01-28 08:19:18,Tofsee
96 c2b4710c6888a5d47befe865c8e6fb19,2018-11-29 20:46:04,2021-08-03 23:37:22,Tofsee
97 decfb48a53789ebe081b88aabb58ee34,2018-12-21 09:06:16,2021-06-14 05:27:16,Adwind
98 08a8a4e85b25ac42e1490bc85cfdb5ce,2019-01-30 02:48:34,2020-10-27 09:50:19,Tofsee
99 c0220cd64849a629397a9cb68f78a0ea,2019-03-24 00:12:32,2021-07-31 00:26:06,Tofsee
100 7a29c223fb122ec64d10f0a159e07996,2019-06-09 22:55:29,2020-10-27 09:50:26,None
101 70a04365be5bbd4653698bebeb43ce68,2019-07-02 06:26:56,2020-05-30 04:19:00,Tofsee
102 d81d654effb94714a4086734fa0adad9,2019-07-16 23:29:02,2020-10-27 09:50:21,Tofsee
103 25d74b7b4b779eb1efd4b31d26d651c6,2019-08-03 20:15:33,2020-07-14 21:43:25,Tofsee
104 fc2299d5b2964cd242c5a2c8c531a5f0,2019-08-09 23:56:32,2021-08-11 02:20:22,Tofsee
105 32926ca3e59f0413d0b98725454594f5,2019-09-12 06:56:10,2021-04-08 19:50:43,Tofsee
106 ffefafdb86336d057eda5fdf02b3d5ce,2019-10-26 07:31:49,2020-07-25 00:14:09,Tofsee
107 8515076cbbca9dce33151b798f782456,2020-12-27 16:53:04,2021-08-11 15:06:36,BitRAT
108 # END (97) entries

1059
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-intel/src/detect.rs Normal file
View File

File diff suppressed because it is too large Load Diff

321
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-intel/src/import.rs Normal file
View File

@ -0,0 +1,321 @@
// ©AngelaMos | 2026
// import.rs
//! Importing the ja4db.com enrichment feed.
//!
//! Unlike the three bundled feeds, ja4db is fetched at install time rather than
//! committed, because its licence is unspecified and the install script is the
//! honest place to pull it. Its records are also known to be dirty: the upstream
//! project tracks an issue where some fingerprint fields hold placeholders or
//! truncated values. So every fingerprint is validated for its kind before it
//! goes in, and the count of rejected rows is reported rather than hidden.
//!
//! The payload is read as untyped JSON rather than a fixed struct. ja4db is a
//! community database whose shape drifts, and a parser that demands an exact
//! schema would reject the whole file the day a field is renamed. Reading each
//! record as a map of optional fields keeps the importer working across those
//! changes and is itself the tolerance the dirty data calls for.
//!
//! ja4db identifies applications, so a record is treated as benign unless its
//! own classification fields name it as malicious. That makes ja4db the benign
//! baseline that the malicious feeds are weighed against, without silently
//! relabelling the handful of malware entries it does carry.
use anyhow::{Context, Result};
use rusqlite::Connection;
use serde_json::Value;
use super::model::{Category, FpKind};
use super::{NewFingerprint, get_or_create_source, insert_fingerprint, refresh_source_count};
const SOURCE_NAME: &str = "ja4db.com";
const SOURCE_URL: &str = "https://ja4db.com/api/read/";
/// The JSON field that carries each fingerprint kind in a ja4db record.
const FIELDS: &[(&str, FpKind)] = &[
("ja4_fingerprint", FpKind::Ja4),
("ja4s_fingerprint", FpKind::Ja4s),
("ja4h_fingerprint", FpKind::Ja4h),
("ja4x_fingerprint", FpKind::Ja4x),
("ja4t_fingerprint", FpKind::Ja4t),
("ja4ts_fingerprint", FpKind::Ja4ts),
];
/// What one import run did.
#[derive(Debug, Clone)]
pub struct ImportSummary {
pub records: usize,
pub imported: usize,
pub skipped: usize,
}
/// Imports a ja4db `/api/read/` JSON array, validating every fingerprint.
pub fn import_ja4db(conn: &mut Connection, json: &str) -> Result<ImportSummary> {
let payload: Value = serde_json::from_str(json).context("parsing ja4db JSON")?;
let records = payload
.as_array()
.context("ja4db payload is not a JSON array")?;
let tx = conn.transaction()?;
let source_id = get_or_create_source(
&tx,
SOURCE_NAME,
Some(SOURCE_URL),
Some("unspecified"),
"fetched",
)?;
let mut imported = 0;
let mut skipped = 0;
for record in records {
let category = classify(record);
let label = build_label(record);
let reference = build_reference(record);
for (field, kind) in FIELDS {
let Some(raw) = record.get(field).and_then(Value::as_str) else {
continue;
};
let value = raw.trim();
if value.is_empty() {
continue;
}
if !valid_fingerprint(*kind, value) {
skipped += 1;
continue;
}
if insert_fingerprint(
&tx,
source_id,
&NewFingerprint {
kind: *kind,
value,
label: &label,
category,
reference: reference.as_deref(),
first_seen: None,
},
)? {
imported += 1;
}
}
}
refresh_source_count(&tx, source_id)?;
tx.commit()?;
Ok(ImportSummary {
records: records.len(),
imported,
skipped,
})
}
/// Builds a display label from the application, library, or device, with the
/// operating system in parentheses when ja4db knows it.
fn build_label(record: &Value) -> String {
let primary = ["application", "library", "device"]
.into_iter()
.filter_map(|key| record.get(key).and_then(Value::as_str))
.map(str::trim)
.find(|name| !name.is_empty());
let os = record
.get("os")
.and_then(Value::as_str)
.map(str::trim)
.filter(|name| !name.is_empty());
match (primary, os) {
(Some(name), Some(os)) => format!("{name} ({os})"),
(Some(name), None) => name.to_string(),
(None, Some(os)) => os.to_string(),
(None, None) => "unknown".to_string(),
}
}
/// Picks a human readable reference for a record, preferring the user agent
/// string and falling back to free form notes.
fn build_reference(record: &Value) -> Option<String> {
for key in ["user_agent_string", "notes"] {
if let Some(text) = record.get(key).and_then(Value::as_str) {
let text = text.trim();
if !text.is_empty() {
return Some(text.to_string());
}
}
}
None
}
/// Classifies a record as benign unless one of its classification fields names
/// it as malicious. Identity fields like the application name are deliberately
/// not scanned, so an app that merely has an alarming name is not relabelled.
fn classify(record: &Value) -> Category {
const MALICIOUS: &[&str] = &[
"malware",
"malicious",
"trojan",
"botnet",
"ransomware",
"stealer",
"cobalt strike",
"backdoor",
"command and control",
];
let mut text = String::new();
for key in ["notes", "classification", "comment", "tags", "category"] {
if let Some(value) = record.get(key).and_then(Value::as_str) {
text.push_str(&value.to_ascii_lowercase());
text.push(' ');
}
}
if MALICIOUS.iter().any(|needle| text.contains(needle)) {
Category::Malware
} else {
Category::Benign
}
}
/// Whether a value has the right shape for its fingerprint kind. The checks are
/// loose enough to admit real data and strict enough to reject the placeholders
/// and truncations that the dirty ja4db rows are made of.
fn valid_fingerprint(kind: FpKind, value: &str) -> bool {
match kind {
FpKind::Ja3 | FpKind::Ja3s => is_hex(value, 32),
FpKind::Ja4 => {
let parts = value.split('_').collect::<Vec<_>>();
parts.len() == 3
&& is_prefix(parts[0], 8, 12)
&& is_hex(parts[1], 12)
&& is_hex(parts[2], 12)
}
FpKind::Ja4s => {
let parts = value.split('_').collect::<Vec<_>>();
parts.len() == 3
&& is_prefix(parts[0], 5, 9)
&& is_hex_between(parts[1], 2, 8)
&& is_hex(parts[2], 12)
}
FpKind::Ja4x => {
let parts = value.split('_').collect::<Vec<_>>();
parts.len() == 3 && parts.iter().all(|part| is_hex(part, 12))
}
FpKind::Ja4t | FpKind::Ja4ts => {
let parts = value.split('_').collect::<Vec<_>>();
(2..=4).contains(&parts.len())
&& !parts[0].is_empty()
&& parts[0].bytes().all(|byte| byte.is_ascii_digit())
}
FpKind::Ja4h => {
let parts = value.split('_').collect::<Vec<_>>();
parts.len() >= 2
&& parts[0].len() >= 8
&& parts[0].bytes().all(|byte| byte.is_ascii_alphanumeric())
}
}
}
fn is_hex(value: &str, len: usize) -> bool {
value.len() == len && value.bytes().all(|byte| byte.is_ascii_hexdigit())
}
fn is_hex_between(value: &str, min: usize, max: usize) -> bool {
(min..=max).contains(&value.len()) && value.bytes().all(|byte| byte.is_ascii_hexdigit())
}
fn is_prefix(value: &str, min: usize, max: usize) -> bool {
(min..=max).contains(&value.len())
&& matches!(value.as_bytes().first(), Some(b't' | b'q' | b'd'))
&& value.bytes().all(|byte| byte.is_ascii_alphanumeric())
}
#[cfg(test)]
mod tests {
use crate::IntelStore;
use crate::model::{FpKind, Verdict};
const SAMPLE: &str = r#"[
{"application":"Chrome","os":"Windows","ja4_fingerprint":"t13d1516h2_8daaf6152771_e5627efa2ab1","user_agent_string":"Mozilla/5.0"},
{"application":"curl","library":"OpenSSL","ja4_fingerprint":"t13d1234h1_aaaaaaaaaaaa_bbbbbbbbbbbb"},
{"application":"Loader","notes":"known malware stealer","ja4_fingerprint":"t10d070600_c50f5591e341_1a3805c3aa63"},
{"application":"Dirty","ja4_fingerprint":"GREASE"},
{"application":"NoFp","ja4_fingerprint":null},
{"application":"Server","ja4s_fingerprint":"t130200_1301_234ea6891581"}
]"#;
#[test]
fn imports_valid_rows_and_counts_dirty_ones() {
let mut store = IntelStore::open_in_memory().unwrap();
let summary = store.import_ja4db(SAMPLE).unwrap();
assert_eq!(summary.records, 6);
assert_eq!(summary.imported, 4);
assert_eq!(summary.skipped, 1);
}
#[test]
fn re_importing_adds_nothing_new() {
let mut store = IntelStore::open_in_memory().unwrap();
store.import_ja4db(SAMPLE).unwrap();
let second = store.import_ja4db(SAMPLE).unwrap();
assert_eq!(second.imported, 0);
assert_eq!(second.skipped, 1);
}
#[test]
fn classification_field_marks_malware() {
let mut store = IntelStore::open_in_memory().unwrap();
store.import_ja4db(SAMPLE).unwrap();
let report = store
.match_fingerprint(FpKind::Ja4, "t10d070600_c50f5591e341_1a3805c3aa63")
.unwrap();
assert_eq!(report.verdict, Verdict::Malicious);
}
#[test]
fn identified_application_is_benign() {
let mut store = IntelStore::open_in_memory().unwrap();
store.import_ja4db(SAMPLE).unwrap();
let report = store
.match_fingerprint(FpKind::Ja4, "t13d1516h2_8daaf6152771_e5627efa2ab1")
.unwrap();
assert_eq!(report.verdict, Verdict::Benign);
assert!(
report
.hits
.iter()
.any(|hit| hit.label == "Chrome (Windows)")
);
}
#[test]
fn a_non_array_payload_is_an_error() {
let mut store = IntelStore::open_in_memory().unwrap();
assert!(store.import_ja4db("{}").is_err());
assert!(store.import_ja4db("not json").is_err());
}
#[test]
fn an_empty_array_imports_nothing() {
let mut store = IntelStore::open_in_memory().unwrap();
let summary = store.import_ja4db("[]").unwrap();
assert_eq!(summary.records, 0);
assert_eq!(summary.imported, 0);
}
#[test]
fn validators_reject_placeholders() {
use super::valid_fingerprint;
assert!(valid_fingerprint(
FpKind::Ja4,
"t13d1516h2_8daaf6152771_e5627efa2ab1"
));
assert!(!valid_fingerprint(FpKind::Ja4, "GREASE"));
assert!(!valid_fingerprint(
FpKind::Ja4,
"t13d1516h2_short_e5627efa2ab1"
));
assert!(!valid_fingerprint(FpKind::Ja4, ""));
assert!(valid_fingerprint(
FpKind::Ja3,
"1aa7bf8b97e540ca5edd75f7b8384bfa"
));
assert!(!valid_fingerprint(FpKind::Ja3, "N/A"));
}
}

365
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-intel/src/lib.rs Normal file
View File

@ -0,0 +1,365 @@
// ©AngelaMos | 2026
// lib.rs
//! The local threat intelligence store.
//!
//! This is the half of the tool that turns a fingerprint into a judgement. It
//! owns a bundled SQLite database, seeds it from three vendored feeds, can pull
//! a fourth feed at install time, and answers the one question the rest of the
//! tool cares about: is this fingerprint known, and is it known to be bad.
//!
//! The store is deliberately synchronous. The capture pipeline that feeds it is
//! a plain loop, and a lookup is a single indexed query, so wrapping it in an
//! async runtime would buy nothing here. The web server, when it arrives, is
//! the place that needs concurrent access, and that is where an async wrapper
//! belongs.
mod detect;
mod import;
mod matcher;
mod model;
mod schema;
mod seed;
mod signal;
use std::path::{Path, PathBuf};
use std::time::{SystemTime, UNIX_EPOCH};
use anyhow::{Context, Result};
use rusqlite::{Connection, params};
use serde::Serialize;
use tlsfp_core::FingerprintEvent;
pub use detect::{Alert, AlertSeverity, DetectConfig, Rule};
pub use import::ImportSummary;
pub use model::{CatalogEntry, Category, FpKind, IntelHit, MatchReport, MatchStrength, Verdict};
pub use seed::{FeedLoad, SeedSummary};
/// A handle to the open intelligence database.
pub struct IntelStore {
conn: Connection,
}
impl IntelStore {
/// Opens or creates the database at `path`, creating any missing parent
/// directories and bringing the schema up to date.
pub fn open(path: &Path) -> Result<Self> {
if let Some(parent) = path.parent() {
if !parent.as_os_str().is_empty() {
std::fs::create_dir_all(parent)
.with_context(|| format!("creating database directory {}", parent.display()))?;
}
}
let conn = Connection::open(path)
.with_context(|| format!("opening database {}", path.display()))?;
conn.execute_batch("PRAGMA foreign_keys = ON; PRAGMA journal_mode = WAL;")?;
// The dashboard runs a read connection alongside a capture writer on this
// same file, so wait on a contended lock rather than failing the query.
conn.busy_timeout(std::time::Duration::from_secs(5))?;
Self::migrate(conn)
}
/// Opens a private in memory database, used by tests and by commands that
/// only need a scratch store.
pub fn open_in_memory() -> Result<Self> {
let conn = Connection::open_in_memory()?;
conn.execute_batch("PRAGMA foreign_keys = ON;")?;
Self::migrate(conn)
}
fn migrate(mut conn: Connection) -> Result<Self> {
schema::apply_migrations(&mut conn).context("applying schema migrations")?;
Ok(Self { conn })
}
/// Loads the three vendored feeds, skipping rows already present so a second
/// call changes nothing.
pub fn seed_bundled(&mut self) -> Result<SeedSummary> {
seed::load_bundled(&mut self.conn)
}
/// Imports a ja4db.com `/api/read/` JSON payload, validating each
/// fingerprint and reporting how many rows were kept and skipped.
pub fn import_ja4db(&mut self, json: &str) -> Result<ImportSummary> {
import::import_ja4db(&mut self.conn, json)
}
/// Looks up one fingerprint and scores the hits into a verdict.
pub fn match_fingerprint(&self, kind: FpKind, value: &str) -> Result<MatchReport> {
matcher::match_one(&self.conn, kind, value)
}
/// Looks up every fingerprint carried by one capture event, returning a
/// report for each kind that found intelligence.
pub fn match_event(&self, event: &FingerprintEvent) -> Result<Vec<MatchReport>> {
let mut reports = Vec::new();
for (kind, value) in matcher::event_fingerprints(event) {
let report = self.match_fingerprint(kind, &value)?;
if report.has_hits() {
reports.push(report);
}
}
Ok(reports)
}
/// Records one capture event and returns every alert its detection rules
/// raised, using the default thresholds.
pub fn detect(&mut self, event: &FingerprintEvent) -> Result<Vec<Alert>> {
self.detect_with(event, &DetectConfig::default())
}
/// Records one capture event under explicit thresholds. The observation and
/// any alerts it raises commit together inside one transaction.
pub fn detect_with(
&mut self,
event: &FingerprintEvent,
config: &DetectConfig,
) -> Result<Vec<Alert>> {
let tx = self.conn.transaction()?;
let alerts = detect::run(&tx, event, config)?;
tx.commit()?;
Ok(alerts)
}
/// The most recent alerts, newest first, for the CLI feed and the dashboard.
pub fn recent_alerts(&self, limit: i64) -> Result<Vec<Alert>> {
detect::recent(&self.conn, limit)
}
/// A count of recorded alerts per rule, for the stats summary.
pub fn alert_counts(&self) -> Result<Vec<(Rule, i64)>> {
detect::counts_by_rule(&self.conn)
}
/// Searches the stored catalogue by a free text substring, optionally
/// narrowed to one fingerprint kind, for the dashboard search box and CLI.
pub fn search(
&self,
query: &str,
kind: Option<FpKind>,
limit: i64,
) -> Result<Vec<CatalogEntry>> {
matcher::search_catalog(&self.conn, query, kind, limit)
}
/// The alerts recorded after `after_id`, oldest first, for a reader tailing
/// the store as new detections land.
pub fn alerts_since(&self, after_id: i64, limit: i64) -> Result<Vec<(i64, Alert)>> {
detect::since(&self.conn, after_id, limit)
}
/// The highest alert id currently stored, the starting point for a tailing
/// reader that wants only alerts raised after it connected.
pub fn latest_alert_id(&self) -> Result<i64> {
detect::max_id(&self.conn)
}
/// Summarises what the store holds, by feed and by category.
pub fn stats(&self) -> Result<Stats> {
let sources = self
.conn
.prepare("SELECT name, kind, license, record_count FROM intel_source ORDER BY name")?
.query_map([], |row| {
Ok(SourceStat {
name: row.get(0)?,
kind: row.get(1)?,
license: row.get(2)?,
records: row.get(3)?,
})
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
let by_category = self
.conn
.prepare(
"SELECT category, count(*) FROM intel_fingerprint GROUP BY category ORDER BY category",
)?
.query_map([], |row| {
Ok(CategoryStat {
category: row.get(0)?,
records: row.get(1)?,
})
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
let total: i64 =
self.conn
.query_row("SELECT count(*) FROM intel_fingerprint", [], |row| {
row.get(0)
})?;
Ok(Stats {
sources,
by_category,
total,
})
}
}
/// A per feed row of the stats summary.
#[derive(Debug, Clone, Serialize)]
pub struct SourceStat {
pub name: String,
pub kind: String,
pub license: Option<String>,
pub records: i64,
}
/// A per category row of the stats summary.
#[derive(Debug, Clone, Serialize)]
pub struct CategoryStat {
pub category: String,
pub records: i64,
}
/// What the store currently holds.
#[derive(Debug, Clone, Serialize)]
pub struct Stats {
pub sources: Vec<SourceStat>,
pub by_category: Vec<CategoryStat>,
pub total: i64,
}
/// The default on disk location of the database, under the XDG data directory
/// when one is set and the home directory otherwise.
pub fn default_db_path() -> PathBuf {
let base = std::env::var_os("XDG_DATA_HOME")
.map(PathBuf::from)
.filter(|path| path.is_absolute())
.or_else(|| {
std::env::var_os("HOME").map(|home| PathBuf::from(home).join(".local").join("share"))
})
.unwrap_or_else(|| PathBuf::from("."));
base.join("tlsfp").join("intel.db")
}
/// Splits a JA4 client fingerprint into its capability prefix and cipher hash,
/// the two parts the partial matcher indexes on. Returns `None` for a value
/// that is not the expected three underscore separated fields.
pub(crate) fn ja4_parts(value: &str) -> Option<(String, String)> {
let mut fields = value.split('_');
let prefix = fields.next()?;
let cipher = fields.next()?;
let extensions = fields.next()?;
if fields.next().is_some() || prefix.is_empty() || cipher.is_empty() || extensions.is_empty() {
return None;
}
Some((prefix.to_string(), cipher.to_string()))
}
/// Inserts a feed by name or updates it if already present, returning its row
/// id. The id is kept stable across re imports so the fingerprints that point
/// at it are never orphaned.
pub(crate) fn get_or_create_source(
conn: &Connection,
name: &str,
url: Option<&str>,
license: Option<&str>,
kind: &str,
) -> rusqlite::Result<i64> {
conn.execute(
"INSERT INTO intel_source (name, url, license, kind, imported_at)
VALUES (?1, ?2, ?3, ?4, ?5)
ON CONFLICT(name) DO UPDATE SET
url = excluded.url,
license = excluded.license,
kind = excluded.kind,
imported_at = excluded.imported_at",
params![name, url, license, kind, unix_now()],
)?;
conn.query_row(
"SELECT id FROM intel_source WHERE name = ?1",
params![name],
|row| row.get(0),
)
}
/// One fingerprint about to be written, shared by the seed loader and the
/// ja4db importer so both compute the JA4 partial match columns identically.
pub(crate) struct NewFingerprint<'a> {
pub kind: FpKind,
pub value: &'a str,
pub label: &'a str,
pub category: Category,
pub reference: Option<&'a str>,
pub first_seen: Option<&'a str>,
}
/// Inserts one fingerprint, lowercasing the value, filling the JA4 partial
/// match columns where they apply, and leaving any existing row untouched.
/// Returns whether a new row was written.
pub(crate) fn insert_fingerprint(
conn: &Connection,
source_id: i64,
fingerprint: &NewFingerprint,
) -> rusqlite::Result<bool> {
let value = fingerprint.value.trim().to_ascii_lowercase();
let (part_a, part_b) = if fingerprint.kind.supports_partial() {
match ja4_parts(&value) {
Some((prefix, cipher)) => (Some(prefix), Some(cipher)),
None => (None, None),
}
} else {
(None, None)
};
let affected = conn.execute(
"INSERT OR IGNORE INTO intel_fingerprint
(fp_kind, value, part_a, part_b, label, category, reference, first_seen, source_id)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)",
params![
fingerprint.kind.as_str(),
value,
part_a,
part_b,
fingerprint.label,
fingerprint.category.as_str(),
fingerprint.reference,
fingerprint.first_seen,
source_id,
],
)?;
Ok(affected == 1)
}
/// Recomputes and stores a feed's record count after its rows are inserted.
pub(crate) fn refresh_source_count(conn: &Connection, source_id: i64) -> rusqlite::Result<()> {
conn.execute(
"UPDATE intel_source
SET record_count = (SELECT count(*) FROM intel_fingerprint WHERE source_id = ?1)
WHERE id = ?1",
params![source_id],
)?;
Ok(())
}
/// The current wall clock time in whole seconds since the Unix epoch, clamped to
/// zero if the clock is set before 1970.
pub(crate) fn unix_now() -> i64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.ok()
.and_then(|d| i64::try_from(d.as_secs()).ok())
.unwrap_or(0)
}
#[cfg(test)]
mod tests {
use super::ja4_parts;
#[test]
fn ja4_parts_splits_three_fields() {
let parts = ja4_parts("t13d1516h2_8daaf6152771_e5627efa2ab1");
assert_eq!(
parts,
Some(("t13d1516h2".to_string(), "8daaf6152771".to_string()))
);
}
#[test]
fn ja4_parts_rejects_wrong_shape() {
assert_eq!(ja4_parts("nounderscores"), None);
assert_eq!(ja4_parts("only_two"), None);
assert_eq!(ja4_parts("a_b_c_d"), None);
assert_eq!(ja4_parts("a__c"), None);
}
}

341
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-intel/src/matcher.rs Normal file
View File

@ -0,0 +1,341 @@
// ©AngelaMos | 2026
// matcher.rs
//! Looking a fingerprint up against the store.
//!
//! Every kind supports an exact match. The JA4 client fingerprint also supports
//! two partial tiers, because its value is three fields joined by underscores:
//! a capability prefix, a hash of the cipher list, and a hash of the extension
//! list. A tool that copies a browser's cipher order but not its full set of
//! extensions produces a different whole fingerprint but the same cipher hash,
//! so matching on the cipher hash alone catches the impersonation that an exact
//! match misses. The three tiers query disjoint rows, so a hit is counted once
//! at its true strength with no later de duplication.
use rusqlite::{Connection, Params, Row, params};
use tlsfp_core::{FingerprintEvent, StreamEvent};
use super::ja4_parts;
use super::model::{CatalogEntry, Category, FpKind, IntelHit, MatchReport, MatchStrength};
const SELECT: &str = "SELECT f.fp_kind, f.value, f.label, f.category, s.name, f.reference
FROM intel_fingerprint f
JOIN intel_source s ON s.id = f.source_id";
/// Looks one fingerprint up and scores every hit into a verdict.
pub fn match_one(conn: &Connection, kind: FpKind, value: &str) -> anyhow::Result<MatchReport> {
let value = value.trim().to_ascii_lowercase();
let mut hits = collect(
conn,
&format!("{SELECT} WHERE f.fp_kind = ?1 AND f.value = ?2"),
params![kind.as_str(), value],
MatchStrength::Exact,
)?;
if kind.supports_partial() {
if let Some((prefix, cipher)) = ja4_parts(&value) {
hits.extend(collect(
conn,
&format!(
"{SELECT} WHERE f.fp_kind = 'ja4' AND f.part_a = ?1 AND f.part_b = ?2 AND f.value != ?3"
),
params![prefix, cipher, value],
MatchStrength::CipherAndPrefix,
)?);
hits.extend(collect(
conn,
&format!("{SELECT} WHERE f.fp_kind = 'ja4' AND f.part_b = ?1 AND f.part_a != ?2"),
params![cipher, prefix],
MatchStrength::CipherOnly,
)?);
}
}
Ok(MatchReport::from_hits(kind, value, hits))
}
/// The fingerprints carried by one capture event, paired with their kind.
pub fn event_fingerprints(event: &FingerprintEvent) -> Vec<(FpKind, String)> {
match &event.event {
StreamEvent::ClientHello { ja3, ja4, .. } => {
vec![
(FpKind::Ja3, ja3.to_string()),
(FpKind::Ja4, ja4.hash.clone()),
]
}
StreamEvent::ServerHello { ja3s, ja4s, .. } => {
vec![
(FpKind::Ja3s, ja3s.to_string()),
(FpKind::Ja4s, ja4s.hash.clone()),
]
}
StreamEvent::Certificate { ja4x } => vec![(FpKind::Ja4x, ja4x.clone())],
StreamEvent::HttpRequest { ja4h, .. } => vec![(FpKind::Ja4h, ja4h.hash.clone())],
StreamEvent::TcpSyn { ja4t } => vec![(FpKind::Ja4t, ja4t.clone())],
StreamEvent::TcpSynAck { ja4ts } => vec![(FpKind::Ja4ts, ja4ts.clone())],
}
}
/// Searches the stored catalogue by a free text substring against fingerprint
/// values and labels, optionally narrowed to one kind. This is the browse path
/// the dashboard search box and the CLI use, distinct from matching an observed
/// value: it returns rows that contain the query, not hits scored against it.
/// LIKE metacharacters in the query are escaped so a literal percent or
/// underscore searches for itself rather than acting as a wildcard.
pub fn search_catalog(
conn: &Connection,
query: &str,
kind: Option<FpKind>,
limit: i64,
) -> anyhow::Result<Vec<CatalogEntry>> {
let needle = format!("%{}%", escape_like(query.trim()));
let select = "SELECT f.fp_kind, f.value, f.label, f.category, s.name, f.reference
FROM intel_fingerprint f
JOIN intel_source s ON s.id = f.source_id
WHERE (f.value LIKE ?1 ESCAPE '\\' OR f.label LIKE ?1 ESCAPE '\\')";
let entries = if let Some(kind) = kind {
let sql = format!("{select} AND f.fp_kind = ?2 ORDER BY f.label, f.value LIMIT ?3");
let mut statement = conn.prepare(&sql)?;
statement
.query_map(params![needle, kind.as_str(), limit], map_catalog_row)?
.collect::<rusqlite::Result<Vec<_>>>()?
} else {
let sql = format!("{select} ORDER BY f.label, f.value LIMIT ?2");
let mut statement = conn.prepare(&sql)?;
statement
.query_map(params![needle, limit], map_catalog_row)?
.collect::<rusqlite::Result<Vec<_>>>()?
};
Ok(entries)
}
fn map_catalog_row(row: &Row) -> rusqlite::Result<CatalogEntry> {
let kind: String = row.get(0)?;
let category: String = row.get(3)?;
Ok(CatalogEntry {
kind: FpKind::from_token(&kind).unwrap_or(FpKind::Ja3),
value: row.get(1)?,
label: row.get(2)?,
category: Category::from_token(&category),
source: row.get(4)?,
reference: row.get(5)?,
})
}
/// Escapes the LIKE metacharacters in user supplied search text so they match
/// as literal characters. Pairs with the `ESCAPE '\'` clause in the query.
fn escape_like(input: &str) -> String {
let mut escaped = String::with_capacity(input.len());
for ch in input.chars() {
if matches!(ch, '\\' | '%' | '_') {
escaped.push('\\');
}
escaped.push(ch);
}
escaped
}
fn collect(
conn: &Connection,
sql: &str,
params: impl Params,
strength: MatchStrength,
) -> rusqlite::Result<Vec<IntelHit>> {
let mut statement = conn.prepare(sql)?;
let hits = statement
.query_map(params, |row| map_row(row, strength))?
.collect::<rusqlite::Result<Vec<_>>>()?;
Ok(hits)
}
fn map_row(row: &Row, strength: MatchStrength) -> rusqlite::Result<IntelHit> {
let kind: String = row.get(0)?;
let category: String = row.get(3)?;
Ok(IntelHit {
kind: FpKind::from_token(&kind).unwrap_or(FpKind::Ja3),
value: row.get(1)?,
label: row.get(2)?,
category: Category::from_token(&category),
source: row.get(4)?,
reference: row.get(5)?,
strength,
})
}
#[cfg(test)]
mod tests {
use super::{event_fingerprints, match_one};
use crate::IntelStore;
use crate::model::{FpKind, MatchStrength, Verdict};
use tlsfp_core::fingerprint::{Ja3, Ja4Family};
use tlsfp_core::{FingerprintEvent, StreamEvent};
fn seeded() -> IntelStore {
let mut store = IntelStore::open_in_memory().unwrap();
store.seed_bundled().unwrap();
store
}
#[test]
fn known_malicious_ja3_is_malicious() {
let store = seeded();
let report = store
.match_fingerprint(FpKind::Ja3, "1aa7bf8b97e540ca5edd75f7b8384bfa")
.unwrap();
assert_eq!(report.verdict, Verdict::Malicious);
assert!(report.hits.iter().any(|hit| hit.label == "TrickBot"));
}
#[test]
fn known_benign_ja3_is_benign() {
let store = seeded();
let report = store
.match_fingerprint(FpKind::Ja3, "c36fb08942cf19508c08d96af22d4ffc")
.unwrap();
assert_eq!(report.verdict, Verdict::Benign);
assert!(report.hits.iter().any(|hit| hit.label == "Safari"));
}
#[test]
fn cross_feed_collision_is_suspicious() {
let store = seeded();
let report = store
.match_fingerprint(FpKind::Ja3, "51a7ad14509fd614c7bb3a50c4982b8c")
.unwrap();
assert_eq!(report.verdict, Verdict::Suspicious);
assert!(report.hits.len() >= 2);
assert!((report.threat_score - 0.5).abs() < 1e-9);
}
#[test]
fn unknown_fingerprint_returns_no_hits() {
let store = seeded();
let report = store
.match_fingerprint(FpKind::Ja3, "00000000000000000000000000000000")
.unwrap();
assert_eq!(report.verdict, Verdict::Unknown);
assert!(!report.has_hits());
}
#[test]
fn case_is_normalised_before_lookup() {
let store = seeded();
let report = store
.match_fingerprint(FpKind::Ja3, "1AA7BF8B97E540CA5EDD75F7B8384BFA")
.unwrap();
assert_eq!(report.verdict, Verdict::Malicious);
}
#[test]
fn exact_ja4_lookup_hits() {
let store = seeded();
let report = store
.match_fingerprint(FpKind::Ja4, "t10d070600_c50f5591e341_1a3805c3aa63")
.unwrap();
assert_eq!(report.verdict, Verdict::Malicious);
assert_eq!(report.hits[0].strength, MatchStrength::Exact);
assert!(report.hits.iter().any(|hit| hit.label == "RedLine Stealer"));
}
#[test]
fn ja4_same_ciphers_different_extensions_is_a_partial_hit() {
let store = seeded();
let report = match_one(
&store.conn,
FpKind::Ja4,
"t13d1516h2_8daaf6152771_ffffffffffff",
)
.unwrap();
assert!(report.has_hits());
assert_eq!(report.hits[0].strength, MatchStrength::CipherAndPrefix);
assert!(report.hits.iter().any(|hit| hit.label == "Google Chrome"));
}
#[test]
fn ja4_same_cipher_list_under_a_different_profile_is_cipher_only() {
let store = seeded();
let report = match_one(
&store.conn,
FpKind::Ja4,
"t13d0000h0_8daaf6152771_000000000000",
)
.unwrap();
assert!(report.has_hits());
assert_eq!(report.hits[0].strength, MatchStrength::CipherOnly);
}
#[test]
fn ja3_does_not_do_partial_matching() {
let store = seeded();
let report = store
.match_fingerprint(FpKind::Ja3, "1aa7bf8b97e540ca5edd75f7b8384bf0")
.unwrap();
assert!(!report.has_hits());
}
#[test]
fn event_fingerprints_pulls_both_client_fingerprints() {
let event = FingerprintEvent {
ts_nanos: 0,
src: "10.0.0.1:1000".parse().unwrap(),
dst: "10.0.0.2:443".parse().unwrap(),
event: StreamEvent::ClientHello {
ja3: Ja3::from_digest([0x1a; 16]),
ja3_raw: "raw".into(),
ja4: Ja4Family::new("t13d1516h2_8daaf6152771_e5627efa2ab1".into(), "raw".into()),
sni: None,
alpn: None,
},
};
let fingerprints = event_fingerprints(&event);
assert_eq!(fingerprints.len(), 2);
assert_eq!(fingerprints[0].0, FpKind::Ja3);
assert_eq!(fingerprints[1].0, FpKind::Ja4);
assert_eq!(fingerprints[1].1, "t13d1516h2_8daaf6152771_e5627efa2ab1");
}
#[test]
fn match_event_only_reports_kinds_with_intel() {
let store = seeded();
let event = FingerprintEvent {
ts_nanos: 0,
src: "10.0.0.1:1000".parse().unwrap(),
dst: "10.0.0.2:443".parse().unwrap(),
event: StreamEvent::ClientHello {
ja3: Ja3::from_digest([0x00; 16]),
ja3_raw: "raw".into(),
ja4: Ja4Family::new("t13d1516h2_8daaf6152771_e5627efa2ab1".into(), "raw".into()),
sni: None,
alpn: None,
},
};
let reports = store.match_event(&event).unwrap();
assert_eq!(reports.len(), 1);
assert_eq!(reports[0].kind, FpKind::Ja4);
assert_eq!(reports[0].verdict, Verdict::Benign);
}
#[test]
fn search_finds_a_label_substring_case_insensitively() {
let store = seeded();
let hits = store.search("trick", None, 20).unwrap();
assert!(hits.iter().any(|entry| entry.label == "TrickBot"));
}
#[test]
fn search_can_narrow_to_one_kind() {
let store = seeded();
let all = store.search("", None, 10_000).unwrap();
let ja3_only = store.search("", Some(FpKind::Ja3), 10_000).unwrap();
assert!(!ja3_only.is_empty());
assert!(ja3_only.iter().all(|entry| entry.kind == FpKind::Ja3));
assert!(ja3_only.len() < all.len());
}
#[test]
fn search_escapes_like_wildcards() {
let store = seeded();
let literal_percent = store.search("%", None, 10).unwrap();
assert!(literal_percent.is_empty());
}
}

496
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-intel/src/model.rs Normal file
View File

@ -0,0 +1,496 @@
// ©AngelaMos | 2026
// model.rs
//! Domain types for the intelligence store.
//!
//! These describe what the store holds (a fingerprint, its kind, and how its
//! label is classified) and what a lookup returns (a set of hits and the
//! verdict they add up to). The scoring lives here too, kept apart from the SQL
//! so it can be unit tested on plain values with no database in the picture.
use std::fmt;
use serde::Serialize;
/// The threat score at or above which a fingerprint is judged malicious.
const MALICIOUS_THRESHOLD: f64 = 0.8;
/// The threat score at or below which a fingerprint is judged benign; between
/// the two thresholds it is suspicious.
const BENIGN_THRESHOLD: f64 = 0.2;
/// How much a suspicious, dual use hit counts toward the threat score, half a
/// malicious vote.
const SUSPICIOUS_VOTE: f64 = 0.5;
/// The share of confidence the best match earns before corroboration, so a lone
/// exact hit is already fairly trusted and agreeing hits raise it the rest of
/// the way without ever exceeding the best match weight.
const CONFIDENCE_FLOOR: f64 = 0.6;
/// The remaining share of confidence that corroborating, aligned hits supply.
const CONFIDENCE_CORROBORATION: f64 = 0.4;
/// Which fingerprint algorithm a stored value belongs to.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum FpKind {
Ja3,
Ja3s,
Ja4,
Ja4s,
Ja4h,
Ja4x,
Ja4t,
Ja4ts,
}
impl FpKind {
/// The lowercase token used for this kind in the database and on the CLI.
pub const fn as_str(self) -> &'static str {
match self {
FpKind::Ja3 => "ja3",
FpKind::Ja3s => "ja3s",
FpKind::Ja4 => "ja4",
FpKind::Ja4s => "ja4s",
FpKind::Ja4h => "ja4h",
FpKind::Ja4x => "ja4x",
FpKind::Ja4t => "ja4t",
FpKind::Ja4ts => "ja4ts",
}
}
/// Parses a kind token, returning `None` for anything unrecognised.
pub fn from_token(token: &str) -> Option<Self> {
Some(match token {
"ja3" => FpKind::Ja3,
"ja3s" => FpKind::Ja3s,
"ja4" => FpKind::Ja4,
"ja4s" => FpKind::Ja4s,
"ja4h" => FpKind::Ja4h,
"ja4x" => FpKind::Ja4x,
"ja4t" => FpKind::Ja4t,
"ja4ts" => FpKind::Ja4ts,
_ => return None,
})
}
/// Whether partial, structure aware matching applies to this kind.
///
/// Only the JA4 client fingerprint carries a cipher list hash and a
/// capability prefix that mean something on their own, so it is the only
/// kind that supports the cipher and prefix match tiers. Everything else is
/// either an opaque digest (JA3) or a single server side value where a
/// partial match would not be informative.
pub const fn supports_partial(self) -> bool {
matches!(self, FpKind::Ja4)
}
}
impl fmt::Display for FpKind {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.as_str())
}
}
/// The classification carried by a stored label, as read from a seed feed.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum Category {
Malware,
C2,
Tool,
Benign,
Os,
Unknown,
}
impl Category {
/// The token used for this category in the database and seed files.
pub const fn as_str(self) -> &'static str {
match self {
Category::Malware => "malware",
Category::C2 => "c2",
Category::Tool => "tool",
Category::Benign => "benign",
Category::Os => "os",
Category::Unknown => "unknown",
}
}
/// Parses a category token, falling back to `Unknown` for anything else so
/// that a dirty feed value never aborts an import.
pub fn from_token(token: &str) -> Self {
match token.trim().to_ascii_lowercase().as_str() {
"malware" => Category::Malware,
"c2" => Category::C2,
"tool" => Category::Tool,
"benign" => Category::Benign,
"os" => Category::Os,
_ => Category::Unknown,
}
}
/// How a hit in this category weighs on the final verdict.
///
/// Command and control and malware are malicious. Dual use tooling, such as
/// Metasploit or a Tor client, is suspicious rather than malicious because
/// its presence is noteworthy but not proof of compromise. Benign and
/// operating system baselines are benign. An unlabelled entry is treated as
/// suspicious, since it was put in the store for some reason.
pub const fn severity(self) -> Severity {
match self {
Category::Malware | Category::C2 => Severity::Malicious,
Category::Tool | Category::Unknown => Severity::Suspicious,
Category::Benign | Category::Os => Severity::Benign,
}
}
}
/// The coarse direction a single hit pushes the verdict.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Severity {
Malicious,
Suspicious,
Benign,
}
/// How closely a stored fingerprint matched the observed one.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum MatchStrength {
/// Every byte of the fingerprint is identical.
Exact,
/// JA4 only: the capability prefix and the cipher hash both match but the
/// extension hash differs, so the same client stack presented a different
/// extension set, often just a different server name.
CipherAndPrefix,
/// JA4 only: the cipher hash matches but the capability prefix differs, so a
/// different protocol or version profile is carrying the same cipher list.
/// This is the tell of a tool that copies a browser cipher order.
CipherOnly,
}
impl MatchStrength {
/// The token used for this strength on the CLI and in JSON.
pub const fn as_str(self) -> &'static str {
match self {
MatchStrength::Exact => "exact",
MatchStrength::CipherAndPrefix => "cipher_and_prefix",
MatchStrength::CipherOnly => "cipher_only",
}
}
/// A weight in the range zero to one expressing how much trust a match of
/// this strength earns when scoring.
pub const fn weight(self) -> f64 {
match self {
MatchStrength::Exact => 1.0,
MatchStrength::CipherAndPrefix => 0.8,
MatchStrength::CipherOnly => 0.55,
}
}
}
/// One stored fingerprint that matched the observed one, with its provenance.
#[derive(Debug, Clone, Serialize)]
pub struct IntelHit {
pub kind: FpKind,
pub value: String,
pub label: String,
pub category: Category,
pub source: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub reference: Option<String>,
pub strength: MatchStrength,
}
/// One stored fingerprint as returned by a catalogue search.
///
/// This is the browse view of the database, a row of intelligence found by a
/// free text query, rather than a hit scored against an observed value. It
/// carries no match strength because nothing was matched against it.
#[derive(Debug, Clone, Serialize)]
pub struct CatalogEntry {
pub kind: FpKind,
pub value: String,
pub label: String,
pub category: Category,
pub source: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub reference: Option<String>,
}
/// The judgement for an observed fingerprint after weighing every hit.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum Verdict {
Malicious,
Suspicious,
Benign,
Unknown,
}
impl Verdict {
pub const fn as_str(self) -> &'static str {
match self {
Verdict::Malicious => "malicious",
Verdict::Suspicious => "suspicious",
Verdict::Benign => "benign",
Verdict::Unknown => "unknown",
}
}
}
impl fmt::Display for Verdict {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.as_str())
}
}
/// The full result of looking up one observed fingerprint.
#[derive(Debug, Clone, Serialize)]
pub struct MatchReport {
pub kind: FpKind,
pub observed: String,
pub verdict: Verdict,
pub threat_score: f64,
pub confidence: f64,
pub hits: Vec<IntelHit>,
}
impl MatchReport {
/// Whether the lookup found any intelligence at all.
pub fn has_hits(&self) -> bool {
!self.hits.is_empty()
}
/// Scores a set of hits into a verdict.
///
/// The threat score follows the prevalence idea from public sandboxes: a
/// fingerprint seen mostly in malicious sources scores high, one seen mostly
/// in benign sources scores low, and one claimed by both lands in the
/// middle. Each hit contributes its match strength as weight, so an exact
/// hash hit counts for more than a partial cipher hit. Suspicious, dual use
/// hits count as half a malicious vote.
///
/// Confidence is separate from the score: it says how sure the verdict is,
/// rising with the strength of the best match and with the number of
/// corroborating hits that agree with the verdict.
pub fn from_hits(kind: FpKind, observed: String, hits: Vec<IntelHit>) -> Self {
if hits.is_empty() {
return Self {
kind,
observed,
verdict: Verdict::Unknown,
threat_score: 0.0,
confidence: 0.0,
hits,
};
}
let mut malicious = 0.0;
let mut suspicious = 0.0;
let mut benign = 0.0;
let mut best = 0.0_f64;
for hit in &hits {
let weight = hit.strength.weight();
best = best.max(weight);
match hit.category.severity() {
Severity::Malicious => malicious += weight,
Severity::Suspicious => suspicious += weight,
Severity::Benign => benign += weight,
}
}
let total = malicious + suspicious + benign;
let threat_score = (malicious + SUSPICIOUS_VOTE * suspicious) / total;
let verdict = if threat_score >= MALICIOUS_THRESHOLD {
Verdict::Malicious
} else if threat_score <= BENIGN_THRESHOLD {
Verdict::Benign
} else {
Verdict::Suspicious
};
let aligned = hits
.iter()
.filter(|hit| verdict_aligns(verdict, hit.category.severity()))
.count();
let corroboration = 1.0 - 1.0 / (1.0 + count_to_f64(aligned));
let confidence = best * (CONFIDENCE_FLOOR + CONFIDENCE_CORROBORATION * corroboration);
Self {
kind,
observed,
verdict,
threat_score,
confidence,
hits,
}
}
}
/// Whether a hit of a given severity supports the chosen verdict, used to count
/// how many hits corroborate the result when scoring confidence.
fn verdict_aligns(verdict: Verdict, severity: Severity) -> bool {
match verdict {
Verdict::Malicious => severity == Severity::Malicious,
Verdict::Benign => severity == Severity::Benign,
Verdict::Suspicious => true,
Verdict::Unknown => false,
}
}
/// Converts a small count to a float without tripping the precision loss lint.
/// Intel hit counts are tiny, so saturating at `u32::MAX` is unreachable.
fn count_to_f64(n: usize) -> f64 {
f64::from(u32::try_from(n).unwrap_or(u32::MAX))
}
#[cfg(test)]
mod tests {
use super::{Category, FpKind, IntelHit, MatchReport, MatchStrength, Severity, Verdict};
fn hit(category: Category, strength: MatchStrength) -> IntelHit {
IntelHit {
kind: FpKind::Ja3,
value: "x".into(),
label: "x".into(),
category,
source: "s".into(),
reference: None,
strength,
}
}
#[test]
fn kind_tokens_round_trip() {
for kind in [
FpKind::Ja3,
FpKind::Ja3s,
FpKind::Ja4,
FpKind::Ja4s,
FpKind::Ja4h,
FpKind::Ja4x,
FpKind::Ja4t,
FpKind::Ja4ts,
] {
assert_eq!(FpKind::from_token(kind.as_str()), Some(kind));
}
assert_eq!(FpKind::from_token("nope"), None);
}
#[test]
fn only_ja4_supports_partial() {
assert!(FpKind::Ja4.supports_partial());
assert!(!FpKind::Ja3.supports_partial());
assert!(!FpKind::Ja4s.supports_partial());
}
#[test]
fn category_severity_mapping() {
assert_eq!(Category::Malware.severity(), Severity::Malicious);
assert_eq!(Category::C2.severity(), Severity::Malicious);
assert_eq!(Category::Tool.severity(), Severity::Suspicious);
assert_eq!(Category::Unknown.severity(), Severity::Suspicious);
assert_eq!(Category::Benign.severity(), Severity::Benign);
assert_eq!(Category::Os.severity(), Severity::Benign);
}
#[test]
fn no_hits_is_unknown() {
let report = MatchReport::from_hits(FpKind::Ja3, "x".into(), vec![]);
assert_eq!(report.verdict, Verdict::Unknown);
assert!(report.threat_score.abs() < 1e-9);
assert!(report.confidence.abs() < 1e-9);
assert!(!report.has_hits());
}
#[test]
fn single_exact_malware_is_malicious() {
let report = MatchReport::from_hits(
FpKind::Ja3,
"x".into(),
vec![hit(Category::Malware, MatchStrength::Exact)],
);
assert_eq!(report.verdict, Verdict::Malicious);
assert!((report.threat_score - 1.0).abs() < 1e-9);
assert!((report.confidence - 0.8).abs() < 1e-9);
}
#[test]
fn single_benign_is_benign() {
let report = MatchReport::from_hits(
FpKind::Ja3,
"x".into(),
vec![hit(Category::Benign, MatchStrength::Exact)],
);
assert_eq!(report.verdict, Verdict::Benign);
assert!((report.threat_score - 0.0).abs() < 1e-9);
assert!((report.confidence - 0.8).abs() < 1e-9);
}
#[test]
fn malicious_and_benign_collision_is_suspicious() {
let report = MatchReport::from_hits(
FpKind::Ja3,
"x".into(),
vec![
hit(Category::Malware, MatchStrength::Exact),
hit(Category::Benign, MatchStrength::Exact),
],
);
assert_eq!(report.verdict, Verdict::Suspicious);
assert!((report.threat_score - 0.5).abs() < 1e-9);
let expected = 1.0 * (0.6 + 0.4 * (1.0 - 1.0 / 3.0));
assert!((report.confidence - expected).abs() < 1e-9);
}
#[test]
fn dual_use_tool_alone_is_suspicious() {
let report = MatchReport::from_hits(
FpKind::Ja3,
"x".into(),
vec![hit(Category::Tool, MatchStrength::Exact)],
);
assert_eq!(report.verdict, Verdict::Suspicious);
assert!((report.threat_score - 0.5).abs() < 1e-9);
}
#[test]
fn partial_cipher_hit_lowers_confidence() {
let exact = MatchReport::from_hits(
FpKind::Ja4,
"x".into(),
vec![hit(Category::Malware, MatchStrength::Exact)],
);
let partial = MatchReport::from_hits(
FpKind::Ja4,
"x".into(),
vec![hit(Category::Malware, MatchStrength::CipherOnly)],
);
assert!(partial.confidence < exact.confidence);
assert_eq!(partial.verdict, Verdict::Malicious);
}
#[test]
fn corroboration_raises_confidence() {
let one = MatchReport::from_hits(
FpKind::Ja3,
"x".into(),
vec![hit(Category::Malware, MatchStrength::Exact)],
);
let three = MatchReport::from_hits(
FpKind::Ja3,
"x".into(),
vec![
hit(Category::Malware, MatchStrength::Exact),
hit(Category::C2, MatchStrength::Exact),
hit(Category::Malware, MatchStrength::Exact),
],
);
assert!(three.confidence > one.confidence);
assert_eq!(three.verdict, Verdict::Malicious);
}
}

167
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-intel/src/schema.rs Normal file
View File

@ -0,0 +1,167 @@
// ©AngelaMos | 2026
// schema.rs
//! The database schema and its migration runner.
//!
//! Versions are tracked in SQLite's own `user_version` header field rather than
//! a side table, so an empty database and a fully migrated one are told apart
//! with a single pragma and no bootstrapping. Each migration is the whole SQL
//! to move from one version to the next, applied inside a transaction so a
//! half applied schema can never be left behind. Running the migrations on an
//! already current database is a no op, which is what lets every command open
//! the store and migrate without checking first.
//!
//! Migration one is the intelligence half: feeds and the fingerprints they
//! carry. Migration two is the detection half: the observations the engine
//! records as it watches traffic and the alerts it raises when a rule fires.
//! Both tables stand on their own, so a detection run needs intelligence loaded
//! only for the rules that consult it.
use rusqlite::Connection;
/// The ordered list of migrations. Index zero moves a fresh database to version
/// one, index one to version two, and so on. Append, never edit in place, or an
/// existing database will silently disagree with a new one.
const MIGRATIONS: &[&str] = &[
r"
CREATE TABLE intel_source (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL UNIQUE,
url TEXT,
license TEXT,
kind TEXT NOT NULL,
imported_at INTEGER NOT NULL,
record_count INTEGER NOT NULL DEFAULT 0
);
CREATE TABLE intel_fingerprint (
id INTEGER PRIMARY KEY,
fp_kind TEXT NOT NULL,
value TEXT NOT NULL,
part_a TEXT,
part_b TEXT,
label TEXT NOT NULL,
category TEXT NOT NULL,
reference TEXT,
first_seen TEXT,
source_id INTEGER NOT NULL REFERENCES intel_source(id) ON DELETE CASCADE,
UNIQUE(fp_kind, value, source_id)
);
CREATE INDEX idx_fp_kind_value ON intel_fingerprint(fp_kind, value);
CREATE INDEX idx_fp_kind_part_b ON intel_fingerprint(fp_kind, part_b);
CREATE INDEX idx_fp_kind_part_a ON intel_fingerprint(fp_kind, part_a);
",
r"
CREATE TABLE observation (
id INTEGER PRIMARY KEY,
ts INTEGER NOT NULL,
ip TEXT NOT NULL,
fp_kind TEXT NOT NULL,
fp_value TEXT NOT NULL,
verdict TEXT,
label TEXT,
category TEXT,
sni TEXT,
host TEXT,
user_agent TEXT,
os_claim TEXT
);
CREATE INDEX idx_obs_ip ON observation(ip);
CREATE INDEX idx_obs_ip_kind ON observation(ip, fp_kind);
CREATE INDEX idx_obs_fp ON observation(fp_kind, fp_value);
CREATE INDEX idx_obs_ts ON observation(ts);
CREATE TABLE alert (
id INTEGER PRIMARY KEY,
ts INTEGER NOT NULL,
rule TEXT NOT NULL,
severity TEXT NOT NULL,
ip TEXT,
fp_kind TEXT,
fp_value TEXT,
title TEXT NOT NULL,
detail TEXT NOT NULL,
score REAL,
observation_id INTEGER REFERENCES observation(id) ON DELETE SET NULL
);
CREATE INDEX idx_alert_ts ON alert(ts);
CREATE INDEX idx_alert_rule ON alert(rule);
CREATE INDEX idx_alert_ip ON alert(ip);
",
];
/// Brings a connection's schema up to the latest version, applying only the
/// migrations it is missing. Safe to call on every open.
pub fn apply_migrations(conn: &mut Connection) -> rusqlite::Result<()> {
let mut version: i64 = conn.pragma_query_value(None, "user_version", |row| row.get(0))?;
while usize::try_from(version).unwrap_or(usize::MAX) < MIGRATIONS.len() {
let index = usize::try_from(version).unwrap_or(usize::MAX);
let tx = conn.transaction()?;
tx.execute_batch(MIGRATIONS[index])?;
tx.pragma_update(None, "user_version", version + 1)?;
tx.commit()?;
version += 1;
}
Ok(())
}
/// The schema version a fully migrated database reports, used by the tests to
/// assert the runner reaches the head of the migration list.
#[cfg(test)]
fn latest_version() -> i64 {
i64::try_from(MIGRATIONS.len()).unwrap_or(i64::MAX)
}
#[cfg(test)]
mod tests {
use super::{MIGRATIONS, apply_migrations, latest_version};
use rusqlite::Connection;
fn user_version(conn: &Connection) -> i64 {
conn.pragma_query_value(None, "user_version", |row| row.get(0))
.unwrap()
}
#[test]
fn migrates_a_fresh_database_to_latest() {
let mut conn = Connection::open_in_memory().unwrap();
assert_eq!(user_version(&conn), 0);
apply_migrations(&mut conn).unwrap();
assert_eq!(user_version(&conn), latest_version());
}
#[test]
fn migrating_twice_is_a_no_op() {
let mut conn = Connection::open_in_memory().unwrap();
apply_migrations(&mut conn).unwrap();
apply_migrations(&mut conn).unwrap();
assert_eq!(user_version(&conn), latest_version());
}
#[test]
fn expected_tables_exist() {
let mut conn = Connection::open_in_memory().unwrap();
apply_migrations(&mut conn).unwrap();
let count: i64 = conn
.query_row(
"SELECT count(*) FROM sqlite_master WHERE type='table' AND name IN ('intel_source','intel_fingerprint','observation','alert')",
[],
|row| row.get(0),
)
.unwrap();
assert_eq!(count, 4);
}
#[test]
fn migration_two_is_appended_not_edited() {
assert!(
MIGRATIONS.len() >= 2,
"detection tables live in migration two"
);
assert!(MIGRATIONS[0].contains("intel_fingerprint"));
assert!(MIGRATIONS[1].contains("observation") && MIGRATIONS[1].contains("alert"));
}
}

360
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-intel/src/seed.rs Normal file
View File

@ -0,0 +1,360 @@
// ©AngelaMos | 2026
// seed.rs
//! Loading the three vendored feeds into the database.
//!
//! The feeds are compiled into the binary, so seeding needs no network. Each
//! feed has its own column layout, so each gets its own small parser, but they
//! share one insert path that computes the JA4 partial match columns and skips
//! rows already present. Parsing uses a real CSV reader rather than splitting on
//! commas, because the salesforce application names are quoted and can contain
//! commas, and its licence header is a quoted field that spans several lines.
//!
//! Every malicious feed row is classified by the feed it came from: abuse.ch
//! SSLBL is a blocklist, so its rows are malware, and the salesforce list is a
//! benign application catalogue, so its rows are benign. The curated file
//! carries an explicit category per row. The point of loading both a malicious
//! and a benign feed is that some hashes appear in both, and the matcher needs
//! to see that disagreement to score it.
use anyhow::Result;
use rusqlite::Connection;
use super::model::{Category, FpKind};
use super::{NewFingerprint, get_or_create_source, insert_fingerprint, refresh_source_count};
const SSLBL: &str = include_str!("../seeds/sslbl-ja3.csv");
const SALESFORCE: &str = include_str!("../seeds/salesforce-osx-nix-ja3.csv");
const CURATED: &str = include_str!("../seeds/curated-c2-intel.csv");
const SSLBL_NAME: &str = "abuse.ch SSLBL";
const SSLBL_URL: &str = "https://sslbl.abuse.ch/blacklist/ja3_fingerprints.csv";
const SALESFORCE_NAME: &str = "salesforce/ja3 osx-nix";
const SALESFORCE_URL: &str = "https://github.com/salesforce/ja3";
const CURATED_NAME: &str = "tlsfp curated";
/// How many rows one feed contributed on a seed run.
#[derive(Debug, Clone)]
pub struct FeedLoad {
pub name: String,
pub inserted: usize,
pub parsed: usize,
}
/// The result of a full seed run across every feed.
#[derive(Debug, Clone)]
pub struct SeedSummary {
pub feeds: Vec<FeedLoad>,
}
impl SeedSummary {
/// Rows newly inserted across all feeds, zero on a repeat seed.
pub fn inserted(&self) -> usize {
self.feeds.iter().map(|feed| feed.inserted).sum()
}
/// Valid rows parsed across all feeds, the same on every seed.
pub fn parsed(&self) -> usize {
self.feeds.iter().map(|feed| feed.parsed).sum()
}
}
/// Loads all three vendored feeds inside a single transaction.
pub fn load_bundled(conn: &mut Connection) -> Result<SeedSummary> {
let tx = conn.transaction()?;
let mut feeds = Vec::new();
let sslbl_id =
get_or_create_source(&tx, SSLBL_NAME, Some(SSLBL_URL), Some("CC0-1.0"), "bundled")?;
let (inserted, parsed) = load_sslbl(&tx, sslbl_id)?;
refresh_source_count(&tx, sslbl_id)?;
feeds.push(FeedLoad {
name: SSLBL_NAME.to_string(),
inserted,
parsed,
});
let sf_id = get_or_create_source(
&tx,
SALESFORCE_NAME,
Some(SALESFORCE_URL),
Some("BSD-3-Clause"),
"bundled",
)?;
let (inserted, parsed) = load_salesforce(&tx, sf_id)?;
refresh_source_count(&tx, sf_id)?;
feeds.push(FeedLoad {
name: SALESFORCE_NAME.to_string(),
inserted,
parsed,
});
let curated_id = get_or_create_source(&tx, CURATED_NAME, None, Some("project"), "bundled")?;
let (inserted, parsed) = load_curated(&tx, curated_id)?;
refresh_source_count(&tx, curated_id)?;
feeds.push(FeedLoad {
name: CURATED_NAME.to_string(),
inserted,
parsed,
});
tx.commit()?;
Ok(SeedSummary { feeds })
}
/// abuse.ch SSLBL: `ja3_md5, Firstseen, Lastseen, Listingreason`, every row a
/// known malicious JA3.
fn load_sslbl(conn: &Connection, source_id: i64) -> Result<(usize, usize)> {
let mut reader = csv::ReaderBuilder::new()
.has_headers(false)
.flexible(true)
.comment(Some(b'#'))
.from_reader(SSLBL.as_bytes());
let mut inserted = 0;
let mut parsed = 0;
for record in reader.records() {
let record = record?;
let value = record.get(0).unwrap_or_default().trim();
if !is_hex_md5(value) {
continue;
}
let first_seen = record.get(1).map(str::trim).filter(|seen| !seen.is_empty());
let label = record
.get(3)
.map(str::trim)
.filter(|reason| !reason.is_empty())
.unwrap_or("unknown");
parsed += 1;
if insert_fingerprint(
conn,
source_id,
&NewFingerprint {
kind: FpKind::Ja3,
value,
label,
category: Category::Malware,
reference: None,
first_seen,
},
)? {
inserted += 1;
}
}
Ok((inserted, parsed))
}
/// salesforce osx-nix: `ja3_md5, "application name(s)"`, every row a benign app.
/// The quoted multi line licence header is the first record and is dropped by
/// the hex check, since its first field is prose rather than a hash.
fn load_salesforce(conn: &Connection, source_id: i64) -> Result<(usize, usize)> {
let mut reader = csv::ReaderBuilder::new()
.has_headers(false)
.flexible(true)
.from_reader(SALESFORCE.as_bytes());
let mut inserted = 0;
let mut parsed = 0;
for record in reader.records() {
let record = record?;
let value = record.get(0).unwrap_or_default().trim();
if !is_hex_md5(value) {
continue;
}
let label = record
.get(1)
.map(str::trim)
.filter(|app| !app.is_empty())
.unwrap_or("unknown");
parsed += 1;
if insert_fingerprint(
conn,
source_id,
&NewFingerprint {
kind: FpKind::Ja3,
value,
label,
category: Category::Benign,
reference: None,
first_seen: None,
},
)? {
inserted += 1;
}
}
Ok((inserted, parsed))
}
/// The curated file: `fp_kind, value, label, category, reference`, each row a
/// hand classified entry from a named source.
fn load_curated(conn: &Connection, source_id: i64) -> Result<(usize, usize)> {
let mut reader = csv::ReaderBuilder::new()
.has_headers(true)
.flexible(true)
.comment(Some(b'#'))
.from_reader(CURATED.as_bytes());
let mut inserted = 0;
let mut parsed = 0;
for record in reader.records() {
let record = record?;
let Some(kind) = record.get(0).map(str::trim).and_then(FpKind::from_token) else {
continue;
};
let value = record.get(1).unwrap_or_default().trim();
if value.is_empty() {
continue;
}
let label = record
.get(2)
.map(str::trim)
.filter(|label| !label.is_empty())
.unwrap_or("unknown");
let category = record
.get(3)
.map_or(Category::Unknown, Category::from_token);
let reference = record.get(4).map(str::trim).filter(|note| !note.is_empty());
parsed += 1;
if insert_fingerprint(
conn,
source_id,
&NewFingerprint {
kind,
value,
label,
category,
reference,
first_seen: None,
},
)? {
inserted += 1;
}
}
Ok((inserted, parsed))
}
/// Whether a string is a lowercase or uppercase 32 character hex digest, the
/// shape of every JA3 value and the test that skips comment and header rows.
fn is_hex_md5(value: &str) -> bool {
value.len() == 32 && value.bytes().all(|byte| byte.is_ascii_hexdigit())
}
#[cfg(test)]
mod tests {
use super::{CURATED_NAME, SALESFORCE_NAME, SSLBL_NAME, load_bundled};
use crate::IntelStore;
use crate::model::{Category, FpKind};
fn category_of(store: &IntelStore, value: &str, source: &str) -> Option<String> {
store
.conn
.query_row(
"SELECT category FROM intel_fingerprint f
JOIN intel_source s ON s.id = f.source_id
WHERE f.value = ?1 AND s.name = ?2",
rusqlite::params![value, source],
|row| row.get(0),
)
.ok()
}
#[test]
fn seeds_every_vendored_row() {
let mut store = IntelStore::open_in_memory().unwrap();
let summary = store.seed_bundled().unwrap();
assert_eq!(summary.parsed(), 97 + 157 + 17);
assert_eq!(summary.inserted(), summary.parsed());
}
#[test]
fn seeding_twice_inserts_nothing_new() {
let mut conn = rusqlite::Connection::open_in_memory().unwrap();
super::super::schema::apply_migrations(&mut conn).unwrap();
let first = load_bundled(&mut conn).unwrap();
let second = load_bundled(&mut conn).unwrap();
assert!(first.inserted() > 0);
assert_eq!(second.inserted(), 0);
assert_eq!(second.parsed(), first.parsed());
}
#[test]
fn sslbl_rows_are_malware_salesforce_rows_are_benign() {
let mut store = IntelStore::open_in_memory().unwrap();
store.seed_bundled().unwrap();
assert_eq!(
category_of(&store, "1aa7bf8b97e540ca5edd75f7b8384bfa", SSLBL_NAME).as_deref(),
Some("malware"),
);
assert_eq!(
category_of(&store, "c36fb08942cf19508c08d96af22d4ffc", SALESFORCE_NAME).as_deref(),
Some("benign"),
);
}
#[test]
fn curated_carries_its_own_categories() {
let mut store = IntelStore::open_in_memory().unwrap();
store.seed_bundled().unwrap();
assert_eq!(
category_of(&store, "72a589da586844d7f0818ce684948eea", CURATED_NAME).as_deref(),
Some("c2"),
);
assert_eq!(
category_of(&store, "8916410db85077a5460817142dcbc8de", CURATED_NAME).as_deref(),
Some("tool"),
);
}
#[test]
fn the_same_hash_can_be_both_malicious_and_benign() {
let mut store = IntelStore::open_in_memory().unwrap();
store.seed_bundled().unwrap();
let collision = "51a7ad14509fd614c7bb3a50c4982b8c";
assert_eq!(
category_of(&store, collision, SSLBL_NAME).as_deref(),
Some("malware"),
);
assert_eq!(
category_of(&store, collision, SALESFORCE_NAME).as_deref(),
Some("benign"),
);
}
#[test]
fn ja4_rows_get_partial_match_columns() {
let mut store = IntelStore::open_in_memory().unwrap();
store.seed_bundled().unwrap();
let parts: (Option<String>, Option<String>) = store
.conn
.query_row(
"SELECT part_a, part_b FROM intel_fingerprint WHERE fp_kind = 'ja4' AND value = ?1",
rusqlite::params!["t13d1516h2_8daaf6152771_e5627efa2ab1"],
|row| Ok((row.get(0)?, row.get(1)?)),
)
.unwrap();
assert_eq!(parts.0.as_deref(), Some("t13d1516h2"));
assert_eq!(parts.1.as_deref(), Some("8daaf6152771"));
let ja3_parts: (Option<String>, Option<String>) = store
.conn
.query_row(
"SELECT part_a, part_b FROM intel_fingerprint WHERE fp_kind = 'ja3' LIMIT 1",
[],
|row| Ok((row.get(0)?, row.get(1)?)),
)
.unwrap();
assert_eq!(ja3_parts, (None, None));
}
#[test]
fn category_tokens_parse() {
assert_eq!(Category::from_token("malware"), Category::Malware);
assert_eq!(Category::from_token(" C2 "), Category::C2);
assert_eq!(Category::from_token("garbage"), Category::Unknown);
}
#[test]
fn fp_kind_tokens_parse() {
assert_eq!(FpKind::from_token("ja4"), Some(FpKind::Ja4));
assert_eq!(FpKind::from_token("zz"), None);
}
}

436
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-intel/src/signal.rs Normal file
View File

@ -0,0 +1,436 @@
// ©AngelaMos | 2026
// signal.rs
//! The classifiers the detection rules read fingerprints and headers through.
//!
//! These are deliberately pure: a string in, a small verdict out, no database
//! and no state. That keeps the judgement calls, which are the part most likely
//! to be wrong, testable against named example values.
//!
//! Two families of classifier live here. The first maps a fingerprint or a
//! User-Agent to a client family, so a request that calls itself a browser can
//! be checked against what its fingerprint actually is. The second maps a JA4T
//! or a User-Agent to a coarse operating-system class, so the operating system
//! a connection claims can be checked against the one its TCP stack reveals.
//!
//! The operating-system heuristic is intentionally coarse: Windows against
//! everything Unix-like. It rests on one signature that is stable across stack
//! versions and well documented: Microsoft Windows does not send the TCP
//! timestamp option (kind 8) on a SYN, while Unix-like stacks do, and within a
//! stack that sends no timestamp, Windows orders the window-scale option before
//! the SACK-permitted option where Linux without timestamps does the reverse.
//! Anything that does not match a known signature is left unclassified rather
//! than guessed, so the mismatch rule never fires on an ambiguous stack.
//!
//! Sources: the JA4T specification and the FoxIO JA4T write-up
//! (blog.foxio.io/ja4t-tcp-fingerprinting), and the p0f v3 SYN signature
//! database it builds on for the per-operating-system option layouts.
use crate::model::Category;
/// Whether `needle` appears in `haystack` as a whole word, bounded by characters
/// that are not ASCII alphanumeric. Short, ambiguous tokens like "tor" and
/// "java" are matched this way so a label such as "Factory" is not read as Tor
/// and a User-Agent mentioning "javascript" is not read as Java.
fn contains_word(haystack: &str, needle: &str) -> bool {
haystack
.split(|c: char| !c.is_ascii_alphanumeric())
.any(|word| word == needle)
}
/// A coarse operating-system class, the resolution the SYN signature can carry
/// without guessing. Finer naming from one packet is not reliable, so the
/// mismatch rule works at this granularity on purpose.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum OsClass {
Windows,
Unix,
}
impl OsClass {
/// A short token for the class, used when describing an alert.
pub const fn as_str(self) -> &'static str {
match self {
OsClass::Windows => "windows",
OsClass::Unix => "unix",
}
}
/// Parses a class token back into a class, the inverse of `as_str`. This
/// reads the operating system an observation already resolved and stored,
/// which is not a User-Agent and must not be run back through the loose
/// User-Agent classifier.
pub fn from_token(token: &str) -> Option<OsClass> {
match token {
"windows" => Some(OsClass::Windows),
"unix" => Some(OsClass::Unix),
_ => None,
}
}
}
/// The operating system a User-Agent string claims, read from the platform
/// token every mainstream browser places near the front of the string.
///
/// Returns `None` when no platform token is recognised, so an unusual or absent
/// User-Agent never produces a false claim to compare against.
#[must_use]
pub fn ua_os_class(user_agent: &str) -> Option<OsClass> {
const UNIX_TOKENS: &[&str] = &[
"android",
"linux",
"mac os x",
"macintosh",
"iphone",
"ipad",
"ipod",
" cros ",
"x11",
"freebsd",
"openbsd",
"netbsd",
];
let ua = user_agent.to_ascii_lowercase();
if ua.contains("windows") {
return Some(OsClass::Windows);
}
if UNIX_TOKENS.iter().any(|token| ua.contains(token)) {
return Some(OsClass::Unix);
}
None
}
/// The operating-system class a JA4T implies from its TCP option layout.
///
/// The JA4T value is `window_options_mss_windowscale`; only the options field,
/// a dash separated list of TCP option kind numbers, is read here. A timestamp
/// option marks a Unix-like stack. Its absence, combined with the window-scale
/// option preceding the SACK-permitted option, marks Windows. Every other
/// layout, including a Unix stack with timestamps disabled, is left
/// unclassified.
#[must_use]
pub fn ja4t_os_class(ja4t: &str) -> Option<OsClass> {
let options = ja4t.split('_').nth(1)?;
if options.is_empty() || options == "0" {
return None;
}
let kinds: Vec<&str> = options.split('-').collect();
if kinds.contains(&"8") {
return Some(OsClass::Unix);
}
let window_scale = kinds.iter().position(|kind| *kind == "3");
let sack = kinds.iter().position(|kind| *kind == "4");
if let (Some(window_scale), Some(sack)) = (window_scale, sack) {
if window_scale < sack {
return Some(OsClass::Windows);
}
}
None
}
/// A client software family, coarse enough that a fingerprint label and a
/// User-Agent can be compared even when they word the same software
/// differently.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Family {
Chrome,
Firefox,
Safari,
Edge,
Opera,
Brave,
Curl,
Wget,
Python,
Go,
OkHttp,
Java,
Tor,
}
impl Family {
/// A short token for the family, used when describing an alert.
pub const fn as_str(self) -> &'static str {
match self {
Family::Chrome => "chrome",
Family::Firefox => "firefox",
Family::Safari => "safari",
Family::Edge => "edge",
Family::Opera => "opera",
Family::Brave => "brave",
Family::Curl => "curl",
Family::Wget => "wget",
Family::Python => "python",
Family::Go => "go-http",
Family::OkHttp => "okhttp",
Family::Java => "java",
Family::Tor => "tor",
}
}
/// Whether this family is a human-driven web browser rather than a script,
/// library, or command line client. The mismatch rule turns on this line:
/// a request that claims a browser but fingerprints as one of the others is
/// the impersonation worth flagging.
pub const fn is_browser(self) -> bool {
matches!(
self,
Family::Chrome
| Family::Firefox
| Family::Safari
| Family::Edge
| Family::Opera
| Family::Brave
)
}
}
/// The family a User-Agent string claims to be.
///
/// Browsers are checked before the engines they embed, because a Chromium
/// derivative carries the Chrome and Safari tokens too, and a script that sets
/// a real browser string would otherwise be read as that browser.
#[must_use]
pub fn ua_family(user_agent: &str) -> Option<Family> {
let ua = user_agent.to_ascii_lowercase();
if ua.contains("edg/") || ua.contains("edga/") || ua.contains("edgios/") {
return Some(Family::Edge);
}
if ua.contains("opr/") || ua.contains("opera") {
return Some(Family::Opera);
}
if ua.contains("brave") {
return Some(Family::Brave);
}
if ua.contains("firefox") || ua.contains("fxios") {
return Some(Family::Firefox);
}
if ua.contains("chrome") || ua.contains("chromium") || ua.contains("crios") {
return Some(Family::Chrome);
}
if ua.contains("safari") {
return Some(Family::Safari);
}
if ua.contains("curl") {
return Some(Family::Curl);
}
if ua.contains("wget") {
return Some(Family::Wget);
}
if ua.contains("python") || ua.contains("urllib") || ua.contains("aiohttp") {
return Some(Family::Python);
}
if ua.contains("go-http-client") {
return Some(Family::Go);
}
if ua.contains("okhttp") {
return Some(Family::OkHttp);
}
if contains_word(&ua, "java") {
return Some(Family::Java);
}
None
}
/// The family an intelligence label names, by keyword.
///
/// This reads the human label a feed attached to a fingerprint, so it
/// recognises the same software the User-Agent classifier does and nothing it
/// cannot name with confidence.
#[must_use]
pub fn label_family(label: &str) -> Option<Family> {
let label = label.to_ascii_lowercase();
if label.contains("edge") {
return Some(Family::Edge);
}
if label.contains("opera") {
return Some(Family::Opera);
}
if label.contains("brave") {
return Some(Family::Brave);
}
if label.contains("firefox") {
return Some(Family::Firefox);
}
if label.contains("chrome") || label.contains("chromium") {
return Some(Family::Chrome);
}
if label.contains("safari") {
return Some(Family::Safari);
}
if label.contains("curl") {
return Some(Family::Curl);
}
if label.contains("wget") {
return Some(Family::Wget);
}
if label.contains("python") || label.contains("requests") || label.contains("urllib") {
return Some(Family::Python);
}
if label.contains("go-http") || label.contains("golang") {
return Some(Family::Go);
}
if label.contains("okhttp") {
return Some(Family::OkHttp);
}
if contains_word(&label, "java") {
return Some(Family::Java);
}
if contains_word(&label, "tor") {
return Some(Family::Tor);
}
None
}
/// Whether an observed fingerprint should be read as a non-browser client.
///
/// A label that names a script or tool says so directly. A category of tool,
/// malware, or command and control says it too, even when the label is just a
/// family name like TrickBot, because none of those are a browser. A benign or
/// unknown category with an unrecognised label is left alone, so the mismatch
/// rule needs a real reason to call something not a browser.
#[must_use]
pub fn label_is_non_browser(label: &str, category: Category) -> bool {
if label_family(label).is_some_and(|family| !family.is_browser()) {
return true;
}
matches!(category, Category::Tool | Category::Malware | Category::C2)
}
/// Whether an observed fingerprint should be read as a browser, used to suppress
/// the mismatch rule when both sides agree the client is a browser.
#[must_use]
pub fn label_is_browser(label: &str) -> bool {
label_family(label).is_some_and(Family::is_browser)
}
#[cfg(test)]
mod tests {
use super::{
Family, OsClass, ja4t_os_class, label_family, label_is_browser, label_is_non_browser,
ua_family, ua_os_class,
};
use crate::model::Category;
#[test]
fn windows_user_agent_is_windows() {
let ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0";
assert_eq!(ua_os_class(ua), Some(OsClass::Windows));
}
#[test]
fn unix_user_agents_are_unix() {
for ua in [
"Mozilla/5.0 (X11; Linux x86_64) Firefox/121.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) Safari/605.1",
"Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X)",
"Mozilla/5.0 (Linux; Android 14) Chrome/120.0",
] {
assert_eq!(ua_os_class(ua), Some(OsClass::Unix), "{ua}");
}
}
#[test]
fn unknown_platform_is_unclassified() {
assert_eq!(ua_os_class("curl/8.4.0"), None);
assert_eq!(ua_os_class(""), None);
}
#[test]
fn ja4t_with_timestamp_is_unix() {
assert_eq!(ja4t_os_class("29200_2-4-8-1-3_1424_7"), Some(OsClass::Unix));
assert_eq!(ja4t_os_class("65535_2-4-8-1-3_1460_6"), Some(OsClass::Unix));
}
#[test]
fn ja4t_windows_layout_is_windows() {
assert_eq!(
ja4t_os_class("64240_2-1-3-1-1-4_1460_8"),
Some(OsClass::Windows)
);
}
#[test]
fn ja4t_ambiguous_layout_is_unclassified() {
assert_eq!(ja4t_os_class("64240_2-4-1-3_1460_7"), None);
assert_eq!(ja4t_os_class("64240_0_0_0"), None);
assert_eq!(ja4t_os_class("nonsense"), None);
}
#[test]
fn windows_scale_byte_is_not_read_as_a_timestamp_option() {
let windows = ja4t_os_class("64240_2-1-3-1-1-4_1460_8");
assert_eq!(windows, Some(OsClass::Windows));
}
#[test]
fn os_class_tokens_round_trip() {
assert_eq!(
OsClass::from_token(OsClass::Windows.as_str()),
Some(OsClass::Windows)
);
assert_eq!(
OsClass::from_token(OsClass::Unix.as_str()),
Some(OsClass::Unix)
);
assert_eq!(OsClass::from_token("plan9"), None);
}
#[test]
fn browser_user_agents_classify() {
assert_eq!(
ua_family("Mozilla/5.0 (Windows NT 10.0) Gecko Firefox/121.0"),
Some(Family::Firefox)
);
assert_eq!(
ua_family(
"Mozilla/5.0 (Macintosh) AppleWebKit/537.36 (KHTML) Chrome/120.0 Safari/537.36"
),
Some(Family::Chrome)
);
assert_eq!(
ua_family("Mozilla/5.0 (Macintosh) AppleWebKit/605.1 Version/17.0 Safari/605.1"),
Some(Family::Safari)
);
}
#[test]
fn edge_and_opera_are_not_read_as_chrome() {
assert_eq!(
ua_family("Mozilla/5.0 (Windows NT 10.0) Chrome/120.0 Safari/537.36 Edg/120.0"),
Some(Family::Edge)
);
assert_eq!(
ua_family("Mozilla/5.0 (Windows NT 10.0) Chrome/120.0 Safari/537.36 OPR/106.0"),
Some(Family::Opera)
);
}
#[test]
fn tool_user_agents_classify() {
assert_eq!(ua_family("curl/8.4.0"), Some(Family::Curl));
assert_eq!(ua_family("python-requests/2.31.0"), Some(Family::Python));
assert_eq!(ua_family("Go-http-client/2.0"), Some(Family::Go));
}
#[test]
fn labels_classify_and_categories_decide_non_browser() {
assert_eq!(label_family("Google Chrome"), Some(Family::Chrome));
assert_eq!(label_family("curl"), Some(Family::Curl));
assert!(label_is_browser("Google Chrome"));
assert!(!label_is_browser("curl"));
assert!(label_is_non_browser("curl", Category::Benign));
assert!(label_is_non_browser("TrickBot", Category::Malware));
assert!(!label_is_non_browser("Google Chrome", Category::Benign));
assert!(!label_is_non_browser("unrecognised", Category::Benign));
}
#[test]
fn ambiguous_substrings_do_not_misclassify() {
assert_eq!(label_family("Factory C2"), None);
assert_eq!(label_family("Tor Browser"), Some(Family::Tor));
assert_eq!(label_family("JavaScript Runtime"), None);
assert_eq!(label_family("Java"), Some(Family::Java));
assert_eq!(ua_family("Mozilla/5.0 javascript engine"), None);
}
}

367
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp-intel/tests/detect.rs Normal file
View File

@ -0,0 +1,367 @@
// ©AngelaMos | 2026
// detect.rs
//! End to end tests for the detection engine.
//!
//! Each test drives a short sequence of hand built fingerprint events through a
//! store and checks that the right rule fires, and just as importantly that the
//! quiet cases stay quiet. The events are synthetic on purpose: the engine
//! consumes events, not packets, so building them directly is the honest seam
//! to test the rules at.
use tlsfp_core::fingerprint::{Ja3, Ja4Family};
use tlsfp_core::{FingerprintEvent, StreamEvent};
use tlsfp_intel::{AlertSeverity, DetectConfig, IntelStore, Rule};
const CURL_JA4: &str = "t13d1234h1_aaaaaaaaaaaa_bbbbbbbbbbbb";
const MALWARE_JA4: &str = "t10d070600_c50f5591e341_1a3805c3aa63";
const CHROME_JA4: &str = "t13d1516h2_8daaf6152771_e5627efa2ab1";
const BROWSER_UA: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36";
const CURL_UA: &str = "curl/8.4.0";
const UNIX_JA4T: &str = "29200_2-4-8-1-3_1424_7";
const WINDOWS_JA4T: &str = "64240_2-1-3-1-1-4_1460_8";
const FEED: &str = r#"[
{"application":"Chrome","os":"Windows","ja4_fingerprint":"t13d1516h2_8daaf6152771_e5627efa2ab1"},
{"application":"curl","library":"OpenSSL","ja4_fingerprint":"t13d1234h1_aaaaaaaaaaaa_bbbbbbbbbbbb"},
{"application":"Loader","notes":"known malware stealer","ja4_fingerprint":"t10d070600_c50f5591e341_1a3805c3aa63"}
]"#;
fn store() -> IntelStore {
let mut store = IntelStore::open_in_memory().unwrap();
store.import_ja4db(FEED).unwrap();
store
}
fn cfg() -> DetectConfig {
DetectConfig {
window_secs: 86_400,
rotation_threshold: 2,
monoculture_threshold: 2,
}
}
fn client_hello(ip: &str, ts: u64, ja4: &str) -> FingerprintEvent {
FingerprintEvent {
ts_nanos: ts,
src: format!("{ip}:1000").parse().unwrap(),
dst: "10.0.0.250:443".parse().unwrap(),
event: StreamEvent::ClientHello {
ja3: Ja3::from_digest([0u8; 16]),
ja3_raw: "raw".into(),
ja4: Ja4Family::new(ja4.into(), "raw".into()),
sni: None,
alpn: None,
},
}
}
fn http_request(ip: &str, ts: u64, ua: &str) -> FingerprintEvent {
FingerprintEvent {
ts_nanos: ts,
src: format!("{ip}:1000").parse().unwrap(),
dst: "10.0.0.250:80".parse().unwrap(),
event: StreamEvent::HttpRequest {
ja4h: Ja4Family::new(
"ge20nn000000_000000000000_000000000000".into(),
"raw".into(),
),
method: "GET".into(),
host: Some("example.com".into()),
user_agent: Some(ua.into()),
},
}
}
fn tcp_syn(ip: &str, ts: u64, ja4t: &str) -> FingerprintEvent {
FingerprintEvent {
ts_nanos: ts,
src: format!("{ip}:1000").parse().unwrap(),
dst: "10.0.0.250:443".parse().unwrap(),
event: StreamEvent::TcpSyn { ja4t: ja4t.into() },
}
}
fn fires(alerts: &[tlsfp_intel::Alert], rule: Rule) -> bool {
alerts.iter().any(|alert| alert.rule == rule)
}
#[test]
fn known_bad_fires_on_malicious_fingerprint() {
let mut store = store();
let alerts = store
.detect_with(
&client_hello("10.0.0.1", 1_000_000_000, MALWARE_JA4),
&cfg(),
)
.unwrap();
let known_bad = alerts
.iter()
.find(|alert| alert.rule == Rule::KnownBad)
.expect("a malicious fingerprint should raise known_bad");
assert_eq!(known_bad.severity, AlertSeverity::High);
assert!(known_bad.detail.contains("Loader"));
assert!(known_bad.detail.contains("prevalence"));
assert_eq!(known_bad.score, Some(1.0));
}
#[test]
fn known_bad_does_not_refire_for_the_same_address_and_fingerprint() {
let mut store = store();
let config = cfg();
let first = store
.detect_with(
&client_hello("10.0.0.1", 1_000_000_000, MALWARE_JA4),
&config,
)
.unwrap();
let second = store
.detect_with(
&client_hello("10.0.0.1", 2_000_000_000, MALWARE_JA4),
&config,
)
.unwrap();
let other = store
.detect_with(
&client_hello("10.0.0.99", 3_000_000_000, MALWARE_JA4),
&config,
)
.unwrap();
assert!(fires(&first, Rule::KnownBad));
assert!(!fires(&second, Rule::KnownBad));
assert!(fires(&other, Rule::KnownBad));
}
#[test]
fn ua_mismatch_browser_claim_over_tool_handshake() {
let mut store = store();
let config = cfg();
store
.detect_with(&client_hello("10.0.0.2", 1_000_000_000, CURL_JA4), &config)
.unwrap();
let alerts = store
.detect_with(
&http_request("10.0.0.2", 2_000_000_000, BROWSER_UA),
&config,
)
.unwrap();
let mismatch = alerts
.iter()
.find(|alert| alert.rule == Rule::UaMismatch)
.expect("a browser User-Agent over a curl handshake is the headline mismatch");
assert_eq!(mismatch.severity, AlertSeverity::High);
assert!(mismatch.detail.to_lowercase().contains("curl"));
assert!(mismatch.title.to_lowercase().contains("chrome"));
}
#[test]
fn ua_mismatch_fires_from_the_tls_side_too() {
let mut store = store();
let config = cfg();
store
.detect_with(
&http_request("10.0.0.3", 1_000_000_000, BROWSER_UA),
&config,
)
.unwrap();
let alerts = store
.detect_with(&client_hello("10.0.0.3", 2_000_000_000, CURL_JA4), &config)
.unwrap();
assert!(fires(&alerts, Rule::UaMismatch));
}
#[test]
fn an_honest_tool_user_agent_does_not_mismatch() {
let mut store = store();
let config = cfg();
store
.detect_with(&client_hello("10.0.0.4", 1_000_000_000, CURL_JA4), &config)
.unwrap();
let alerts = store
.detect_with(&http_request("10.0.0.4", 2_000_000_000, CURL_UA), &config)
.unwrap();
assert!(!fires(&alerts, Rule::UaMismatch));
}
#[test]
fn a_real_browser_does_not_mismatch_its_own_handshake() {
let mut store = store();
let config = cfg();
store
.detect_with(
&client_hello("10.0.0.5", 1_000_000_000, CHROME_JA4),
&config,
)
.unwrap();
let alerts = store
.detect_with(
&http_request("10.0.0.5", 2_000_000_000, BROWSER_UA),
&config,
)
.unwrap();
assert!(!fires(&alerts, Rule::UaMismatch));
}
#[test]
fn os_mismatch_windows_claim_over_unix_stack() {
let mut store = store();
let config = cfg();
store
.detect_with(&tcp_syn("10.0.0.6", 1_000_000_000, UNIX_JA4T), &config)
.unwrap();
let alerts = store
.detect_with(
&http_request("10.0.0.6", 2_000_000_000, BROWSER_UA),
&config,
)
.unwrap();
let mismatch = alerts
.iter()
.find(|alert| alert.rule == Rule::OsMismatch)
.expect("a Windows User-Agent over a Unix SYN should raise os_mismatch");
assert_eq!(mismatch.severity, AlertSeverity::Medium);
assert!(mismatch.title.contains("windows"));
assert!(mismatch.title.contains("unix"));
}
#[test]
fn os_mismatch_fires_from_the_syn_side_too() {
let mut store = store();
let config = cfg();
store
.detect_with(
&http_request("10.0.0.7", 1_000_000_000, BROWSER_UA),
&config,
)
.unwrap();
let alerts = store
.detect_with(&tcp_syn("10.0.0.7", 2_000_000_000, UNIX_JA4T), &config)
.unwrap();
assert!(fires(&alerts, Rule::OsMismatch));
}
#[test]
fn a_consistent_operating_system_does_not_mismatch() {
let mut store = store();
let config = cfg();
store
.detect_with(&tcp_syn("10.0.0.8", 1_000_000_000, WINDOWS_JA4T), &config)
.unwrap();
let alerts = store
.detect_with(
&http_request("10.0.0.8", 2_000_000_000, BROWSER_UA),
&config,
)
.unwrap();
assert!(!fires(&alerts, Rule::OsMismatch));
}
#[test]
fn first_seen_fires_once_per_fingerprint() {
let mut store = store();
let config = cfg();
let first = store
.detect_with(
&client_hello("10.0.0.9", 1_000_000_000, CHROME_JA4),
&config,
)
.unwrap();
assert!(fires(&first, Rule::FirstSeen));
let again = store
.detect_with(
&client_hello("10.0.0.10", 2_000_000_000, CHROME_JA4),
&config,
)
.unwrap();
assert!(!fires(&again, Rule::FirstSeen));
}
#[test]
fn rotation_fires_when_one_address_cycles_fingerprints() {
let mut store = store();
let config = cfg();
let ip = "10.0.0.11";
let first = store
.detect_with(
&client_hello(ip, 1_000_000_000, "t13d1516h2_111111111111_222222222222"),
&config,
)
.unwrap();
let second = store
.detect_with(
&client_hello(ip, 2_000_000_000, "t13d1516h2_333333333333_444444444444"),
&config,
)
.unwrap();
let third = store
.detect_with(
&client_hello(ip, 3_000_000_000, "t13d1516h2_555555555555_666666666666"),
&config,
)
.unwrap();
assert!(!fires(&first, Rule::FpRotation));
assert!(!fires(&second, Rule::FpRotation));
assert!(fires(&third, Rule::FpRotation));
}
#[test]
fn monoculture_fires_when_one_fingerprint_spans_addresses() {
let mut store = store();
let config = cfg();
let ja4 = "t13d1516h2_777777777777_888888888888";
let first = store
.detect_with(&client_hello("10.1.0.1", 1_000_000_000, ja4), &config)
.unwrap();
let second = store
.detect_with(&client_hello("10.1.0.2", 2_000_000_000, ja4), &config)
.unwrap();
let third = store
.detect_with(&client_hello("10.1.0.3", 3_000_000_000, ja4), &config)
.unwrap();
assert!(!fires(&first, Rule::Monoculture));
assert!(!fires(&second, Rule::Monoculture));
assert!(fires(&third, Rule::Monoculture));
}
#[test]
fn alerts_persist_and_round_trip_through_the_store() {
let mut store = store();
store
.detect_with(
&client_hello("10.2.0.1", 1_000_000_000, MALWARE_JA4),
&cfg(),
)
.unwrap();
let recent = store.recent_alerts(10).unwrap();
assert!(
recent
.iter()
.any(|alert| alert.rule == Rule::KnownBad && alert.severity == AlertSeverity::High)
);
let counts = store.alert_counts().unwrap();
assert!(
counts
.iter()
.any(|(rule, count)| *rule == Rule::KnownBad && *count >= 1)
);
}
#[test]
fn correlation_respects_the_time_window() {
let mut store = store();
let config = DetectConfig {
window_secs: 1,
rotation_threshold: 2,
monoculture_threshold: 2,
};
store
.detect_with(&client_hello("10.3.0.1", 0, CURL_JA4), &config)
.unwrap();
let alerts = store
.detect_with(
&http_request("10.3.0.1", 10_000_000_000, BROWSER_UA),
&config,
)
.unwrap();
assert!(!fires(&alerts, Rule::UaMismatch));
}

43
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp/Cargo.toml Normal file
View File

@ -0,0 +1,43 @@
# ©AngelaMos | 2026
# Cargo.toml
[package]
name = "tlsfp"
description = "JA3/JA4 TLS fingerprinting tool with live capture, intel matching, and anomaly detection"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
authors.workspace = true
repository.workspace = true
default-run = "tlsfp"
[[bin]]
name = "tlsfp"
path = "src/main.rs"
[lints]
workspace = true
[dependencies]
tlsfp-core.workspace = true
tlsfp-intel.workspace = true
anyhow.workspace = true
tracing.workspace = true
tracing-subscriber.workspace = true
pcap-parser.workspace = true
pcap.workspace = true
tokio.workspace = true
tokio-stream.workspace = true
flume.workspace = true
axum.workspace = true
tower-http.workspace = true
tower.workspace = true
serde.workspace = true
serde_json.workspace = true
csv.workspace = true
clap.workspace = true
etherparse.workspace = true
smallvec.workspace = true
thiserror.workspace = true
rustix.workspace = true

987
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp/src/cli.rs Normal file
View File

@ -0,0 +1,987 @@
// ©AngelaMos | 2026
// cli.rs
use std::io::Write as _;
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};
use anyhow::{Context, Result};
use clap::{Parser, Subcommand};
use tracing_subscriber::EnvFilter;
use tlsfp_core::{FingerprintEvent, PcapFileSource, Pipeline, PipelineConfig, SourceError};
use crate::live::{DEFAULT_BPF_FILTER, LiveConfig, LiveSource};
use crate::report::ReportBuilder;
use crate::serve::{self, ServeConfig, Source};
use tlsfp_intel::{Alert, FpKind, IntelStore, MatchReport, MatchStrength, default_db_path};
/// How many alerts `intel alerts` shows when no count is given, and the floor a
/// zero or negative count is raised to.
const DEFAULT_ALERT_LIMIT: i64 = 50;
/// JA3/JA4 TLS fingerprinting tool.
///
/// Fingerprints TLS clients and servers from live capture or packet captures,
/// matches them against a local intelligence database, and flags anomalies such
/// as a fingerprint that disagrees with its own User-Agent.
#[derive(Debug, Parser)]
#[command(name = "tlsfp", version, about, long_about = None)]
pub struct Cli {
/// Increase log verbosity (repeat for more detail).
#[arg(short, long, global = true, action = clap::ArgAction::Count)]
pub verbose: u8,
#[command(subcommand)]
pub command: Command,
}
#[derive(Debug, Subcommand)]
pub enum Command {
/// Fingerprint every TLS and QUIC handshake in a packet capture file.
Pcap {
/// Path to a pcap or pcapng file.
path: PathBuf,
/// Emit one JSON object per event instead of readable lines.
#[arg(long)]
json: bool,
/// Match each fingerprint against the intelligence database.
#[arg(long)]
intel: bool,
/// Run the detection rules, recording observations and raising alerts.
#[arg(long)]
detect: bool,
/// Print a single forensic summary of the whole capture instead of one
/// line per handshake. When an intelligence database is present it also
/// folds in match verdicts and detection alerts.
#[arg(long)]
report: bool,
/// How many rows each ranked section of the report shows.
#[arg(long, default_value_t = crate::report::DEFAULT_TOP)]
top: usize,
/// Path to the intelligence database, defaulting to the data directory.
#[arg(long)]
db: Option<PathBuf>,
},
/// Capture live from a network interface and fingerprint in real time.
///
/// Live capture opens a raw socket, which an unprivileged user cannot
/// do. Rather than running the whole tool as root, grant the binary the
/// two capabilities libpcap needs:
///
/// sudo setcap cap_net_raw,cap_net_admin=eip "$(command -v tlsfp)"
///
/// cap_net_raw opens the capture socket, cap_net_admin covers
/// promiscuous mode, and =eip marks both permitted, inheritable, and
/// effective when the binary runs. File capabilities live on the binary
/// itself, so repeat the grant after every rebuild, and point it at
/// target/debug/tlsfp or target/release/tlsfp when running from cargo.
///
/// The default --filter keeps all TCP (TLS lives on any port, JA4T
/// wants the SYNs) plus UDP 443 for QUIC, and drops everything else in
/// the kernel. Tighten it on busy links, for example:
///
/// tlsfp live eth0 --filter "tcp port 443"
///
/// Stop with ctrl-c: the first one drains and prints final counters, a
/// second one exits immediately.
#[command(verbatim_doc_comment)]
Live {
/// Interface name, for example eth0, or any for every interface.
interface: String,
/// Emit one JSON object per event instead of readable lines.
#[arg(long)]
json: bool,
/// BPF filter compiled into the kernel before capture begins.
#[arg(long, default_value = DEFAULT_BPF_FILTER)]
filter: String,
/// Capture only traffic the interface would normally receive
/// instead of switching it to promiscuous mode.
#[arg(long)]
no_promisc: bool,
/// Match each fingerprint against the intelligence database.
#[arg(long)]
intel: bool,
/// Run the detection rules, recording observations and raising alerts.
#[arg(long)]
detect: bool,
/// Path to the intelligence database, defaulting to the data directory.
#[arg(long)]
db: Option<PathBuf>,
},
/// Serve the web dashboard and HTTP API.
///
/// The dashboard's live stream is fed one of three ways. With --replay it
/// pumps a capture file through the pipeline as a synthetic feed, paced and
/// optionally looping, which needs no privileges and is the simplest way to
/// see the dashboard alive. With --live it captures from an interface in
/// process, the same as `tlsfp live` but rendered in the browser. With
/// neither, the stream tails the database, so a separate
/// `tlsfp live --detect` writing to the same store surfaces here.
Serve {
/// Address to bind, for example 127.0.0.1:8080.
#[arg(default_value = "127.0.0.1:8080")]
bind: String,
/// Path to the intelligence database, defaulting to the data directory.
#[arg(long)]
db: Option<PathBuf>,
/// Directory of built dashboard assets to serve.
#[arg(long, default_value = "frontend/dist")]
web: PathBuf,
/// Replay this capture file as the live feed.
#[arg(long, value_name = "PCAP")]
replay: Option<PathBuf>,
/// Loop the replayed capture instead of stopping at its end.
#[arg(long = "loop")]
repeat: bool,
/// Pause between replayed events, in milliseconds.
#[arg(long, default_value_t = 400)]
interval_ms: u64,
/// Capture live from this interface as the feed instead of replaying.
#[arg(long, value_name = "IFACE")]
live: Option<String>,
/// BPF filter compiled into the kernel for live capture.
#[arg(long, default_value = DEFAULT_BPF_FILTER)]
filter: String,
/// Capture only the interface's own traffic instead of promiscuous mode.
#[arg(long)]
no_promisc: bool,
},
/// Manage the local threat intelligence database.
Intel {
#[command(subcommand)]
action: IntelCommand,
},
}
/// The subcommands under `tlsfp intel`.
#[derive(Debug, Subcommand)]
pub enum IntelCommand {
/// Create the database if needed and load the three bundled feeds.
Seed {
/// Path to the intelligence database, defaulting to the data directory.
#[arg(long)]
db: Option<PathBuf>,
},
/// Import a ja4db.com JSON export, validating every record on the way in.
Import {
/// Path to the JSON file, or - to read standard input.
path: PathBuf,
/// Path to the intelligence database, defaulting to the data directory.
#[arg(long)]
db: Option<PathBuf>,
},
/// Look up one fingerprint and print its verdict.
Lookup {
/// Fingerprint kind: ja3, ja3s, ja4, ja4s, ja4h, ja4x, ja4t, or ja4ts.
kind: String,
/// The fingerprint value to look up.
value: String,
/// Emit the report as JSON instead of readable lines.
#[arg(long)]
json: bool,
/// Path to the intelligence database, defaulting to the data directory.
#[arg(long)]
db: Option<PathBuf>,
},
/// Show what the database holds, by feed and by category.
Stats {
/// Emit the summary as JSON instead of readable lines.
#[arg(long)]
json: bool,
/// Path to the intelligence database, defaulting to the data directory.
#[arg(long)]
db: Option<PathBuf>,
},
/// Show the most recent alerts the detection rules have raised.
Alerts {
/// Emit the alerts as JSON instead of readable lines.
#[arg(long)]
json: bool,
/// How many alerts to show, newest first.
#[arg(long, default_value_t = DEFAULT_ALERT_LIMIT)]
limit: i64,
/// Path to the intelligence database, defaulting to the data directory.
#[arg(long)]
db: Option<PathBuf>,
},
}
impl Cli {
pub fn init_tracing(&self) {
let default = match self.verbose {
0 => "tlsfp=info",
1 => "tlsfp=debug",
_ => "tlsfp=trace",
};
let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(default));
tracing_subscriber::fmt()
.with_env_filter(filter)
.with_writer(std::io::stderr)
.init();
}
pub fn run(self) -> Result<()> {
match self.command {
Command::Pcap {
path,
json,
intel,
detect,
report,
top,
db,
} => run_pcap(&path, json, intel, detect, report, top, db.as_deref()),
Command::Live {
interface,
json,
filter,
no_promisc,
intel,
detect,
db,
} => run_live(
&interface,
json,
filter,
!no_promisc,
intel,
detect,
db.as_deref(),
),
Command::Serve {
bind,
db,
web,
replay,
repeat,
interval_ms,
live,
filter,
no_promisc,
} => run_serve(
bind,
db,
web,
replay,
repeat,
interval_ms,
live,
filter,
no_promisc,
),
Command::Intel { action } => action.run(),
}
}
}
impl IntelCommand {
fn run(self) -> Result<()> {
match self {
IntelCommand::Seed { db } => run_intel_seed(db),
IntelCommand::Import { path, db } => run_intel_import(&path, db),
IntelCommand::Lookup {
kind,
value,
json,
db,
} => run_intel_lookup(&kind, &value, json, db),
IntelCommand::Stats { json, db } => run_intel_stats(json, db),
IntelCommand::Alerts { json, limit, db } => run_intel_alerts(json, limit, db),
}
}
}
/// Fingerprints a capture file and prints one event per line on stdout.
///
/// The summary goes to the log rather than stdout so that piping the output
/// into a tool sees only events, while a human still learns how much of the
/// capture was readable and whether the file was cut short mid packet.
#[allow(clippy::fn_params_excessive_bools)]
fn run_pcap(
path: &Path,
json: bool,
intel: bool,
detect: bool,
report: bool,
top: usize,
db: Option<&Path>,
) -> Result<()> {
if report {
return run_pcap_report(path, json, top, db);
}
let mut source = PcapFileSource::open(path)
.with_context(|| format!("cannot open capture {}", path.display()))?;
let mut store = open_for_run(intel, detect, db)?;
let mut pipeline = Pipeline::new(PipelineConfig::default());
let stdout = std::io::stdout().lock();
let mut out = std::io::BufWriter::new(stdout);
let mut write_failure: Option<std::io::Error> = None;
pipeline.run(&mut source, |event| {
if write_failure.is_some() {
return;
}
let reports = if intel {
enrich(store.as_ref(), &event)
} else {
Vec::new()
};
let alerts = detect_event(store.as_mut(), detect, &event);
if let Err(error) = write_event(&mut out, &event, reports, alerts, json) {
write_failure = Some(error);
}
})?;
if let Some(error) = write_failure {
return Err(anyhow::Error::from(error).context("writing events to stdout"));
}
out.flush().context("flushing events to stdout")?;
let counters = pipeline.counters();
tracing::info!(
frames = counters.frames,
tcp_segments = counters.tcp_segments,
udp_datagrams = counters.udp_datagrams,
events = counters.events,
flows = counters.flows_created,
quic_initials = counters.quic_initials,
quic_decrypted = counters.quic_decrypted,
quic_version_unsupported = counters.quic_version_unsupported,
tls_handshakes = counters.tls_handshakes_fingerprinted,
unfinished_tls_streams = counters.unfinished_tls_streams,
tls_miss_rate = counters.tls_miss_rate(),
segments_dropped = counters.segments_dropped,
"capture processed"
);
if source.truncated() {
tracing::warn!("capture file ended mid packet; the tail was not read");
}
Ok(())
}
/// Reads a whole capture and prints one forensic summary instead of a stream
/// of events.
///
/// Unlike the streaming path, the report builds an in process picture of every
/// endpoint, fingerprint, and miss before printing, so it folds intelligence
/// and detection in automatically whenever a database is present rather than
/// asking for the flags. With no database it falls back to a pure fingerprint
/// inventory. The intel lookups and detection writes are the same calls the
/// streaming path makes, so the report can never disagree with a live sensor
/// pointed at the same store.
fn run_pcap_report(path: &Path, json: bool, top: usize, db: Option<&Path>) -> Result<()> {
let mut source = PcapFileSource::open(path)
.with_context(|| format!("cannot open capture {}", path.display()))?;
let mut store = open_for_report(db)?;
let enabled = store.is_some();
let mut pipeline = Pipeline::new(PipelineConfig::default());
let mut builder = ReportBuilder::new(&path.display().to_string());
let started = Instant::now();
pipeline.run(&mut source, |event| {
let reports = if enabled {
enrich(store.as_ref(), &event)
} else {
Vec::new()
};
let alerts = detect_event(store.as_mut(), enabled, &event);
builder.observe(&event, &reports, &alerts);
})?;
let elapsed = started.elapsed();
let report = builder.finish(*pipeline.counters(), source.truncated(), elapsed, top);
let stdout = std::io::stdout().lock();
let mut out = std::io::BufWriter::new(stdout);
if json {
serde_json::to_writer_pretty(&mut out, &report).context("writing report as JSON")?;
writeln!(out).context("writing report as JSON")?;
} else {
write!(out, "{}", report.render_text()).context("writing report")?;
}
out.flush().context("flushing the report to stdout")?;
Ok(())
}
/// Opens the store for a report run. A report folds in intelligence and
/// detection whenever a database is there to answer, so this opens an existing
/// database but never creates one: a forensic read of a capture should not
/// quietly leave a new database behind, and with none present the report is
/// still a complete fingerprint inventory.
fn open_for_report(db: Option<&Path>) -> Result<Option<IntelStore>> {
let path = db.map_or_else(default_db_path, Path::to_path_buf);
if path.exists() {
tracing::info!(
path = %path.display(),
"matching and recording detections against the intelligence database"
);
return Ok(Some(open_or_create(&path)?));
}
tracing::info!(
path = %path.display(),
"no intelligence database found; reporting fingerprints only, run 'tlsfp intel seed' to add verdicts"
);
Ok(None)
}
/// Builds the dashboard configuration from the command line and starts the
/// server. The live feed is a replayed capture, a live interface, or, by
/// default, the alerts an external sensor tails into the same database.
#[allow(clippy::too_many_arguments, clippy::fn_params_excessive_bools)]
fn run_serve(
bind: String,
db: Option<PathBuf>,
web: PathBuf,
replay: Option<PathBuf>,
repeat: bool,
interval_ms: u64,
live: Option<String>,
filter: String,
no_promisc: bool,
) -> Result<()> {
if replay.is_some() && live.is_some() {
anyhow::bail!("choose either --replay or --live as the live feed, not both");
}
let db = resolve_db(db);
let source = if let Some(path) = replay {
if !path.exists() {
anyhow::bail!("replay capture {} does not exist", path.display());
}
Source::Replay {
path,
looping: repeat,
interval: Duration::from_millis(interval_ms),
}
} else if let Some(interface) = live {
Source::Live {
interface,
filter,
promiscuous: !no_promisc,
}
} else {
Source::Tail
};
serve::run(ServeConfig {
bind,
db,
web,
source,
})
}
/// Captures from an interface until ctrl-c and fingerprints in real time.
///
/// The capture itself runs on a dedicated OS thread inside [`LiveSource`];
/// this function owns the tokio side of the bridge. The runtime is built
/// here rather than in main so the file path stays a plain synchronous
/// program.
#[allow(clippy::too_many_arguments, clippy::fn_params_excessive_bools)]
fn run_live(
interface: &str,
json: bool,
filter: String,
promiscuous: bool,
intel: bool,
detect: bool,
db: Option<&Path>,
) -> Result<()> {
let config = LiveConfig {
filter,
promiscuous,
};
let store = open_for_run(intel, detect, db)?;
let source = LiveSource::open(interface, &config)?;
tracing::info!(interface, filter = %config.filter, "live capture started");
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.context("building the async runtime")?;
runtime.block_on(drive_live(source, json, intel, detect, store))
}
/// Drains the live source through the same pipeline the file path uses.
///
/// Events flush per line so the stream is followable as it happens. The
/// first ctrl-c asks the capture thread to stop and lets the channel
/// drain, which makes the final counters trustworthy; a second ctrl-c
/// exits without ceremony. A closed stdout pipe is a normal way for a
/// live session to end, so it stops the capture instead of reporting an
/// error.
async fn drive_live(
mut source: LiveSource,
json: bool,
intel: bool,
detect: bool,
mut store: Option<IntelStore>,
) -> Result<()> {
let stop = source.stop_handle();
tokio::spawn(async move {
if tokio::signal::ctrl_c().await.is_ok() {
tracing::info!("ctrl-c received; draining capture");
stop.stop();
}
if tokio::signal::ctrl_c().await.is_ok() {
std::process::exit(130);
}
});
let mut pipeline = Pipeline::new(PipelineConfig::default());
let stdout = std::io::stdout().lock();
let mut out = std::io::BufWriter::new(stdout);
let mut write_failure: Option<std::io::Error> = None;
let capture_failure: Option<SourceError> = loop {
let received = source.next_frame_async().await;
let frame = match received {
Ok(Some(frame)) => frame,
Ok(None) => break None,
Err(error) => break Some(error),
};
pipeline.feed(&frame, &mut |event| {
write_live_event(
&mut out,
&mut store,
intel,
detect,
&event,
json,
&mut write_failure,
);
});
if write_failure.is_some() {
break None;
}
};
pipeline.finish();
let counters = pipeline.counters();
tracing::info!(
frames = counters.frames,
tcp_segments = counters.tcp_segments,
udp_datagrams = counters.udp_datagrams,
events = counters.events,
flows = counters.flows_created,
quic_initials = counters.quic_initials,
quic_decrypted = counters.quic_decrypted,
quic_version_unsupported = counters.quic_version_unsupported,
tls_handshakes = counters.tls_handshakes_fingerprinted,
unfinished_tls_streams = counters.unfinished_tls_streams,
tls_miss_rate = counters.tls_miss_rate(),
segments_dropped = counters.segments_dropped,
"live capture stopped"
);
if let Some(stats) = source.close() {
tracing::info!(
received = stats.received,
kernel_dropped = stats.kernel_dropped,
interface_dropped = stats.interface_dropped,
"kernel capture statistics"
);
if stats.kernel_dropped > 0 || stats.interface_dropped > 0 {
tracing::warn!(
"the kernel dropped frames; an absent fingerprint is not proof of an absent handshake"
);
}
}
match (write_failure, capture_failure) {
(Some(error), _) if error.kind() == std::io::ErrorKind::BrokenPipe => Ok(()),
(Some(error), _) => Err(anyhow::Error::from(error).context("writing events to stdout")),
(None, Some(error)) => Err(anyhow::Error::from(error).context("live capture failed")),
(None, None) => Ok(()),
}
}
/// Writes one event and flushes it immediately, recording the first
/// failure instead of panicking inside the pipeline's sink. Later calls
/// become no-ops once a write has failed.
#[allow(clippy::too_many_arguments)]
fn write_live_event(
out: &mut impl std::io::Write,
store: &mut Option<IntelStore>,
intel: bool,
detect: bool,
event: &FingerprintEvent,
json: bool,
failure: &mut Option<std::io::Error>,
) {
if failure.is_some() {
return;
}
let reports = if intel {
enrich(store.as_ref(), event)
} else {
Vec::new()
};
let alerts = detect_event(store.as_mut(), detect, event);
let result = write_event(out, event, reports, alerts, json).and_then(|()| out.flush());
if let Err(error) = result {
*failure = Some(error);
}
}
/// One event plus any intelligence that matched it, the shape both the file and
/// the live path serialise. The intel field is omitted when nothing matched, so
/// a run without enrichment produces exactly the same JSON as before.
#[derive(serde::Serialize)]
struct EnrichedEvent<'a> {
#[serde(flatten)]
event: &'a FingerprintEvent,
#[serde(rename = "intel", skip_serializing_if = "Vec::is_empty")]
reports: Vec<MatchReport>,
#[serde(rename = "alerts", skip_serializing_if = "Vec::is_empty")]
alerts: Vec<Alert>,
}
/// Looks every fingerprint in an event up against the store, returning the
/// reports that found intelligence. A lookup error degrades to no enrichment
/// with a warning rather than ending the capture.
fn enrich(store: Option<&IntelStore>, event: &FingerprintEvent) -> Vec<MatchReport> {
let Some(store) = store else {
return Vec::new();
};
match store.match_event(event) {
Ok(reports) => reports,
Err(error) => {
tracing::warn!(%error, "intelligence lookup failed for an event");
Vec::new()
}
}
}
/// Runs the detection rules for one event when detection is enabled, recording
/// the observation and any alerts. A per-event failure degrades to a warning so
/// one bad record cannot end the capture.
fn detect_event(
store: Option<&mut IntelStore>,
detect: bool,
event: &FingerprintEvent,
) -> Vec<Alert> {
if !detect {
return Vec::new();
}
let Some(store) = store else {
return Vec::new();
};
match store.detect(event) {
Ok(alerts) => alerts,
Err(error) => {
tracing::warn!(%error, "detection failed for an event");
Vec::new()
}
}
}
/// Writes one event, as JSON or as a readable line, followed by any intel.
fn write_event(
out: &mut impl std::io::Write,
event: &FingerprintEvent,
reports: Vec<MatchReport>,
alerts: Vec<Alert>,
json: bool,
) -> std::io::Result<()> {
if json {
let enriched = EnrichedEvent {
event,
reports,
alerts,
};
serde_json::to_writer(&mut *out, &enriched).map_err(std::io::Error::from)?;
writeln!(out)
} else {
writeln!(out, "{event}")?;
write_intel_lines(out, &reports)?;
write_alert_lines(out, &alerts)
}
}
/// Writes one indented pair of lines per alert beneath its event: the rule and
/// severity that name it, then the evidence that tripped it.
fn write_alert_lines(out: &mut impl std::io::Write, alerts: &[Alert]) -> std::io::Result<()> {
for alert in alerts {
writeln!(
out,
" alert [{}] {}: {}",
alert.severity.as_str(),
alert.rule.as_str(),
alert.title,
)?;
writeln!(out, " {}", alert.detail)?;
}
Ok(())
}
/// Writes one indented line per intel report beneath its event.
fn write_intel_lines(
out: &mut impl std::io::Write,
reports: &[MatchReport],
) -> std::io::Result<()> {
for report in reports {
let labels = report
.hits
.iter()
.map(|hit| {
if hit.strength == MatchStrength::Exact {
format!("{} ({})", hit.label, hit.source)
} else {
format!("{} ({}, {})", hit.label, hit.source, hit.strength.as_str())
}
})
.collect::<Vec<_>>()
.join(", ");
writeln!(
out,
" intel {}={} score={:.2} confidence={:.2} {labels}",
report.kind.as_str(),
report.verdict.as_str(),
report.threat_score,
report.confidence,
)?;
}
Ok(())
}
/// Opens the store for a capture run. Enrichment that finds no database runs
/// without annotation rather than failing. Detection needs somewhere to record
/// observations, so it creates the database, warning that known-bad matching
/// stays dark until the feeds are seeded.
fn open_for_run(intel: bool, detect: bool, db: Option<&Path>) -> Result<Option<IntelStore>> {
if !intel && !detect {
return Ok(None);
}
let path = db.map_or_else(default_db_path, Path::to_path_buf);
if !path.exists() {
if detect {
tracing::warn!(
path = %path.display(),
"no intelligence database found; creating one to record detections, run 'tlsfp intel seed' to enable known-bad matching"
);
return Ok(Some(open_or_create(&path)?));
}
tracing::warn!(
path = %path.display(),
"no intelligence database found; run 'tlsfp intel seed' first, continuing without it"
);
return Ok(None);
}
Ok(Some(open_or_create(&path)?))
}
/// Resolves the database path from the flag or the default data directory.
fn resolve_db(db: Option<PathBuf>) -> PathBuf {
db.unwrap_or_else(default_db_path)
}
/// Opens or creates a store at `path`, used by the commands allowed to build
/// the database from scratch.
fn open_or_create(path: &Path) -> Result<IntelStore> {
IntelStore::open(path)
.with_context(|| format!("opening intelligence database {}", path.display()))
}
/// Opens a store that is expected to already exist, with a hint to seed first.
fn open_existing(path: &Path) -> Result<IntelStore> {
if !path.exists() {
anyhow::bail!(
"no intelligence database at {}; run 'tlsfp intel seed' first",
path.display()
);
}
open_or_create(path)
}
fn run_intel_seed(db: Option<PathBuf>) -> Result<()> {
let path = resolve_db(db);
let mut store = open_or_create(&path)?;
let summary = store.seed_bundled()?;
println!(
"seeded {} fingerprints into {}",
summary.parsed(),
path.display()
);
for feed in &summary.feeds {
println!(
" {:<24} {} new, {} total",
feed.name, feed.inserted, feed.parsed
);
}
Ok(())
}
fn run_intel_import(path: &Path, db: Option<PathBuf>) -> Result<()> {
let json = read_input(path)?;
let mut store = open_or_create(&resolve_db(db))?;
let summary = store.import_ja4db(&json)?;
println!(
"imported {} fingerprints from {} ja4db records, {} skipped as invalid",
summary.imported, summary.records, summary.skipped
);
Ok(())
}
fn run_intel_lookup(kind: &str, value: &str, json: bool, db: Option<PathBuf>) -> Result<()> {
let kind = FpKind::from_token(&kind.to_ascii_lowercase()).with_context(|| {
format!(
"unknown fingerprint kind '{kind}'; expected ja3, ja3s, ja4, ja4s, ja4h, ja4x, ja4t, or ja4ts"
)
})?;
let store = open_existing(&resolve_db(db))?;
let report = store.match_fingerprint(kind, value)?;
if json {
let stdout = std::io::stdout().lock();
serde_json::to_writer_pretty(stdout, &report).context("writing report as JSON")?;
println!();
} else {
print_report(&report);
}
Ok(())
}
fn run_intel_stats(json: bool, db: Option<PathBuf>) -> Result<()> {
let store = open_existing(&resolve_db(db))?;
let stats = store.stats()?;
if json {
let stdout = std::io::stdout().lock();
serde_json::to_writer_pretty(stdout, &stats).context("writing stats as JSON")?;
println!();
return Ok(());
}
println!("{} fingerprints total", stats.total);
println!("feeds:");
for source in &stats.sources {
println!(
" {:<24} {:<8} {:<14} {}",
source.name,
source.kind,
source.license.as_deref().unwrap_or("-"),
source.records,
);
}
println!("by category:");
for category in &stats.by_category {
println!(" {:<10} {}", category.category, category.records);
}
Ok(())
}
fn run_intel_alerts(json: bool, limit: i64, db: Option<PathBuf>) -> Result<()> {
let store = open_existing(&resolve_db(db))?;
let limit = if limit <= 0 {
DEFAULT_ALERT_LIMIT
} else {
limit
};
let alerts = store.recent_alerts(limit)?;
if json {
let stdout = std::io::stdout().lock();
serde_json::to_writer_pretty(stdout, &alerts).context("writing alerts as JSON")?;
println!();
return Ok(());
}
if alerts.is_empty() {
println!("no alerts recorded; run a capture with --detect first");
return Ok(());
}
for alert in &alerts {
print_alert(alert);
}
let counts = store.alert_counts()?;
if !counts.is_empty() {
println!("by rule:");
for (rule, count) in &counts {
println!(" {:<14} {count}", rule.as_str());
}
}
Ok(())
}
/// Prints one alert as readable lines: a header naming when, how urgent, which
/// rule, and the subject, then the evidence beneath it.
fn print_alert(alert: &Alert) {
let secs = alert.ts_nanos / 1_000_000_000;
let millis = alert.ts_nanos % 1_000_000_000 / 1_000_000;
let target = alert.ip.as_deref().unwrap_or("-");
println!(
"{secs}.{millis:03} [{}] {} {target} {}",
alert.severity.as_str(),
alert.rule.as_str(),
alert.title,
);
println!(" {}", alert.detail);
}
/// Prints a lookup report as readable lines.
fn print_report(report: &MatchReport) {
println!(
"{} {} => {} (threat {:.2}, confidence {:.2})",
report.kind.as_str(),
report.observed,
report.verdict.as_str(),
report.threat_score,
report.confidence,
);
if report.hits.is_empty() {
println!(" no intelligence on this fingerprint");
return;
}
for hit in &report.hits {
println!(
" {:<8} {:<18} {:<22} {}",
hit.category.as_str(),
hit.strength.as_str(),
hit.label,
hit.source,
);
if let Some(reference) = &hit.reference {
println!(" {reference}");
}
}
}
/// Reads the importer's input from a file or from standard input for `-`.
fn read_input(path: &Path) -> Result<String> {
if path.as_os_str() == "-" {
let mut buffer = String::new();
std::io::Read::read_to_string(&mut std::io::stdin(), &mut buffer)
.context("reading ja4db JSON from standard input")?;
Ok(buffer)
} else {
std::fs::read_to_string(path)
.with_context(|| format!("reading ja4db JSON from {}", path.display()))
}
}

562
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp/src/live.rs Normal file
View File

@ -0,0 +1,562 @@
// ©AngelaMos | 2026
// live.rs
//! Live capture behind the same [`PacketSource`] seam the file path uses.
//!
//! libpcap blocks, tokio must not, and the two meet here. A dedicated OS
//! thread owns the activated capture handle and does nothing but pull
//! frames and push them into a bounded channel. A `std::thread` rather
//! than `spawn_blocking` because an indefinite capture loop parked on the
//! blocking pool would pin one of its slots forever. The channel is
//! bounded so a slow consumer backs pressure up into the kernel ring
//! buffer, where overload becomes a counted drop in [`CaptureStats`]
//! instead of unbounded memory growth here. The consumer side is
//! [`LiveSource`]: a synchronous [`PacketSource`] for anything that wants
//! to block, and [`LiveSource::next_frame_async`] for the tokio side of
//! the bridge. Both feed the exact pipeline the pcap file path uses.
//!
//! The capture handle runs non-blocking and the thread waits on `poll`
//! with a bounded timeout, not on libpcap's own read timeout. That read
//! timeout cannot be relied on to bound a read: on a silent interface
//! several platforms never start its timer, so a `next_packet` call would
//! park until the next frame arrives, which on an idle link can be never.
//! Owning the wait through `poll` is what lets the loop notice a stop
//! request promptly no matter how quiet the wire is.
use std::io::ErrorKind;
use std::sync::Arc;
use std::sync::OnceLock;
use std::sync::atomic::{AtomicBool, Ordering};
use std::thread::JoinHandle;
use pcap::{Active, Capture, Device};
use rustix::event::{PollFd, PollFlags, Timespec, poll};
use smallvec::SmallVec;
use thiserror::Error;
use tlsfp_core::{PacketSource, RawFrame, SourceError};
/// Kernel side prefilter applied before frames ever reach userspace.
///
/// Every TCP segment stays: TLS runs on any port, JA4T wants the SYNs, and
/// JA4H wants cleartext HTTP wherever it appears. UDP narrows to 443,
/// where QUIC lives. Everything else, ARP, ICMP, DNS, mDNS, DHCP, drops in
/// the kernel for the cost of a BPF program, not a context switch.
pub const DEFAULT_BPF_FILTER: &str = "tcp or (udp and port 443)";
/// Full frames, never clipped. A truncated segment would punch a hole in
/// TCP reassembly and silently end fingerprinting for that stream, and
/// offloads like GRO can hand the capture socket frames far beyond the
/// wire MTU.
const SNAPLEN: i32 = 65_535;
/// How long the capture thread waits in `poll` before looping back to
/// re-check the stop flag. This is the upper bound on shutdown latency on
/// an idle interface, and it is paid only as latency, never as busy work.
const POLL_TIMEOUT_MILLIS: i64 = 100;
/// Kernel capture buffer. This is the shock absorber while the channel is
/// full: bursts queue here, and only when it overflows does the kernel
/// drop, visibly, into [`CaptureStats::kernel_dropped`].
const KERNEL_BUFFER_BYTES: i32 = 4 * 1024 * 1024;
/// Frames in flight between the capture thread and the consumer. Bounded
/// so the bridge applies backpressure instead of growing without limit.
const CHANNEL_CAPACITY: usize = 512;
/// Frames at or under this size live inline in the channel message, no
/// heap allocation per frame. Sized to cover an ethernet frame at the
/// usual 1500 MTU plus VLAN tags; offload jumbos spill to the heap.
const INLINE_FRAME_BYTES: usize = 2048;
const NANOS_PER_SECOND: u64 = 1_000_000_000;
const NANOS_PER_MILLI: i64 = 1_000_000;
const NANOS_PER_MICRO: u64 = 1_000;
/// Errors that prevent a live capture from starting.
#[derive(Debug, Error)]
pub enum LiveError {
#[error("{0}")]
Open(String),
#[error("invalid BPF filter {filter:?}: {source}")]
Filter { filter: String, source: pcap::Error },
#[error("failed to start the capture thread: {0}")]
Spawn(std::io::Error),
}
/// Knobs for a live capture session.
#[derive(Debug, Clone)]
pub struct LiveConfig {
/// BPF program text compiled into the kernel before capture begins.
pub filter: String,
/// Whether to ask the interface for traffic beyond its own addresses.
pub promiscuous: bool,
}
impl Default for LiveConfig {
fn default() -> Self {
Self {
filter: DEFAULT_BPF_FILTER.to_owned(),
promiscuous: true,
}
}
}
/// Final tallies from the kernel side of a finished capture.
///
/// `kernel_dropped` is the honesty number: frames the kernel discarded
/// because the consumer fell behind. When it is nonzero, an absence of
/// fingerprints is not evidence of an absence of handshakes.
#[derive(Debug, Clone, Copy, Default)]
pub struct CaptureStats {
pub received: u64,
pub kernel_dropped: u64,
pub interface_dropped: u64,
}
/// Asks the capture thread to stop, from any thread or task.
///
/// The request is honored within one poll timeout: the capture loop
/// checks the flag between packets, and an idle interface wakes it every
/// [`POLL_TIMEOUT_MILLIS`]. Relaxed ordering is enough because the flag
/// carries no data of its own: every captured frame crosses the channel
/// and every error crosses the `OnceLock`, each of which synchronizes
/// itself, and the join in [`LiveSource::close`] is the final barrier.
#[derive(Debug, Clone)]
pub struct StopHandle(Arc<AtomicBool>);
impl StopHandle {
pub fn stop(&self) {
self.0.store(true, Ordering::Relaxed);
}
}
/// One frame copied out of the kernel buffer, owned so it can cross the
/// thread boundary.
struct CapturedFrame {
ts_nanos: u64,
data: SmallVec<[u8; INLINE_FRAME_BYTES]>,
}
/// Frames from a live interface, behind the [`PacketSource`] seam.
///
/// Construction activates the interface, installs the BPF filter, and
/// spawns the capture thread. From then on this type is the receiving end
/// of the bridge: [`PacketSource::next_frame`] blocks for synchronous
/// consumers, [`LiveSource::next_frame_async`] suspends for tokio ones,
/// and both hand out frames borrowing from an internal staging buffer
/// exactly like [`tlsfp_core::PcapFileSource`] does.
pub struct LiveSource {
receiver: flume::Receiver<CapturedFrame>,
staged: CapturedFrame,
link_type: i32,
stop: Arc<AtomicBool>,
failure: Arc<OnceLock<String>>,
failure_reported: bool,
thread: Option<JoinHandle<Option<CaptureStats>>>,
}
impl LiveSource {
/// Opens an interface by name and starts capturing immediately.
pub fn open(interface: &str, config: &LiveConfig) -> Result<Self, LiveError> {
let inactive = Capture::from_device(interface)
.map_err(|error| open_error(interface, &error))?
.promisc(config.promiscuous)
.snaplen(SNAPLEN)
.immediate_mode(true)
.buffer_size(KERNEL_BUFFER_BYTES);
let mut capture = inactive
.open()
.map_err(|error| open_error(interface, &error))?;
capture
.filter(&config.filter, true)
.map_err(|source| LiveError::Filter {
filter: config.filter.clone(),
source,
})?;
let capture = capture
.setnonblock()
.map_err(|error| open_error(interface, &error))?;
let link_type = capture.get_datalink().0;
let (sender, receiver) = flume::bounded(CHANNEL_CAPACITY);
let stop = Arc::new(AtomicBool::new(false));
let failure = Arc::new(OnceLock::new());
let thread = std::thread::Builder::new()
.name("tlsfp-capture".to_owned())
.spawn({
let stop = Arc::clone(&stop);
let failure = Arc::clone(&failure);
move || capture_loop(capture, &sender, &stop, &failure)
})
.map_err(LiveError::Spawn)?;
Ok(Self {
receiver,
staged: CapturedFrame {
ts_nanos: 0,
data: SmallVec::new(),
},
link_type,
stop,
failure,
failure_reported: false,
thread: Some(thread),
})
}
/// Returns a handle that requests a graceful stop of the capture.
pub fn stop_handle(&self) -> StopHandle {
StopHandle(Arc::clone(&self.stop))
}
/// The async twin of [`PacketSource::next_frame`] for the tokio side
/// of the bridge: suspends instead of blocking, then stages and
/// returns the frame through the same internals.
pub async fn next_frame_async(&mut self) -> Result<Option<RawFrame<'_>>, SourceError> {
let received = self.receiver.recv_async().await;
match received {
Ok(frame) => Ok(Some(self.stage(frame))),
Err(_) => self.drained(),
}
}
/// Stops the capture, waits for the thread to finish, and returns the
/// kernel's final counts. `None` when the statistics could not be
/// read.
pub fn close(mut self) -> Option<CaptureStats> {
self.stop.store(true, Ordering::Relaxed);
let thread = self.thread.take();
drop(self);
thread.and_then(|handle| handle.join().ok()).flatten()
}
fn stage(&mut self, frame: CapturedFrame) -> RawFrame<'_> {
self.staged = frame;
RawFrame {
ts_nanos: self.staged.ts_nanos,
link_type: self.link_type,
data: &self.staged.data,
}
}
/// Maps channel disconnection to the trait's vocabulary: a recorded
/// capture failure surfaces as an error exactly once, then the source
/// reads as exhausted; a clean shutdown reads as exhausted from the
/// start.
fn drained(&mut self) -> Result<Option<RawFrame<'_>>, SourceError> {
if self.failure_reported {
return Ok(None);
}
match self.failure.get() {
Some(message) => {
self.failure_reported = true;
Err(SourceError::Capture(message.clone()))
}
None => Ok(None),
}
}
}
impl PacketSource for LiveSource {
fn next_frame(&mut self) -> Result<Option<RawFrame<'_>>, SourceError> {
match self.receiver.recv() {
Ok(frame) => Ok(Some(self.stage(frame))),
Err(_) => self.drained(),
}
}
}
/// Dropping the source signals the capture thread instead of joining it.
/// The receiver field drops right after, which unblocks a capture thread
/// parked on a full channel, so the thread always exits on its own within
/// one poll timeout. [`LiveSource::close`] is the path that also waits
/// and collects statistics.
impl Drop for LiveSource {
fn drop(&mut self) {
self.stop.store(true, Ordering::Relaxed);
}
}
/// The capture thread: wait for readability, drain the ring, repeat.
///
/// Each turn polls the capture fd with a bounded timeout. A timeout is the
/// heartbeat that lets an idle interface still notice a stop request. When
/// the fd is readable the loop drains every queued packet before polling
/// again, since one readiness signal can cover a burst. A full channel
/// makes `send` block, which stops the draining, which lets the kernel
/// buffer absorb the burst and count what it sheds. The non-blocking
/// handle reports an empty ring as `TimeoutExpired`, the signal to go wait
/// again. An error or hangup latched on the fd, the mark of an interface
/// going down, is drained of any last packets and then ends the loop,
/// because `poll` keeps reporting a latched condition as ready and would
/// otherwise spin a core over an empty ring. Any other capture or poll
/// error is recorded for the consumer and ends the loop.
fn capture_loop(
mut capture: Capture<Active>,
sender: &flume::Sender<CapturedFrame>,
stop: &AtomicBool,
failure: &OnceLock<String>,
) -> Option<CaptureStats> {
let timeout = Timespec {
tv_sec: 0,
tv_nsec: POLL_TIMEOUT_MILLIS * NANOS_PER_MILLI,
};
'capture: while !stop.load(Ordering::Relaxed) {
let revents = {
let mut fds = [PollFd::new(&capture, PollFlags::IN)];
match poll(&mut fds, Some(&timeout)) {
Ok(0) | Err(rustix::io::Errno::INTR) => None,
Ok(_) => Some(fds[0].revents()),
Err(error) => {
let _ = failure.set(format!("waiting on the capture failed: {error}"));
break 'capture;
}
}
};
let Some(revents) = revents else {
continue;
};
loop {
if stop.load(Ordering::Relaxed) {
break 'capture;
}
match capture.next_packet() {
Ok(packet) => {
let frame = CapturedFrame {
ts_nanos: timeval_to_nanos(
packet.header.ts.tv_sec,
packet.header.ts.tv_usec,
),
data: SmallVec::from_slice(packet.data),
};
if sender.send(frame).is_err() {
break 'capture;
}
}
Err(pcap::Error::TimeoutExpired) => break,
Err(error) => {
let _ = failure.set(error.to_string());
break 'capture;
}
}
}
if revents.intersects(PollFlags::ERR | PollFlags::HUP | PollFlags::NVAL) {
let _ = failure.set("the capture interface reported an error or hangup".to_owned());
break 'capture;
}
}
capture.stats().ok().map(|stat| CaptureStats {
received: u64::from(stat.received),
kernel_dropped: u64::from(stat.dropped),
interface_dropped: u64::from(stat.if_dropped),
})
}
/// Converts a capture timestamp to nanoseconds since the epoch, the unit
/// [`RawFrame`] carries. Generic over the integer widths because libc's
/// timeval fields differ across platforms. Saturates rather than wraps,
/// and clamps timestamps from before the epoch to zero.
fn timeval_to_nanos(sec: impl Into<i64>, micros: impl Into<i64>) -> u64 {
let sec = u64::try_from(sec.into()).unwrap_or(0);
let micros = u64::try_from(micros.into()).unwrap_or(0);
sec.saturating_mul(NANOS_PER_SECOND)
.saturating_add(micros.saturating_mul(NANOS_PER_MICRO))
}
/// Builds the open failure message an operator can act on: the underlying
/// error, the capability grant when the cause is permissions, and the
/// interfaces libpcap can actually see.
fn open_error(interface: &str, error: &pcap::Error) -> LiveError {
use std::fmt::Write as _;
let mut message = format!("cannot open capture on {interface}: {error}");
if is_permission_denied(error) {
let binary = std::env::current_exe().map_or_else(
|_| "$(command -v tlsfp)".to_owned(),
|path| path.display().to_string(),
);
let _ = write!(
message,
"\nlive capture needs CAP_NET_RAW; grant it once per build with:\n sudo setcap cap_net_raw,cap_net_admin=eip {binary}"
);
}
if let Ok(devices) = Device::list() {
if !devices.is_empty() {
let names: Vec<String> = devices.into_iter().map(|device| device.name).collect();
let _ = write!(message, "\navailable interfaces: {}", names.join(", "));
}
}
LiveError::Open(message)
}
/// libpcap has no structured permission error; it reports activation
/// failures as text. The two phrasings Linux produces both name the
/// problem, which is enough to decide whether the setcap hint belongs in
/// the message.
fn is_permission_denied(error: &pcap::Error) -> bool {
match error {
pcap::Error::PcapError(message) => {
let lower = message.to_ascii_lowercase();
lower.contains("permission") || lower.contains("not permitted")
}
pcap::Error::IoError(kind) => *kind == ErrorKind::PermissionDenied,
_ => false,
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use std::sync::OnceLock;
use std::sync::atomic::AtomicBool;
use etherparse::PacketBuilder;
use smallvec::SmallVec;
use tlsfp_core::{PacketSource, Pipeline, PipelineConfig, SourceError};
use super::{CapturedFrame, LiveConfig, LiveSource, timeval_to_nanos};
fn source_from_parts(
receiver: flume::Receiver<CapturedFrame>,
failure: Arc<OnceLock<String>>,
) -> LiveSource {
LiveSource {
receiver,
staged: CapturedFrame {
ts_nanos: 0,
data: SmallVec::new(),
},
link_type: 1,
stop: Arc::new(AtomicBool::new(false)),
failure,
failure_reported: false,
thread: None,
}
}
fn frame(ts_nanos: u64, data: &[u8]) -> CapturedFrame {
CapturedFrame {
ts_nanos,
data: SmallVec::from_slice(data),
}
}
#[test]
fn frames_cross_the_bridge_in_order_and_end_cleanly() {
let (sender, receiver) = flume::bounded(8);
let producer = std::thread::spawn(move || {
for i in 0..100u64 {
sender.send(frame(i, &i.to_be_bytes())).unwrap();
}
});
let mut source = source_from_parts(receiver, Arc::new(OnceLock::new()));
for i in 0..100u64 {
let staged = source.next_frame().unwrap().unwrap();
assert_eq!(staged.ts_nanos, i);
assert_eq!(staged.link_type, 1);
assert_eq!(staged.data, i.to_be_bytes());
}
assert!(source.next_frame().unwrap().is_none());
producer.join().unwrap();
}
#[test]
fn capture_failure_surfaces_once_then_reads_exhausted() {
let (sender, receiver) = flume::bounded::<CapturedFrame>(1);
let failure = Arc::new(OnceLock::new());
failure.set("the interface went away".to_owned()).unwrap();
drop(sender);
let mut source = source_from_parts(receiver, failure);
let error = source.next_frame().unwrap_err();
assert!(matches!(
error,
SourceError::Capture(message) if message.contains("went away")
));
assert!(source.next_frame().unwrap().is_none());
}
#[test]
fn live_frames_feed_the_same_pipeline_as_files() {
let request = b"GET / HTTP/1.1\r\nHost: example.com\r\nAccept: */*\r\n\r\n";
let builder = PacketBuilder::ethernet2([1; 6], [2; 6])
.ipv4([10, 0, 0, 1], [10, 0, 0, 2], 64)
.tcp(40000, 80, 1000, 64240);
let mut bytes = Vec::with_capacity(builder.size(request.len()));
builder.write(&mut bytes, request).unwrap();
let (sender, receiver) = flume::bounded(8);
sender.send(frame(7, &bytes)).unwrap();
drop(sender);
let mut source = source_from_parts(receiver, Arc::new(OnceLock::new()));
let mut pipeline = Pipeline::new(PipelineConfig::default());
let mut events = Vec::new();
pipeline
.run(&mut source, |event| events.push(event))
.unwrap();
assert_eq!(events.len(), 1);
assert!(events[0].to_string().contains("http_request"));
assert_eq!(events[0].src.to_string(), "10.0.0.1:40000");
assert_eq!(pipeline.counters().tcp_segments, 1);
}
#[tokio::test]
async fn async_bridge_yields_frames_then_reads_exhausted() {
let (sender, receiver) = flume::bounded(2);
let mut source = source_from_parts(receiver, Arc::new(OnceLock::new()));
sender.send(frame(1, &[0xab])).unwrap();
drop(sender);
let staged = source.next_frame_async().await.unwrap().unwrap();
assert_eq!(staged.ts_nanos, 1);
assert_eq!(staged.data, [0xab]);
assert!(source.next_frame_async().await.unwrap().is_none());
}
#[tokio::test]
async fn async_bridge_surfaces_capture_failure() {
let (sender, receiver) = flume::bounded::<CapturedFrame>(1);
let failure = Arc::new(OnceLock::new());
failure.set("device vanished".to_owned()).unwrap();
drop(sender);
let mut source = source_from_parts(receiver, failure);
let error = source.next_frame_async().await.unwrap_err();
assert!(matches!(error, SourceError::Capture(_)));
assert!(source.next_frame_async().await.unwrap().is_none());
}
#[test]
fn timeval_conversion_scales_clamps_and_saturates() {
assert_eq!(timeval_to_nanos(1i64, 500_000i64), 1_500_000_000);
assert_eq!(timeval_to_nanos(0i64, 7i64), 7_000);
assert_eq!(timeval_to_nanos(-5i64, 0i64), 0);
assert_eq!(timeval_to_nanos(0i64, -1i64), 0);
assert_eq!(timeval_to_nanos(i64::MAX, 999_999i64), u64::MAX);
}
#[test]
#[ignore = "needs CAP_NET_RAW or root: sudo -E cargo test -p tlsfp -- --ignored"]
fn loopback_capture_sees_a_syn() {
let config = LiveConfig {
filter: "tcp and port 39999".to_owned(),
promiscuous: false,
};
let mut source = LiveSource::open("lo", &config).unwrap();
let _ = std::net::TcpStream::connect(("127.0.0.1", 39999));
let seen = source.next_frame().unwrap().expect("a SYN on loopback");
assert!(!seen.data.is_empty());
let stats = source.close().expect("final capture statistics");
assert!(stats.received >= 1);
}
}

18
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp/src/main.rs Normal file
View File

@ -0,0 +1,18 @@
// ©AngelaMos | 2026
// main.rs
mod cli;
mod live;
mod report;
mod serve;
use anyhow::Result;
use clap::Parser;
use crate::cli::Cli;
fn main() -> Result<()> {
let cli = Cli::parse();
cli.init_tracing();
cli.run()
}

727
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp/src/report.rs Normal file
View File

@ -0,0 +1,727 @@
// ©AngelaMos | 2026
// report.rs
//! The forensic batch report for a capture file.
//!
//! Where the streaming `pcap` output prints one line per handshake as it is
//! read, the report holds the whole capture in mind and answers the questions
//! an analyst asks after the fact: who spoke, what they presented, which
//! fingerprints the intelligence database recognised, what the detection rules
//! flagged, and, just as important, how much of the capture the tool could not
//! read. The last part is the honesty section: a miss rate and a throughput
//! that let a reader tell a clean capture from a clipped one before trusting an
//! absence of fingerprints.
//!
//! The aggregation is fed one event at a time so a multi gigabyte capture never
//! has to be held in memory at once; only the distinct fingerprints, endpoints,
//! and names survive between events. The [`Report`] it produces serialises
//! straight to JSON and renders to aligned text, and the builder is unit tested
//! on synthetic events with no capture file in the picture.
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
use std::fmt::Write as _;
use std::net::IpAddr;
use std::time::Duration;
use serde::Serialize;
use tlsfp_core::{Counters, FingerprintEvent, StreamEvent};
use tlsfp_intel::{Alert, MatchReport, Verdict};
/// How many rows each ranked section shows by default.
pub const DEFAULT_TOP: usize = 15;
/// One endpoint's running inventory while the capture is being read.
#[derive(Default)]
struct EndpointAgg {
events: u64,
ja4: BTreeSet<String>,
ja4s: BTreeSet<String>,
ja4t: BTreeSet<String>,
ja4h: BTreeSet<String>,
ja4x: BTreeSet<String>,
sni: BTreeSet<String>,
user_agents: BTreeSet<String>,
worst_verdict: Option<Verdict>,
alerts: u64,
}
/// One intelligence finding's running aggregate, keyed by fingerprint.
struct IntelAgg {
verdict: Verdict,
threat_score: f64,
label: String,
source: String,
endpoints: BTreeSet<String>,
}
/// Accumulates a capture into a [`Report`], one event at a time.
pub struct ReportBuilder {
source: String,
first_ts: Option<u64>,
last_ts: Option<u64>,
endpoints: HashMap<IpAddr, EndpointAgg>,
by_kind: BTreeMap<&'static str, u64>,
ja4_counts: HashMap<String, u64>,
ja4_endpoints: HashMap<String, HashSet<IpAddr>>,
sni_counts: HashMap<String, u64>,
intel: HashMap<(String, String), IntelAgg>,
alerts: Vec<Alert>,
}
impl ReportBuilder {
#[must_use]
pub fn new(source: &str) -> Self {
Self {
source: source.to_owned(),
first_ts: None,
last_ts: None,
endpoints: HashMap::new(),
by_kind: BTreeMap::new(),
ja4_counts: HashMap::new(),
ja4_endpoints: HashMap::new(),
sni_counts: HashMap::new(),
intel: HashMap::new(),
alerts: Vec::new(),
}
}
/// Folds one event, with any intelligence and alerts it produced, into the
/// running aggregates.
pub fn observe(&mut self, event: &FingerprintEvent, reports: &[MatchReport], alerts: &[Alert]) {
self.first_ts = Some(
self.first_ts
.map_or(event.ts_nanos, |t| t.min(event.ts_nanos)),
);
self.last_ts = Some(
self.last_ts
.map_or(event.ts_nanos, |t| t.max(event.ts_nanos)),
);
let ip = event.src.ip();
*self.by_kind.entry(kind_label(&event.event)).or_insert(0) += 1;
let endpoint = self.endpoints.entry(ip).or_default();
endpoint.events += 1;
match &event.event {
StreamEvent::ClientHello { ja4, sni, .. } => {
endpoint.ja4.insert(ja4.hash.clone());
*self.ja4_counts.entry(ja4.hash.clone()).or_insert(0) += 1;
self.ja4_endpoints
.entry(ja4.hash.clone())
.or_default()
.insert(ip);
if let Some(name) = sni {
endpoint.sni.insert(name.clone());
*self.sni_counts.entry(name.clone()).or_insert(0) += 1;
}
}
StreamEvent::ServerHello { ja4s, .. } => {
endpoint.ja4s.insert(ja4s.hash.clone());
}
StreamEvent::Certificate { ja4x } => {
endpoint.ja4x.insert(ja4x.clone());
}
StreamEvent::HttpRequest {
ja4h, user_agent, ..
} => {
endpoint.ja4h.insert(ja4h.hash.clone());
if let Some(agent) = user_agent {
endpoint.user_agents.insert(agent.clone());
}
}
StreamEvent::TcpSyn { ja4t } => {
endpoint.ja4t.insert(ja4t.clone());
}
StreamEvent::TcpSynAck { .. } => {}
}
for report in reports {
endpoint.worst_verdict = Some(worse(endpoint.worst_verdict, report.verdict));
let key = (report.kind.as_str().to_owned(), report.observed.clone());
let best = report
.hits
.iter()
.max_by(|a, b| a.strength.weight().total_cmp(&b.strength.weight()));
let agg = self.intel.entry(key).or_insert_with(|| IntelAgg {
verdict: report.verdict,
threat_score: report.threat_score,
label: best.map_or_else(|| "-".to_owned(), |hit| hit.label.clone()),
source: best.map_or_else(|| "-".to_owned(), |hit| hit.source.clone()),
endpoints: BTreeSet::new(),
});
agg.threat_score = agg.threat_score.max(report.threat_score);
agg.endpoints.insert(ip.to_string());
}
endpoint.alerts += u64::try_from(alerts.len()).unwrap_or(u64::MAX);
self.alerts.extend_from_slice(alerts);
}
/// Closes the books and produces the report from the running aggregates,
/// the pipeline's final counters, and the wall clock the run took.
#[must_use]
pub fn finish(
self,
counters: Counters,
truncated: bool,
elapsed: Duration,
top: usize,
) -> Report {
let duration_secs = match (self.first_ts, self.last_ts) {
(Some(first), Some(last)) => nanos_to_secs(last.saturating_sub(first)),
_ => 0.0,
};
let mut endpoints: Vec<EndpointSummary> = self
.endpoints
.into_iter()
.map(|(ip, agg)| EndpointSummary {
ip: ip.to_string(),
events: agg.events,
ja4: agg.ja4.into_iter().collect(),
ja4s: agg.ja4s.into_iter().collect(),
ja4t: agg.ja4t.into_iter().collect(),
ja4h: agg.ja4h.into_iter().collect(),
ja4x: agg.ja4x.into_iter().collect(),
sni: agg.sni.into_iter().collect(),
user_agents: agg.user_agents.into_iter().collect(),
verdict: agg.worst_verdict.map(|v| v.as_str().to_owned()),
alerts: agg.alerts,
})
.collect();
endpoints.sort_by(|a, b| b.events.cmp(&a.events).then_with(|| a.ip.cmp(&b.ip)));
let top_ja4 = rank(&self.ja4_counts, top, |value| FpCount {
value: value.clone(),
count: self.ja4_counts[value],
endpoints: self.ja4_endpoints.get(value).map_or(0, HashSet::len),
});
let top_sni = rank(&self.sni_counts, top, |value| NameCount {
name: value.clone(),
count: self.sni_counts[value],
});
let by_kind = self
.by_kind
.iter()
.map(|(kind, count)| KindCount {
kind: (*kind).to_owned(),
count: *count,
})
.collect();
let mut intel: Vec<IntelFinding> = self
.intel
.into_iter()
.map(|((kind, value), agg)| IntelFinding {
kind,
value,
verdict: agg.verdict.as_str().to_owned(),
threat_score: agg.threat_score,
label: agg.label,
source: agg.source,
endpoints: agg.endpoints.into_iter().collect(),
})
.collect();
intel.sort_by(|a, b| {
b.threat_score
.total_cmp(&a.threat_score)
.then_with(|| a.value.cmp(&b.value))
});
intel.truncate(top);
let alerts = AlertSummary::from_alerts(&self.alerts, top);
Report {
capture: CaptureSummary {
source: self.source,
frames: counters.frames,
bytes: counters.bytes,
tcp_segments: counters.tcp_segments,
udp_datagrams: counters.udp_datagrams,
events: counters.events,
flows: counters.flows_created,
duration_secs,
truncated,
},
distribution: Distribution {
by_kind,
top_ja4,
top_sni,
},
endpoints,
intel,
alerts,
coverage: Coverage::derive(&counters, elapsed),
}
}
}
/// The full forensic report, ready to serialise or render.
#[derive(Debug, Serialize)]
pub struct Report {
pub capture: CaptureSummary,
pub distribution: Distribution,
pub endpoints: Vec<EndpointSummary>,
pub intel: Vec<IntelFinding>,
pub alerts: AlertSummary,
pub coverage: Coverage,
}
#[derive(Debug, Serialize)]
pub struct CaptureSummary {
pub source: String,
pub frames: u64,
pub bytes: u64,
pub tcp_segments: u64,
pub udp_datagrams: u64,
pub events: u64,
pub flows: u64,
pub duration_secs: f64,
pub truncated: bool,
}
#[derive(Debug, Serialize)]
pub struct Distribution {
pub by_kind: Vec<KindCount>,
pub top_ja4: Vec<FpCount>,
pub top_sni: Vec<NameCount>,
}
#[derive(Debug, Serialize)]
pub struct KindCount {
pub kind: String,
pub count: u64,
}
#[derive(Debug, Serialize)]
pub struct FpCount {
pub value: String,
pub count: u64,
pub endpoints: usize,
}
#[derive(Debug, Serialize)]
pub struct NameCount {
pub name: String,
pub count: u64,
}
#[derive(Debug, Serialize)]
pub struct EndpointSummary {
pub ip: String,
pub events: u64,
pub ja4: Vec<String>,
pub ja4s: Vec<String>,
pub ja4t: Vec<String>,
pub ja4h: Vec<String>,
pub ja4x: Vec<String>,
pub sni: Vec<String>,
pub user_agents: Vec<String>,
pub verdict: Option<String>,
pub alerts: u64,
}
#[derive(Debug, Serialize)]
pub struct IntelFinding {
pub kind: String,
pub value: String,
pub verdict: String,
pub threat_score: f64,
pub label: String,
pub source: String,
pub endpoints: Vec<String>,
}
#[derive(Debug, Serialize)]
pub struct AlertSummary {
pub total: u64,
pub by_rule: Vec<RuleCount>,
pub recent: Vec<Alert>,
}
impl AlertSummary {
fn from_alerts(alerts: &[Alert], top: usize) -> Self {
let mut by_rule: BTreeMap<&'static str, u64> = BTreeMap::new();
for alert in alerts {
*by_rule.entry(alert.rule.as_str()).or_insert(0) += 1;
}
let mut counts: Vec<RuleCount> = by_rule
.into_iter()
.map(|(rule, count)| RuleCount {
rule: rule.to_owned(),
count,
})
.collect();
counts.sort_by(|a, b| b.count.cmp(&a.count).then_with(|| a.rule.cmp(&b.rule)));
let recent = alerts.iter().rev().take(top).cloned().collect();
Self {
total: u64::try_from(alerts.len()).unwrap_or(u64::MAX),
by_rule: counts,
recent,
}
}
}
#[derive(Debug, Serialize)]
pub struct RuleCount {
pub rule: String,
pub count: u64,
}
#[derive(Debug, Serialize)]
pub struct Coverage {
pub counters: Counters,
pub tls_miss_rate: f64,
pub quic_decrypt_rate: f64,
pub events_per_sec: f64,
pub frames_per_sec: f64,
pub megabits_per_sec: f64,
}
impl Coverage {
#[allow(clippy::cast_precision_loss)]
fn derive(counters: &Counters, elapsed: Duration) -> Self {
let secs = elapsed.as_secs_f64();
let per_sec = |n: u64| if secs > 0.0 { n as f64 / secs } else { 0.0 };
let quic_decrypt_rate = if counters.quic_initials == 0 {
0.0
} else {
counters.quic_decrypted as f64 / counters.quic_initials as f64
};
Self {
counters: *counters,
tls_miss_rate: counters.tls_miss_rate(),
quic_decrypt_rate,
events_per_sec: per_sec(counters.events),
frames_per_sec: per_sec(counters.frames),
megabits_per_sec: if secs > 0.0 {
(counters.bytes as f64 * 8.0) / 1_000_000.0 / secs
} else {
0.0
},
}
}
}
impl Report {
/// Renders the report as aligned, sectioned text for a terminal reader.
#[must_use]
pub fn render_text(&self) -> String {
let mut out = String::new();
self.render_capture(&mut out);
self.render_distribution(&mut out);
self.render_endpoints(&mut out);
self.render_intel(&mut out);
self.render_alerts(&mut out);
self.render_coverage(&mut out);
out
}
fn render_capture(&self, out: &mut String) {
let c = &self.capture;
let _ = writeln!(out, "== capture ==");
let _ = writeln!(out, " source {}", c.source);
let _ = writeln!(out, " frames {}", c.frames);
let _ = writeln!(out, " bytes {}", c.bytes);
let _ = writeln!(out, " tcp segments {}", c.tcp_segments);
let _ = writeln!(out, " udp datagrams {}", c.udp_datagrams);
let _ = writeln!(out, " flows {}", c.flows);
let _ = writeln!(out, " fingerprints {}", c.events);
let _ = writeln!(out, " capture span {:.3}s", c.duration_secs);
if c.truncated {
let _ = writeln!(out, " truncated yes (file ended mid packet)");
}
}
fn render_distribution(&self, out: &mut String) {
let d = &self.distribution;
let _ = writeln!(out, "\n== fingerprints by kind ==");
if d.by_kind.is_empty() {
let _ = writeln!(out, " none");
}
for row in &d.by_kind {
let _ = writeln!(out, " {:<16} {}", row.kind, row.count);
}
let _ = writeln!(out, "\n== top client ja4 ==");
if d.top_ja4.is_empty() {
let _ = writeln!(out, " none");
}
for row in &d.top_ja4 {
let _ = writeln!(
out,
" {:<40} {:>5} across {} endpoint(s)",
row.value, row.count, row.endpoints
);
}
if !d.top_sni.is_empty() {
let _ = writeln!(out, "\n== top server names ==");
for row in &d.top_sni {
let _ = writeln!(out, " {:<40} {:>5}", row.name, row.count);
}
}
}
fn render_endpoints(&self, out: &mut String) {
let _ = writeln!(out, "\n== endpoints ==");
if self.endpoints.is_empty() {
let _ = writeln!(out, " none");
}
for endpoint in &self.endpoints {
let verdict = endpoint.verdict.as_deref().unwrap_or("-");
let _ = writeln!(
out,
" {} ({} event(s), verdict {}, {} alert(s))",
endpoint.ip, endpoint.events, verdict, endpoint.alerts
);
write_list(out, "ja4", &endpoint.ja4);
write_list(out, "ja4s", &endpoint.ja4s);
write_list(out, "ja4t", &endpoint.ja4t);
write_list(out, "ja4h", &endpoint.ja4h);
write_list(out, "ja4x", &endpoint.ja4x);
write_list(out, "sni", &endpoint.sni);
write_list(out, "ua", &endpoint.user_agents);
}
}
fn render_intel(&self, out: &mut String) {
let _ = writeln!(out, "\n== intelligence ==");
if self.intel.is_empty() {
let _ = writeln!(out, " no fingerprints matched the database");
return;
}
for finding in &self.intel {
let _ = writeln!(
out,
" [{}] {} {} (threat {:.2})",
finding.verdict, finding.kind, finding.value, finding.threat_score
);
let _ = writeln!(
out,
" {} ({}), {} endpoint(s)",
finding.label,
finding.source,
finding.endpoints.len()
);
}
}
fn render_alerts(&self, out: &mut String) {
let _ = writeln!(out, "\n== alerts ==");
if self.alerts.total == 0 {
let _ = writeln!(out, " none raised");
return;
}
let _ = writeln!(out, " {} total", self.alerts.total);
for row in &self.alerts.by_rule {
let _ = writeln!(out, " {:<14} {}", row.rule, row.count);
}
let _ = writeln!(out, " most recent:");
for alert in &self.alerts.recent {
let target = alert.ip.as_deref().unwrap_or("-");
let _ = writeln!(
out,
" [{}] {} {} {}",
alert.severity.as_str(),
alert.rule.as_str(),
target,
alert.title
);
}
}
fn render_coverage(&self, out: &mut String) {
let c = &self.coverage;
let _ = writeln!(out, "\n== coverage ==");
let _ = writeln!(
out,
" tls miss rate {:.1}% ({} read, {} clipped)",
c.tls_miss_rate * 100.0,
c.counters.tls_handshakes_fingerprinted,
c.counters.unfinished_tls_streams
);
let _ = writeln!(out, " streams capped {}", c.counters.streams_capped);
let _ = writeln!(out, " segments dropped {}", c.counters.segments_dropped);
let _ = writeln!(
out,
" quic initials {} ({} decrypted, {:.0}%, {} unsupported version)",
c.counters.quic_initials,
c.counters.quic_decrypted,
c.quic_decrypt_rate * 100.0,
c.counters.quic_version_unsupported
);
let _ = writeln!(
out,
" throughput {:.0} fp/s, {:.0} frames/s, {:.1} Mb/s",
c.events_per_sec, c.frames_per_sec, c.megabits_per_sec
);
}
}
/// Writes one labelled line listing a set's values, skipping an empty set.
fn write_list(out: &mut String, label: &str, values: &[String]) {
if values.is_empty() {
return;
}
let _ = writeln!(out, " {:<5} {}", label, values.join(", "));
}
/// The snake case name a stream event carries, matching its serialised tag.
fn kind_label(event: &StreamEvent) -> &'static str {
match event {
StreamEvent::ClientHello { .. } => "client_hello",
StreamEvent::ServerHello { .. } => "server_hello",
StreamEvent::Certificate { .. } => "certificate",
StreamEvent::HttpRequest { .. } => "http_request",
StreamEvent::TcpSyn { .. } => "tcp_syn",
StreamEvent::TcpSynAck { .. } => "tcp_syn_ack",
}
}
/// Ranks the keys of a count map by count descending then key ascending,
/// keeping the top `n` and mapping each through `build`.
fn rank<T>(counts: &HashMap<String, u64>, n: usize, build: impl Fn(&String) -> T) -> Vec<T> {
let mut keys: Vec<&String> = counts.keys().collect();
keys.sort_by(|a, b| counts[*b].cmp(&counts[*a]).then_with(|| a.cmp(b)));
keys.into_iter().take(n).map(build).collect()
}
/// The more alarming of two verdicts, used to fold an endpoint's worst case.
fn worse(current: Option<Verdict>, next: Verdict) -> Verdict {
let rank = |v: Verdict| match v {
Verdict::Malicious => 3,
Verdict::Suspicious => 2,
Verdict::Unknown => 1,
Verdict::Benign => 0,
};
match current {
Some(existing) if rank(existing) >= rank(next) => existing,
_ => next,
}
}
/// Converts whole nanoseconds to fractional seconds without the precision loss
/// lint, accepting that a capture span beyond `u32::MAX` seconds is unreachable.
#[allow(clippy::cast_precision_loss)]
fn nanos_to_secs(nanos: u64) -> f64 {
nanos as f64 / 1_000_000_000.0
}
#[cfg(test)]
mod tests {
use std::time::Duration;
use tlsfp_core::{Counters, FingerprintEvent, Ja3, Ja4Family, StreamEvent};
use tlsfp_intel::{Alert, AlertSeverity, FpKind, IntelHit, MatchReport, MatchStrength, Rule};
use super::ReportBuilder;
fn client_hello(ip: &str, ja4: &str, sni: &str) -> FingerprintEvent {
FingerprintEvent {
ts_nanos: 1_000_000_000,
src: format!("{ip}:40000").parse().unwrap(),
dst: "10.0.0.2:443".parse().unwrap(),
event: StreamEvent::ClientHello {
ja3: Ja3::from_digest([0u8; 16]),
ja3_raw: String::new(),
ja4: Ja4Family::new(ja4.to_owned(), String::new()),
sni: Some(sni.to_owned()),
alpn: None,
},
}
}
#[test]
fn aggregates_endpoints_and_distribution() {
let mut builder = ReportBuilder::new("test.pcap");
builder.observe(
&client_hello("10.0.0.1", "t13d1516h2_aaaa_bbbb", "a.example"),
&[],
&[],
);
builder.observe(
&client_hello("10.0.0.1", "t13d1516h2_aaaa_bbbb", "b.example"),
&[],
&[],
);
builder.observe(
&client_hello("10.0.0.9", "t13d1516h2_aaaa_bbbb", "a.example"),
&[],
&[],
);
let report = builder.finish(Counters::default(), false, Duration::from_millis(10), 15);
assert_eq!(report.endpoints.len(), 2);
assert_eq!(report.endpoints[0].ip, "10.0.0.1");
assert_eq!(report.endpoints[0].events, 2);
assert_eq!(report.distribution.top_ja4.len(), 1);
assert_eq!(report.distribution.top_ja4[0].count, 3);
assert_eq!(report.distribution.top_ja4[0].endpoints, 2);
assert_eq!(report.distribution.top_sni[0].count, 2);
}
#[test]
fn folds_intel_and_alerts_into_worst_verdict() {
let mut builder = ReportBuilder::new("test.pcap");
let event = client_hello("10.0.0.1", "t13d1516h2_aaaa_bbbb", "evil.example");
let report = MatchReport::from_hits(
FpKind::Ja4,
"t13d1516h2_aaaa_bbbb".to_owned(),
vec![IntelHit {
kind: FpKind::Ja4,
value: "t13d1516h2_aaaa_bbbb".to_owned(),
label: "Cobalt Strike".to_owned(),
category: tlsfp_intel::Category::Malware,
source: "test-feed".to_owned(),
reference: None,
strength: MatchStrength::Exact,
}],
);
let alert = Alert {
ts_nanos: 1_000_000_000,
rule: Rule::KnownBad,
severity: AlertSeverity::Critical,
ip: Some("10.0.0.1".to_owned()),
fp_kind: Some(FpKind::Ja4),
fp_value: Some("t13d1516h2_aaaa_bbbb".to_owned()),
title: "known bad fingerprint".to_owned(),
detail: "matched test-feed".to_owned(),
score: Some(1.0),
};
builder.observe(&event, &[report], &[alert]);
let built = builder.finish(Counters::default(), false, Duration::from_millis(5), 15);
assert_eq!(built.endpoints[0].verdict.as_deref(), Some("malicious"));
assert_eq!(built.endpoints[0].alerts, 1);
assert_eq!(built.intel.len(), 1);
assert_eq!(built.intel[0].label, "Cobalt Strike");
assert_eq!(built.alerts.total, 1);
assert_eq!(built.alerts.by_rule[0].rule, "known_bad");
let text = built.render_text();
assert!(text.contains("== coverage =="));
assert!(text.contains("Cobalt Strike"));
}
#[test]
fn miss_rate_and_throughput_reach_the_report() {
let counters = Counters {
frames: 1000,
bytes: 1_000_000,
events: 200,
tls_handshakes_fingerprinted: 3,
unfinished_tls_streams: 1,
..Counters::default()
};
let builder = ReportBuilder::new("test.pcap");
let report = builder.finish(counters, true, Duration::from_secs(1), 15);
assert!((report.coverage.tls_miss_rate - 0.25).abs() < 1e-9);
assert!((report.coverage.events_per_sec - 200.0).abs() < 1e-9);
assert!(report.capture.truncated);
assert!(report.render_text().contains("truncated"));
}
}

671
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp/src/serve.rs Normal file
View File

@ -0,0 +1,671 @@
// ©AngelaMos | 2026
// serve.rs
//! The web dashboard and HTTP API.
//!
//! This is the one place in the tool that needs concurrent access to the
//! intelligence store, and the one place the store's deliberate synchrony has
//! to be reconciled with an async runtime. The reconciliation is small on
//! purpose: the store stays a plain blocking handle, and every request that
//! touches it does so inside [`tokio::task::spawn_blocking`], so a slow query
//! parks a blocking thread rather than stalling the reactor. A single
//! `Mutex` serialises access, which is the right shape for a dashboard whose
//! queries are short and whose writers, when present, are one capture loop.
//!
//! The live stream is a `broadcast` channel. Whatever feeds the dashboard, a
//! replayed capture, a live interface, or new alerts tailed from the database,
//! publishes a pre-serialised line onto the channel, and every connected
//! browser's Server-Sent Events response subscribes to it. The stream route is
//! kept off the compression layer on purpose: a compressor buffers to find
//! runs worth packing, and buffering is the one thing a live event stream
//! cannot tolerate.
use std::convert::Infallible;
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex, MutexGuard};
use std::thread::JoinHandle;
use std::time::Duration;
use anyhow::{Context, Result};
use axum::Json;
use axum::Router;
use axum::extract::{Query, State};
use axum::http::{StatusCode, header};
use axum::response::sse::{Event, KeepAlive, Sse};
use axum::response::{IntoResponse, Response};
use axum::routing::get;
use serde::{Deserialize, Serialize};
use tokio::sync::broadcast;
use tokio_stream::StreamExt;
use tokio_stream::wrappers::BroadcastStream;
use tower_http::compression::CompressionLayer;
use tower_http::services::{ServeDir, ServeFile};
use tower_http::trace::TraceLayer;
use tlsfp_core::{FingerprintEvent, PcapFileSource, Pipeline, PipelineConfig};
use tlsfp_intel::{Alert, CatalogEntry, FpKind, IntelStore, MatchReport, Rule, Stats};
use crate::live::{LiveConfig, LiveSource, StopHandle};
/// How many serialised live lines the broadcast channel holds before the
/// slowest subscriber starts missing the oldest. A browser that lags past this
/// skips the gap rather than stalling every other client.
const STREAM_BUFFER: usize = 1024;
/// How often the keep-alive comment is sent on an idle stream, below the
/// common sixty second proxy idle timeout so a quiet link stays open.
const KEEPALIVE_SECS: u64 = 15;
/// How often the database tail poller checks for alerts raised by an external
/// sensor writing to the same store.
const TAIL_POLL_MILLIS: u64 = 1000;
/// The largest export or alert page served in one response, a guard against a
/// query string asking the store for everything at once.
const MAX_PAGE: i64 = 100_000;
/// The default alert page when a request does not ask for a size.
const DEFAULT_PAGE: i64 = 200;
/// The largest batch the database tail drains per poll. It carries the last id
/// forward between ticks, so a backlog catches up over several polls rather
/// than blocking one task on a single huge read.
const TAIL_PAGE: i64 = 256;
/// Where the live events shown on the dashboard come from.
pub enum Source {
/// Nothing in process; the stream tails the database for alerts an external
/// `tlsfp live --detect` writes to the same store.
Tail,
/// Replay a capture file as a synthetic live feed, optionally looping, with
/// a pause between events so the stream is watchable.
Replay {
path: PathBuf,
looping: bool,
interval: Duration,
},
/// Capture live from an interface in process, detecting and broadcasting as
/// it goes.
Live {
interface: String,
filter: String,
promiscuous: bool,
},
}
/// Everything the server needs to start.
pub struct ServeConfig {
pub bind: String,
pub db: PathBuf,
pub web: PathBuf,
pub source: Source,
}
/// Shared state every handler reads through. The store is the read side of the
/// database behind a mutex; the sender is the live channel handlers subscribe
/// to.
struct AppState {
store: Mutex<IntelStore>,
tx: broadcast::Sender<Arc<str>>,
}
impl AppState {
/// Locks the store, recovering the guard if a previous handler panicked
/// while holding it, since a poisoned read connection is still usable.
fn store(&self) -> MutexGuard<'_, IntelStore> {
self.store
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
}
type Shared = Arc<AppState>;
/// One message on the live stream: either a freshly fingerprinted flow with its
/// intelligence and any alerts it raised, or a standalone alert the database
/// tail surfaced. Serialised once by the producer and sent as an opaque line,
/// so a thousand subscribers cost one serialisation, not a thousand.
#[derive(Serialize)]
#[serde(tag = "type", rename_all = "snake_case")]
enum LiveMessage {
Flow {
event: FingerprintEvent,
#[serde(skip_serializing_if = "Vec::is_empty")]
intel: Vec<MatchReport>,
#[serde(skip_serializing_if = "Vec::is_empty")]
alerts: Vec<Alert>,
},
Alert {
alert: Alert,
},
}
/// Runs the dashboard until interrupted. Builds its own runtime so the rest of
/// the program stays a synchronous command line tool, the same split the live
/// capture path uses.
pub fn run(config: ServeConfig) -> Result<()> {
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.context("building the async runtime")?;
runtime.block_on(serve(config))
}
async fn serve(config: ServeConfig) -> Result<()> {
let store = IntelStore::open(&config.db)
.with_context(|| format!("opening intelligence database {}", config.db.display()))?;
if store.latest_alert_id().unwrap_or(0) == 0 {
tracing::info!(
"the store holds no alerts yet; run a capture with --detect, or start serve with --replay or --live to populate the stream"
);
}
let (tx, _) = broadcast::channel::<Arc<str>>(STREAM_BUFFER);
let state: Shared = Arc::new(AppState {
store: Mutex::new(store),
tx: tx.clone(),
});
let shutdown = Arc::new(AtomicBool::new(false));
let mut capture = spawn_source(&config, tx, Arc::clone(&shutdown), Arc::clone(&state))?;
let app = router(&config.web, Arc::clone(&state));
let listener = tokio::net::TcpListener::bind(&config.bind)
.await
.with_context(|| format!("binding {}", config.bind))?;
let addr = listener.local_addr().context("reading the bound address")?;
tracing::info!(%addr, web = %config.web.display(), "dashboard listening on http://{addr}");
let result = axum::serve(listener, app.into_make_service())
.with_graceful_shutdown(shutdown_signal())
.await
.context("serving the dashboard");
shutdown.store(true, Ordering::Relaxed);
capture.stop();
result
}
/// Builds the route table. The data API and the static files are compressed;
/// the event stream is merged in afterwards so the compression layer never
/// touches it.
fn router(web: &std::path::Path, state: Shared) -> Router {
let data_api = Router::new()
.route("/stats", get(stats))
.route("/alerts", get(alerts))
.route("/search", get(search))
.route("/export", get(export))
.route("/health", get(health))
.layer(CompressionLayer::new());
let stream_api = Router::new().route("/stream", get(stream));
let api = data_api.merge(stream_api).with_state(state);
let index = web.join("index.html");
let static_files = ServeDir::new(web)
.append_index_html_on_directories(true)
.fallback(ServeFile::new(index));
Router::new()
.nest("/api", api)
.fallback_service(static_files)
.layer(TraceLayer::new_for_http())
}
/// A count of alerts attributed to one rule, the shape the distribution chart
/// reads.
#[derive(Serialize)]
struct RuleCount {
rule: Rule,
count: i64,
}
/// The summary the dashboard header and distribution chart draw from: what the
/// intelligence store holds, and how the alerts raised so far break down by
/// rule.
#[derive(Serialize)]
struct StatsResponse {
intel: Stats,
alerts_by_rule: Vec<RuleCount>,
alert_total: i64,
}
async fn stats(State(state): State<Shared>) -> Result<Json<StatsResponse>, ApiError> {
let response = tokio::task::spawn_blocking(move || -> Result<StatsResponse> {
let store = state.store();
let intel = store.stats()?;
let counts = store.alert_counts()?;
let alert_total = counts.iter().map(|(_, count)| count).sum();
let alerts_by_rule = counts
.into_iter()
.map(|(rule, count)| RuleCount { rule, count })
.collect();
Ok(StatsResponse {
intel,
alerts_by_rule,
alert_total,
})
})
.await??;
Ok(Json(response))
}
#[derive(Deserialize)]
struct AlertsQuery {
limit: Option<i64>,
}
async fn alerts(
State(state): State<Shared>,
Query(query): Query<AlertsQuery>,
) -> Result<Json<Vec<Alert>>, ApiError> {
let limit = page_size(query.limit);
let alerts = tokio::task::spawn_blocking(move || state.store().recent_alerts(limit)).await??;
Ok(Json(alerts))
}
#[derive(Deserialize)]
struct SearchQuery {
q: Option<String>,
kind: Option<String>,
limit: Option<i64>,
}
async fn search(
State(state): State<Shared>,
Query(query): Query<SearchQuery>,
) -> Result<Json<Vec<CatalogEntry>>, ApiError> {
let kind = match query.kind.as_deref().filter(|token| !token.is_empty()) {
Some(token) => Some(FpKind::from_token(&token.to_ascii_lowercase()).ok_or_else(|| {
ApiError::bad_request(format!(
"unknown fingerprint kind '{token}'; expected ja3, ja3s, ja4, ja4s, ja4h, ja4x, ja4t, or ja4ts"
))
})?),
None => None,
};
let needle = query.q.unwrap_or_default();
let limit = page_size(query.limit);
let entries =
tokio::task::spawn_blocking(move || state.store().search(&needle, kind, limit)).await??;
Ok(Json(entries))
}
#[derive(Deserialize)]
struct ExportQuery {
format: Option<String>,
limit: Option<i64>,
}
async fn export(
State(state): State<Shared>,
Query(query): Query<ExportQuery>,
) -> Result<Response, ApiError> {
let limit = query.limit.unwrap_or(MAX_PAGE).clamp(1, MAX_PAGE);
let csv = matches!(query.format.as_deref(), Some("csv"));
let alerts = tokio::task::spawn_blocking(move || state.store().recent_alerts(limit)).await??;
if csv {
let body = alerts_to_csv(&alerts)?;
Ok((
[
(header::CONTENT_TYPE, "text/csv; charset=utf-8"),
(
header::CONTENT_DISPOSITION,
"attachment; filename=\"tlsfp-alerts.csv\"",
),
],
body,
)
.into_response())
} else {
let body = serde_json::to_vec_pretty(&alerts).context("serialising alerts as JSON")?;
Ok((
[
(header::CONTENT_TYPE, "application/json"),
(
header::CONTENT_DISPOSITION,
"attachment; filename=\"tlsfp-alerts.json\"",
),
],
body,
)
.into_response())
}
}
async fn health() -> impl IntoResponse {
Json(serde_json::json!({ "status": "ok" }))
}
/// The Server-Sent Events stream. Subscribes to the broadcast channel and
/// forwards every line as an event, dropping the gap when a lagging client
/// falls behind rather than tearing its connection down.
async fn stream(
State(state): State<Shared>,
) -> Sse<impl tokio_stream::Stream<Item = Result<Event, Infallible>>> {
let receiver = state.tx.subscribe();
let events = BroadcastStream::new(receiver).filter_map(|item| match item {
Ok(line) => Some(Ok(Event::default().data(line.as_ref()))),
Err(_lagged) => None,
});
Sse::new(events).keep_alive(
KeepAlive::new()
.interval(Duration::from_secs(KEEPALIVE_SECS))
.text("keep-alive"),
)
}
/// Clamps a requested page size into a sane range, defaulting an absent or
/// non-positive request to a readable page.
fn page_size(requested: Option<i64>) -> i64 {
match requested {
Some(value) if value > 0 => value.min(MAX_PAGE),
_ => DEFAULT_PAGE,
}
}
/// Renders alerts as CSV, quoting through the csv writer so a comma or newline
/// inside a detail string can never break the row structure.
fn alerts_to_csv(alerts: &[Alert]) -> Result<Vec<u8>> {
let mut writer = csv::Writer::from_writer(Vec::new());
writer.write_record([
"ts_nanos", "rule", "severity", "ip", "fp_kind", "fp_value", "title", "detail", "score",
])?;
for alert in alerts {
writer.write_record([
alert.ts_nanos.to_string(),
alert.rule.as_str().to_string(),
alert.severity.as_str().to_string(),
alert.ip.clone().unwrap_or_default(),
alert
.fp_kind
.map(|kind| kind.as_str().to_string())
.unwrap_or_default(),
alert.fp_value.clone().unwrap_or_default(),
alert.title.clone(),
alert.detail.clone(),
alert.score.map(|s| format!("{s:.4}")).unwrap_or_default(),
])?;
}
writer.into_inner().context("finishing the CSV export")
}
/// A handle to whatever is feeding the live stream, so the server can stop it
/// on shutdown. A replay or tail loop watches the shared flag; a live capture
/// also needs its blocking read woken, which is what the stop handle does.
struct Capture {
shutdown: Arc<AtomicBool>,
live_stop: Option<StopHandle>,
handle: Option<JoinHandle<()>>,
}
impl Capture {
fn stop(&mut self) {
self.shutdown.store(true, Ordering::Relaxed);
if let Some(stop) = &self.live_stop {
stop.stop();
}
if let Some(handle) = self.handle.take() {
let _ = handle.join();
}
}
}
/// Starts whatever produces the live stream for this run and returns a handle
/// to stop it. The replay and live producers each own a private write
/// connection to the same database, so detection persists while the API reads
/// through its own connection; SQLite's write-ahead log lets the two coexist.
fn spawn_source(
config: &ServeConfig,
tx: broadcast::Sender<Arc<str>>,
shutdown: Arc<AtomicBool>,
state: Shared,
) -> Result<Capture> {
match &config.source {
Source::Tail => {
spawn_tail(state, tx, Arc::clone(&shutdown));
Ok(Capture {
shutdown,
live_stop: None,
handle: None,
})
}
Source::Replay {
path,
looping,
interval,
} => {
let path = path.clone();
let db = config.db.clone();
let looping = *looping;
let interval = *interval;
let stop = Arc::clone(&shutdown);
let handle = std::thread::Builder::new()
.name("tlsfp-replay".to_owned())
.spawn(move || replay_loop(&path, &db, &tx, &stop, looping, interval))
.context("starting the replay thread")?;
Ok(Capture {
shutdown,
live_stop: None,
handle: Some(handle),
})
}
Source::Live {
interface,
filter,
promiscuous,
} => {
let live_config = LiveConfig {
filter: filter.clone(),
promiscuous: *promiscuous,
};
let source = LiveSource::open(interface, &live_config)
.with_context(|| format!("opening live capture on {interface}"))?;
let live_stop = source.stop_handle();
let db = config.db.clone();
let stop = Arc::clone(&shutdown);
let interface = interface.clone();
tracing::info!(interface, filter = %live_config.filter, "serving live capture");
let handle = std::thread::Builder::new()
.name("tlsfp-live".to_owned())
.spawn(move || live_loop(source, &db, &tx, &stop))
.context("starting the live capture thread")?;
Ok(Capture {
shutdown,
live_stop: Some(live_stop),
handle: Some(handle),
})
}
}
}
/// Replays a capture file as a synthetic live feed, pausing between events so
/// the stream reads at a human pace and looping when asked.
fn replay_loop(
path: &std::path::Path,
db: &std::path::Path,
tx: &broadcast::Sender<Arc<str>>,
shutdown: &AtomicBool,
looping: bool,
interval: Duration,
) {
let mut store = match IntelStore::open(db) {
Ok(store) => store,
Err(error) => {
tracing::error!(%error, "replay: cannot open the intelligence database");
return;
}
};
tracing::info!(path = %path.display(), looping, "serving replayed capture");
loop {
let mut source = match PcapFileSource::open(path) {
Ok(source) => source,
Err(error) => {
tracing::error!(%error, path = %path.display(), "replay: cannot open the capture");
return;
}
};
let mut pipeline = Pipeline::new(PipelineConfig::default());
let run = pipeline.run(&mut source, |event| {
if shutdown.load(Ordering::Relaxed) {
return;
}
broadcast_flow(&mut store, tx, event);
if !interval.is_zero() {
std::thread::sleep(interval);
}
});
if let Err(error) = run {
tracing::error!(%error, "replay: reading the capture failed");
return;
}
if shutdown.load(Ordering::Relaxed) || !looping {
return;
}
}
}
/// Drives a live capture through the same pipeline the file path uses, on this
/// dedicated thread, detecting and broadcasting until the stop handle wakes the
/// blocking read.
fn live_loop(
mut source: LiveSource,
db: &std::path::Path,
tx: &broadcast::Sender<Arc<str>>,
shutdown: &AtomicBool,
) {
let mut store = match IntelStore::open(db) {
Ok(store) => store,
Err(error) => {
tracing::error!(%error, "live: cannot open the intelligence database");
return;
}
};
let mut pipeline = Pipeline::new(PipelineConfig::default());
let run = pipeline.run(&mut source, |event| {
if shutdown.load(Ordering::Relaxed) {
return;
}
broadcast_flow(&mut store, tx, event);
});
if let Err(error) = run {
tracing::warn!(%error, "live capture ended");
}
}
/// Tails the database for alerts written by an external sensor and broadcasts
/// each new one. Used when nothing is captured in process: a `tlsfp live
/// --detect` writing to the same store shows up on the dashboard without the
/// server holding a capture socket of its own.
fn spawn_tail(state: Shared, tx: broadcast::Sender<Arc<str>>, shutdown: Arc<AtomicBool>) {
tokio::spawn(async move {
let start = Arc::clone(&state);
let mut last_id =
tokio::task::spawn_blocking(move || start.store().latest_alert_id().unwrap_or(0))
.await
.unwrap_or(0);
let mut ticker = tokio::time::interval(Duration::from_millis(TAIL_POLL_MILLIS));
loop {
ticker.tick().await;
if shutdown.load(Ordering::Relaxed) {
return;
}
let reader = Arc::clone(&state);
let since = last_id;
let fresh = tokio::task::spawn_blocking(move || {
reader
.store()
.alerts_since(since, TAIL_PAGE)
.unwrap_or_default()
})
.await
.unwrap_or_default();
for (id, alert) in fresh {
last_id = last_id.max(id);
let message = LiveMessage::Alert { alert };
if let Ok(line) = serde_json::to_string(&message) {
let _ = tx.send(Arc::from(line));
}
}
}
});
}
/// Enriches one event, runs the detection rules over it, and broadcasts the
/// result as a single line. A serialisation failure is dropped rather than
/// allowed to end the capture, since one bad line is not worth a dark stream.
fn broadcast_flow(
store: &mut IntelStore,
tx: &broadcast::Sender<Arc<str>>,
event: FingerprintEvent,
) {
let intel = store.match_event(&event).unwrap_or_default();
let alerts = store.detect(&event).unwrap_or_default();
let message = LiveMessage::Flow {
event,
intel,
alerts,
};
if let Ok(line) = serde_json::to_string(&message) {
let _ = tx.send(Arc::from(line));
}
}
/// Resolves when the process is asked to stop, the trigger for axum's graceful
/// shutdown.
async fn shutdown_signal() {
if let Err(error) = tokio::signal::ctrl_c().await {
tracing::error!(%error, "failed to listen for ctrl-c; serving until killed");
std::future::pending::<()>().await;
}
tracing::info!("shutting down the dashboard");
}
/// The error type every handler returns. Failures become a 500 with a JSON
/// body; a bad request is the one client-caused case that is reported as a 400.
struct ApiError {
status: StatusCode,
message: String,
}
impl ApiError {
fn bad_request(message: impl Into<String>) -> Self {
Self {
status: StatusCode::BAD_REQUEST,
message: message.into(),
}
}
}
impl IntoResponse for ApiError {
fn into_response(self) -> Response {
if self.status.is_server_error() {
tracing::error!(error = %self.message, "request failed");
}
(
self.status,
Json(serde_json::json!({ "error": self.message })),
)
.into_response()
}
}
impl From<anyhow::Error> for ApiError {
fn from(error: anyhow::Error) -> Self {
Self {
status: StatusCode::INTERNAL_SERVER_ERROR,
message: error.to_string(),
}
}
}
impl From<tokio::task::JoinError> for ApiError {
fn from(error: tokio::task::JoinError) -> Self {
Self {
status: StatusCode::INTERNAL_SERVER_ERROR,
message: format!("request task failed: {error}"),
}
}
}

118
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/crates/tlsfp/tests/report.rs Normal file
View File

@ -0,0 +1,118 @@
// ©AngelaMos | 2026
// report.rs
//! End to end coverage of the forensic report mode.
//!
//! These drive the real binary against a vendored capture the way an analyst
//! would, then assert on what it prints. The capture is read with a database
//! path that does not exist, so the report runs as a pure fingerprint inventory
//! with no intelligence side effects and no dependence on a seeded store, which
//! keeps the test hermetic.
use std::path::PathBuf;
use std::process::Command;
/// The vendored capture used throughout: a browser session with TLS, QUIC, and
/// certificate handshakes, enough to exercise every section of the report.
fn capture() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../testdata/pcap/tls-handshake.pcapng")
}
/// A database path under the temp directory that is guaranteed not to exist, so
/// the report stays a pure inventory and never writes a store behind the test.
fn absent_db() -> PathBuf {
std::env::temp_dir().join(format!(
"tlsfp-report-test-{}-absent.db",
std::process::id()
))
}
fn run(args: &[&str]) -> std::process::Output {
Command::new(env!("CARGO_BIN_EXE_tlsfp"))
.args(args)
.output()
.expect("running the tlsfp binary")
}
#[test]
fn text_report_has_every_section() {
let db = absent_db();
let output = run(&[
"pcap",
capture().to_str().unwrap(),
"--report",
"--db",
db.to_str().unwrap(),
]);
assert!(output.status.success(), "report run failed");
let text = String::from_utf8(output.stdout).expect("utf8 report");
for section in [
"== capture ==",
"== fingerprints by kind ==",
"== top client ja4 ==",
"== endpoints ==",
"== intelligence ==",
"== alerts ==",
"== coverage ==",
] {
assert!(text.contains(section), "missing section {section}");
}
assert!(text.contains("t13d1516h2_8daaf6152771_e5627efa2ab1"));
assert!(text.contains("q13d0310h3_55b375c5d22e_cd85d2d88918"));
assert!(text.contains("tls miss rate"));
assert!(text.contains("fp/s"));
assert!(!db.exists(), "report must not create a database");
}
#[test]
fn json_report_is_well_formed() {
let db = absent_db();
let output = run(&[
"pcap",
capture().to_str().unwrap(),
"--report",
"--json",
"--db",
db.to_str().unwrap(),
]);
assert!(output.status.success(), "json report run failed");
let value: serde_json::Value =
serde_json::from_slice(&output.stdout).expect("report is valid JSON");
assert!(value["capture"]["frames"].as_u64().unwrap() > 0);
assert!(
value["coverage"]["counters"]["tls_handshakes_fingerprinted"]
.as_u64()
.unwrap()
> 0
);
assert!(value["coverage"]["events_per_sec"].as_f64().unwrap() > 0.0);
assert!(
!value["distribution"]["top_ja4"]
.as_array()
.unwrap()
.is_empty()
);
let endpoints = value["endpoints"].as_array().unwrap();
assert!(endpoints.iter().any(|e| e["ip"] == "192.168.1.168"));
}
#[test]
fn top_flag_caps_ranked_rows() {
let db = absent_db();
let output = run(&[
"pcap",
capture().to_str().unwrap(),
"--report",
"--json",
"--top",
"2",
"--db",
db.to_str().unwrap(),
]);
assert!(output.status.success(), "capped report run failed");
let value: serde_json::Value =
serde_json::from_slice(&output.stdout).expect("report is valid JSON");
assert!(value["distribution"]["top_sni"].as_array().unwrap().len() <= 2);
}

30
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/deny.toml Normal file
View File

@ -0,0 +1,30 @@
# ©AngelaMos | 2026
# deny.toml
[graph]
targets = ["x86_64-unknown-linux-gnu"]
all-features = true
[advisories]
version = 2
yanked = "deny"
[licenses]
version = 2
confidence-threshold = 0.93
allow = [
"MIT",
"Apache-2.0",
"BSD-2-Clause",
"BSD-3-Clause",
"ISC",
"Zlib",
"Unicode-3.0",
"Unicode-DFS-2016",
"MPL-2.0",
"CDLA-Permissive-2.0",
]
[bans]
multiple-versions = "warn"
wildcards = "deny"

80
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/dev.compose.yml Normal file
View File

@ -0,0 +1,80 @@
# =============================================================================
# ©AngelaMos | 2026
# dev.compose.yml
# =============================================================================
# Development compose: nginx fronts the Vite dev server (HMR) and proxies /api
# to the tlsfp backend. The backend lives under the `backend` profile and is
# not started by default until the serve command is implemented.
# docker compose -f dev.compose.yml up
# docker compose -f dev.compose.yml --profile backend up
# Uses .env.development
# =============================================================================
name: ${APP_NAME:-tlsfp}-dev
services:
nginx:
image: nginx:1.27-alpine
container_name: ${APP_NAME:-tlsfp}-nginx-dev
ports:
- "${NGINX_HOST_PORT:-48418}:80"
volumes:
- ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
- ./infra/nginx/dev.nginx:/etc/nginx/conf.d/default.conf:ro
depends_on:
frontend:
condition: service_started
networks:
- app
restart: unless-stopped
frontend:
build:
context: ./frontend
dockerfile: ../infra/docker/vite.dev
container_name: ${APP_NAME:-tlsfp}-frontend-dev
command: sh -c "pnpm install && exec pnpm dev --host 0.0.0.0"
ports:
- "${FRONTEND_HOST_PORT:-46494}:5173"
volumes:
- ./frontend:/app
- frontend_modules:/app/node_modules
environment:
- CI=true
- VITE_API_URL=${VITE_API_URL:-/api}
- VITE_API_TARGET=${VITE_API_TARGET:-http://tlsfp:8080}
- VITE_APP_TITLE=${VITE_APP_TITLE:-JA3/JA4 TLS Fingerprinting (Dev)}
networks:
- app
restart: unless-stopped
tlsfp:
build:
context: .
dockerfile: infra/docker/tlsfp.dev
container_name: ${APP_NAME:-tlsfp}-backend-dev
entrypoint: ["/bin/sh", "-c"]
command:
- |
cargo run --bin tlsfp -- intel seed --db /data/intel.db || true
exec cargo watch -x 'run --bin tlsfp -- serve 0.0.0.0:8080 --db /data/intel.db --replay /workspace/testdata/pcap/tls-alpn-h2.pcap --loop'
profiles:
- backend
environment:
- RUST_LOG=${RUST_LOG:-tlsfp=debug}
volumes:
- .:/workspace
- tlsfp_target:/workspace/target
- tlsfp_data:/data
networks:
- app
restart: unless-stopped
networks:
app:
driver: bridge
volumes:
frontend_modules:
tlsfp_target:
tlsfp_data:

15
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/frontend/.dockerignore Normal file
View File

@ -0,0 +1,15 @@
node_modules
build
dist
.git
.gitignore
*.md
.env*
.vscode
.idea
*.log
npm-debug.log*
pnpm-debug.log*
.DS_Store
coverage
.nyc_output

26
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/frontend/.gitignore vendored Normal file
View File

@ -0,0 +1,26 @@
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
pnpm-debug.log*
lerna-debug.log*
node_modules
.pnpm-store
dist
dist-ssr
*.local
.vite
# Editor directories and files
.vscode/*
!.vscode/extensions.json
.idea
.DS_Store
*.suo
*.ntvs*
*.njsproj
*.sln
*.sw?

4
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/frontend/.npmrc Normal file
View File

@ -0,0 +1,4 @@
# ©AngelaMos | 2026
# .npmrc
strict-dep-builds=false
auto-install-peers=true

22
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/frontend/.stylelintignore Normal file
View File

@ -0,0 +1,22 @@
# ©AngelaMos | 2025
# .stylelintignore
# Dependencies
node_modules/
# Production builds
dist/
build/
out/
# JS/TS files
**/*.js
**/*.jsx
**/*.ts
**/*.tsx
# Generated files
*.min.css
# Error system styles - ignore from linting
src/core/app/_toastStyles.scss

103
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/frontend/biome.json Normal file
View File

@ -0,0 +1,103 @@
{
"$schema": "https://biomejs.dev/schemas/2.3.8/schema.json",
"vcs": {
"enabled": true,
"clientKind": "git",
"useIgnoreFile": true
},
"files": {
"includes": [
"**/*.ts",
"**/*.tsx",
"**/*.js",
"**/*.jsx",
"**/*.json",
"!**/node_modules",
"!**/.pnpm-store",
"!**/dist"
]
},
"formatter": {
"enabled": true,
"indentStyle": "space",
"indentWidth": 2,
"lineWidth": 82,
"lineEnding": "lf"
},
"javascript": {
"formatter": {
"quoteStyle": "single",
"jsxQuoteStyle": "double",
"semicolons": "asNeeded",
"trailingCommas": "es5",
"arrowParentheses": "always"
}
},
"linter": {
"enabled": true,
"rules": {
"recommended": true,
"complexity": {
"noExcessiveCognitiveComplexity": {
"level": "error",
"options": { "maxAllowedComplexity": 25 }
},
"noForEach": "off",
"useLiteralKeys": "off"
},
"correctness": {
"noUnusedVariables": "error",
"noUnusedImports": "error",
"useExhaustiveDependencies": "warn",
"useHookAtTopLevel": "error",
"noUndeclaredVariables": "error"
},
"style": {
"useImportType": "error",
"useConst": "error",
"useTemplate": "error",
"useSelfClosingElements": "error",
"useFragmentSyntax": "error",
"noNonNullAssertion": "error",
"useConsistentArrayType": {
"level": "error",
"options": { "syntax": "shorthand" }
},
"useNamingConvention": "off"
},
"suspicious": {
"noExplicitAny": "error",
"noDebugger": "error",
"noConsole": "warn",
"noArrayIndexKey": "warn",
"noAssignInExpressions": "error",
"noDoubleEquals": "error",
"noRedeclare": "error",
"noVar": "error"
},
"security": {
"noDangerouslySetInnerHtml": "error"
},
"a11y": {
"useAltText": "error",
"useAnchorContent": "error",
"useKeyWithClickEvents": "error",
"noStaticElementInteractions": "error",
"useButtonType": "error",
"useValidAnchor": "error"
}
}
},
"overrides": [
{
"includes": ["src/main.tsx"],
"linter": {
"rules": {
"style": {
"noNonNullAssertion": "off"
}
}
}
}
]
}

48
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/frontend/index.html Normal file
View File

@ -0,0 +1,48 @@
<!doctype html>
<html lang="en">
<head>
<!--
JA3/JA4 TLS Fingerprinting
Author(s): © AngelaMos, CarterPerez-dev
-->
<meta charset="UTF-8" />
<link
rel="icon"
type="image/x-icon"
href="/assets/favicon.ico"
/>
<link
rel="apple-touch-icon"
type="image/png"
href="/assets/apple-touch-icon.png"
/>
<link
rel="manifest"
href="/assets/site.webmanifest"
/>
<meta
name="viewport"
content="width=device-width, initial-scale=1.0"
/>
<meta
http-equiv="Content-Security-Policy"
content="default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval'; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob:; font-src 'self'; connect-src 'self'; form-action 'self'; worker-src 'self' blob:;"
/>
<title>MKUtra Valedictorian</title>
<meta
name="description"
content="*Cracked*"
/>
<meta
name="author"
content=" ©AngelaMos | 2026 | CarterPerez-dev"
/>
</head>
<body>
<div id="root"></div>
<script
type="module"
src="/src/main.tsx"
></script>
</body>
</html>

48
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/frontend/package.json Normal file
View File

@ -0,0 +1,48 @@
{
"name": "tlsfp",
"private": true,
"version": "1.0.0",
"type": "module",
"scripts": {
"dev": "vite",
"build": "tsc -b && vite build",
"preview": "vite preview",
"lint": "biome check .",
"lint:fix": "biome check --write .",
"format": "biome format --write .",
"typecheck": "tsc -b",
"lint:scss": "stylelint '**/*.scss'",
"lint:scss:fix": "stylelint '**/*.scss' --fix"
},
"dependencies": {
"@tanstack/react-query": "^5.90.12",
"axios": "^1.13.0",
"react": "^19.2.1",
"react-dom": "^19.2.0",
"react-error-boundary": "^6.0.0",
"react-router-dom": "^7.1.1",
"sonner": "^2.0.7",
"zod": "^4.3.6",
"zustand": "^5.0.9"
},
"devDependencies": {
"@biomejs/biome": "^2.3.8",
"@tanstack/react-query-devtools": "^5.91.1",
"@types/node": "^24.10.2",
"@types/react": "^19.2.7",
"@types/react-dom": "^19.2.3",
"@vitejs/plugin-react": "^5.1.1",
"sass": "^1.95.0",
"stylelint": "^16.26.1",
"stylelint-config-prettier-scss": "^1.0.0",
"stylelint-config-standard-scss": "^16.0.0",
"typescript": "~5.9.3",
"vite": "npm:rolldown-vite@7.2.5",
"vite-tsconfig-paths": "^5.1.0"
},
"pnpm": {
"overrides": {
"vite": "npm:rolldown-vite@7.2.5"
}
}
}

2582
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/frontend/pnpm-lock.yaml Normal file
View File

File diff suppressed because it is too large Load Diff

BIN
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/frontend/public/assets/android-chrome-192x192.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 56 KiB

BIN
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/frontend/public/assets/android-chrome-512x512.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 234 KiB

BIN
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/frontend/public/assets/apple-touch-icon.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 51 KiB

BIN
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/frontend/public/assets/favicon-16x16.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 901 B

BIN
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/frontend/public/assets/favicon-32x32.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.8 KiB

BIN
PROJECTS/intermediate/ja3-ja4-tls-fingerprinting/frontend/public/assets/favicon.ico Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 15 KiB

Some files were not shown because too many files have changed in this diff Show More